blob: bcca791edf327ad02bbfdee833cf979c1d6f44c1 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001"""BitBake Persistent Data Store
2
3Used to store data in a central location such that other threads/tasks can
4access them at some future date. Acts as a convenience wrapper around sqlite,
5currently, providing a key/value store accessed by 'domain'.
6"""
7
8# Copyright (C) 2007 Richard Purdie
9# Copyright (C) 2010 Chris Larson <chris_larson@mentor.com>
10#
Brad Bishopc342db32019-05-15 21:57:59 -040011# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -050012#
Patrick Williamsc124f4f2015-09-15 14:41:29 -050013
14import collections
Andrew Geissler5199d832021-09-24 16:47:35 -050015import collections.abc
Andrew Geisslerc9f78652020-09-18 14:11:35 -050016import contextlib
17import functools
Patrick Williamsc124f4f2015-09-15 14:41:29 -050018import logging
19import os.path
Andrew Geisslerc9f78652020-09-18 14:11:35 -050020import sqlite3
Patrick Williamsc124f4f2015-09-15 14:41:29 -050021import sys
Andrew Geissler5199d832021-09-24 16:47:35 -050022from collections.abc import Mapping
Patrick Williamsc124f4f2015-09-15 14:41:29 -050023
24sqlversion = sqlite3.sqlite_version_info
25if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
26 raise Exception("sqlite3 version 3.3.0 or later is required.")
27
28
29logger = logging.getLogger("BitBake.PersistData")
Patrick Williamsc124f4f2015-09-15 14:41:29 -050030
Andrew Geisslerc9f78652020-09-18 14:11:35 -050031@functools.total_ordering
Andrew Geissler5199d832021-09-24 16:47:35 -050032class SQLTable(collections.abc.MutableMapping):
Brad Bishop19323692019-04-05 15:28:33 -040033 class _Decorators(object):
34 @staticmethod
35 def retry(*, reconnect=True):
36 """
37 Decorator that restarts a function if a database locked sqlite
38 exception occurs. If reconnect is True, the database connection
39 will be closed and reopened each time a failure occurs
40 """
41 def retry_wrapper(f):
42 def wrap_func(self, *args, **kwargs):
43 # Reconnect if necessary
44 if self.connection is None and reconnect:
45 self.reconnect()
46
47 count = 0
48 while True:
49 try:
50 return f(self, *args, **kwargs)
51 except sqlite3.OperationalError as exc:
52 if count < 500 and ('is locked' in str(exc) or 'locking protocol' in str(exc)):
53 count = count + 1
54 if reconnect:
55 self.reconnect()
56 continue
57 raise
58 return wrap_func
59 return retry_wrapper
60
61 @staticmethod
62 def transaction(f):
63 """
64 Decorator that starts a database transaction and creates a database
65 cursor for performing queries. If no exception is thrown, the
Andrew Geissler7e0e3c02022-02-25 20:34:39 +000066 database results are committed. If an exception occurs, the database
Brad Bishop19323692019-04-05 15:28:33 -040067 is rolled back. In all cases, the cursor is closed after the
68 function ends.
69
70 Note that the cursor is passed as an extra argument to the function
71 after `self` and before any of the normal arguments
72 """
73 def wrap_func(self, *args, **kwargs):
74 # Context manager will COMMIT the database on success,
75 # or ROLLBACK on an exception
76 with self.connection:
77 # Automatically close the cursor when done
78 with contextlib.closing(self.connection.cursor()) as cursor:
79 return f(self, cursor, *args, **kwargs)
80 return wrap_func
81
Patrick Williamsc124f4f2015-09-15 14:41:29 -050082 """Object representing a table/domain in the database"""
83 def __init__(self, cachefile, table):
84 self.cachefile = cachefile
85 self.table = table
Patrick Williamsc124f4f2015-09-15 14:41:29 -050086
Brad Bishop19323692019-04-05 15:28:33 -040087 self.connection = None
88 self._execute_single("CREATE TABLE IF NOT EXISTS %s(key TEXT PRIMARY KEY NOT NULL, value TEXT);" % table)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050089
Brad Bishop19323692019-04-05 15:28:33 -040090 @_Decorators.retry(reconnect=False)
91 @_Decorators.transaction
92 def _setup_database(self, cursor):
93 cursor.execute("pragma synchronous = off;")
94 # Enable WAL and keep the autocheckpoint length small (the default is
95 # usually 1000). Persistent caches are usually read-mostly, so keeping
96 # this short will keep readers running quickly
97 cursor.execute("pragma journal_mode = WAL;")
98 cursor.execute("pragma wal_autocheckpoint = 100;")
99
100 def reconnect(self):
101 if self.connection is not None:
102 self.connection.close()
103 self.connection = sqlite3.connect(self.cachefile, timeout=5)
104 self.connection.text_factory = str
105 self._setup_database()
106
107 @_Decorators.retry()
108 @_Decorators.transaction
109 def _execute_single(self, cursor, *query):
110 """
111 Executes a single query and discards the results. This correctly closes
112 the database cursor when finished
113 """
114 cursor.execute(*query)
115
116 @_Decorators.retry()
117 def _row_iter(self, f, *query):
118 """
119 Helper function that returns a row iterator. Each time __next__ is
120 called on the iterator, the provided function is evaluated to determine
121 the return value
122 """
123 class CursorIter(object):
124 def __init__(self, cursor):
125 self.cursor = cursor
126
127 def __iter__(self):
128 return self
129
130 def __next__(self):
131 row = self.cursor.fetchone()
132 if row is None:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500133 self.cursor.close()
Brad Bishop19323692019-04-05 15:28:33 -0400134 raise StopIteration
135 return f(row)
136
137 def __enter__(self):
138 return self
139
140 def __exit__(self, typ, value, traceback):
141 self.cursor.close()
142 return False
143
144 cursor = self.connection.cursor()
145 try:
146 cursor.execute(*query)
147 return CursorIter(cursor)
148 except:
149 cursor.close()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500150
151 def __enter__(self):
Brad Bishop19323692019-04-05 15:28:33 -0400152 self.connection.__enter__()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500153 return self
154
155 def __exit__(self, *excinfo):
Brad Bishop19323692019-04-05 15:28:33 -0400156 self.connection.__exit__(*excinfo)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500157
Brad Bishop19323692019-04-05 15:28:33 -0400158 @_Decorators.retry()
159 @_Decorators.transaction
160 def __getitem__(self, cursor, key):
161 cursor.execute("SELECT * from %s where key=?;" % self.table, [key])
162 row = cursor.fetchone()
163 if row is not None:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500164 return row[1]
165 raise KeyError(key)
166
Brad Bishop19323692019-04-05 15:28:33 -0400167 @_Decorators.retry()
168 @_Decorators.transaction
169 def __delitem__(self, cursor, key):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500170 if key not in self:
171 raise KeyError(key)
Brad Bishop19323692019-04-05 15:28:33 -0400172 cursor.execute("DELETE from %s where key=?;" % self.table, [key])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500173
Brad Bishop19323692019-04-05 15:28:33 -0400174 @_Decorators.retry()
175 @_Decorators.transaction
176 def __setitem__(self, cursor, key, value):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600177 if not isinstance(key, str):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500178 raise TypeError('Only string keys are supported')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600179 elif not isinstance(value, str):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500180 raise TypeError('Only string values are supported')
181
Andrew Geissler82c905d2020-04-13 13:39:40 -0500182 # Ensure the entire transaction (including SELECT) executes under write lock
183 cursor.execute("BEGIN EXCLUSIVE")
184
Brad Bishop19323692019-04-05 15:28:33 -0400185 cursor.execute("SELECT * from %s where key=?;" % self.table, [key])
186 row = cursor.fetchone()
187 if row is not None:
188 cursor.execute("UPDATE %s SET value=? WHERE key=?;" % self.table, [value, key])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500189 else:
Brad Bishop19323692019-04-05 15:28:33 -0400190 cursor.execute("INSERT into %s(key, value) values (?, ?);" % self.table, [key, value])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500191
Brad Bishop19323692019-04-05 15:28:33 -0400192 @_Decorators.retry()
193 @_Decorators.transaction
194 def __contains__(self, cursor, key):
195 cursor.execute('SELECT * from %s where key=?;' % self.table, [key])
196 return cursor.fetchone() is not None
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500197
Brad Bishop19323692019-04-05 15:28:33 -0400198 @_Decorators.retry()
199 @_Decorators.transaction
200 def __len__(self, cursor):
201 cursor.execute("SELECT COUNT(key) FROM %s;" % self.table)
202 row = cursor.fetchone()
203 if row is not None:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500204 return row[0]
205
206 def __iter__(self):
Brad Bishop19323692019-04-05 15:28:33 -0400207 return self._row_iter(lambda row: row[0], "SELECT key from %s;" % self.table)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500208
209 def __lt__(self, other):
210 if not isinstance(other, Mapping):
Patrick Williams03907ee2022-05-01 06:28:52 -0500211 raise NotImplementedError()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500212
213 return len(self) < len(other)
214
215 def get_by_pattern(self, pattern):
Brad Bishop19323692019-04-05 15:28:33 -0400216 return self._row_iter(lambda row: row[1], "SELECT * FROM %s WHERE key LIKE ?;" %
217 self.table, [pattern])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500218
219 def values(self):
220 return list(self.itervalues())
221
222 def itervalues(self):
Brad Bishop19323692019-04-05 15:28:33 -0400223 return self._row_iter(lambda row: row[0], "SELECT value FROM %s;" %
224 self.table)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500225
226 def items(self):
227 return list(self.iteritems())
228
229 def iteritems(self):
Brad Bishop19323692019-04-05 15:28:33 -0400230 return self._row_iter(lambda row: (row[0], row[1]), "SELECT * FROM %s;" %
231 self.table)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500232
Brad Bishop19323692019-04-05 15:28:33 -0400233 @_Decorators.retry()
234 @_Decorators.transaction
235 def clear(self, cursor):
236 cursor.execute("DELETE FROM %s;" % self.table)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500237
238 def has_key(self, key):
239 return key in self
240
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500241def persist(domain, d):
242 """Convenience factory for SQLTable objects based upon metadata"""
243 import bb.utils
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500244 cachedir = (d.getVar("PERSISTENT_DIR") or
245 d.getVar("CACHE"))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500246 if not cachedir:
247 logger.critical("Please set the 'PERSISTENT_DIR' or 'CACHE' variable")
248 sys.exit(1)
249
250 bb.utils.mkdirhier(cachedir)
251 cachefile = os.path.join(cachedir, "bb_persist_data.sqlite3")
Andrew Geisslerc5535c92023-01-27 16:10:19 -0600252
253 try:
254 return SQLTable(cachefile, domain)
255 except sqlite3.OperationalError:
256 # Sqlite fails to open database when its path is too long.
257 # After testing, 504 is the biggest path length that can be opened by
258 # sqlite.
259 # Note: This code is called before sanity.bbclass and its path length
260 # check
261 max_len = 504
262 if len(cachefile) > max_len:
263 logger.critical("The path of the cache file is too long "
264 "({0} chars > {1}) to be opened by sqlite! "
265 "Your cache file is \"{2}\"".format(
266 len(cachefile),
267 max_len,
268 cachefile))
269 sys.exit(1)
270 else:
271 raise