reset upstream subtrees to HEAD
Reset the following subtrees on HEAD:
poky: 8217b477a1(master)
meta-xilinx: 64aa3d35ae(master)
meta-openembedded: 0435c9e193(master)
meta-raspberrypi: 490a4441ac(master)
meta-security: cb6d1c85ee(master)
Squashed patches:
meta-phosphor: drop systemd 239 patches
meta-phosphor: mrw-api: use correct install path
Change-Id: I268e2646d9174ad305630c6bbd3fbc1a6105f43d
Signed-off-by: Brad Bishop <bradleyb@fuzziesquirrel.com>
diff --git a/poky/bitbake/lib/bb/persist_data.py b/poky/bitbake/lib/bb/persist_data.py
index bef7018..0d44100 100644
--- a/poky/bitbake/lib/bb/persist_data.py
+++ b/poky/bitbake/lib/bb/persist_data.py
@@ -29,6 +29,7 @@
from bb.compat import total_ordering
from collections import Mapping
import sqlite3
+import contextlib
sqlversion = sqlite3.sqlite_version_info
if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
@@ -36,84 +37,181 @@
logger = logging.getLogger("BitBake.PersistData")
-if hasattr(sqlite3, 'enable_shared_cache'):
- try:
- sqlite3.enable_shared_cache(True)
- except sqlite3.OperationalError:
- pass
-
@total_ordering
class SQLTable(collections.MutableMapping):
+ class _Decorators(object):
+ @staticmethod
+ def retry(*, reconnect=True):
+ """
+ Decorator that restarts a function if a database locked sqlite
+ exception occurs. If reconnect is True, the database connection
+ will be closed and reopened each time a failure occurs
+ """
+ def retry_wrapper(f):
+ def wrap_func(self, *args, **kwargs):
+ # Reconnect if necessary
+ if self.connection is None and reconnect:
+ self.reconnect()
+
+ count = 0
+ while True:
+ try:
+ return f(self, *args, **kwargs)
+ except sqlite3.OperationalError as exc:
+ if count < 500 and ('is locked' in str(exc) or 'locking protocol' in str(exc)):
+ count = count + 1
+ if reconnect:
+ self.reconnect()
+ continue
+ raise
+ return wrap_func
+ return retry_wrapper
+
+ @staticmethod
+ def transaction(f):
+ """
+ Decorator that starts a database transaction and creates a database
+ cursor for performing queries. If no exception is thrown, the
+ database results are commited. If an exception occurs, the database
+ is rolled back. In all cases, the cursor is closed after the
+ function ends.
+
+ Note that the cursor is passed as an extra argument to the function
+ after `self` and before any of the normal arguments
+ """
+ def wrap_func(self, *args, **kwargs):
+ # Context manager will COMMIT the database on success,
+ # or ROLLBACK on an exception
+ with self.connection:
+ # Automatically close the cursor when done
+ with contextlib.closing(self.connection.cursor()) as cursor:
+ return f(self, cursor, *args, **kwargs)
+ return wrap_func
+
"""Object representing a table/domain in the database"""
def __init__(self, cachefile, table):
self.cachefile = cachefile
self.table = table
- self.cursor = connect(self.cachefile)
- self._execute("CREATE TABLE IF NOT EXISTS %s(key TEXT, value TEXT);"
- % table)
+ self.connection = None
+ self._execute_single("CREATE TABLE IF NOT EXISTS %s(key TEXT PRIMARY KEY NOT NULL, value TEXT);" % table)
- def _execute(self, *query):
- """Execute a query, waiting to acquire a lock if necessary"""
- count = 0
- while True:
- try:
- return self.cursor.execute(*query)
- except sqlite3.OperationalError as exc:
- if 'database is locked' in str(exc) and count < 500:
- count = count + 1
+ @_Decorators.retry(reconnect=False)
+ @_Decorators.transaction
+ def _setup_database(self, cursor):
+ cursor.execute("pragma synchronous = off;")
+ # Enable WAL and keep the autocheckpoint length small (the default is
+ # usually 1000). Persistent caches are usually read-mostly, so keeping
+ # this short will keep readers running quickly
+ cursor.execute("pragma journal_mode = WAL;")
+ cursor.execute("pragma wal_autocheckpoint = 100;")
+
+ def reconnect(self):
+ if self.connection is not None:
+ self.connection.close()
+ self.connection = sqlite3.connect(self.cachefile, timeout=5)
+ self.connection.text_factory = str
+ self._setup_database()
+
+ @_Decorators.retry()
+ @_Decorators.transaction
+ def _execute_single(self, cursor, *query):
+ """
+ Executes a single query and discards the results. This correctly closes
+ the database cursor when finished
+ """
+ cursor.execute(*query)
+
+ @_Decorators.retry()
+ def _row_iter(self, f, *query):
+ """
+ Helper function that returns a row iterator. Each time __next__ is
+ called on the iterator, the provided function is evaluated to determine
+ the return value
+ """
+ class CursorIter(object):
+ def __init__(self, cursor):
+ self.cursor = cursor
+
+ def __iter__(self):
+ return self
+
+ def __next__(self):
+ row = self.cursor.fetchone()
+ if row is None:
self.cursor.close()
- self.cursor = connect(self.cachefile)
- continue
- raise
+ raise StopIteration
+ return f(row)
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, typ, value, traceback):
+ self.cursor.close()
+ return False
+
+ cursor = self.connection.cursor()
+ try:
+ cursor.execute(*query)
+ return CursorIter(cursor)
+ except:
+ cursor.close()
def __enter__(self):
- self.cursor.__enter__()
+ self.connection.__enter__()
return self
def __exit__(self, *excinfo):
- self.cursor.__exit__(*excinfo)
+ self.connection.__exit__(*excinfo)
- def __getitem__(self, key):
- data = self._execute("SELECT * from %s where key=?;" %
- self.table, [key])
- for row in data:
+ @_Decorators.retry()
+ @_Decorators.transaction
+ def __getitem__(self, cursor, key):
+ cursor.execute("SELECT * from %s where key=?;" % self.table, [key])
+ row = cursor.fetchone()
+ if row is not None:
return row[1]
raise KeyError(key)
- def __delitem__(self, key):
+ @_Decorators.retry()
+ @_Decorators.transaction
+ def __delitem__(self, cursor, key):
if key not in self:
raise KeyError(key)
- self._execute("DELETE from %s where key=?;" % self.table, [key])
+ cursor.execute("DELETE from %s where key=?;" % self.table, [key])
- def __setitem__(self, key, value):
+ @_Decorators.retry()
+ @_Decorators.transaction
+ def __setitem__(self, cursor, key, value):
if not isinstance(key, str):
raise TypeError('Only string keys are supported')
elif not isinstance(value, str):
raise TypeError('Only string values are supported')
- data = self._execute("SELECT * from %s where key=?;" %
- self.table, [key])
- exists = len(list(data))
- if exists:
- self._execute("UPDATE %s SET value=? WHERE key=?;" % self.table,
- [value, key])
+ cursor.execute("SELECT * from %s where key=?;" % self.table, [key])
+ row = cursor.fetchone()
+ if row is not None:
+ cursor.execute("UPDATE %s SET value=? WHERE key=?;" % self.table, [value, key])
else:
- self._execute("INSERT into %s(key, value) values (?, ?);" %
- self.table, [key, value])
+ cursor.execute("INSERT into %s(key, value) values (?, ?);" % self.table, [key, value])
- def __contains__(self, key):
- return key in set(self)
+ @_Decorators.retry()
+ @_Decorators.transaction
+ def __contains__(self, cursor, key):
+ cursor.execute('SELECT * from %s where key=?;' % self.table, [key])
+ return cursor.fetchone() is not None
- def __len__(self):
- data = self._execute("SELECT COUNT(key) FROM %s;" % self.table)
- for row in data:
+ @_Decorators.retry()
+ @_Decorators.transaction
+ def __len__(self, cursor):
+ cursor.execute("SELECT COUNT(key) FROM %s;" % self.table)
+ row = cursor.fetchone()
+ if row is not None:
return row[0]
def __iter__(self):
- data = self._execute("SELECT key FROM %s;" % self.table)
- return (row[0] for row in data)
+ return self._row_iter(lambda row: row[0], "SELECT key from %s;" % self.table)
def __lt__(self, other):
if not isinstance(other, Mapping):
@@ -122,25 +220,27 @@
return len(self) < len(other)
def get_by_pattern(self, pattern):
- data = self._execute("SELECT * FROM %s WHERE key LIKE ?;" %
- self.table, [pattern])
- return [row[1] for row in data]
+ return self._row_iter(lambda row: row[1], "SELECT * FROM %s WHERE key LIKE ?;" %
+ self.table, [pattern])
def values(self):
return list(self.itervalues())
def itervalues(self):
- data = self._execute("SELECT value FROM %s;" % self.table)
- return (row[0] for row in data)
+ return self._row_iter(lambda row: row[0], "SELECT value FROM %s;" %
+ self.table)
def items(self):
return list(self.iteritems())
def iteritems(self):
- return self._execute("SELECT * FROM %s;" % self.table)
+ return self._row_iter(lambda row: (row[0], row[1]), "SELECT * FROM %s;" %
+ self.table)
- def clear(self):
- self._execute("DELETE FROM %s;" % self.table)
+ @_Decorators.retry()
+ @_Decorators.transaction
+ def clear(self, cursor):
+ cursor.execute("DELETE FROM %s;" % self.table)
def has_key(self, key):
return key in self
@@ -194,12 +294,6 @@
"""
del self.data[domain][key]
-def connect(database):
- connection = sqlite3.connect(database, timeout=5, isolation_level=None)
- connection.execute("pragma synchronous = off;")
- connection.text_factory = str
- return connection
-
def persist(domain, d):
"""Convenience factory for SQLTable objects based upon metadata"""
import bb.utils