blob: e45042324e7328335c0b8c3a3fe70f7667036081 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001"""BitBake Persistent Data Store
2
3Used to store data in a central location such that other threads/tasks can
4access them at some future date. Acts as a convenience wrapper around sqlite,
5currently, providing a key/value store accessed by 'domain'.
6"""
7
8# Copyright (C) 2007 Richard Purdie
9# Copyright (C) 2010 Chris Larson <chris_larson@mentor.com>
10#
11# This program is free software; you can redistribute it and/or modify
12# it under the terms of the GNU General Public License version 2 as
13# published by the Free Software Foundation.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License along
21# with this program; if not, write to the Free Software Foundation, Inc.,
22# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
23
24import collections
25import logging
26import os.path
27import sys
28import warnings
29from bb.compat import total_ordering
30from collections import Mapping
31
32try:
33 import sqlite3
34except ImportError:
35 from pysqlite2 import dbapi2 as sqlite3
36
37sqlversion = sqlite3.sqlite_version_info
38if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
39 raise Exception("sqlite3 version 3.3.0 or later is required.")
40
41
42logger = logging.getLogger("BitBake.PersistData")
43if hasattr(sqlite3, 'enable_shared_cache'):
44 try:
45 sqlite3.enable_shared_cache(True)
46 except sqlite3.OperationalError:
47 pass
48
49
50@total_ordering
51class SQLTable(collections.MutableMapping):
52 """Object representing a table/domain in the database"""
53 def __init__(self, cachefile, table):
54 self.cachefile = cachefile
55 self.table = table
56 self.cursor = connect(self.cachefile)
57
58 self._execute("CREATE TABLE IF NOT EXISTS %s(key TEXT, value TEXT);"
59 % table)
60
61 def _execute(self, *query):
62 """Execute a query, waiting to acquire a lock if necessary"""
63 count = 0
64 while True:
65 try:
66 return self.cursor.execute(*query)
67 except sqlite3.OperationalError as exc:
68 if 'database is locked' in str(exc) and count < 500:
69 count = count + 1
70 self.cursor.close()
71 self.cursor = connect(self.cachefile)
72 continue
73 raise
74
75 def __enter__(self):
76 self.cursor.__enter__()
77 return self
78
79 def __exit__(self, *excinfo):
80 self.cursor.__exit__(*excinfo)
81
82 def __getitem__(self, key):
83 data = self._execute("SELECT * from %s where key=?;" %
84 self.table, [key])
85 for row in data:
86 return row[1]
87 raise KeyError(key)
88
89 def __delitem__(self, key):
90 if key not in self:
91 raise KeyError(key)
92 self._execute("DELETE from %s where key=?;" % self.table, [key])
93
94 def __setitem__(self, key, value):
95 if not isinstance(key, basestring):
96 raise TypeError('Only string keys are supported')
97 elif not isinstance(value, basestring):
98 raise TypeError('Only string values are supported')
99
100 data = self._execute("SELECT * from %s where key=?;" %
101 self.table, [key])
102 exists = len(list(data))
103 if exists:
104 self._execute("UPDATE %s SET value=? WHERE key=?;" % self.table,
105 [value, key])
106 else:
107 self._execute("INSERT into %s(key, value) values (?, ?);" %
108 self.table, [key, value])
109
110 def __contains__(self, key):
111 return key in set(self)
112
113 def __len__(self):
114 data = self._execute("SELECT COUNT(key) FROM %s;" % self.table)
115 for row in data:
116 return row[0]
117
118 def __iter__(self):
119 data = self._execute("SELECT key FROM %s;" % self.table)
120 return (row[0] for row in data)
121
122 def __lt__(self, other):
123 if not isinstance(other, Mapping):
124 raise NotImplemented
125
126 return len(self) < len(other)
127
128 def get_by_pattern(self, pattern):
129 data = self._execute("SELECT * FROM %s WHERE key LIKE ?;" %
130 self.table, [pattern])
131 return [row[1] for row in data]
132
133 def values(self):
134 return list(self.itervalues())
135
136 def itervalues(self):
137 data = self._execute("SELECT value FROM %s;" % self.table)
138 return (row[0] for row in data)
139
140 def items(self):
141 return list(self.iteritems())
142
143 def iteritems(self):
144 return self._execute("SELECT * FROM %s;" % self.table)
145
146 def clear(self):
147 self._execute("DELETE FROM %s;" % self.table)
148
149 def has_key(self, key):
150 return key in self
151
152
153class PersistData(object):
154 """Deprecated representation of the bitbake persistent data store"""
155 def __init__(self, d):
156 warnings.warn("Use of PersistData is deprecated. Please use "
157 "persist(domain, d) instead.",
158 category=DeprecationWarning,
159 stacklevel=2)
160
161 self.data = persist(d)
162 logger.debug(1, "Using '%s' as the persistent data cache",
163 self.data.filename)
164
165 def addDomain(self, domain):
166 """
167 Add a domain (pending deprecation)
168 """
169 return self.data[domain]
170
171 def delDomain(self, domain):
172 """
173 Removes a domain and all the data it contains
174 """
175 del self.data[domain]
176
177 def getKeyValues(self, domain):
178 """
179 Return a list of key + value pairs for a domain
180 """
181 return self.data[domain].items()
182
183 def getValue(self, domain, key):
184 """
185 Return the value of a key for a domain
186 """
187 return self.data[domain][key]
188
189 def setValue(self, domain, key, value):
190 """
191 Sets the value of a key for a domain
192 """
193 self.data[domain][key] = value
194
195 def delValue(self, domain, key):
196 """
197 Deletes a key/value pair
198 """
199 del self.data[domain][key]
200
201def connect(database):
202 connection = sqlite3.connect(database, timeout=5, isolation_level=None)
203 connection.execute("pragma synchronous = off;")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500204 connection.text_factory = str
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500205 return connection
206
207def persist(domain, d):
208 """Convenience factory for SQLTable objects based upon metadata"""
209 import bb.utils
210 cachedir = (d.getVar("PERSISTENT_DIR", True) or
211 d.getVar("CACHE", True))
212 if not cachedir:
213 logger.critical("Please set the 'PERSISTENT_DIR' or 'CACHE' variable")
214 sys.exit(1)
215
216 bb.utils.mkdirhier(cachedir)
217 cachefile = os.path.join(cachedir, "bb_persist_data.sqlite3")
218 return SQLTable(cachefile, domain)