blob: 117d8c052c1721be29a8e446fec0308062739d63 [file] [log] [blame]
Brad Bishopc342db32019-05-15 21:57:59 -04001#
2# SPDX-License-Identifier: GPL-2.0-only
3#
4
Patrick Williamsc124f4f2015-09-15 14:41:29 -05005import logging
6import os.path
7import errno
8import prserv
9import time
10
11try:
12 import sqlite3
13except ImportError:
14 from pysqlite2 import dbapi2 as sqlite3
15
16logger = logging.getLogger("BitBake.PRserv")
17
18sqlversion = sqlite3.sqlite_version_info
19if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
20 raise Exception("sqlite3 version 3.3.0 or later is required.")
21
22#
23# "No History" mode - for a given query tuple (version, pkgarch, checksum),
24# the returned value will be the largest among all the values of the same
25# (version, pkgarch). This means the PR value returned can NOT be decremented.
26#
27# "History" mode - Return a new higher value for previously unseen query
28# tuple (version, pkgarch, checksum), otherwise return historical value.
29# Value can decrement if returning to a previous build.
30#
31
32class PRTable(object):
33 def __init__(self, conn, table, nohist):
34 self.conn = conn
35 self.nohist = nohist
36 self.dirty = False
37 if nohist:
38 self.table = "%s_nohist" % table
39 else:
40 self.table = "%s_hist" % table
41
42 self._execute("CREATE TABLE IF NOT EXISTS %s \
43 (version TEXT NOT NULL, \
44 pkgarch TEXT NOT NULL, \
45 checksum TEXT NOT NULL, \
46 value INTEGER, \
47 PRIMARY KEY (version, pkgarch, checksum));" % self.table)
48
49 def _execute(self, *query):
50 """Execute a query, waiting to acquire a lock if necessary"""
51 start = time.time()
52 end = start + 20
53 while True:
54 try:
55 return self.conn.execute(*query)
56 except sqlite3.OperationalError as exc:
57 if 'is locked' in str(exc) and end > time.time():
58 continue
59 raise exc
60
61 def sync(self):
62 self.conn.commit()
63 self._execute("BEGIN EXCLUSIVE TRANSACTION")
64
65 def sync_if_dirty(self):
66 if self.dirty:
67 self.sync()
68 self.dirty = False
69
70 def _getValueHist(self, version, pkgarch, checksum):
71 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
72 (version, pkgarch, checksum))
73 row=data.fetchone()
74 if row != None:
75 return row[0]
76 else:
77 #no value found, try to insert
78 try:
79 self._execute("INSERT INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1,0) from %s where version=? AND pkgarch=?));"
80 % (self.table,self.table),
81 (version,pkgarch, checksum,version, pkgarch))
82 except sqlite3.IntegrityError as exc:
83 logger.error(str(exc))
84
85 self.dirty = True
86
87 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
88 (version, pkgarch, checksum))
89 row=data.fetchone()
90 if row != None:
91 return row[0]
92 else:
93 raise prserv.NotFoundError
94
95 def _getValueNohist(self, version, pkgarch, checksum):
96 data=self._execute("SELECT value FROM %s \
97 WHERE version=? AND pkgarch=? AND checksum=? AND \
98 value >= (select max(value) from %s where version=? AND pkgarch=?);"
99 % (self.table, self.table),
100 (version, pkgarch, checksum, version, pkgarch))
101 row=data.fetchone()
102 if row != None:
103 return row[0]
104 else:
105 #no value found, try to insert
106 try:
107 self._execute("INSERT OR REPLACE INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1,0) from %s where version=? AND pkgarch=?));"
108 % (self.table,self.table),
109 (version, pkgarch, checksum, version, pkgarch))
110 except sqlite3.IntegrityError as exc:
111 logger.error(str(exc))
112 self.conn.rollback()
113
114 self.dirty = True
115
116 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
117 (version, pkgarch, checksum))
118 row=data.fetchone()
119 if row != None:
120 return row[0]
121 else:
122 raise prserv.NotFoundError
123
124 def getValue(self, version, pkgarch, checksum):
125 if self.nohist:
126 return self._getValueNohist(version, pkgarch, checksum)
127 else:
128 return self._getValueHist(version, pkgarch, checksum)
129
130 def _importHist(self, version, pkgarch, checksum, value):
131 val = None
132 data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
133 (version, pkgarch, checksum))
134 row = data.fetchone()
135 if row != None:
136 val=row[0]
137 else:
138 #no value found, try to insert
139 try:
140 self._execute("INSERT INTO %s VALUES (?, ?, ?, ?);" % (self.table),
141 (version, pkgarch, checksum, value))
142 except sqlite3.IntegrityError as exc:
143 logger.error(str(exc))
144
145 self.dirty = True
146
147 data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
148 (version, pkgarch, checksum))
149 row = data.fetchone()
150 if row != None:
151 val = row[0]
152 return val
153
154 def _importNohist(self, version, pkgarch, checksum, value):
155 try:
156 #try to insert
157 self._execute("INSERT INTO %s VALUES (?, ?, ?, ?);" % (self.table),
158 (version, pkgarch, checksum,value))
159 except sqlite3.IntegrityError as exc:
160 #already have the record, try to update
161 try:
162 self._execute("UPDATE %s SET value=? WHERE version=? AND pkgarch=? AND checksum=? AND value<?"
163 % (self.table),
164 (value,version,pkgarch,checksum,value))
165 except sqlite3.IntegrityError as exc:
166 logger.error(str(exc))
167
168 self.dirty = True
169
170 data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=? AND value>=?;" % self.table,
171 (version,pkgarch,checksum,value))
172 row=data.fetchone()
173 if row != None:
174 return row[0]
175 else:
176 return None
177
178 def importone(self, version, pkgarch, checksum, value):
179 if self.nohist:
180 return self._importNohist(version, pkgarch, checksum, value)
181 else:
182 return self._importHist(version, pkgarch, checksum, value)
183
184 def export(self, version, pkgarch, checksum, colinfo):
185 metainfo = {}
186 #column info
187 if colinfo:
188 metainfo['tbl_name'] = self.table
189 metainfo['core_ver'] = prserv.__version__
190 metainfo['col_info'] = []
191 data = self._execute("PRAGMA table_info(%s);" % self.table)
192 for row in data:
193 col = {}
194 col['name'] = row['name']
195 col['type'] = row['type']
196 col['notnull'] = row['notnull']
197 col['dflt_value'] = row['dflt_value']
198 col['pk'] = row['pk']
199 metainfo['col_info'].append(col)
200
201 #data info
202 datainfo = []
203
204 if self.nohist:
205 sqlstmt = "SELECT T1.version, T1.pkgarch, T1.checksum, T1.value FROM %s as T1, \
206 (SELECT version,pkgarch,max(value) as maxvalue FROM %s GROUP BY version,pkgarch) as T2 \
207 WHERE T1.version=T2.version AND T1.pkgarch=T2.pkgarch AND T1.value=T2.maxvalue " % (self.table, self.table)
208 else:
209 sqlstmt = "SELECT * FROM %s as T1 WHERE 1=1 " % self.table
210 sqlarg = []
211 where = ""
212 if version:
213 where += "AND T1.version=? "
214 sqlarg.append(str(version))
215 if pkgarch:
216 where += "AND T1.pkgarch=? "
217 sqlarg.append(str(pkgarch))
218 if checksum:
219 where += "AND T1.checksum=? "
220 sqlarg.append(str(checksum))
221
222 sqlstmt += where + ";"
223
224 if len(sqlarg):
225 data = self._execute(sqlstmt, tuple(sqlarg))
226 else:
227 data = self._execute(sqlstmt)
228 for row in data:
229 if row['version']:
230 col = {}
231 col['version'] = row['version']
232 col['pkgarch'] = row['pkgarch']
233 col['checksum'] = row['checksum']
234 col['value'] = row['value']
235 datainfo.append(col)
236 return (metainfo, datainfo)
237
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500238 def dump_db(self, fd):
239 writeCount = 0
240 for line in self.conn.iterdump():
241 writeCount = writeCount + len(line) + 1
242 fd.write(line)
243 fd.write('\n')
244 return writeCount
245
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500246class PRData(object):
247 """Object representing the PR database"""
248 def __init__(self, filename, nohist=True):
249 self.filename=os.path.abspath(filename)
250 self.nohist=nohist
251 #build directory hierarchy
252 try:
253 os.makedirs(os.path.dirname(self.filename))
254 except OSError as e:
255 if e.errno != errno.EEXIST:
256 raise e
257 self.connection=sqlite3.connect(self.filename, isolation_level="EXCLUSIVE", check_same_thread = False)
258 self.connection.row_factory=sqlite3.Row
259 self.connection.execute("pragma synchronous = off;")
Brad Bishop08902b02019-08-20 09:16:51 -0400260 self.connection.execute("PRAGMA journal_mode = MEMORY;")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500261 self._tables={}
262
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500263 def disconnect(self):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500264 self.connection.close()
265
266 def __getitem__(self,tblname):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600267 if not isinstance(tblname, str):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500268 raise TypeError("tblname argument must be a string, not '%s'" %
269 type(tblname))
270 if tblname in self._tables:
271 return self._tables[tblname]
272 else:
273 tableobj = self._tables[tblname] = PRTable(self.connection, tblname, self.nohist)
274 return tableobj
275
276 def __delitem__(self, tblname):
277 if tblname in self._tables:
278 del self._tables[tblname]
279 logger.info("drop table %s" % (tblname))
280 self.connection.execute("DROP TABLE IF EXISTS %s;" % tblname)