blob: 495d09f39baf46307977864cf4dce216e55ab95a [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001import logging
2import os.path
3import errno
4import prserv
5import time
6
7try:
8 import sqlite3
9except ImportError:
10 from pysqlite2 import dbapi2 as sqlite3
11
12logger = logging.getLogger("BitBake.PRserv")
13
14sqlversion = sqlite3.sqlite_version_info
15if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
16 raise Exception("sqlite3 version 3.3.0 or later is required.")
17
18#
19# "No History" mode - for a given query tuple (version, pkgarch, checksum),
20# the returned value will be the largest among all the values of the same
21# (version, pkgarch). This means the PR value returned can NOT be decremented.
22#
23# "History" mode - Return a new higher value for previously unseen query
24# tuple (version, pkgarch, checksum), otherwise return historical value.
25# Value can decrement if returning to a previous build.
26#
27
28class PRTable(object):
29 def __init__(self, conn, table, nohist):
30 self.conn = conn
31 self.nohist = nohist
32 self.dirty = False
33 if nohist:
34 self.table = "%s_nohist" % table
35 else:
36 self.table = "%s_hist" % table
37
38 self._execute("CREATE TABLE IF NOT EXISTS %s \
39 (version TEXT NOT NULL, \
40 pkgarch TEXT NOT NULL, \
41 checksum TEXT NOT NULL, \
42 value INTEGER, \
43 PRIMARY KEY (version, pkgarch, checksum));" % self.table)
44
45 def _execute(self, *query):
46 """Execute a query, waiting to acquire a lock if necessary"""
47 start = time.time()
48 end = start + 20
49 while True:
50 try:
51 return self.conn.execute(*query)
52 except sqlite3.OperationalError as exc:
53 if 'is locked' in str(exc) and end > time.time():
54 continue
55 raise exc
56
57 def sync(self):
58 self.conn.commit()
59 self._execute("BEGIN EXCLUSIVE TRANSACTION")
60
61 def sync_if_dirty(self):
62 if self.dirty:
63 self.sync()
64 self.dirty = False
65
66 def _getValueHist(self, version, pkgarch, checksum):
67 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
68 (version, pkgarch, checksum))
69 row=data.fetchone()
70 if row != None:
71 return row[0]
72 else:
73 #no value found, try to insert
74 try:
75 self._execute("INSERT INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1,0) from %s where version=? AND pkgarch=?));"
76 % (self.table,self.table),
77 (version,pkgarch, checksum,version, pkgarch))
78 except sqlite3.IntegrityError as exc:
79 logger.error(str(exc))
80
81 self.dirty = True
82
83 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
84 (version, pkgarch, checksum))
85 row=data.fetchone()
86 if row != None:
87 return row[0]
88 else:
89 raise prserv.NotFoundError
90
91 def _getValueNohist(self, version, pkgarch, checksum):
92 data=self._execute("SELECT value FROM %s \
93 WHERE version=? AND pkgarch=? AND checksum=? AND \
94 value >= (select max(value) from %s where version=? AND pkgarch=?);"
95 % (self.table, self.table),
96 (version, pkgarch, checksum, version, pkgarch))
97 row=data.fetchone()
98 if row != None:
99 return row[0]
100 else:
101 #no value found, try to insert
102 try:
103 self._execute("INSERT OR REPLACE INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1,0) from %s where version=? AND pkgarch=?));"
104 % (self.table,self.table),
105 (version, pkgarch, checksum, version, pkgarch))
106 except sqlite3.IntegrityError as exc:
107 logger.error(str(exc))
108 self.conn.rollback()
109
110 self.dirty = True
111
112 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
113 (version, pkgarch, checksum))
114 row=data.fetchone()
115 if row != None:
116 return row[0]
117 else:
118 raise prserv.NotFoundError
119
120 def getValue(self, version, pkgarch, checksum):
121 if self.nohist:
122 return self._getValueNohist(version, pkgarch, checksum)
123 else:
124 return self._getValueHist(version, pkgarch, checksum)
125
126 def _importHist(self, version, pkgarch, checksum, value):
127 val = None
128 data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
129 (version, pkgarch, checksum))
130 row = data.fetchone()
131 if row != None:
132 val=row[0]
133 else:
134 #no value found, try to insert
135 try:
136 self._execute("INSERT INTO %s VALUES (?, ?, ?, ?);" % (self.table),
137 (version, pkgarch, checksum, value))
138 except sqlite3.IntegrityError as exc:
139 logger.error(str(exc))
140
141 self.dirty = True
142
143 data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
144 (version, pkgarch, checksum))
145 row = data.fetchone()
146 if row != None:
147 val = row[0]
148 return val
149
150 def _importNohist(self, version, pkgarch, checksum, value):
151 try:
152 #try to insert
153 self._execute("INSERT INTO %s VALUES (?, ?, ?, ?);" % (self.table),
154 (version, pkgarch, checksum,value))
155 except sqlite3.IntegrityError as exc:
156 #already have the record, try to update
157 try:
158 self._execute("UPDATE %s SET value=? WHERE version=? AND pkgarch=? AND checksum=? AND value<?"
159 % (self.table),
160 (value,version,pkgarch,checksum,value))
161 except sqlite3.IntegrityError as exc:
162 logger.error(str(exc))
163
164 self.dirty = True
165
166 data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=? AND value>=?;" % self.table,
167 (version,pkgarch,checksum,value))
168 row=data.fetchone()
169 if row != None:
170 return row[0]
171 else:
172 return None
173
174 def importone(self, version, pkgarch, checksum, value):
175 if self.nohist:
176 return self._importNohist(version, pkgarch, checksum, value)
177 else:
178 return self._importHist(version, pkgarch, checksum, value)
179
180 def export(self, version, pkgarch, checksum, colinfo):
181 metainfo = {}
182 #column info
183 if colinfo:
184 metainfo['tbl_name'] = self.table
185 metainfo['core_ver'] = prserv.__version__
186 metainfo['col_info'] = []
187 data = self._execute("PRAGMA table_info(%s);" % self.table)
188 for row in data:
189 col = {}
190 col['name'] = row['name']
191 col['type'] = row['type']
192 col['notnull'] = row['notnull']
193 col['dflt_value'] = row['dflt_value']
194 col['pk'] = row['pk']
195 metainfo['col_info'].append(col)
196
197 #data info
198 datainfo = []
199
200 if self.nohist:
201 sqlstmt = "SELECT T1.version, T1.pkgarch, T1.checksum, T1.value FROM %s as T1, \
202 (SELECT version,pkgarch,max(value) as maxvalue FROM %s GROUP BY version,pkgarch) as T2 \
203 WHERE T1.version=T2.version AND T1.pkgarch=T2.pkgarch AND T1.value=T2.maxvalue " % (self.table, self.table)
204 else:
205 sqlstmt = "SELECT * FROM %s as T1 WHERE 1=1 " % self.table
206 sqlarg = []
207 where = ""
208 if version:
209 where += "AND T1.version=? "
210 sqlarg.append(str(version))
211 if pkgarch:
212 where += "AND T1.pkgarch=? "
213 sqlarg.append(str(pkgarch))
214 if checksum:
215 where += "AND T1.checksum=? "
216 sqlarg.append(str(checksum))
217
218 sqlstmt += where + ";"
219
220 if len(sqlarg):
221 data = self._execute(sqlstmt, tuple(sqlarg))
222 else:
223 data = self._execute(sqlstmt)
224 for row in data:
225 if row['version']:
226 col = {}
227 col['version'] = row['version']
228 col['pkgarch'] = row['pkgarch']
229 col['checksum'] = row['checksum']
230 col['value'] = row['value']
231 datainfo.append(col)
232 return (metainfo, datainfo)
233
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500234 def dump_db(self, fd):
235 writeCount = 0
236 for line in self.conn.iterdump():
237 writeCount = writeCount + len(line) + 1
238 fd.write(line)
239 fd.write('\n')
240 return writeCount
241
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500242class PRData(object):
243 """Object representing the PR database"""
244 def __init__(self, filename, nohist=True):
245 self.filename=os.path.abspath(filename)
246 self.nohist=nohist
247 #build directory hierarchy
248 try:
249 os.makedirs(os.path.dirname(self.filename))
250 except OSError as e:
251 if e.errno != errno.EEXIST:
252 raise e
253 self.connection=sqlite3.connect(self.filename, isolation_level="EXCLUSIVE", check_same_thread = False)
254 self.connection.row_factory=sqlite3.Row
255 self.connection.execute("pragma synchronous = off;")
256 self.connection.execute("PRAGMA journal_mode = WAL;")
257 self._tables={}
258
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500259 def disconnect(self):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500260 self.connection.close()
261
262 def __getitem__(self,tblname):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600263 if not isinstance(tblname, str):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500264 raise TypeError("tblname argument must be a string, not '%s'" %
265 type(tblname))
266 if tblname in self._tables:
267 return self._tables[tblname]
268 else:
269 tableobj = self._tables[tblname] = PRTable(self.connection, tblname, self.nohist)
270 return tableobj
271
272 def __delitem__(self, tblname):
273 if tblname in self._tables:
274 del self._tables[tblname]
275 logger.info("drop table %s" % (tblname))
276 self.connection.execute("DROP TABLE IF EXISTS %s;" % tblname)