blob: b1856846b6a2389e6b4cc7313a43421eebd8e352 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# Report significant differences in the buildhistory repository since a specific revision
2#
Brad Bishop6e60e8b2018-02-01 10:27:11 -05003# Copyright (C) 2012-2013, 2016-2017 Intel Corporation
Patrick Williamsc124f4f2015-09-15 14:41:29 -05004# Author: Paul Eggleton <paul.eggleton@linux.intel.com>
5#
Brad Bishopc342db32019-05-15 21:57:59 -04006# SPDX-License-Identifier: GPL-2.0-only
7#
Patrick Williamsc124f4f2015-09-15 14:41:29 -05008# Note: requires GitPython 0.3.1+
9#
10# You can use this from the command line by running scripts/buildhistory-diff
11#
12
13import sys
14import os.path
15import difflib
16import git
17import re
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080018import shlex
Brad Bishop6e60e8b2018-02-01 10:27:11 -050019import hashlib
20import collections
Patrick Williamsc124f4f2015-09-15 14:41:29 -050021import bb.utils
Brad Bishop6e60e8b2018-02-01 10:27:11 -050022import bb.tinfoil
Patrick Williamsc124f4f2015-09-15 14:41:29 -050023
24
25# How to display fields
26list_fields = ['DEPENDS', 'RPROVIDES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RREPLACES', 'RCONFLICTS', 'FILES', 'FILELIST', 'USER_CLASSES', 'IMAGE_CLASSES', 'IMAGE_FEATURES', 'IMAGE_LINGUAS', 'IMAGE_INSTALL', 'BAD_RECOMMENDATIONS', 'PACKAGE_EXCLUDE']
27list_order_fields = ['PACKAGES']
28defaultval_map = {'PKG': 'PKG', 'PKGE': 'PE', 'PKGV': 'PV', 'PKGR': 'PR'}
29numeric_fields = ['PKGSIZE', 'IMAGESIZE']
30# Fields to monitor
31monitor_fields = ['RPROVIDES', 'RDEPENDS', 'RRECOMMENDS', 'RREPLACES', 'RCONFLICTS', 'PACKAGES', 'FILELIST', 'PKGSIZE', 'IMAGESIZE', 'PKG']
32ver_monitor_fields = ['PKGE', 'PKGV', 'PKGR']
33# Percentage change to alert for numeric fields
34monitor_numeric_threshold = 10
35# Image files to monitor (note that image-info.txt is handled separately)
36img_monitor_files = ['installed-package-names.txt', 'files-in-image.txt']
Patrick Williamsc124f4f2015-09-15 14:41:29 -050037
Brad Bishop316dfdd2018-06-25 12:45:53 -040038colours = {
39 'colour_default': '',
40 'colour_add': '',
41 'colour_remove': '',
42}
43
44def init_colours(use_colours):
45 global colours
46 if use_colours:
47 colours = {
48 'colour_default': '\033[0m',
49 'colour_add': '\033[1;32m',
50 'colour_remove': '\033[1;31m',
51 }
52 else:
53 colours = {
54 'colour_default': '',
55 'colour_add': '',
56 'colour_remove': '',
57 }
Patrick Williamsc124f4f2015-09-15 14:41:29 -050058
59class ChangeRecord:
60 def __init__(self, path, fieldname, oldvalue, newvalue, monitored):
61 self.path = path
62 self.fieldname = fieldname
63 self.oldvalue = oldvalue
64 self.newvalue = newvalue
65 self.monitored = monitored
Patrick Williamsc124f4f2015-09-15 14:41:29 -050066 self.filechanges = None
67
68 def __str__(self):
69 return self._str_internal(True)
70
71 def _str_internal(self, outer):
72 if outer:
73 if '/image-files/' in self.path:
74 prefix = '%s: ' % self.path.split('/image-files/')[0]
75 else:
76 prefix = '%s: ' % self.path
77 else:
78 prefix = ''
79
80 def pkglist_combine(depver):
81 pkglist = []
Patrick Williamsc0f7c042017-02-23 20:41:17 -060082 for k,v in depver.items():
Patrick Williamsc124f4f2015-09-15 14:41:29 -050083 if v:
84 pkglist.append("%s (%s)" % (k,v))
85 else:
86 pkglist.append(k)
87 return pkglist
88
Brad Bishop6e60e8b2018-02-01 10:27:11 -050089 def detect_renamed_dirs(aitems, bitems):
90 adirs = set(map(os.path.dirname, aitems))
91 bdirs = set(map(os.path.dirname, bitems))
92 files_ab = [(name, sorted(os.path.basename(item) for item in aitems if os.path.dirname(item) == name)) \
93 for name in adirs - bdirs]
94 files_ba = [(name, sorted(os.path.basename(item) for item in bitems if os.path.dirname(item) == name)) \
95 for name in bdirs - adirs]
Brad Bishop316dfdd2018-06-25 12:45:53 -040096 renamed_dirs = []
97 for dir1, files1 in files_ab:
98 rename = False
99 for dir2, files2 in files_ba:
100 if files1 == files2 and not rename:
101 renamed_dirs.append((dir1,dir2))
102 # Make sure that we don't use this (dir, files) pair again.
103 files_ba.remove((dir2,files2))
104 # If a dir has already been found to have a rename, stop and go no further.
105 rename = True
106
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500107 # remove files that belong to renamed dirs from aitems and bitems
108 for dir1, dir2 in renamed_dirs:
109 aitems = [item for item in aitems if os.path.dirname(item) not in (dir1, dir2)]
110 bitems = [item for item in bitems if os.path.dirname(item) not in (dir1, dir2)]
111 return renamed_dirs, aitems, bitems
112
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500113 if self.fieldname in list_fields or self.fieldname in list_order_fields:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500114 renamed_dirs = []
Brad Bishop316dfdd2018-06-25 12:45:53 -0400115 changed_order = False
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500116 if self.fieldname in ['RPROVIDES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RREPLACES', 'RCONFLICTS']:
117 (depvera, depverb) = compare_pkg_lists(self.oldvalue, self.newvalue)
118 aitems = pkglist_combine(depvera)
119 bitems = pkglist_combine(depverb)
120 else:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500121 if self.fieldname == 'FILELIST':
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800122 aitems = shlex.split(self.oldvalue)
123 bitems = shlex.split(self.newvalue)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500124 renamed_dirs, aitems, bitems = detect_renamed_dirs(aitems, bitems)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800125 else:
126 aitems = self.oldvalue.split()
127 bitems = self.newvalue.split()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500128
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500129 removed = list(set(aitems) - set(bitems))
130 added = list(set(bitems) - set(aitems))
131
Brad Bishopc342db32019-05-15 21:57:59 -0400132 if not removed and not added and self.fieldname in ['RPROVIDES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RREPLACES', 'RCONFLICTS']:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400133 depvera = bb.utils.explode_dep_versions2(self.oldvalue, sort=False)
134 depverb = bb.utils.explode_dep_versions2(self.newvalue, sort=False)
135 for i, j in zip(depvera.items(), depverb.items()):
136 if i[0] != j[0]:
137 changed_order = True
138 break
139
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500140 lines = []
141 if renamed_dirs:
142 for dfrom, dto in renamed_dirs:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400143 lines.append('directory renamed {colour_remove}{}{colour_default} -> {colour_add}{}{colour_default}'.format(dfrom, dto, **colours))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500144 if removed or added:
145 if removed and not bitems:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400146 lines.append('removed all items "{colour_remove}{}{colour_default}"'.format(' '.join(removed), **colours))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500147 else:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500148 if removed:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400149 lines.append('removed "{colour_remove}{value}{colour_default}"'.format(value=' '.join(removed), **colours))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500150 if added:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400151 lines.append('added "{colour_add}{value}{colour_default}"'.format(value=' '.join(added), **colours))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500152 else:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500153 lines.append('changed order')
154
Brad Bishop316dfdd2018-06-25 12:45:53 -0400155 if not (removed or added or changed_order):
156 out = ''
157 else:
158 out = '%s: %s' % (self.fieldname, ', '.join(lines))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500159
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500160 elif self.fieldname in numeric_fields:
161 aval = int(self.oldvalue or 0)
162 bval = int(self.newvalue or 0)
163 if aval != 0:
164 percentchg = ((bval - aval) / float(aval)) * 100
165 else:
166 percentchg = 100
Brad Bishop316dfdd2018-06-25 12:45:53 -0400167 out = '{} changed from {colour_remove}{}{colour_default} to {colour_add}{}{colour_default} ({}{:.0f}%)'.format(self.fieldname, self.oldvalue or "''", self.newvalue or "''", '+' if percentchg > 0 else '', percentchg, **colours)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500168 elif self.fieldname in defaultval_map:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400169 out = '{} changed from {colour_remove}{}{colour_default} to {colour_add}{}{colour_default}'.format(self.fieldname, self.oldvalue, self.newvalue, **colours)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500170 if self.fieldname == 'PKG' and '[default]' in self.newvalue:
171 out += ' - may indicate debian renaming failure'
172 elif self.fieldname in ['pkg_preinst', 'pkg_postinst', 'pkg_prerm', 'pkg_postrm']:
173 if self.oldvalue and self.newvalue:
174 out = '%s changed:\n ' % self.fieldname
175 elif self.newvalue:
176 out = '%s added:\n ' % self.fieldname
177 elif self.oldvalue:
178 out = '%s cleared:\n ' % self.fieldname
179 alines = self.oldvalue.splitlines()
180 blines = self.newvalue.splitlines()
181 diff = difflib.unified_diff(alines, blines, self.fieldname, self.fieldname, lineterm='')
182 out += '\n '.join(list(diff)[2:])
183 out += '\n --'
Brad Bishop96ff1982019-08-19 13:50:42 -0400184 elif self.fieldname in img_monitor_files or '/image-files/' in self.path or self.fieldname == "sysroot":
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500185 if self.filechanges or (self.oldvalue and self.newvalue):
186 fieldname = self.fieldname
187 if '/image-files/' in self.path:
188 fieldname = os.path.join('/' + self.path.split('/image-files/')[1], self.fieldname)
189 out = 'Changes to %s:\n ' % fieldname
190 else:
191 if outer:
192 prefix = 'Changes to %s ' % self.path
193 out = '(%s):\n ' % self.fieldname
194 if self.filechanges:
195 out += '\n '.join(['%s' % i for i in self.filechanges])
196 else:
197 alines = self.oldvalue.splitlines()
198 blines = self.newvalue.splitlines()
199 diff = difflib.unified_diff(alines, blines, fieldname, fieldname, lineterm='')
200 out += '\n '.join(list(diff))
201 out += '\n --'
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500202 else:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500203 out = ''
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500204 else:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400205 out = '{} changed from "{colour_remove}{}{colour_default}" to "{colour_add}{}{colour_default}"'.format(self.fieldname, self.oldvalue, self.newvalue, **colours)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500206
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500207 return '%s%s' % (prefix, out) if out else ''
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500208
209class FileChange:
210 changetype_add = 'A'
211 changetype_remove = 'R'
212 changetype_type = 'T'
213 changetype_perms = 'P'
214 changetype_ownergroup = 'O'
215 changetype_link = 'L'
Andrew Geisslerd25ed322020-06-27 00:28:28 -0500216 changetype_move = 'M'
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500217
218 def __init__(self, path, changetype, oldvalue = None, newvalue = None):
219 self.path = path
220 self.changetype = changetype
221 self.oldvalue = oldvalue
222 self.newvalue = newvalue
223
224 def _ftype_str(self, ftype):
225 if ftype == '-':
226 return 'file'
227 elif ftype == 'd':
228 return 'directory'
229 elif ftype == 'l':
230 return 'symlink'
231 elif ftype == 'c':
232 return 'char device'
233 elif ftype == 'b':
234 return 'block device'
235 elif ftype == 'p':
236 return 'fifo'
237 elif ftype == 's':
238 return 'socket'
239 else:
240 return 'unknown (%s)' % ftype
241
242 def __str__(self):
243 if self.changetype == self.changetype_add:
244 return '%s was added' % self.path
245 elif self.changetype == self.changetype_remove:
246 return '%s was removed' % self.path
247 elif self.changetype == self.changetype_type:
248 return '%s changed type from %s to %s' % (self.path, self._ftype_str(self.oldvalue), self._ftype_str(self.newvalue))
249 elif self.changetype == self.changetype_perms:
250 return '%s changed permissions from %s to %s' % (self.path, self.oldvalue, self.newvalue)
251 elif self.changetype == self.changetype_ownergroup:
252 return '%s changed owner/group from %s to %s' % (self.path, self.oldvalue, self.newvalue)
253 elif self.changetype == self.changetype_link:
254 return '%s changed symlink target from %s to %s' % (self.path, self.oldvalue, self.newvalue)
Andrew Geisslerd25ed322020-06-27 00:28:28 -0500255 elif self.changetype == self.changetype_move:
256 return '%s moved to %s' % (self.path, self.oldvalue)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500257 else:
258 return '%s changed (unknown)' % self.path
259
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500260def blob_to_dict(blob):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600261 alines = [line for line in blob.data_stream.read().decode('utf-8').splitlines()]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500262 adict = {}
263 for line in alines:
264 splitv = [i.strip() for i in line.split('=',1)]
265 if len(splitv) > 1:
266 adict[splitv[0]] = splitv[1]
267 return adict
268
269
270def file_list_to_dict(lines):
271 adict = {}
272 for line in lines:
273 # Leave the last few fields intact so we handle file names containing spaces
274 splitv = line.split(None,4)
275 # Grab the path and remove the leading .
276 path = splitv[4][1:].strip()
277 # Handle symlinks
278 if(' -> ' in path):
279 target = path.split(' -> ')[1]
280 path = path.split(' -> ')[0]
281 adict[path] = splitv[0:3] + [target]
282 else:
283 adict[path] = splitv[0:3]
284 return adict
285
Andrew Geisslerd25ed322020-06-27 00:28:28 -0500286numeric_removal = str.maketrans('0123456789', 'XXXXXXXXXX')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500287
Brad Bishop96ff1982019-08-19 13:50:42 -0400288def compare_file_lists(alines, blines, compare_ownership=True):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500289 adict = file_list_to_dict(alines)
290 bdict = file_list_to_dict(blines)
291 filechanges = []
Andrew Geisslerd25ed322020-06-27 00:28:28 -0500292 additions = []
293 removals = []
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600294 for path, splitv in adict.items():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500295 newsplitv = bdict.pop(path, None)
296 if newsplitv:
297 # Check type
298 oldvalue = splitv[0][0]
299 newvalue = newsplitv[0][0]
300 if oldvalue != newvalue:
301 filechanges.append(FileChange(path, FileChange.changetype_type, oldvalue, newvalue))
Brad Bishop96ff1982019-08-19 13:50:42 -0400302
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500303 # Check permissions
304 oldvalue = splitv[0][1:]
305 newvalue = newsplitv[0][1:]
306 if oldvalue != newvalue:
307 filechanges.append(FileChange(path, FileChange.changetype_perms, oldvalue, newvalue))
Brad Bishop96ff1982019-08-19 13:50:42 -0400308
309 if compare_ownership:
310 # Check owner/group
311 oldvalue = '%s/%s' % (splitv[1], splitv[2])
312 newvalue = '%s/%s' % (newsplitv[1], newsplitv[2])
313 if oldvalue != newvalue:
314 filechanges.append(FileChange(path, FileChange.changetype_ownergroup, oldvalue, newvalue))
315
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500316 # Check symlink target
317 if newsplitv[0][0] == 'l':
318 if len(splitv) > 3:
319 oldvalue = splitv[3]
320 else:
321 oldvalue = None
322 newvalue = newsplitv[3]
323 if oldvalue != newvalue:
324 filechanges.append(FileChange(path, FileChange.changetype_link, oldvalue, newvalue))
325 else:
Andrew Geisslerd25ed322020-06-27 00:28:28 -0500326 removals.append(path)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500327
328 # Whatever is left over has been added
329 for path in bdict:
Andrew Geisslerd25ed322020-06-27 00:28:28 -0500330 additions.append(path)
331
332 # Rather than print additions and removals, its nicer to print file 'moves'
333 # where names or paths are similar.
334 revmap_remove = {}
335 for removal in removals:
336 translated = removal.translate(numeric_removal)
337 if translated not in revmap_remove:
338 revmap_remove[translated] = []
339 revmap_remove[translated].append(removal)
340
341 #
342 # We want to detect renames of large trees of files like
343 # /lib/modules/5.4.40-yocto-standard to /lib/modules/5.4.43-yocto-standard
344 #
345 renames = {}
346 for addition in additions.copy():
347 if addition not in additions:
348 continue
349 translated = addition.translate(numeric_removal)
350 if translated in revmap_remove:
351 if len(revmap_remove[translated]) != 1:
352 continue
353 removal = revmap_remove[translated][0]
354 commondir = addition.split("/")
355 commondir2 = removal.split("/")
356 idx = None
357 for i in range(len(commondir)):
358 if commondir[i] != commondir2[i]:
359 idx = i
360 break
361 commondir = "/".join(commondir[:i+1])
362 commondir2 = "/".join(commondir2[:i+1])
363 # If the common parent is in one dict and not the other its likely a rename
364 # so iterate through those files and process as such
365 if commondir2 not in bdict and commondir not in adict:
366 if commondir not in renames:
367 renames[commondir] = commondir2
368 for addition2 in additions.copy():
369 if addition2.startswith(commondir):
370 removal2 = addition2.replace(commondir, commondir2)
371 if removal2 in removals:
372 additions.remove(addition2)
373 removals.remove(removal2)
374 continue
375 filechanges.append(FileChange(removal, FileChange.changetype_move, addition))
Andrew Geissler6ce62a22020-11-30 19:58:47 -0600376 if addition in additions:
377 additions.remove(addition)
378 if removal in removals:
379 removals.remove(removal)
Andrew Geisslerd25ed322020-06-27 00:28:28 -0500380 for rename in renames:
381 filechanges.append(FileChange(renames[rename], FileChange.changetype_move, rename))
382
383 for addition in additions:
384 filechanges.append(FileChange(addition, FileChange.changetype_add))
385 for removal in removals:
386 filechanges.append(FileChange(removal, FileChange.changetype_remove))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500387
388 return filechanges
389
390
391def compare_lists(alines, blines):
392 removed = list(set(alines) - set(blines))
393 added = list(set(blines) - set(alines))
394
395 filechanges = []
396 for pkg in removed:
397 filechanges.append(FileChange(pkg, FileChange.changetype_remove))
398 for pkg in added:
399 filechanges.append(FileChange(pkg, FileChange.changetype_add))
400
401 return filechanges
402
403
404def compare_pkg_lists(astr, bstr):
405 depvera = bb.utils.explode_dep_versions2(astr)
406 depverb = bb.utils.explode_dep_versions2(bstr)
407
408 # Strip out changes where the version has increased
409 remove = []
410 for k in depvera:
411 if k in depverb:
412 dva = depvera[k]
413 dvb = depverb[k]
414 if dva and dvb and len(dva) == len(dvb):
415 # Since length is the same, sort so that prefixes (e.g. >=) will line up
416 dva.sort()
417 dvb.sort()
418 removeit = True
419 for dvai, dvbi in zip(dva, dvb):
420 if dvai != dvbi:
421 aiprefix = dvai.split(' ')[0]
422 biprefix = dvbi.split(' ')[0]
423 if aiprefix == biprefix and aiprefix in ['>=', '=']:
424 if bb.utils.vercmp(bb.utils.split_version(dvai), bb.utils.split_version(dvbi)) > 0:
425 removeit = False
426 break
427 else:
428 removeit = False
429 break
430 if removeit:
431 remove.append(k)
432
433 for k in remove:
434 depvera.pop(k)
435 depverb.pop(k)
436
437 return (depvera, depverb)
438
439
440def compare_dict_blobs(path, ablob, bblob, report_all, report_ver):
441 adict = blob_to_dict(ablob)
442 bdict = blob_to_dict(bblob)
443
444 pkgname = os.path.basename(path)
445
446 defaultvals = {}
447 defaultvals['PKG'] = pkgname
448 defaultvals['PKGE'] = '0'
449
450 changes = []
451 keys = list(set(adict.keys()) | set(bdict.keys()) | set(defaultval_map.keys()))
452 for key in keys:
453 astr = adict.get(key, '')
454 bstr = bdict.get(key, '')
455 if key in ver_monitor_fields:
456 monitored = report_ver or astr or bstr
457 else:
458 monitored = key in monitor_fields
459 mapped_key = defaultval_map.get(key, '')
460 if mapped_key:
461 if not astr:
462 astr = '%s [default]' % adict.get(mapped_key, defaultvals.get(key, ''))
463 if not bstr:
464 bstr = '%s [default]' % bdict.get(mapped_key, defaultvals.get(key, ''))
465
466 if astr != bstr:
467 if (not report_all) and key in numeric_fields:
468 aval = int(astr or 0)
469 bval = int(bstr or 0)
470 if aval != 0:
471 percentchg = ((bval - aval) / float(aval)) * 100
472 else:
473 percentchg = 100
474 if abs(percentchg) < monitor_numeric_threshold:
475 continue
476 elif (not report_all) and key in list_fields:
Brad Bishop64c979e2019-11-04 13:55:29 -0500477 if key == "FILELIST" and (path.endswith("-dbg") or path.endswith("-src")) and bstr.strip() != '':
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500478 continue
479 if key in ['RPROVIDES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RREPLACES', 'RCONFLICTS']:
480 (depvera, depverb) = compare_pkg_lists(astr, bstr)
481 if depvera == depverb:
482 continue
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800483 if key == 'FILELIST':
484 alist = shlex.split(astr)
485 blist = shlex.split(bstr)
486 else:
487 alist = astr.split()
488 blist = bstr.split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500489 alist.sort()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500490 blist.sort()
491 # We don't care about the removal of self-dependencies
492 if pkgname in alist and not pkgname in blist:
493 alist.remove(pkgname)
494 if ' '.join(alist) == ' '.join(blist):
495 continue
496
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600497 if key == 'PKGR' and not report_all:
498 vers = []
499 # strip leading 'r' and dots
500 for ver in (astr.split()[0], bstr.split()[0]):
501 if ver.startswith('r'):
502 ver = ver[1:]
503 vers.append(ver.replace('.', ''))
504 maxlen = max(len(vers[0]), len(vers[1]))
505 try:
506 # pad with '0' and convert to int
507 vers = [int(ver.ljust(maxlen, '0')) for ver in vers]
508 except ValueError:
509 pass
510 else:
511 # skip decrements and increments
512 if abs(vers[0] - vers[1]) == 1:
513 continue
514
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500515 chg = ChangeRecord(path, key, astr, bstr, monitored)
516 changes.append(chg)
517 return changes
518
519
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500520def compare_siglists(a_blob, b_blob, taskdiff=False):
521 # FIXME collapse down a recipe's tasks?
522 alines = a_blob.data_stream.read().decode('utf-8').splitlines()
523 blines = b_blob.data_stream.read().decode('utf-8').splitlines()
524 keys = []
525 pnmap = {}
526 def readsigs(lines):
527 sigs = {}
528 for line in lines:
529 linesplit = line.split()
530 if len(linesplit) > 2:
531 sigs[linesplit[0]] = linesplit[2]
532 if not linesplit[0] in keys:
533 keys.append(linesplit[0])
534 pnmap[linesplit[1]] = linesplit[0].rsplit('.', 1)[0]
535 return sigs
536 adict = readsigs(alines)
537 bdict = readsigs(blines)
538 out = []
539
540 changecount = 0
541 addcount = 0
542 removecount = 0
543 if taskdiff:
544 with bb.tinfoil.Tinfoil() as tinfoil:
545 tinfoil.prepare(config_only=True)
546
547 changes = collections.OrderedDict()
548
549 def compare_hashfiles(pn, taskname, hash1, hash2):
550 hashes = [hash1, hash2]
551 hashfiles = bb.siggen.find_siginfo(pn, taskname, hashes, tinfoil.config_data)
552
553 if not taskname:
554 (pn, taskname) = pn.rsplit('.', 1)
555 pn = pnmap.get(pn, pn)
556 desc = '%s.%s' % (pn, taskname)
557
558 if len(hashfiles) == 0:
559 out.append("Unable to find matching sigdata for %s with hashes %s or %s" % (desc, hash1, hash2))
560 elif not hash1 in hashfiles:
561 out.append("Unable to find matching sigdata for %s with hash %s" % (desc, hash1))
562 elif not hash2 in hashfiles:
563 out.append("Unable to find matching sigdata for %s with hash %s" % (desc, hash2))
564 else:
565 out2 = bb.siggen.compare_sigfiles(hashfiles[hash1], hashfiles[hash2], recursecb, collapsed=True)
566 for line in out2:
567 m = hashlib.sha256()
568 m.update(line.encode('utf-8'))
569 entry = changes.get(m.hexdigest(), (line, []))
570 if desc not in entry[1]:
571 changes[m.hexdigest()] = (line, entry[1] + [desc])
572
573 # Define recursion callback
574 def recursecb(key, hash1, hash2):
575 compare_hashfiles(key, None, hash1, hash2)
576 return []
577
578 for key in keys:
579 siga = adict.get(key, None)
580 sigb = bdict.get(key, None)
581 if siga is not None and sigb is not None and siga != sigb:
582 changecount += 1
583 (pn, taskname) = key.rsplit('.', 1)
584 compare_hashfiles(pn, taskname, siga, sigb)
585 elif siga is None:
586 addcount += 1
587 elif sigb is None:
588 removecount += 1
589 for key, item in changes.items():
590 line, tasks = item
591 if len(tasks) == 1:
592 desc = tasks[0]
593 elif len(tasks) == 2:
594 desc = '%s and %s' % (tasks[0], tasks[1])
595 else:
596 desc = '%s and %d others' % (tasks[-1], len(tasks)-1)
597 out.append('%s: %s' % (desc, line))
598 else:
599 for key in keys:
600 siga = adict.get(key, None)
601 sigb = bdict.get(key, None)
602 if siga is not None and sigb is not None and siga != sigb:
603 out.append('%s changed from %s to %s' % (key, siga, sigb))
604 changecount += 1
605 elif siga is None:
606 out.append('%s was added' % key)
607 addcount += 1
608 elif sigb is None:
609 out.append('%s was removed' % key)
610 removecount += 1
611 out.append('Summary: %d tasks added, %d tasks removed, %d tasks modified (%.1f%%)' % (addcount, removecount, changecount, (changecount / float(len(bdict)) * 100)))
612 return '\n'.join(out)
613
614
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500615def process_changes(repopath, revision1, revision2='HEAD', report_all=False, report_ver=False,
616 sigs=False, sigsdiff=False, exclude_path=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500617 repo = git.Repo(repopath)
618 assert repo.bare == False
619 commit = repo.commit(revision1)
620 diff = commit.diff(revision2)
621
622 changes = []
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500623
624 if sigs or sigsdiff:
625 for d in diff.iter_change_type('M'):
626 if d.a_blob.path == 'siglist.txt':
627 changes.append(compare_siglists(d.a_blob, d.b_blob, taskdiff=sigsdiff))
628 return changes
629
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500630 for d in diff.iter_change_type('M'):
631 path = os.path.dirname(d.a_blob.path)
632 if path.startswith('packages/'):
633 filename = os.path.basename(d.a_blob.path)
634 if filename == 'latest':
635 changes.extend(compare_dict_blobs(path, d.a_blob, d.b_blob, report_all, report_ver))
636 elif filename.startswith('latest.'):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600637 chg = ChangeRecord(path, filename, d.a_blob.data_stream.read().decode('utf-8'), d.b_blob.data_stream.read().decode('utf-8'), True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500638 changes.append(chg)
Brad Bishop96ff1982019-08-19 13:50:42 -0400639 elif filename == 'sysroot':
640 alines = d.a_blob.data_stream.read().decode('utf-8').splitlines()
641 blines = d.b_blob.data_stream.read().decode('utf-8').splitlines()
642 filechanges = compare_file_lists(alines,blines, compare_ownership=False)
643 if filechanges:
644 chg = ChangeRecord(path, filename, None, None, True)
645 chg.filechanges = filechanges
646 changes.append(chg)
647
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500648 elif path.startswith('images/'):
649 filename = os.path.basename(d.a_blob.path)
650 if filename in img_monitor_files:
651 if filename == 'files-in-image.txt':
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600652 alines = d.a_blob.data_stream.read().decode('utf-8').splitlines()
653 blines = d.b_blob.data_stream.read().decode('utf-8').splitlines()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500654 filechanges = compare_file_lists(alines,blines)
655 if filechanges:
656 chg = ChangeRecord(path, filename, None, None, True)
657 chg.filechanges = filechanges
658 changes.append(chg)
659 elif filename == 'installed-package-names.txt':
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600660 alines = d.a_blob.data_stream.read().decode('utf-8').splitlines()
661 blines = d.b_blob.data_stream.read().decode('utf-8').splitlines()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500662 filechanges = compare_lists(alines,blines)
663 if filechanges:
664 chg = ChangeRecord(path, filename, None, None, True)
665 chg.filechanges = filechanges
666 changes.append(chg)
667 else:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600668 chg = ChangeRecord(path, filename, d.a_blob.data_stream.read().decode('utf-8'), d.b_blob.data_stream.read().decode('utf-8'), True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500669 changes.append(chg)
670 elif filename == 'image-info.txt':
671 changes.extend(compare_dict_blobs(path, d.a_blob, d.b_blob, report_all, report_ver))
672 elif '/image-files/' in path:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600673 chg = ChangeRecord(path, filename, d.a_blob.data_stream.read().decode('utf-8'), d.b_blob.data_stream.read().decode('utf-8'), True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500674 changes.append(chg)
675
676 # Look for added preinst/postinst/prerm/postrm
677 # (without reporting newly added recipes)
678 addedpkgs = []
679 addedchanges = []
680 for d in diff.iter_change_type('A'):
681 path = os.path.dirname(d.b_blob.path)
682 if path.startswith('packages/'):
683 filename = os.path.basename(d.b_blob.path)
684 if filename == 'latest':
685 addedpkgs.append(path)
686 elif filename.startswith('latest.'):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600687 chg = ChangeRecord(path, filename[7:], '', d.b_blob.data_stream.read().decode('utf-8'), True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500688 addedchanges.append(chg)
689 for chg in addedchanges:
690 found = False
691 for pkg in addedpkgs:
692 if chg.path.startswith(pkg):
693 found = True
694 break
695 if not found:
696 changes.append(chg)
697
698 # Look for cleared preinst/postinst/prerm/postrm
699 for d in diff.iter_change_type('D'):
700 path = os.path.dirname(d.a_blob.path)
701 if path.startswith('packages/'):
702 filename = os.path.basename(d.a_blob.path)
703 if filename != 'latest' and filename.startswith('latest.'):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600704 chg = ChangeRecord(path, filename[7:], d.a_blob.data_stream.read().decode('utf-8'), '', True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500705 changes.append(chg)
706
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500707 # filter out unwanted paths
708 if exclude_path:
709 for chg in changes:
710 if chg.filechanges:
711 fchgs = []
712 for fchg in chg.filechanges:
713 for epath in exclude_path:
714 if fchg.path.startswith(epath):
715 break
716 else:
717 fchgs.append(fchg)
718 chg.filechanges = fchgs
719
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500720 if report_all:
721 return changes
722 else:
723 return [chg for chg in changes if chg.monitored]