blob: 6f4b268119b903c459358caefb20a4ead285f0b2 [file] [log] [blame]
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001#!/usr/bin/env python3
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002# ex:ts=4:sw=4:sts=4:et
3# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
4
5# Copyright (c) 2013 Wind River Systems, Inc.
6#
Brad Bishopc342db32019-05-15 21:57:59 -04007# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -05008#
Patrick Williamsc124f4f2015-09-15 14:41:29 -05009
Patrick Williamsc124f4f2015-09-15 14:41:29 -050010import os
11import sys
12import getopt
13import shutil
14import re
15import warnings
16import subprocess
Patrick Williamsc0f7c042017-02-23 20:41:17 -060017import argparse
Patrick Williamsc124f4f2015-09-15 14:41:29 -050018
19scripts_path = os.path.abspath(os.path.dirname(os.path.abspath(sys.argv[0])))
20lib_path = scripts_path + '/lib'
21sys.path = sys.path + [lib_path]
22
23import scriptpath
24
25# Figure out where is the bitbake/lib/bb since we need bb.siggen and bb.process
26bitbakepath = scriptpath.add_bitbake_lib_path()
27if not bitbakepath:
28 sys.stderr.write("Unable to find bitbake by searching parent directory of this script or PATH\n")
29 sys.exit(1)
Patrick Williamsc0f7c042017-02-23 20:41:17 -060030scriptpath.add_oe_lib_path()
31import argparse_oe
Patrick Williamsc124f4f2015-09-15 14:41:29 -050032
33import bb.siggen
34import bb.process
35
36# Match the stamp's filename
37# group(1): PE_PV (may no PE)
38# group(2): PR
39# group(3): TASK
40# group(4): HASH
41stamp_re = re.compile("(?P<pv>.*)-(?P<pr>r\d+)\.(?P<task>do_\w+)\.(?P<hash>[^\.]*)")
42sigdata_re = re.compile(".*\.sigdata\..*")
43
44def gen_dict(stamps):
45 """
46 Generate the dict from the stamps dir.
47 The output dict format is:
48 {fake_f: {pn: PN, pv: PV, pr: PR, task: TASK, path: PATH}}
49 Where:
50 fake_f: pv + task + hash
51 path: the path to the stamp file
52 """
53 # The member of the sub dict (A "path" will be appended below)
54 sub_mem = ("pv", "pr", "task")
55 d = {}
56 for dirpath, _, files in os.walk(stamps):
57 for f in files:
58 # The "bitbake -S" would generate ".sigdata", but no "_setscene".
59 fake_f = re.sub('_setscene.', '.', f)
60 fake_f = re.sub('.sigdata', '', fake_f)
61 subdict = {}
62 tmp = stamp_re.match(fake_f)
63 if tmp:
64 for i in sub_mem:
65 subdict[i] = tmp.group(i)
66 if len(subdict) != 0:
67 pn = os.path.basename(dirpath)
68 subdict['pn'] = pn
69 # The path will be used by os.stat() and bb.siggen
70 subdict['path'] = dirpath + "/" + f
71 fake_f = tmp.group('pv') + tmp.group('task') + tmp.group('hash')
72 d[fake_f] = subdict
73 return d
74
75# Re-construct the dict
76def recon_dict(dict_in):
77 """
78 The output dict format is:
79 {pn_task: {pv: PV, pr: PR, path: PATH}}
80 """
81 dict_out = {}
82 for k in dict_in.keys():
83 subdict = {}
84 # The key
85 pn_task = "%s_%s" % (dict_in.get(k).get('pn'), dict_in.get(k).get('task'))
86 # If more than one stamps are found, use the latest one.
87 if pn_task in dict_out:
88 full_path_pre = dict_out.get(pn_task).get('path')
89 full_path_cur = dict_in.get(k).get('path')
90 if os.stat(full_path_pre).st_mtime > os.stat(full_path_cur).st_mtime:
91 continue
92 subdict['pv'] = dict_in.get(k).get('pv')
93 subdict['pr'] = dict_in.get(k).get('pr')
94 subdict['path'] = dict_in.get(k).get('path')
95 dict_out[pn_task] = subdict
96
97 return dict_out
98
99def split_pntask(s):
100 """
101 Split the pn_task in to (pn, task) and return it
102 """
103 tmp = re.match("(.*)_(do_.*)", s)
104 return (tmp.group(1), tmp.group(2))
105
106
107def print_added(d_new = None, d_old = None):
108 """
109 Print the newly added tasks
110 """
111 added = {}
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600112 for k in list(d_new.keys()):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500113 if k not in d_old:
114 # Add the new one to added dict, and remove it from
115 # d_new, so the remaining ones are the changed ones
116 added[k] = d_new.get(k)
117 del(d_new[k])
118
119 if not added:
120 return 0
121
122 # Format the output, the dict format is:
123 # {pn: task1, task2 ...}
124 added_format = {}
125 counter = 0
126 for k in added.keys():
127 pn, task = split_pntask(k)
128 if pn in added_format:
129 # Append the value
130 added_format[pn] = "%s %s" % (added_format.get(pn), task)
131 else:
132 added_format[pn] = task
133 counter += 1
134 print("=== Newly added tasks: (%s tasks)" % counter)
135 for k in added_format.keys():
136 print(" %s: %s" % (k, added_format.get(k)))
137
138 return counter
139
140def print_vrchanged(d_new = None, d_old = None, vr = None):
141 """
142 Print the pv or pr changed tasks.
143 The arg "vr" is "pv" or "pr"
144 """
145 pvchanged = {}
146 counter = 0
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600147 for k in list(d_new.keys()):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500148 if d_new.get(k).get(vr) != d_old.get(k).get(vr):
149 counter += 1
150 pn, task = split_pntask(k)
151 if pn not in pvchanged:
152 # Format the output, we only print pn (no task) since
153 # all the tasks would be changed when pn or pr changed,
154 # the dict format is:
155 # {pn: pv/pr_old -> pv/pr_new}
156 pvchanged[pn] = "%s -> %s" % (d_old.get(k).get(vr), d_new.get(k).get(vr))
157 del(d_new[k])
158
159 if not pvchanged:
160 return 0
161
162 print("\n=== %s changed: (%s tasks)" % (vr.upper(), counter))
163 for k in pvchanged.keys():
164 print(" %s: %s" % (k, pvchanged.get(k)))
165
166 return counter
167
168def print_depchanged(d_new = None, d_old = None, verbose = False):
169 """
170 Print the dependency changes
171 """
172 depchanged = {}
173 counter = 0
174 for k in d_new.keys():
175 counter += 1
176 pn, task = split_pntask(k)
177 if (verbose):
178 full_path_old = d_old.get(k).get("path")
179 full_path_new = d_new.get(k).get("path")
180 # No counter since it is not ready here
181 if sigdata_re.match(full_path_old) and sigdata_re.match(full_path_new):
182 output = bb.siggen.compare_sigfiles(full_path_old, full_path_new)
183 if output:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500184 print("\n=== The verbose changes of %s.%s:" % (pn, task))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500185 print('\n'.join(output))
186 else:
187 # Format the output, the format is:
188 # {pn: task1, task2, ...}
189 if pn in depchanged:
190 depchanged[pn] = "%s %s" % (depchanged.get(pn), task)
191 else:
192 depchanged[pn] = task
193
194 if len(depchanged) > 0:
195 print("\n=== Dependencies changed: (%s tasks)" % counter)
196 for k in depchanged.keys():
197 print(" %s: %s" % (k, depchanged[k]))
198
199 return counter
200
201
202def main():
203 """
204 Print what will be done between the current and last builds:
205 1) Run "STAMPS_DIR=<path> bitbake -S recipe" to re-generate the stamps
206 2) Figure out what are newly added and changed, can't figure out
207 what are removed since we can't know the previous stamps
208 clearly, for example, if there are several builds, we can't know
209 which stamps the last build has used exactly.
210 3) Use bb.siggen.compare_sigfiles to diff the old and new stamps
211 """
212
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600213 parser = argparse_oe.ArgumentParser(usage = """%(prog)s [options] [package ...]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500214print what will be done between the current and last builds, for example:
215
216 $ bitbake core-image-sato
217 # Edit the recipes
218 $ bitbake-whatchanged core-image-sato
219
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500220The changes will be printed.
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500221
222Note:
223 The amount of tasks is not accurate when the task is "do_build" since
224 it usually depends on other tasks.
225 The "nostamp" task is not included.
226"""
227)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600228 parser.add_argument("recipe", help="recipe to check")
229 parser.add_argument("-v", "--verbose", help = "print the verbose changes", action = "store_true")
230 args = parser.parse_args()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500231
232 # Get the STAMPS_DIR
233 print("Figuring out the STAMPS_DIR ...")
234 cmdline = "bitbake -e | sed -ne 's/^STAMPS_DIR=\"\(.*\)\"/\\1/p'"
235 try:
236 stampsdir, err = bb.process.run(cmdline)
237 except:
238 raise
239 if not stampsdir:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600240 print("ERROR: No STAMPS_DIR found for '%s'" % args.recipe, file=sys.stderr)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500241 return 2
242 stampsdir = stampsdir.rstrip("\n")
243 if not os.path.isdir(stampsdir):
244 print("ERROR: stamps directory \"%s\" not found!" % stampsdir, file=sys.stderr)
245 return 2
246
247 # The new stamps dir
248 new_stampsdir = stampsdir + ".bbs"
249 if os.path.exists(new_stampsdir):
250 print("ERROR: %s already exists!" % new_stampsdir, file=sys.stderr)
251 return 2
252
253 try:
254 # Generate the new stamps dir
255 print("Generating the new stamps ... (need several minutes)")
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600256 cmdline = "STAMPS_DIR=%s bitbake -S none %s" % (new_stampsdir, args.recipe)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500257 # FIXME
258 # The "bitbake -S" may fail, not fatal error, the stamps will still
259 # be generated, this might be a bug of "bitbake -S".
260 try:
261 bb.process.run(cmdline)
262 except Exception as exc:
263 print(exc)
264
265 # The dict for the new and old stamps.
266 old_dict = gen_dict(stampsdir)
267 new_dict = gen_dict(new_stampsdir)
268
269 # Remove the same one from both stamps.
270 cnt_unchanged = 0
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600271 for k in list(new_dict.keys()):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500272 if k in old_dict:
273 cnt_unchanged += 1
274 del(new_dict[k])
275 del(old_dict[k])
276
277 # Re-construct the dict to easily find out what is added or changed.
278 # The dict format is:
279 # {pn_task: {pv: PV, pr: PR, path: PATH}}
280 new_recon = recon_dict(new_dict)
281 old_recon = recon_dict(old_dict)
282
283 del new_dict
284 del old_dict
285
286 # Figure out what are changed, the new_recon would be changed
287 # by the print_xxx function.
288 # Newly added
289 cnt_added = print_added(new_recon, old_recon)
290
291 # PV (including PE) and PR changed
292 # Let the bb.siggen handle them if verbose
293 cnt_rv = {}
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600294 if not args.verbose:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500295 for i in ('pv', 'pr'):
296 cnt_rv[i] = print_vrchanged(new_recon, old_recon, i)
297
298 # Dependencies changed (use bitbake-diffsigs)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600299 cnt_dep = print_depchanged(new_recon, old_recon, args.verbose)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500300
301 total_changed = cnt_added + (cnt_rv.get('pv') or 0) + (cnt_rv.get('pr') or 0) + cnt_dep
302
303 print("\n=== Summary: (%s changed, %s unchanged)" % (total_changed, cnt_unchanged))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600304 if args.verbose:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500305 print("Newly added: %s\nDependencies changed: %s\n" % \
306 (cnt_added, cnt_dep))
307 else:
308 print("Newly added: %s\nPV changed: %s\nPR changed: %s\nDependencies changed: %s\n" % \
309 (cnt_added, cnt_rv.get('pv') or 0, cnt_rv.get('pr') or 0, cnt_dep))
310 except:
311 print("ERROR occurred!")
312 raise
313 finally:
314 # Remove the newly generated stamps dir
315 if os.path.exists(new_stampsdir):
316 print("Removing the newly generated stamps dir ...")
317 shutil.rmtree(new_stampsdir)
318
319if __name__ == "__main__":
320 sys.exit(main())