Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 2 | # ex:ts=4:sw=4:sts=4:et |
| 3 | # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- |
| 4 | |
| 5 | # Copyright (c) 2013 Wind River Systems, Inc. |
| 6 | # |
| 7 | # This program is free software; you can redistribute it and/or modify |
| 8 | # it under the terms of the GNU General Public License version 2 as |
| 9 | # published by the Free Software Foundation. |
| 10 | # |
| 11 | # This program is distributed in the hope that it will be useful, |
| 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. |
| 14 | # See the GNU General Public License for more details. |
| 15 | # |
| 16 | # You should have received a copy of the GNU General Public License |
| 17 | # along with this program; if not, write to the Free Software |
| 18 | # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA |
| 19 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 20 | import os |
| 21 | import sys |
| 22 | import getopt |
| 23 | import shutil |
| 24 | import re |
| 25 | import warnings |
| 26 | import subprocess |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 27 | import argparse |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 28 | |
| 29 | scripts_path = os.path.abspath(os.path.dirname(os.path.abspath(sys.argv[0]))) |
| 30 | lib_path = scripts_path + '/lib' |
| 31 | sys.path = sys.path + [lib_path] |
| 32 | |
| 33 | import scriptpath |
| 34 | |
| 35 | # Figure out where is the bitbake/lib/bb since we need bb.siggen and bb.process |
| 36 | bitbakepath = scriptpath.add_bitbake_lib_path() |
| 37 | if not bitbakepath: |
| 38 | sys.stderr.write("Unable to find bitbake by searching parent directory of this script or PATH\n") |
| 39 | sys.exit(1) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 40 | scriptpath.add_oe_lib_path() |
| 41 | import argparse_oe |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 42 | |
| 43 | import bb.siggen |
| 44 | import bb.process |
| 45 | |
| 46 | # Match the stamp's filename |
| 47 | # group(1): PE_PV (may no PE) |
| 48 | # group(2): PR |
| 49 | # group(3): TASK |
| 50 | # group(4): HASH |
| 51 | stamp_re = re.compile("(?P<pv>.*)-(?P<pr>r\d+)\.(?P<task>do_\w+)\.(?P<hash>[^\.]*)") |
| 52 | sigdata_re = re.compile(".*\.sigdata\..*") |
| 53 | |
| 54 | def gen_dict(stamps): |
| 55 | """ |
| 56 | Generate the dict from the stamps dir. |
| 57 | The output dict format is: |
| 58 | {fake_f: {pn: PN, pv: PV, pr: PR, task: TASK, path: PATH}} |
| 59 | Where: |
| 60 | fake_f: pv + task + hash |
| 61 | path: the path to the stamp file |
| 62 | """ |
| 63 | # The member of the sub dict (A "path" will be appended below) |
| 64 | sub_mem = ("pv", "pr", "task") |
| 65 | d = {} |
| 66 | for dirpath, _, files in os.walk(stamps): |
| 67 | for f in files: |
| 68 | # The "bitbake -S" would generate ".sigdata", but no "_setscene". |
| 69 | fake_f = re.sub('_setscene.', '.', f) |
| 70 | fake_f = re.sub('.sigdata', '', fake_f) |
| 71 | subdict = {} |
| 72 | tmp = stamp_re.match(fake_f) |
| 73 | if tmp: |
| 74 | for i in sub_mem: |
| 75 | subdict[i] = tmp.group(i) |
| 76 | if len(subdict) != 0: |
| 77 | pn = os.path.basename(dirpath) |
| 78 | subdict['pn'] = pn |
| 79 | # The path will be used by os.stat() and bb.siggen |
| 80 | subdict['path'] = dirpath + "/" + f |
| 81 | fake_f = tmp.group('pv') + tmp.group('task') + tmp.group('hash') |
| 82 | d[fake_f] = subdict |
| 83 | return d |
| 84 | |
| 85 | # Re-construct the dict |
| 86 | def recon_dict(dict_in): |
| 87 | """ |
| 88 | The output dict format is: |
| 89 | {pn_task: {pv: PV, pr: PR, path: PATH}} |
| 90 | """ |
| 91 | dict_out = {} |
| 92 | for k in dict_in.keys(): |
| 93 | subdict = {} |
| 94 | # The key |
| 95 | pn_task = "%s_%s" % (dict_in.get(k).get('pn'), dict_in.get(k).get('task')) |
| 96 | # If more than one stamps are found, use the latest one. |
| 97 | if pn_task in dict_out: |
| 98 | full_path_pre = dict_out.get(pn_task).get('path') |
| 99 | full_path_cur = dict_in.get(k).get('path') |
| 100 | if os.stat(full_path_pre).st_mtime > os.stat(full_path_cur).st_mtime: |
| 101 | continue |
| 102 | subdict['pv'] = dict_in.get(k).get('pv') |
| 103 | subdict['pr'] = dict_in.get(k).get('pr') |
| 104 | subdict['path'] = dict_in.get(k).get('path') |
| 105 | dict_out[pn_task] = subdict |
| 106 | |
| 107 | return dict_out |
| 108 | |
| 109 | def split_pntask(s): |
| 110 | """ |
| 111 | Split the pn_task in to (pn, task) and return it |
| 112 | """ |
| 113 | tmp = re.match("(.*)_(do_.*)", s) |
| 114 | return (tmp.group(1), tmp.group(2)) |
| 115 | |
| 116 | |
| 117 | def print_added(d_new = None, d_old = None): |
| 118 | """ |
| 119 | Print the newly added tasks |
| 120 | """ |
| 121 | added = {} |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 122 | for k in list(d_new.keys()): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 123 | if k not in d_old: |
| 124 | # Add the new one to added dict, and remove it from |
| 125 | # d_new, so the remaining ones are the changed ones |
| 126 | added[k] = d_new.get(k) |
| 127 | del(d_new[k]) |
| 128 | |
| 129 | if not added: |
| 130 | return 0 |
| 131 | |
| 132 | # Format the output, the dict format is: |
| 133 | # {pn: task1, task2 ...} |
| 134 | added_format = {} |
| 135 | counter = 0 |
| 136 | for k in added.keys(): |
| 137 | pn, task = split_pntask(k) |
| 138 | if pn in added_format: |
| 139 | # Append the value |
| 140 | added_format[pn] = "%s %s" % (added_format.get(pn), task) |
| 141 | else: |
| 142 | added_format[pn] = task |
| 143 | counter += 1 |
| 144 | print("=== Newly added tasks: (%s tasks)" % counter) |
| 145 | for k in added_format.keys(): |
| 146 | print(" %s: %s" % (k, added_format.get(k))) |
| 147 | |
| 148 | return counter |
| 149 | |
| 150 | def print_vrchanged(d_new = None, d_old = None, vr = None): |
| 151 | """ |
| 152 | Print the pv or pr changed tasks. |
| 153 | The arg "vr" is "pv" or "pr" |
| 154 | """ |
| 155 | pvchanged = {} |
| 156 | counter = 0 |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 157 | for k in list(d_new.keys()): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 158 | if d_new.get(k).get(vr) != d_old.get(k).get(vr): |
| 159 | counter += 1 |
| 160 | pn, task = split_pntask(k) |
| 161 | if pn not in pvchanged: |
| 162 | # Format the output, we only print pn (no task) since |
| 163 | # all the tasks would be changed when pn or pr changed, |
| 164 | # the dict format is: |
| 165 | # {pn: pv/pr_old -> pv/pr_new} |
| 166 | pvchanged[pn] = "%s -> %s" % (d_old.get(k).get(vr), d_new.get(k).get(vr)) |
| 167 | del(d_new[k]) |
| 168 | |
| 169 | if not pvchanged: |
| 170 | return 0 |
| 171 | |
| 172 | print("\n=== %s changed: (%s tasks)" % (vr.upper(), counter)) |
| 173 | for k in pvchanged.keys(): |
| 174 | print(" %s: %s" % (k, pvchanged.get(k))) |
| 175 | |
| 176 | return counter |
| 177 | |
| 178 | def print_depchanged(d_new = None, d_old = None, verbose = False): |
| 179 | """ |
| 180 | Print the dependency changes |
| 181 | """ |
| 182 | depchanged = {} |
| 183 | counter = 0 |
| 184 | for k in d_new.keys(): |
| 185 | counter += 1 |
| 186 | pn, task = split_pntask(k) |
| 187 | if (verbose): |
| 188 | full_path_old = d_old.get(k).get("path") |
| 189 | full_path_new = d_new.get(k).get("path") |
| 190 | # No counter since it is not ready here |
| 191 | if sigdata_re.match(full_path_old) and sigdata_re.match(full_path_new): |
| 192 | output = bb.siggen.compare_sigfiles(full_path_old, full_path_new) |
| 193 | if output: |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 194 | print("\n=== The verbose changes of %s.%s:" % (pn, task)) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 195 | print('\n'.join(output)) |
| 196 | else: |
| 197 | # Format the output, the format is: |
| 198 | # {pn: task1, task2, ...} |
| 199 | if pn in depchanged: |
| 200 | depchanged[pn] = "%s %s" % (depchanged.get(pn), task) |
| 201 | else: |
| 202 | depchanged[pn] = task |
| 203 | |
| 204 | if len(depchanged) > 0: |
| 205 | print("\n=== Dependencies changed: (%s tasks)" % counter) |
| 206 | for k in depchanged.keys(): |
| 207 | print(" %s: %s" % (k, depchanged[k])) |
| 208 | |
| 209 | return counter |
| 210 | |
| 211 | |
| 212 | def main(): |
| 213 | """ |
| 214 | Print what will be done between the current and last builds: |
| 215 | 1) Run "STAMPS_DIR=<path> bitbake -S recipe" to re-generate the stamps |
| 216 | 2) Figure out what are newly added and changed, can't figure out |
| 217 | what are removed since we can't know the previous stamps |
| 218 | clearly, for example, if there are several builds, we can't know |
| 219 | which stamps the last build has used exactly. |
| 220 | 3) Use bb.siggen.compare_sigfiles to diff the old and new stamps |
| 221 | """ |
| 222 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 223 | parser = argparse_oe.ArgumentParser(usage = """%(prog)s [options] [package ...] |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 224 | print what will be done between the current and last builds, for example: |
| 225 | |
| 226 | $ bitbake core-image-sato |
| 227 | # Edit the recipes |
| 228 | $ bitbake-whatchanged core-image-sato |
| 229 | |
| 230 | The changes will be printed" |
| 231 | |
| 232 | Note: |
| 233 | The amount of tasks is not accurate when the task is "do_build" since |
| 234 | it usually depends on other tasks. |
| 235 | The "nostamp" task is not included. |
| 236 | """ |
| 237 | ) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 238 | parser.add_argument("recipe", help="recipe to check") |
| 239 | parser.add_argument("-v", "--verbose", help = "print the verbose changes", action = "store_true") |
| 240 | args = parser.parse_args() |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 241 | |
| 242 | # Get the STAMPS_DIR |
| 243 | print("Figuring out the STAMPS_DIR ...") |
| 244 | cmdline = "bitbake -e | sed -ne 's/^STAMPS_DIR=\"\(.*\)\"/\\1/p'" |
| 245 | try: |
| 246 | stampsdir, err = bb.process.run(cmdline) |
| 247 | except: |
| 248 | raise |
| 249 | if not stampsdir: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 250 | print("ERROR: No STAMPS_DIR found for '%s'" % args.recipe, file=sys.stderr) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 251 | return 2 |
| 252 | stampsdir = stampsdir.rstrip("\n") |
| 253 | if not os.path.isdir(stampsdir): |
| 254 | print("ERROR: stamps directory \"%s\" not found!" % stampsdir, file=sys.stderr) |
| 255 | return 2 |
| 256 | |
| 257 | # The new stamps dir |
| 258 | new_stampsdir = stampsdir + ".bbs" |
| 259 | if os.path.exists(new_stampsdir): |
| 260 | print("ERROR: %s already exists!" % new_stampsdir, file=sys.stderr) |
| 261 | return 2 |
| 262 | |
| 263 | try: |
| 264 | # Generate the new stamps dir |
| 265 | print("Generating the new stamps ... (need several minutes)") |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 266 | cmdline = "STAMPS_DIR=%s bitbake -S none %s" % (new_stampsdir, args.recipe) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 267 | # FIXME |
| 268 | # The "bitbake -S" may fail, not fatal error, the stamps will still |
| 269 | # be generated, this might be a bug of "bitbake -S". |
| 270 | try: |
| 271 | bb.process.run(cmdline) |
| 272 | except Exception as exc: |
| 273 | print(exc) |
| 274 | |
| 275 | # The dict for the new and old stamps. |
| 276 | old_dict = gen_dict(stampsdir) |
| 277 | new_dict = gen_dict(new_stampsdir) |
| 278 | |
| 279 | # Remove the same one from both stamps. |
| 280 | cnt_unchanged = 0 |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 281 | for k in list(new_dict.keys()): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 282 | if k in old_dict: |
| 283 | cnt_unchanged += 1 |
| 284 | del(new_dict[k]) |
| 285 | del(old_dict[k]) |
| 286 | |
| 287 | # Re-construct the dict to easily find out what is added or changed. |
| 288 | # The dict format is: |
| 289 | # {pn_task: {pv: PV, pr: PR, path: PATH}} |
| 290 | new_recon = recon_dict(new_dict) |
| 291 | old_recon = recon_dict(old_dict) |
| 292 | |
| 293 | del new_dict |
| 294 | del old_dict |
| 295 | |
| 296 | # Figure out what are changed, the new_recon would be changed |
| 297 | # by the print_xxx function. |
| 298 | # Newly added |
| 299 | cnt_added = print_added(new_recon, old_recon) |
| 300 | |
| 301 | # PV (including PE) and PR changed |
| 302 | # Let the bb.siggen handle them if verbose |
| 303 | cnt_rv = {} |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 304 | if not args.verbose: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 305 | for i in ('pv', 'pr'): |
| 306 | cnt_rv[i] = print_vrchanged(new_recon, old_recon, i) |
| 307 | |
| 308 | # Dependencies changed (use bitbake-diffsigs) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 309 | cnt_dep = print_depchanged(new_recon, old_recon, args.verbose) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 310 | |
| 311 | total_changed = cnt_added + (cnt_rv.get('pv') or 0) + (cnt_rv.get('pr') or 0) + cnt_dep |
| 312 | |
| 313 | print("\n=== Summary: (%s changed, %s unchanged)" % (total_changed, cnt_unchanged)) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 314 | if args.verbose: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 315 | print("Newly added: %s\nDependencies changed: %s\n" % \ |
| 316 | (cnt_added, cnt_dep)) |
| 317 | else: |
| 318 | print("Newly added: %s\nPV changed: %s\nPR changed: %s\nDependencies changed: %s\n" % \ |
| 319 | (cnt_added, cnt_rv.get('pv') or 0, cnt_rv.get('pr') or 0, cnt_dep)) |
| 320 | except: |
| 321 | print("ERROR occurred!") |
| 322 | raise |
| 323 | finally: |
| 324 | # Remove the newly generated stamps dir |
| 325 | if os.path.exists(new_stampsdir): |
| 326 | print("Removing the newly generated stamps dir ...") |
| 327 | shutil.rmtree(new_stampsdir) |
| 328 | |
| 329 | if __name__ == "__main__": |
| 330 | sys.exit(main()) |