blob: 21bde7e15618c28bd423da25b67e3a1ff7ec316d [file] [log] [blame]
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001#!/usr/bin/python3
2#
3# Examine build performance test results
4#
5# Copyright (c) 2017, Intel Corporation.
6#
Brad Bishopc342db32019-05-15 21:57:59 -04007# SPDX-License-Identifier: GPL-2.0-only
Brad Bishop6e60e8b2018-02-01 10:27:11 -05008#
Brad Bishopc342db32019-05-15 21:57:59 -04009
Brad Bishop6e60e8b2018-02-01 10:27:11 -050010import argparse
11import json
12import logging
13import os
14import re
15import sys
16from collections import namedtuple, OrderedDict
17from operator import attrgetter
18from xml.etree import ElementTree as ET
19
20# Import oe libs
21scripts_path = os.path.dirname(os.path.realpath(__file__))
22sys.path.append(os.path.join(scripts_path, 'lib'))
23import scriptpath
24from build_perf import print_table
25from build_perf.report import (metadata_xml_to_json, results_xml_to_json,
Brad Bishopd7bf8c12018-02-25 22:55:05 -050026 aggregate_data, aggregate_metadata, measurement_stats,
27 AggregateTestData)
Brad Bishop6e60e8b2018-02-01 10:27:11 -050028from build_perf import html
Brad Bishopd7bf8c12018-02-25 22:55:05 -050029from buildstats import BuildStats, diff_buildstats, BSVerDiff
Brad Bishop6e60e8b2018-02-01 10:27:11 -050030
31scriptpath.add_oe_lib_path()
32
Brad Bishopd7bf8c12018-02-25 22:55:05 -050033from oeqa.utils.git import GitRepo, GitError
Andrew Geissler99467da2019-02-25 18:54:23 -060034import oeqa.utils.gitarchive as gitarchive
Brad Bishop6e60e8b2018-02-01 10:27:11 -050035
36
37# Setup logging
38logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
39log = logging.getLogger('oe-build-perf-report')
40
Brad Bishopd7bf8c12018-02-25 22:55:05 -050041def list_test_revs(repo, tag_name, verbosity, **kwargs):
Brad Bishop6e60e8b2018-02-01 10:27:11 -050042 """Get list of all tested revisions"""
Brad Bishopd7bf8c12018-02-25 22:55:05 -050043 valid_kwargs = dict([(k, v) for k, v in kwargs.items() if v is not None])
44
Andrew Geissler99467da2019-02-25 18:54:23 -060045 fields, revs = gitarchive.get_test_runs(log, repo, tag_name, **valid_kwargs)
Brad Bishop6e60e8b2018-02-01 10:27:11 -050046 ignore_fields = ['tag_number']
Brad Bishopd7bf8c12018-02-25 22:55:05 -050047 if verbosity < 2:
48 extra_fields = ['COMMITS', 'TEST RUNS']
49 ignore_fields.extend(['commit_number', 'commit'])
50 else:
51 extra_fields = ['TEST RUNS']
52
Brad Bishop6e60e8b2018-02-01 10:27:11 -050053 print_fields = [i for i, f in enumerate(fields) if f not in ignore_fields]
54
55 # Sort revs
Brad Bishopd7bf8c12018-02-25 22:55:05 -050056 rows = [[fields[i].upper() for i in print_fields] + extra_fields]
57
58 prev = [''] * len(print_fields)
59 prev_commit = None
60 commit_cnt = 0
61 commit_field = fields.index('commit')
Brad Bishop6e60e8b2018-02-01 10:27:11 -050062 for rev in revs:
63 # Only use fields that we want to print
Brad Bishopd7bf8c12018-02-25 22:55:05 -050064 cols = [rev[i] for i in print_fields]
Brad Bishop6e60e8b2018-02-01 10:27:11 -050065
Brad Bishopd7bf8c12018-02-25 22:55:05 -050066
67 if cols != prev:
68 commit_cnt = 1
69 test_run_cnt = 1
70 new_row = [''] * (len(print_fields) + len(extra_fields))
71
Brad Bishop6e60e8b2018-02-01 10:27:11 -050072 for i in print_fields:
Brad Bishopd7bf8c12018-02-25 22:55:05 -050073 if cols[i] != prev[i]:
Brad Bishop6e60e8b2018-02-01 10:27:11 -050074 break
Brad Bishopd7bf8c12018-02-25 22:55:05 -050075 new_row[i:-len(extra_fields)] = cols[i:]
Brad Bishop6e60e8b2018-02-01 10:27:11 -050076 rows.append(new_row)
77 else:
Brad Bishopd7bf8c12018-02-25 22:55:05 -050078 if rev[commit_field] != prev_commit:
79 commit_cnt += 1
80 test_run_cnt += 1
81
82 if verbosity < 2:
83 new_row[-2] = commit_cnt
84 new_row[-1] = test_run_cnt
85 prev = cols
86 prev_commit = rev[commit_field]
Brad Bishop6e60e8b2018-02-01 10:27:11 -050087
88 print_table(rows)
89
Brad Bishop6e60e8b2018-02-01 10:27:11 -050090def is_xml_format(repo, commit):
91 """Check if the commit contains xml (or json) data"""
92 if repo.rev_parse(commit + ':results.xml'):
93 log.debug("Detected report in xml format in %s", commit)
94 return True
95 else:
96 log.debug("No xml report in %s, assuming json formatted results", commit)
97 return False
98
99def read_results(repo, tags, xml=True):
100 """Read result files from repo"""
101
102 def parse_xml_stream(data):
103 """Parse multiple concatenated XML objects"""
104 objs = []
105 xml_d = ""
106 for line in data.splitlines():
107 if xml_d and line.startswith('<?xml version='):
108 objs.append(ET.fromstring(xml_d))
109 xml_d = line
110 else:
111 xml_d += line
112 objs.append(ET.fromstring(xml_d))
113 return objs
114
115 def parse_json_stream(data):
116 """Parse multiple concatenated JSON objects"""
117 objs = []
118 json_d = ""
119 for line in data.splitlines():
120 if line == '}{':
121 json_d += '}'
122 objs.append(json.loads(json_d, object_pairs_hook=OrderedDict))
123 json_d = '{'
124 else:
125 json_d += line
126 objs.append(json.loads(json_d, object_pairs_hook=OrderedDict))
127 return objs
128
129 num_revs = len(tags)
130
131 # Optimize by reading all data with one git command
132 log.debug("Loading raw result data from %d tags, %s...", num_revs, tags[0])
133 if xml:
134 git_objs = [tag + ':metadata.xml' for tag in tags] + [tag + ':results.xml' for tag in tags]
135 data = parse_xml_stream(repo.run_cmd(['show'] + git_objs + ['--']))
136 return ([metadata_xml_to_json(e) for e in data[0:num_revs]],
137 [results_xml_to_json(e) for e in data[num_revs:]])
138 else:
139 git_objs = [tag + ':metadata.json' for tag in tags] + [tag + ':results.json' for tag in tags]
140 data = parse_json_stream(repo.run_cmd(['show'] + git_objs + ['--']))
141 return data[0:num_revs], data[num_revs:]
142
143
144def get_data_item(data, key):
145 """Nested getitem lookup"""
146 for k in key.split('.'):
147 data = data[k]
148 return data
149
150
151def metadata_diff(metadata_l, metadata_r):
152 """Prepare a metadata diff for printing"""
153 keys = [('Hostname', 'hostname', 'hostname'),
154 ('Branch', 'branch', 'layers.meta.branch'),
155 ('Commit number', 'commit_num', 'layers.meta.commit_count'),
156 ('Commit', 'commit', 'layers.meta.commit'),
157 ('Number of test runs', 'testrun_count', 'testrun_count')
158 ]
159
160 def _metadata_diff(key):
161 """Diff metadata from two test reports"""
162 try:
163 val1 = get_data_item(metadata_l, key)
164 except KeyError:
165 val1 = '(N/A)'
166 try:
167 val2 = get_data_item(metadata_r, key)
168 except KeyError:
169 val2 = '(N/A)'
170 return val1, val2
171
172 metadata = OrderedDict()
173 for title, key, key_json in keys:
174 value_l, value_r = _metadata_diff(key_json)
175 metadata[key] = {'title': title,
176 'value_old': value_l,
177 'value': value_r}
178 return metadata
179
180
181def print_diff_report(metadata_l, data_l, metadata_r, data_r):
182 """Print differences between two data sets"""
183
184 # First, print general metadata
185 print("\nTEST METADATA:\n==============")
186 meta_diff = metadata_diff(metadata_l, metadata_r)
187 rows = []
188 row_fmt = ['{:{wid}} ', '{:<{wid}} ', '{:<{wid}}']
189 rows = [['', 'CURRENT COMMIT', 'COMPARING WITH']]
190 for key, val in meta_diff.items():
191 # Shorten commit hashes
192 if key == 'commit':
193 rows.append([val['title'] + ':', val['value'][:20], val['value_old'][:20]])
194 else:
195 rows.append([val['title'] + ':', val['value'], val['value_old']])
196 print_table(rows, row_fmt)
197
198
199 # Print test results
200 print("\nTEST RESULTS:\n=============")
201
202 tests = list(data_l['tests'].keys())
203 # Append tests that are only present in 'right' set
204 tests += [t for t in list(data_r['tests'].keys()) if t not in tests]
205
206 # Prepare data to be printed
207 rows = []
208 row_fmt = ['{:8}', '{:{wid}}', '{:{wid}}', ' {:>{wid}}', ' {:{wid}} ', '{:{wid}}',
209 ' {:>{wid}}', ' {:>{wid}}']
210 num_cols = len(row_fmt)
211 for test in tests:
212 test_l = data_l['tests'][test] if test in data_l['tests'] else None
213 test_r = data_r['tests'][test] if test in data_r['tests'] else None
214 pref = ' '
215 if test_l is None:
216 pref = '+'
217 elif test_r is None:
218 pref = '-'
219 descr = test_l['description'] if test_l else test_r['description']
220 heading = "{} {}: {}".format(pref, test, descr)
221
222 rows.append([heading])
223
224 # Generate the list of measurements
225 meas_l = test_l['measurements'] if test_l else {}
226 meas_r = test_r['measurements'] if test_r else {}
227 measurements = list(meas_l.keys())
228 measurements += [m for m in list(meas_r.keys()) if m not in measurements]
229
230 for meas in measurements:
231 m_pref = ' '
232 if meas in meas_l:
233 stats_l = measurement_stats(meas_l[meas], 'l.')
234 else:
235 stats_l = measurement_stats(None, 'l.')
236 m_pref = '+'
237 if meas in meas_r:
238 stats_r = measurement_stats(meas_r[meas], 'r.')
239 else:
240 stats_r = measurement_stats(None, 'r.')
241 m_pref = '-'
242 stats = stats_l.copy()
243 stats.update(stats_r)
244
245 absdiff = stats['val_cls'](stats['r.mean'] - stats['l.mean'])
246 reldiff = "{:+.1f} %".format(absdiff * 100 / stats['l.mean'])
247 if stats['r.mean'] > stats['l.mean']:
248 absdiff = '+' + str(absdiff)
249 else:
250 absdiff = str(absdiff)
251 rows.append(['', m_pref, stats['name'] + ' ' + stats['quantity'],
252 str(stats['l.mean']), '->', str(stats['r.mean']),
253 absdiff, reldiff])
254 rows.append([''] * num_cols)
255
256 print_table(rows, row_fmt)
257
258 print()
259
260
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500261class BSSummary(object):
262 def __init__(self, bs1, bs2):
263 self.tasks = {'count': bs2.num_tasks,
264 'change': '{:+d}'.format(bs2.num_tasks - bs1.num_tasks)}
265 self.top_consumer = None
266 self.top_decrease = None
267 self.top_increase = None
268 self.ver_diff = OrderedDict()
269
270 tasks_diff = diff_buildstats(bs1, bs2, 'cputime')
271
272 # Get top consumers of resources
273 tasks_diff = sorted(tasks_diff, key=attrgetter('value2'))
274 self.top_consumer = tasks_diff[-5:]
275
276 # Get biggest increase and decrease in resource usage
277 tasks_diff = sorted(tasks_diff, key=attrgetter('absdiff'))
278 self.top_decrease = tasks_diff[0:5]
279 self.top_increase = tasks_diff[-5:]
280
281 # Compare recipe versions and prepare data for display
282 ver_diff = BSVerDiff(bs1, bs2)
283 if ver_diff:
284 if ver_diff.new:
285 self.ver_diff['New recipes'] = [(n, r.evr) for n, r in ver_diff.new.items()]
286 if ver_diff.dropped:
287 self.ver_diff['Dropped recipes'] = [(n, r.evr) for n, r in ver_diff.dropped.items()]
288 if ver_diff.echanged:
289 self.ver_diff['Epoch changed'] = [(n, "{} &rarr; {}".format(r.left.evr, r.right.evr)) for n, r in ver_diff.echanged.items()]
290 if ver_diff.vchanged:
291 self.ver_diff['Version changed'] = [(n, "{} &rarr; {}".format(r.left.version, r.right.version)) for n, r in ver_diff.vchanged.items()]
292 if ver_diff.rchanged:
293 self.ver_diff['Revision changed'] = [(n, "{} &rarr; {}".format(r.left.evr, r.right.evr)) for n, r in ver_diff.rchanged.items()]
294
295
296def print_html_report(data, id_comp, buildstats):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500297 """Print report in html format"""
298 # Handle metadata
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500299 metadata = metadata_diff(data[id_comp].metadata, data[-1].metadata)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500300
301 # Generate list of tests
302 tests = []
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500303 for test in data[-1].results['tests'].keys():
304 test_r = data[-1].results['tests'][test]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500305 new_test = {'name': test_r['name'],
306 'description': test_r['description'],
307 'status': test_r['status'],
308 'measurements': [],
309 'err_type': test_r.get('err_type'),
310 }
311 # Limit length of err output shown
312 if 'message' in test_r:
313 lines = test_r['message'].splitlines()
314 if len(lines) > 20:
315 new_test['message'] = '...\n' + '\n'.join(lines[-20:])
316 else:
317 new_test['message'] = test_r['message']
318
319
320 # Generate the list of measurements
321 for meas in test_r['measurements'].keys():
322 meas_r = test_r['measurements'][meas]
323 meas_type = 'time' if meas_r['type'] == 'sysres' else 'size'
324 new_meas = {'name': meas_r['name'],
325 'legend': meas_r['legend'],
326 'description': meas_r['name'] + ' ' + meas_type,
327 }
328 samples = []
329
330 # Run through all revisions in our data
331 for meta, test_data in data:
332 if (not test in test_data['tests'] or
333 not meas in test_data['tests'][test]['measurements']):
334 samples.append(measurement_stats(None))
335 continue
336 test_i = test_data['tests'][test]
337 meas_i = test_i['measurements'][meas]
338 commit_num = get_data_item(meta, 'layers.meta.commit_count')
339 samples.append(measurement_stats(meas_i))
340 samples[-1]['commit_num'] = commit_num
341
342 absdiff = samples[-1]['val_cls'](samples[-1]['mean'] - samples[id_comp]['mean'])
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800343 reldiff = absdiff * 100 / samples[id_comp]['mean']
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500344 new_meas['absdiff'] = absdiff
345 new_meas['absdiff_str'] = str(absdiff) if absdiff < 0 else '+' + str(absdiff)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800346 new_meas['reldiff'] = reldiff
347 new_meas['reldiff_str'] = "{:+.1f} %".format(reldiff)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500348 new_meas['samples'] = samples
349 new_meas['value'] = samples[-1]
350 new_meas['value_type'] = samples[-1]['val_cls']
351
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500352 # Compare buildstats
353 bs_key = test + '.' + meas
Andrew Geissler99467da2019-02-25 18:54:23 -0600354 rev = str(metadata['commit_num']['value'])
355 comp_rev = str(metadata['commit_num']['value_old'])
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500356 if (rev in buildstats and bs_key in buildstats[rev] and
357 comp_rev in buildstats and bs_key in buildstats[comp_rev]):
358 new_meas['buildstats'] = BSSummary(buildstats[comp_rev][bs_key],
359 buildstats[rev][bs_key])
360
361
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500362 new_test['measurements'].append(new_meas)
363 tests.append(new_test)
364
365 # Chart options
366 chart_opts = {'haxis': {'min': get_data_item(data[0][0], 'layers.meta.commit_count'),
367 'max': get_data_item(data[-1][0], 'layers.meta.commit_count')}
368 }
369
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500370 print(html.template.render(title="Build Perf Test Report",
371 metadata=metadata, test_data=tests,
372 chart_opts=chart_opts))
373
374
375def get_buildstats(repo, notes_ref, revs, outdir=None):
376 """Get the buildstats from git notes"""
377 full_ref = 'refs/notes/' + notes_ref
378 if not repo.rev_parse(full_ref):
379 log.error("No buildstats found, please try running "
380 "'git fetch origin %s:%s' to fetch them from the remote",
381 full_ref, full_ref)
382 return
383
384 missing = False
385 buildstats = {}
386 log.info("Parsing buildstats from 'refs/notes/%s'", notes_ref)
387 for rev in revs:
388 buildstats[rev.commit_number] = {}
389 log.debug('Dumping buildstats for %s (%s)', rev.commit_number,
390 rev.commit)
391 for tag in rev.tags:
392 log.debug(' %s', tag)
393 try:
394 bs_all = json.loads(repo.run_cmd(['notes', '--ref', notes_ref,
395 'show', tag + '^0']))
396 except GitError:
397 log.warning("Buildstats not found for %s", tag)
398 bs_all = {}
399 missing = True
400
401 for measurement, bs in bs_all.items():
402 # Write out onto disk
403 if outdir:
404 tag_base, run_id = tag.rsplit('/', 1)
405 tag_base = tag_base.replace('/', '_')
406 bs_dir = os.path.join(outdir, measurement, tag_base)
407 if not os.path.exists(bs_dir):
408 os.makedirs(bs_dir)
409 with open(os.path.join(bs_dir, run_id + '.json'), 'w') as f:
410 json.dump(bs, f, indent=2)
411
412 # Read buildstats into a dict
413 _bs = BuildStats.from_json(bs)
414 if measurement not in buildstats[rev.commit_number]:
415 buildstats[rev.commit_number][measurement] = _bs
416 else:
417 buildstats[rev.commit_number][measurement].aggregate(_bs)
418
419 if missing:
420 log.info("Buildstats were missing for some test runs, please "
421 "run 'git fetch origin %s:%s' and try again",
422 full_ref, full_ref)
423
424 return buildstats
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500425
426
427def auto_args(repo, args):
428 """Guess arguments, if not defined by the user"""
429 # Get the latest commit in the repo
430 log.debug("Guessing arguments from the latest commit")
431 msg = repo.run_cmd(['log', '-1', '--branches', '--remotes', '--format=%b'])
432 for line in msg.splitlines():
433 split = line.split(':', 1)
434 if len(split) != 2:
435 continue
436
437 key = split[0]
438 val = split[1].strip()
Andrew Geissler99467da2019-02-25 18:54:23 -0600439 if key == 'hostname' and not args.hostname:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500440 log.debug("Using hostname %s", val)
441 args.hostname = val
Andrew Geissler99467da2019-02-25 18:54:23 -0600442 elif key == 'branch' and not args.branch:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500443 log.debug("Using branch %s", val)
444 args.branch = val
445
446
447def parse_args(argv):
448 """Parse command line arguments"""
449 description = """
450Examine build performance test results from a Git repository"""
451 parser = argparse.ArgumentParser(
452 formatter_class=argparse.ArgumentDefaultsHelpFormatter,
453 description=description)
454
455 parser.add_argument('--debug', '-d', action='store_true',
456 help="Verbose logging")
457 parser.add_argument('--repo', '-r', required=True,
458 help="Results repository (local git clone)")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500459 parser.add_argument('--list', '-l', action='count',
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500460 help="List available test runs")
461 parser.add_argument('--html', action='store_true',
462 help="Generate report in html format")
463 group = parser.add_argument_group('Tag and revision')
464 group.add_argument('--tag-name', '-t',
465 default='{hostname}/{branch}/{machine}/{commit_number}-g{commit}/{tag_number}',
466 help="Tag name (pattern) for finding results")
467 group.add_argument('--hostname', '-H')
Andrew Geissler99467da2019-02-25 18:54:23 -0600468 group.add_argument('--branch', '-B', default='master', help="Branch to find commit in")
469 group.add_argument('--branch2', help="Branch to find comparision revisions in")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500470 group.add_argument('--machine', default='qemux86')
471 group.add_argument('--history-length', default=25, type=int,
472 help="Number of tested revisions to plot in html report")
473 group.add_argument('--commit',
474 help="Revision to search for")
475 group.add_argument('--commit-number',
476 help="Revision number to search for, redundant if "
477 "--commit is specified")
478 group.add_argument('--commit2',
479 help="Revision to compare with")
480 group.add_argument('--commit-number2',
481 help="Revision number to compare with, redundant if "
482 "--commit2 is specified")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500483 parser.add_argument('--dump-buildstats', nargs='?', const='.',
484 help="Dump buildstats of the tests")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500485
486 return parser.parse_args(argv)
487
488
489def main(argv=None):
490 """Script entry point"""
491 args = parse_args(argv)
492 if args.debug:
493 log.setLevel(logging.DEBUG)
494
495 repo = GitRepo(args.repo)
496
497 if args.list:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500498 list_test_revs(repo, args.tag_name, args.list, hostname=args.hostname)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500499 return 0
500
501 # Determine hostname which to use
502 if not args.hostname:
503 auto_args(repo, args)
504
Andrew Geissler99467da2019-02-25 18:54:23 -0600505 revs = gitarchive.get_test_revs(log, repo, args.tag_name, hostname=args.hostname,
506 branch=args.branch, machine=args.machine)
507 if args.branch2:
508 revs2 = gitarchive.get_test_revs(log, repo, args.tag_name, hostname=args.hostname,
509 branch=args.branch2, machine=args.machine)
510 if not len(revs2):
511 log.error("No revisions found to compare against")
512 return 1
513 if not len(revs):
514 log.error("No revision to report on found")
515 return 1
516 else:
517 if len(revs) < 2:
518 log.error("Only %d tester revisions found, unable to generate report" % len(revs))
519 return 1
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500520
521 # Pick revisions
522 if args.commit:
523 if args.commit_number:
524 log.warning("Ignoring --commit-number as --commit was specified")
Andrew Geissler99467da2019-02-25 18:54:23 -0600525 index1 = gitarchive.rev_find(revs, 'commit', args.commit)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500526 elif args.commit_number:
Andrew Geissler99467da2019-02-25 18:54:23 -0600527 index1 = gitarchive.rev_find(revs, 'commit_number', args.commit_number)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500528 else:
529 index1 = len(revs) - 1
530
Andrew Geissler99467da2019-02-25 18:54:23 -0600531 if args.branch2:
532 revs2.append(revs[index1])
533 index1 = len(revs2) - 1
534 revs = revs2
535
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500536 if args.commit2:
537 if args.commit_number2:
538 log.warning("Ignoring --commit-number2 as --commit2 was specified")
Andrew Geissler99467da2019-02-25 18:54:23 -0600539 index2 = gitarchive.rev_find(revs, 'commit', args.commit2)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500540 elif args.commit_number2:
Andrew Geissler99467da2019-02-25 18:54:23 -0600541 index2 = gitarchive.rev_find(revs, 'commit_number', args.commit_number2)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500542 else:
543 if index1 > 0:
544 index2 = index1 - 1
Andrew Geissler99467da2019-02-25 18:54:23 -0600545 # Find the closest matching commit number for comparision
546 # In future we could check the commit is a common ancestor and
547 # continue back if not but this good enough for now
548 while index2 > 0 and revs[index2].commit_number > revs[index1].commit_number:
549 index2 = index2 - 1
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500550 else:
551 log.error("Unable to determine the other commit, use "
552 "--commit2 or --commit-number2 to specify it")
553 return 1
554
555 index_l = min(index1, index2)
556 index_r = max(index1, index2)
557
558 rev_l = revs[index_l]
559 rev_r = revs[index_r]
560 log.debug("Using 'left' revision %s (%s), %s test runs:\n %s",
561 rev_l.commit_number, rev_l.commit, len(rev_l.tags),
562 '\n '.join(rev_l.tags))
563 log.debug("Using 'right' revision %s (%s), %s test runs:\n %s",
564 rev_r.commit_number, rev_r.commit, len(rev_r.tags),
565 '\n '.join(rev_r.tags))
566
567 # Check report format used in the repo (assume all reports in the same fmt)
568 xml = is_xml_format(repo, revs[index_r].tags[-1])
569
570 if args.html:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500571 index_0 = max(0, min(index_l, index_r - args.history_length))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500572 rev_range = range(index_0, index_r + 1)
573 else:
574 # We do not need range of commits for text report (no graphs)
575 index_0 = index_l
576 rev_range = (index_l, index_r)
577
578 # Read raw data
579 log.debug("Reading %d revisions, starting from %s (%s)",
580 len(rev_range), revs[index_0].commit_number, revs[index_0].commit)
581 raw_data = [read_results(repo, revs[i].tags, xml) for i in rev_range]
582
583 data = []
584 for raw_m, raw_d in raw_data:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500585 data.append(AggregateTestData(aggregate_metadata(raw_m),
586 aggregate_data(raw_d)))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500587
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500588 # Read buildstats only when needed
589 buildstats = None
590 if args.dump_buildstats or args.html:
591 outdir = 'oe-build-perf-buildstats' if args.dump_buildstats else None
592 notes_ref = 'buildstats/{}/{}/{}'.format(args.hostname, args.branch,
593 args.machine)
594 buildstats = get_buildstats(repo, notes_ref, [rev_l, rev_r], outdir)
595
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500596 # Print report
597 if not args.html:
Brad Bishop00111322018-04-01 22:23:53 -0400598 print_diff_report(data[0].metadata, data[0].results,
599 data[1].metadata, data[1].results)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500600 else:
Brad Bishop00111322018-04-01 22:23:53 -0400601 # Re-map 'left' list index to the data table where index_0 maps to 0
602 print_html_report(data, index_l - index_0, buildstats)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500603
604 return 0
605
606if __name__ == "__main__":
607 sys.exit(main())