blob: 0bd05f44ef0292aee204a064e70f6558293816de [file] [log] [blame]
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001#!/usr/bin/python3
2#
3# Examine build performance test results
4#
5# Copyright (c) 2017, Intel Corporation.
6#
7# This program is free software; you can redistribute it and/or modify it
8# under the terms and conditions of the GNU General Public License,
9# version 2, as published by the Free Software Foundation.
10#
11# This program is distributed in the hope it will be useful, but WITHOUT
12# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14# more details.
15#
16import argparse
17import json
18import logging
19import os
20import re
21import sys
22from collections import namedtuple, OrderedDict
23from operator import attrgetter
24from xml.etree import ElementTree as ET
25
26# Import oe libs
27scripts_path = os.path.dirname(os.path.realpath(__file__))
28sys.path.append(os.path.join(scripts_path, 'lib'))
29import scriptpath
30from build_perf import print_table
31from build_perf.report import (metadata_xml_to_json, results_xml_to_json,
Brad Bishopd7bf8c12018-02-25 22:55:05 -050032 aggregate_data, aggregate_metadata, measurement_stats,
33 AggregateTestData)
Brad Bishop6e60e8b2018-02-01 10:27:11 -050034from build_perf import html
Brad Bishopd7bf8c12018-02-25 22:55:05 -050035from buildstats import BuildStats, diff_buildstats, BSVerDiff
Brad Bishop6e60e8b2018-02-01 10:27:11 -050036
37scriptpath.add_oe_lib_path()
38
Brad Bishopd7bf8c12018-02-25 22:55:05 -050039from oeqa.utils.git import GitRepo, GitError
Brad Bishop6e60e8b2018-02-01 10:27:11 -050040
41
42# Setup logging
43logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
44log = logging.getLogger('oe-build-perf-report')
45
46
47# Container class for tester revisions
48TestedRev = namedtuple('TestedRev', 'commit commit_number tags')
49
50
51def get_test_runs(repo, tag_name, **kwargs):
52 """Get a sorted list of test runs, matching given pattern"""
53 # First, get field names from the tag name pattern
54 field_names = [m.group(1) for m in re.finditer(r'{(\w+)}', tag_name)]
55 undef_fields = [f for f in field_names if f not in kwargs.keys()]
56
57 # Fields for formatting tag name pattern
58 str_fields = dict([(f, '*') for f in field_names])
59 str_fields.update(kwargs)
60
61 # Get a list of all matching tags
62 tag_pattern = tag_name.format(**str_fields)
63 tags = repo.run_cmd(['tag', '-l', tag_pattern]).splitlines()
64 log.debug("Found %d tags matching pattern '%s'", len(tags), tag_pattern)
65
66 # Parse undefined fields from tag names
67 str_fields = dict([(f, r'(?P<{}>[\w\-.()]+)'.format(f)) for f in field_names])
68 str_fields['branch'] = r'(?P<branch>[\w\-.()/]+)'
69 str_fields['commit'] = '(?P<commit>[0-9a-f]{7,40})'
70 str_fields['commit_number'] = '(?P<commit_number>[0-9]{1,7})'
71 str_fields['tag_number'] = '(?P<tag_number>[0-9]{1,5})'
72 # escape parenthesis in fields in order to not messa up the regexp
73 fixed_fields = dict([(k, v.replace('(', r'\(').replace(')', r'\)')) for k, v in kwargs.items()])
74 str_fields.update(fixed_fields)
75 tag_re = re.compile(tag_name.format(**str_fields))
76
77 # Parse fields from tags
78 revs = []
79 for tag in tags:
80 m = tag_re.match(tag)
81 groups = m.groupdict()
82 revs.append([groups[f] for f in undef_fields] + [tag])
83
84 # Return field names and a sorted list of revs
85 return undef_fields, sorted(revs)
86
Brad Bishopd7bf8c12018-02-25 22:55:05 -050087def list_test_revs(repo, tag_name, verbosity, **kwargs):
Brad Bishop6e60e8b2018-02-01 10:27:11 -050088 """Get list of all tested revisions"""
Brad Bishopd7bf8c12018-02-25 22:55:05 -050089 valid_kwargs = dict([(k, v) for k, v in kwargs.items() if v is not None])
90
91 fields, revs = get_test_runs(repo, tag_name, **valid_kwargs)
Brad Bishop6e60e8b2018-02-01 10:27:11 -050092 ignore_fields = ['tag_number']
Brad Bishopd7bf8c12018-02-25 22:55:05 -050093 if verbosity < 2:
94 extra_fields = ['COMMITS', 'TEST RUNS']
95 ignore_fields.extend(['commit_number', 'commit'])
96 else:
97 extra_fields = ['TEST RUNS']
98
Brad Bishop6e60e8b2018-02-01 10:27:11 -050099 print_fields = [i for i, f in enumerate(fields) if f not in ignore_fields]
100
101 # Sort revs
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500102 rows = [[fields[i].upper() for i in print_fields] + extra_fields]
103
104 prev = [''] * len(print_fields)
105 prev_commit = None
106 commit_cnt = 0
107 commit_field = fields.index('commit')
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500108 for rev in revs:
109 # Only use fields that we want to print
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500110 cols = [rev[i] for i in print_fields]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500111
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500112
113 if cols != prev:
114 commit_cnt = 1
115 test_run_cnt = 1
116 new_row = [''] * (len(print_fields) + len(extra_fields))
117
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500118 for i in print_fields:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500119 if cols[i] != prev[i]:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500120 break
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500121 new_row[i:-len(extra_fields)] = cols[i:]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500122 rows.append(new_row)
123 else:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500124 if rev[commit_field] != prev_commit:
125 commit_cnt += 1
126 test_run_cnt += 1
127
128 if verbosity < 2:
129 new_row[-2] = commit_cnt
130 new_row[-1] = test_run_cnt
131 prev = cols
132 prev_commit = rev[commit_field]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500133
134 print_table(rows)
135
136def get_test_revs(repo, tag_name, **kwargs):
137 """Get list of all tested revisions"""
138 fields, runs = get_test_runs(repo, tag_name, **kwargs)
139
140 revs = {}
141 commit_i = fields.index('commit')
142 commit_num_i = fields.index('commit_number')
143 for run in runs:
144 commit = run[commit_i]
145 commit_num = run[commit_num_i]
146 tag = run[-1]
147 if not commit in revs:
148 revs[commit] = TestedRev(commit, commit_num, [tag])
149 else:
150 assert commit_num == revs[commit].commit_number, "Commit numbers do not match"
151 revs[commit].tags.append(tag)
152
153 # Return in sorted table
154 revs = sorted(revs.values(), key=attrgetter('commit_number'))
155 log.debug("Found %d tested revisions:\n %s", len(revs),
156 "\n ".join(['{} ({})'.format(rev.commit_number, rev.commit) for rev in revs]))
157 return revs
158
159def rev_find(revs, attr, val):
160 """Search from a list of TestedRev"""
161 for i, rev in enumerate(revs):
162 if getattr(rev, attr) == val:
163 return i
164 raise ValueError("Unable to find '{}' value '{}'".format(attr, val))
165
166def is_xml_format(repo, commit):
167 """Check if the commit contains xml (or json) data"""
168 if repo.rev_parse(commit + ':results.xml'):
169 log.debug("Detected report in xml format in %s", commit)
170 return True
171 else:
172 log.debug("No xml report in %s, assuming json formatted results", commit)
173 return False
174
175def read_results(repo, tags, xml=True):
176 """Read result files from repo"""
177
178 def parse_xml_stream(data):
179 """Parse multiple concatenated XML objects"""
180 objs = []
181 xml_d = ""
182 for line in data.splitlines():
183 if xml_d and line.startswith('<?xml version='):
184 objs.append(ET.fromstring(xml_d))
185 xml_d = line
186 else:
187 xml_d += line
188 objs.append(ET.fromstring(xml_d))
189 return objs
190
191 def parse_json_stream(data):
192 """Parse multiple concatenated JSON objects"""
193 objs = []
194 json_d = ""
195 for line in data.splitlines():
196 if line == '}{':
197 json_d += '}'
198 objs.append(json.loads(json_d, object_pairs_hook=OrderedDict))
199 json_d = '{'
200 else:
201 json_d += line
202 objs.append(json.loads(json_d, object_pairs_hook=OrderedDict))
203 return objs
204
205 num_revs = len(tags)
206
207 # Optimize by reading all data with one git command
208 log.debug("Loading raw result data from %d tags, %s...", num_revs, tags[0])
209 if xml:
210 git_objs = [tag + ':metadata.xml' for tag in tags] + [tag + ':results.xml' for tag in tags]
211 data = parse_xml_stream(repo.run_cmd(['show'] + git_objs + ['--']))
212 return ([metadata_xml_to_json(e) for e in data[0:num_revs]],
213 [results_xml_to_json(e) for e in data[num_revs:]])
214 else:
215 git_objs = [tag + ':metadata.json' for tag in tags] + [tag + ':results.json' for tag in tags]
216 data = parse_json_stream(repo.run_cmd(['show'] + git_objs + ['--']))
217 return data[0:num_revs], data[num_revs:]
218
219
220def get_data_item(data, key):
221 """Nested getitem lookup"""
222 for k in key.split('.'):
223 data = data[k]
224 return data
225
226
227def metadata_diff(metadata_l, metadata_r):
228 """Prepare a metadata diff for printing"""
229 keys = [('Hostname', 'hostname', 'hostname'),
230 ('Branch', 'branch', 'layers.meta.branch'),
231 ('Commit number', 'commit_num', 'layers.meta.commit_count'),
232 ('Commit', 'commit', 'layers.meta.commit'),
233 ('Number of test runs', 'testrun_count', 'testrun_count')
234 ]
235
236 def _metadata_diff(key):
237 """Diff metadata from two test reports"""
238 try:
239 val1 = get_data_item(metadata_l, key)
240 except KeyError:
241 val1 = '(N/A)'
242 try:
243 val2 = get_data_item(metadata_r, key)
244 except KeyError:
245 val2 = '(N/A)'
246 return val1, val2
247
248 metadata = OrderedDict()
249 for title, key, key_json in keys:
250 value_l, value_r = _metadata_diff(key_json)
251 metadata[key] = {'title': title,
252 'value_old': value_l,
253 'value': value_r}
254 return metadata
255
256
257def print_diff_report(metadata_l, data_l, metadata_r, data_r):
258 """Print differences between two data sets"""
259
260 # First, print general metadata
261 print("\nTEST METADATA:\n==============")
262 meta_diff = metadata_diff(metadata_l, metadata_r)
263 rows = []
264 row_fmt = ['{:{wid}} ', '{:<{wid}} ', '{:<{wid}}']
265 rows = [['', 'CURRENT COMMIT', 'COMPARING WITH']]
266 for key, val in meta_diff.items():
267 # Shorten commit hashes
268 if key == 'commit':
269 rows.append([val['title'] + ':', val['value'][:20], val['value_old'][:20]])
270 else:
271 rows.append([val['title'] + ':', val['value'], val['value_old']])
272 print_table(rows, row_fmt)
273
274
275 # Print test results
276 print("\nTEST RESULTS:\n=============")
277
278 tests = list(data_l['tests'].keys())
279 # Append tests that are only present in 'right' set
280 tests += [t for t in list(data_r['tests'].keys()) if t not in tests]
281
282 # Prepare data to be printed
283 rows = []
284 row_fmt = ['{:8}', '{:{wid}}', '{:{wid}}', ' {:>{wid}}', ' {:{wid}} ', '{:{wid}}',
285 ' {:>{wid}}', ' {:>{wid}}']
286 num_cols = len(row_fmt)
287 for test in tests:
288 test_l = data_l['tests'][test] if test in data_l['tests'] else None
289 test_r = data_r['tests'][test] if test in data_r['tests'] else None
290 pref = ' '
291 if test_l is None:
292 pref = '+'
293 elif test_r is None:
294 pref = '-'
295 descr = test_l['description'] if test_l else test_r['description']
296 heading = "{} {}: {}".format(pref, test, descr)
297
298 rows.append([heading])
299
300 # Generate the list of measurements
301 meas_l = test_l['measurements'] if test_l else {}
302 meas_r = test_r['measurements'] if test_r else {}
303 measurements = list(meas_l.keys())
304 measurements += [m for m in list(meas_r.keys()) if m not in measurements]
305
306 for meas in measurements:
307 m_pref = ' '
308 if meas in meas_l:
309 stats_l = measurement_stats(meas_l[meas], 'l.')
310 else:
311 stats_l = measurement_stats(None, 'l.')
312 m_pref = '+'
313 if meas in meas_r:
314 stats_r = measurement_stats(meas_r[meas], 'r.')
315 else:
316 stats_r = measurement_stats(None, 'r.')
317 m_pref = '-'
318 stats = stats_l.copy()
319 stats.update(stats_r)
320
321 absdiff = stats['val_cls'](stats['r.mean'] - stats['l.mean'])
322 reldiff = "{:+.1f} %".format(absdiff * 100 / stats['l.mean'])
323 if stats['r.mean'] > stats['l.mean']:
324 absdiff = '+' + str(absdiff)
325 else:
326 absdiff = str(absdiff)
327 rows.append(['', m_pref, stats['name'] + ' ' + stats['quantity'],
328 str(stats['l.mean']), '->', str(stats['r.mean']),
329 absdiff, reldiff])
330 rows.append([''] * num_cols)
331
332 print_table(rows, row_fmt)
333
334 print()
335
336
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500337class BSSummary(object):
338 def __init__(self, bs1, bs2):
339 self.tasks = {'count': bs2.num_tasks,
340 'change': '{:+d}'.format(bs2.num_tasks - bs1.num_tasks)}
341 self.top_consumer = None
342 self.top_decrease = None
343 self.top_increase = None
344 self.ver_diff = OrderedDict()
345
346 tasks_diff = diff_buildstats(bs1, bs2, 'cputime')
347
348 # Get top consumers of resources
349 tasks_diff = sorted(tasks_diff, key=attrgetter('value2'))
350 self.top_consumer = tasks_diff[-5:]
351
352 # Get biggest increase and decrease in resource usage
353 tasks_diff = sorted(tasks_diff, key=attrgetter('absdiff'))
354 self.top_decrease = tasks_diff[0:5]
355 self.top_increase = tasks_diff[-5:]
356
357 # Compare recipe versions and prepare data for display
358 ver_diff = BSVerDiff(bs1, bs2)
359 if ver_diff:
360 if ver_diff.new:
361 self.ver_diff['New recipes'] = [(n, r.evr) for n, r in ver_diff.new.items()]
362 if ver_diff.dropped:
363 self.ver_diff['Dropped recipes'] = [(n, r.evr) for n, r in ver_diff.dropped.items()]
364 if ver_diff.echanged:
365 self.ver_diff['Epoch changed'] = [(n, "{} &rarr; {}".format(r.left.evr, r.right.evr)) for n, r in ver_diff.echanged.items()]
366 if ver_diff.vchanged:
367 self.ver_diff['Version changed'] = [(n, "{} &rarr; {}".format(r.left.version, r.right.version)) for n, r in ver_diff.vchanged.items()]
368 if ver_diff.rchanged:
369 self.ver_diff['Revision changed'] = [(n, "{} &rarr; {}".format(r.left.evr, r.right.evr)) for n, r in ver_diff.rchanged.items()]
370
371
372def print_html_report(data, id_comp, buildstats):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500373 """Print report in html format"""
374 # Handle metadata
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500375 metadata = metadata_diff(data[id_comp].metadata, data[-1].metadata)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500376
377 # Generate list of tests
378 tests = []
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500379 for test in data[-1].results['tests'].keys():
380 test_r = data[-1].results['tests'][test]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500381 new_test = {'name': test_r['name'],
382 'description': test_r['description'],
383 'status': test_r['status'],
384 'measurements': [],
385 'err_type': test_r.get('err_type'),
386 }
387 # Limit length of err output shown
388 if 'message' in test_r:
389 lines = test_r['message'].splitlines()
390 if len(lines) > 20:
391 new_test['message'] = '...\n' + '\n'.join(lines[-20:])
392 else:
393 new_test['message'] = test_r['message']
394
395
396 # Generate the list of measurements
397 for meas in test_r['measurements'].keys():
398 meas_r = test_r['measurements'][meas]
399 meas_type = 'time' if meas_r['type'] == 'sysres' else 'size'
400 new_meas = {'name': meas_r['name'],
401 'legend': meas_r['legend'],
402 'description': meas_r['name'] + ' ' + meas_type,
403 }
404 samples = []
405
406 # Run through all revisions in our data
407 for meta, test_data in data:
408 if (not test in test_data['tests'] or
409 not meas in test_data['tests'][test]['measurements']):
410 samples.append(measurement_stats(None))
411 continue
412 test_i = test_data['tests'][test]
413 meas_i = test_i['measurements'][meas]
414 commit_num = get_data_item(meta, 'layers.meta.commit_count')
415 samples.append(measurement_stats(meas_i))
416 samples[-1]['commit_num'] = commit_num
417
418 absdiff = samples[-1]['val_cls'](samples[-1]['mean'] - samples[id_comp]['mean'])
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800419 reldiff = absdiff * 100 / samples[id_comp]['mean']
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500420 new_meas['absdiff'] = absdiff
421 new_meas['absdiff_str'] = str(absdiff) if absdiff < 0 else '+' + str(absdiff)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800422 new_meas['reldiff'] = reldiff
423 new_meas['reldiff_str'] = "{:+.1f} %".format(reldiff)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500424 new_meas['samples'] = samples
425 new_meas['value'] = samples[-1]
426 new_meas['value_type'] = samples[-1]['val_cls']
427
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500428 # Compare buildstats
429 bs_key = test + '.' + meas
430 rev = metadata['commit_num']['value']
431 comp_rev = metadata['commit_num']['value_old']
432 if (rev in buildstats and bs_key in buildstats[rev] and
433 comp_rev in buildstats and bs_key in buildstats[comp_rev]):
434 new_meas['buildstats'] = BSSummary(buildstats[comp_rev][bs_key],
435 buildstats[rev][bs_key])
436
437
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500438 new_test['measurements'].append(new_meas)
439 tests.append(new_test)
440
441 # Chart options
442 chart_opts = {'haxis': {'min': get_data_item(data[0][0], 'layers.meta.commit_count'),
443 'max': get_data_item(data[-1][0], 'layers.meta.commit_count')}
444 }
445
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500446 print(html.template.render(title="Build Perf Test Report",
447 metadata=metadata, test_data=tests,
448 chart_opts=chart_opts))
449
450
451def get_buildstats(repo, notes_ref, revs, outdir=None):
452 """Get the buildstats from git notes"""
453 full_ref = 'refs/notes/' + notes_ref
454 if not repo.rev_parse(full_ref):
455 log.error("No buildstats found, please try running "
456 "'git fetch origin %s:%s' to fetch them from the remote",
457 full_ref, full_ref)
458 return
459
460 missing = False
461 buildstats = {}
462 log.info("Parsing buildstats from 'refs/notes/%s'", notes_ref)
463 for rev in revs:
464 buildstats[rev.commit_number] = {}
465 log.debug('Dumping buildstats for %s (%s)', rev.commit_number,
466 rev.commit)
467 for tag in rev.tags:
468 log.debug(' %s', tag)
469 try:
470 bs_all = json.loads(repo.run_cmd(['notes', '--ref', notes_ref,
471 'show', tag + '^0']))
472 except GitError:
473 log.warning("Buildstats not found for %s", tag)
474 bs_all = {}
475 missing = True
476
477 for measurement, bs in bs_all.items():
478 # Write out onto disk
479 if outdir:
480 tag_base, run_id = tag.rsplit('/', 1)
481 tag_base = tag_base.replace('/', '_')
482 bs_dir = os.path.join(outdir, measurement, tag_base)
483 if not os.path.exists(bs_dir):
484 os.makedirs(bs_dir)
485 with open(os.path.join(bs_dir, run_id + '.json'), 'w') as f:
486 json.dump(bs, f, indent=2)
487
488 # Read buildstats into a dict
489 _bs = BuildStats.from_json(bs)
490 if measurement not in buildstats[rev.commit_number]:
491 buildstats[rev.commit_number][measurement] = _bs
492 else:
493 buildstats[rev.commit_number][measurement].aggregate(_bs)
494
495 if missing:
496 log.info("Buildstats were missing for some test runs, please "
497 "run 'git fetch origin %s:%s' and try again",
498 full_ref, full_ref)
499
500 return buildstats
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500501
502
503def auto_args(repo, args):
504 """Guess arguments, if not defined by the user"""
505 # Get the latest commit in the repo
506 log.debug("Guessing arguments from the latest commit")
507 msg = repo.run_cmd(['log', '-1', '--branches', '--remotes', '--format=%b'])
508 for line in msg.splitlines():
509 split = line.split(':', 1)
510 if len(split) != 2:
511 continue
512
513 key = split[0]
514 val = split[1].strip()
515 if key == 'hostname':
516 log.debug("Using hostname %s", val)
517 args.hostname = val
518 elif key == 'branch':
519 log.debug("Using branch %s", val)
520 args.branch = val
521
522
523def parse_args(argv):
524 """Parse command line arguments"""
525 description = """
526Examine build performance test results from a Git repository"""
527 parser = argparse.ArgumentParser(
528 formatter_class=argparse.ArgumentDefaultsHelpFormatter,
529 description=description)
530
531 parser.add_argument('--debug', '-d', action='store_true',
532 help="Verbose logging")
533 parser.add_argument('--repo', '-r', required=True,
534 help="Results repository (local git clone)")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500535 parser.add_argument('--list', '-l', action='count',
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500536 help="List available test runs")
537 parser.add_argument('--html', action='store_true',
538 help="Generate report in html format")
539 group = parser.add_argument_group('Tag and revision')
540 group.add_argument('--tag-name', '-t',
541 default='{hostname}/{branch}/{machine}/{commit_number}-g{commit}/{tag_number}',
542 help="Tag name (pattern) for finding results")
543 group.add_argument('--hostname', '-H')
544 group.add_argument('--branch', '-B', default='master')
545 group.add_argument('--machine', default='qemux86')
546 group.add_argument('--history-length', default=25, type=int,
547 help="Number of tested revisions to plot in html report")
548 group.add_argument('--commit',
549 help="Revision to search for")
550 group.add_argument('--commit-number',
551 help="Revision number to search for, redundant if "
552 "--commit is specified")
553 group.add_argument('--commit2',
554 help="Revision to compare with")
555 group.add_argument('--commit-number2',
556 help="Revision number to compare with, redundant if "
557 "--commit2 is specified")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500558 parser.add_argument('--dump-buildstats', nargs='?', const='.',
559 help="Dump buildstats of the tests")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500560
561 return parser.parse_args(argv)
562
563
564def main(argv=None):
565 """Script entry point"""
566 args = parse_args(argv)
567 if args.debug:
568 log.setLevel(logging.DEBUG)
569
570 repo = GitRepo(args.repo)
571
572 if args.list:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500573 list_test_revs(repo, args.tag_name, args.list, hostname=args.hostname)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500574 return 0
575
576 # Determine hostname which to use
577 if not args.hostname:
578 auto_args(repo, args)
579
580 revs = get_test_revs(repo, args.tag_name, hostname=args.hostname,
581 branch=args.branch, machine=args.machine)
582 if len(revs) < 2:
583 log.error("%d tester revisions found, unable to generate report",
584 len(revs))
585 return 1
586
587 # Pick revisions
588 if args.commit:
589 if args.commit_number:
590 log.warning("Ignoring --commit-number as --commit was specified")
591 index1 = rev_find(revs, 'commit', args.commit)
592 elif args.commit_number:
593 index1 = rev_find(revs, 'commit_number', args.commit_number)
594 else:
595 index1 = len(revs) - 1
596
597 if args.commit2:
598 if args.commit_number2:
599 log.warning("Ignoring --commit-number2 as --commit2 was specified")
600 index2 = rev_find(revs, 'commit', args.commit2)
601 elif args.commit_number2:
602 index2 = rev_find(revs, 'commit_number', args.commit_number2)
603 else:
604 if index1 > 0:
605 index2 = index1 - 1
606 else:
607 log.error("Unable to determine the other commit, use "
608 "--commit2 or --commit-number2 to specify it")
609 return 1
610
611 index_l = min(index1, index2)
612 index_r = max(index1, index2)
613
614 rev_l = revs[index_l]
615 rev_r = revs[index_r]
616 log.debug("Using 'left' revision %s (%s), %s test runs:\n %s",
617 rev_l.commit_number, rev_l.commit, len(rev_l.tags),
618 '\n '.join(rev_l.tags))
619 log.debug("Using 'right' revision %s (%s), %s test runs:\n %s",
620 rev_r.commit_number, rev_r.commit, len(rev_r.tags),
621 '\n '.join(rev_r.tags))
622
623 # Check report format used in the repo (assume all reports in the same fmt)
624 xml = is_xml_format(repo, revs[index_r].tags[-1])
625
626 if args.html:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500627 index_0 = max(0, min(index_l, index_r - args.history_length))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500628 rev_range = range(index_0, index_r + 1)
629 else:
630 # We do not need range of commits for text report (no graphs)
631 index_0 = index_l
632 rev_range = (index_l, index_r)
633
634 # Read raw data
635 log.debug("Reading %d revisions, starting from %s (%s)",
636 len(rev_range), revs[index_0].commit_number, revs[index_0].commit)
637 raw_data = [read_results(repo, revs[i].tags, xml) for i in rev_range]
638
639 data = []
640 for raw_m, raw_d in raw_data:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500641 data.append(AggregateTestData(aggregate_metadata(raw_m),
642 aggregate_data(raw_d)))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500643
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500644 # Read buildstats only when needed
645 buildstats = None
646 if args.dump_buildstats or args.html:
647 outdir = 'oe-build-perf-buildstats' if args.dump_buildstats else None
648 notes_ref = 'buildstats/{}/{}/{}'.format(args.hostname, args.branch,
649 args.machine)
650 buildstats = get_buildstats(repo, notes_ref, [rev_l, rev_r], outdir)
651
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500652 # Print report
653 if not args.html:
Brad Bishop00111322018-04-01 22:23:53 -0400654 print_diff_report(data[0].metadata, data[0].results,
655 data[1].metadata, data[1].results)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500656 else:
Brad Bishop00111322018-04-01 22:23:53 -0400657 # Re-map 'left' list index to the data table where index_0 maps to 0
658 print_html_report(data, index_l - index_0, buildstats)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500659
660 return 0
661
662if __name__ == "__main__":
663 sys.exit(main())