Gitiles
Code Review Sign In
gerrit.openbmc.org / stefanberger / openbmc / 0dd04f33864280128a3d2869833d56fddad804d2 / . / poky / scripts / lib / resulttool / regression.py
blob: bdf531dedf16d6cec2f39d1dcce9405c0fc67775 [file] [log] [blame]
Brad Bishop40320b12019-03-26 16:08:25 -0400[diff] [blame]1# resulttool - regression analysis
2#
3# Copyright (c) 2019, Intel Corporation.
4# Copyright (c) 2019, Linux Foundation
5#
6# This program is free software; you can redistribute it and/or modify it
7# under the terms and conditions of the GNU General Public License,
8# version 2, as published by the Free Software Foundation.
9#
10# This program is distributed in the hope it will be useful, but WITHOUT
11# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13# more details.
14#
15import resulttool.resultutils as resultutils
16import json
17
18from oeqa.utils.git import GitRepo
19import oeqa.utils.gitarchive as gitarchive
20
21def compare_result(logger, base_name, target_name, base_result, target_result):
22 base_result = base_result.get('result')
23 target_result = target_result.get('result')
24 result = {}
25 if base_result and target_result:
26 for k in base_result:
27 base_testcase = base_result[k]
28 base_status = base_testcase.get('status')
29 if base_status:
30 target_testcase = target_result.get(k, {})
31 target_status = target_testcase.get('status')
32 if base_status != target_status:
33 result[k] = {'base': base_status, 'target': target_status}
34 else:
35 logger.error('Failed to retrieved base test case status: %s' % k)
36 if result:
37 resultstring = "Regression: %s\n %s\n" % (base_name, target_name)
38 for k in sorted(result):
39 resultstring += ' %s: %s -> %s\n' % (k, result[k]['base'], result[k]['target'])
40 else:
41 resultstring = "Match: %s\n %s" % (base_name, target_name)
42 return result, resultstring
43
44def get_results(logger, source):
45 return resultutils.load_resultsdata(source, configmap=resultutils.regression_map)
46
47def regression(args, logger):
48 base_results = get_results(logger, args.base_result)
49 target_results = get_results(logger, args.target_result)
50
51 regression_common(args, logger, base_results, target_results)
52
53def regression_common(args, logger, base_results, target_results):
54 if args.base_result_id:
55 base_results = resultutils.filter_resultsdata(base_results, args.base_result_id)
56 if args.target_result_id:
57 target_results = resultutils.filter_resultsdata(target_results, args.target_result_id)
58
59 matches = []
60 regressions = []
61 notfound = []
62
63 for a in base_results:
64 if a in target_results:
65 base = list(base_results[a].keys())
66 target = list(target_results[a].keys())
67 # We may have multiple base/targets which are for different configurations. Start by
68 # removing any pairs which match
69 for c in base.copy():
70 for b in target.copy():
71 res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b])
72 if not res:
73 matches.append(resstr)
74 base.remove(c)
75 target.remove(b)
76 break
77 # Should only now see regressions, we may not be able to match multiple pairs directly
78 for c in base:
79 for b in target:
80 res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b])
81 if res:
82 regressions.append(resstr)
83 else:
84 notfound.append("%s not found in target" % a)
85 print("\n".join(sorted(matches)))
86 print("\n".join(sorted(regressions)))
87 print("\n".join(sorted(notfound)))
88
89 return 0
90
91def regression_git(args, logger):
92 base_results = {}
93 target_results = {}
94
95 tag_name = "{branch}/{commit_number}-g{commit}/{tag_number}"
96 repo = GitRepo(args.repo)
97
98 revs = gitarchive.get_test_revs(logger, repo, tag_name, branch=args.branch)
99
100 if args.branch2:
101 revs2 = gitarchive.get_test_revs(logger, repo, tag_name, branch=args.branch2)
102 if not len(revs2):
103 logger.error("No revisions found to compare against")
104 return 1
105 if not len(revs):
106 logger.error("No revision to report on found")
107 return 1
108 else:
109 if len(revs) < 2:
110 logger.error("Only %d tester revisions found, unable to generate report" % len(revs))
111 return 1
112
113 # Pick revisions
114 if args.commit:
115 if args.commit_number:
116 logger.warning("Ignoring --commit-number as --commit was specified")
117 index1 = gitarchive.rev_find(revs, 'commit', args.commit)
118 elif args.commit_number:
119 index1 = gitarchive.rev_find(revs, 'commit_number', args.commit_number)
120 else:
121 index1 = len(revs) - 1
122
123 if args.branch2:
124 revs2.append(revs[index1])
125 index1 = len(revs2) - 1
126 revs = revs2
127
128 if args.commit2:
129 if args.commit_number2:
130 logger.warning("Ignoring --commit-number2 as --commit2 was specified")
131 index2 = gitarchive.rev_find(revs, 'commit', args.commit2)
132 elif args.commit_number2:
133 index2 = gitarchive.rev_find(revs, 'commit_number', args.commit_number2)
134 else:
135 if index1 > 0:
136 index2 = index1 - 1
137 # Find the closest matching commit number for comparision
138 # In future we could check the commit is a common ancestor and
139 # continue back if not but this good enough for now
140 while index2 > 0 and revs[index2].commit_number > revs[index1].commit_number:
141 index2 = index2 - 1
142 else:
143 logger.error("Unable to determine the other commit, use "
144 "--commit2 or --commit-number2 to specify it")
145 return 1
146
147 logger.info("Comparing:\n%s\nto\n%s\n" % (revs[index1], revs[index2]))
148
149 base_results = resultutils.git_get_result(repo, revs[index1][2])
150 target_results = resultutils.git_get_result(repo, revs[index2][2])
151
152 regression_common(args, logger, base_results, target_results)
153
154 return 0
155
156def register_commands(subparsers):
157 """Register subcommands from this plugin"""
158
159 parser_build = subparsers.add_parser('regression', help='regression file/directory analysis',
160 description='regression analysis comparing the base set of results to the target results',
161 group='analysis')
162 parser_build.set_defaults(func=regression)
163 parser_build.add_argument('base_result',
164 help='base result file/directory for the comparison')
165 parser_build.add_argument('target_result',
166 help='target result file/directory to compare with')
167 parser_build.add_argument('-b', '--base-result-id', default='',
168 help='(optional) filter the base results to this result ID')
169 parser_build.add_argument('-t', '--target-result-id', default='',
170 help='(optional) filter the target results to this result ID')
171
172 parser_build = subparsers.add_parser('regression-git', help='regression git analysis',
173 description='regression analysis comparing base result set to target '
174 'result set',
175 group='analysis')
176 parser_build.set_defaults(func=regression_git)
177 parser_build.add_argument('repo',
178 help='the git repository containing the data')
179 parser_build.add_argument('-b', '--base-result-id', default='',
180 help='(optional) default select regression based on configurations unless base result '
181 'id was provided')
182 parser_build.add_argument('-t', '--target-result-id', default='',
183 help='(optional) default select regression based on configurations unless target result '
184 'id was provided')
185
186 parser_build.add_argument('--branch', '-B', default='master', help="Branch to find commit in")
187 parser_build.add_argument('--branch2', help="Branch to find comparision revisions in")
188 parser_build.add_argument('--commit', help="Revision to search for")
189 parser_build.add_argument('--commit-number', help="Revision number to search for, redundant if --commit is specified")
190 parser_build.add_argument('--commit2', help="Revision to compare with")
191 parser_build.add_argument('--commit-number2', help="Revision number to compare with, redundant if --commit2 is specified")
192