Gitiles
Code Review Sign In
gerrit.openbmc.org / mdmillerii / openbmc / b58112e5af484d9314f6a4487b1bc8b292f952ef / . / poky / meta / lib / oeqa / selftest / cases / resulttooltests.py
blob: c3303f3fbbee270a819da46b20524c2acd89ee5e [file] [log] [blame]
Brad Bishopc342db32019-05-15 21:57:59 -0400[diff] [blame]1#
Patrick Williams92b42cb2022-09-03 06:53:57 -0500[diff] [blame]2# Copyright OpenEmbedded Contributors
3#
Brad Bishopc342db32019-05-15 21:57:59 -0400[diff] [blame]4# SPDX-License-Identifier: MIT
5#
6
Brad Bishop40320b12019-03-26 16:08:25 -0400[diff] [blame]7import os
8import sys
9basepath = os.path.abspath(os.path.dirname(__file__) + '/../../../../../')
10lib_path = basepath + '/scripts/lib'
11sys.path = sys.path + [lib_path]
12from resulttool.report import ResultsTextReport
13from resulttool import regression as regression
14from resulttool import resultutils as resultutils
15from oeqa.selftest.case import OESelftestTestCase
16
17class ResultToolTests(OESelftestTestCase):
18 base_results_data = {'base_result1': {'configuration': {"TEST_TYPE": "runtime",
19 "TESTSERIES": "series1",
20 "IMAGE_BASENAME": "image",
21 "IMAGE_PKGTYPE": "ipk",
22 "DISTRO": "mydistro",
23 "MACHINE": "qemux86"},
24 'result': {}},
25 'base_result2': {'configuration': {"TEST_TYPE": "runtime",
26 "TESTSERIES": "series1",
27 "IMAGE_BASENAME": "image",
28 "IMAGE_PKGTYPE": "ipk",
29 "DISTRO": "mydistro",
30 "MACHINE": "qemux86-64"},
31 'result': {}}}
32 target_results_data = {'target_result1': {'configuration': {"TEST_TYPE": "runtime",
33 "TESTSERIES": "series1",
34 "IMAGE_BASENAME": "image",
35 "IMAGE_PKGTYPE": "ipk",
36 "DISTRO": "mydistro",
37 "MACHINE": "qemux86"},
38 'result': {}},
39 'target_result2': {'configuration': {"TEST_TYPE": "runtime",
40 "TESTSERIES": "series1",
41 "IMAGE_BASENAME": "image",
42 "IMAGE_PKGTYPE": "ipk",
43 "DISTRO": "mydistro",
44 "MACHINE": "qemux86"},
45 'result': {}},
46 'target_result3': {'configuration': {"TEST_TYPE": "runtime",
47 "TESTSERIES": "series1",
48 "IMAGE_BASENAME": "image",
49 "IMAGE_PKGTYPE": "ipk",
50 "DISTRO": "mydistro",
51 "MACHINE": "qemux86-64"},
52 'result': {}}}
53
54 def test_report_can_aggregate_test_result(self):
55 result_data = {'result': {'test1': {'status': 'PASSED'},
56 'test2': {'status': 'PASSED'},
57 'test3': {'status': 'FAILED'},
58 'test4': {'status': 'ERROR'},
59 'test5': {'status': 'SKIPPED'}}}
60 report = ResultsTextReport()
Brad Bishop15ae2502019-06-18 21:44:24 -0400[diff] [blame]61 result_report = report.get_aggregated_test_result(None, result_data, 'DummyMachine')
Brad Bishop40320b12019-03-26 16:08:25 -0400[diff] [blame]62 self.assertTrue(result_report['passed'] == 2, msg="Passed count not correct:%s" % result_report['passed'])
63 self.assertTrue(result_report['failed'] == 2, msg="Failed count not correct:%s" % result_report['failed'])
64 self.assertTrue(result_report['skipped'] == 1, msg="Skipped count not correct:%s" % result_report['skipped'])
65
66 def test_regression_can_get_regression_base_target_pair(self):
67
68 results = {}
69 resultutils.append_resultsdata(results, ResultToolTests.base_results_data)
70 resultutils.append_resultsdata(results, ResultToolTests.target_results_data)
71 self.assertTrue('target_result1' in results['runtime/mydistro/qemux86/image'], msg="Pair not correct:%s" % results)
72 self.assertTrue('target_result3' in results['runtime/mydistro/qemux86-64/image'], msg="Pair not correct:%s" % results)
73
Andrew Geissler6aa7eec2023-03-03 12:41:14 -0600[diff] [blame]74 def test_regression_can_get_regression_result(self):
Brad Bishop40320b12019-03-26 16:08:25 -0400[diff] [blame]75 base_result_data = {'result': {'test1': {'status': 'PASSED'},
76 'test2': {'status': 'PASSED'},
77 'test3': {'status': 'FAILED'},
78 'test4': {'status': 'ERROR'},
79 'test5': {'status': 'SKIPPED'}}}
80 target_result_data = {'result': {'test1': {'status': 'PASSED'},
81 'test2': {'status': 'FAILED'},
82 'test3': {'status': 'PASSED'},
83 'test4': {'status': 'ERROR'},
84 'test5': {'status': 'SKIPPED'}}}
85 result, text = regression.compare_result(self.logger, "BaseTestRunName", "TargetTestRunName", base_result_data, target_result_data)
86 self.assertTrue(result['test2']['base'] == 'PASSED',
87 msg="regression not correct:%s" % result['test2']['base'])
88 self.assertTrue(result['test2']['target'] == 'FAILED',
89 msg="regression not correct:%s" % result['test2']['target'])
90 self.assertTrue(result['test3']['base'] == 'FAILED',
91 msg="regression not correct:%s" % result['test3']['base'])
92 self.assertTrue(result['test3']['target'] == 'PASSED',
93 msg="regression not correct:%s" % result['test3']['target'])
94
95 def test_merge_can_merged_results(self):
96 results = {}
97 resultutils.append_resultsdata(results, ResultToolTests.base_results_data, configmap=resultutils.flatten_map)
98 resultutils.append_resultsdata(results, ResultToolTests.target_results_data, configmap=resultutils.flatten_map)
99 self.assertEqual(len(results[''].keys()), 5, msg="Flattened results not correct %s" % str(results))
100
Andrew Geissler6aa7eec2023-03-03 12:41:14 -0600[diff] [blame]101 def test_results_without_metadata_can_be_compared(self):
102 base_configuration = {"configuration": {
103 "TEST_TYPE": "oeselftest",
104 "TESTSERIES": "series1",
105 "IMAGE_BASENAME": "image",
106 "IMAGE_PKGTYPE": "ipk",
107 "DISTRO": "mydistro",
108 "MACHINE": "qemux86",
109 "STARTTIME": 1672527600
110 }, "result": {}}
111 target_configuration = {"configuration": {
112 "TEST_TYPE": "oeselftest",
113 "TESTSERIES": "series1",
114 "IMAGE_BASENAME": "image",
115 "IMAGE_PKGTYPE": "ipk",
116 "DISTRO": "mydistro",
117 "MACHINE": "qemux86",
118 "STARTTIME": 1672527600
119 }, "result": {}}
120 self.assertTrue(regression.can_be_compared(self.logger, base_configuration, target_configuration),
121 msg="incorrect metadata filtering, tests without metadata should be compared")
122
123 def test_target_result_with_missing_metadata_can_not_be_compared(self):
124 base_configuration = {"configuration": {
125 "TEST_TYPE": "oeselftest",
126 "TESTSERIES": "series1",
127 "IMAGE_BASENAME": "image",
128 "IMAGE_PKGTYPE": "ipk",
129 "DISTRO": "mydistro",
130 "MACHINE": "qemux86",
131 "OESELFTEST_METADATA": {
132 "run_all_tests": True,
133 "run_tests": None,
134 "skips": None,
135 "machine": None,
136 "select_tags": ["toolchain-user", "toolchain-system"],
137 "exclude_tags": None
138 }}, "result": {}}
139 target_configuration = {"configuration": {"TEST_TYPE": "oeselftest",
140 "TESTSERIES": "series1",
141 "IMAGE_BASENAME": "image",
142 "IMAGE_PKGTYPE": "ipk",
143 "DISTRO": "mydistro",
144 "MACHINE": "qemux86",
145 "STARTTIME": 1672527600
146 }, "result": {}}
147 self.assertFalse(regression.can_be_compared(self.logger, base_configuration, target_configuration),
148 msg="incorrect metadata filtering, tests should not be compared")
149
150 def test_results_with_matching_metadata_can_be_compared(self):
151 base_configuration = {"configuration": {
152 "TEST_TYPE": "oeselftest",
153 "TESTSERIES": "series1",
154 "IMAGE_BASENAME": "image",
155 "IMAGE_PKGTYPE": "ipk",
156 "DISTRO": "mydistro",
157 "MACHINE": "qemux86",
158 "STARTTIME": 1672527600,
159 "OESELFTEST_METADATA": {"run_all_tests": True,
160 "run_tests": None,
161 "skips": None,
162 "machine": None,
163 "select_tags": ["toolchain-user", "toolchain-system"],
164 "exclude_tags": None}
165 }, "result": {}}
166 target_configuration = {"configuration": {
167 "TEST_TYPE": "oeselftest",
168 "TESTSERIES": "series1",
169 "IMAGE_BASENAME": "image",
170 "IMAGE_PKGTYPE": "ipk",
171 "DISTRO": "mydistro",
172 "MACHINE": "qemux86",
173 "STARTTIME": 1672527600,
174 "OESELFTEST_METADATA": {"run_all_tests": True,
175 "run_tests": None,
176 "skips": None,
177 "machine": None,
178 "select_tags": ["toolchain-user", "toolchain-system"],
179 "exclude_tags": None}
180 }, "result": {}}
181 self.assertTrue(regression.can_be_compared(self.logger, base_configuration, target_configuration),
182 msg="incorrect metadata filtering, tests with matching metadata should be compared")
183
184 def test_results_with_mismatching_metadata_can_not_be_compared(self):
185 base_configuration = {"configuration": {
186 "TEST_TYPE": "oeselftest",
187 "TESTSERIES": "series1",
188 "IMAGE_BASENAME": "image",
189 "IMAGE_PKGTYPE": "ipk",
190 "DISTRO": "mydistro",
191 "MACHINE": "qemux86",
192 "STARTTIME": 1672527600,
193 "OESELFTEST_METADATA": {"run_all_tests": True,
194 "run_tests": None,
195 "skips": None,
196 "machine": None,
197 "select_tags": ["toolchain-user", "toolchain-system"],
198 "exclude_tags": None}
199 }, "result": {}}
200 target_configuration = {"configuration": {
201 "TEST_TYPE": "oeselftest",
202 "TESTSERIES": "series1",
203 "IMAGE_BASENAME": "image",
204 "IMAGE_PKGTYPE": "ipk",
205 "DISTRO": "mydistro",
206 "MACHINE": "qemux86",
207 "STARTTIME": 1672527600,
208 "OESELFTEST_METADATA": {"run_all_tests": True,
209 "run_tests": None,
210 "skips": None,
211 "machine": None,
212 "select_tags": ["machine"],
213 "exclude_tags": None}
214 }, "result": {}}
215 self.assertFalse(regression.can_be_compared(self.logger, base_configuration, target_configuration),
216 msg="incorrect metadata filtering, tests with mismatching metadata should not be compared")
217
218 def test_metadata_matching_is_only_checked_for_relevant_test_type(self):
219 base_configuration = {"configuration": {"TEST_TYPE": "runtime",
220 "TESTSERIES": "series1",
221 "IMAGE_BASENAME": "image",
222 "IMAGE_PKGTYPE": "ipk",
223 "DISTRO": "mydistro",
224 "MACHINE": "qemux86",
225 "STARTTIME": 1672527600,
226 "OESELFTEST_METADATA": {"run_all_tests": True,
227 "run_tests": None,
228 "skips": None,
229 "machine": None,
230 "select_tags": ["toolchain-user", "toolchain-system"],
231 "exclude_tags": None}}, "result": {}}
232 target_configuration = {"configuration": {"TEST_TYPE": "runtime",
233 "TESTSERIES": "series1",
234 "IMAGE_BASENAME": "image",
235 "IMAGE_PKGTYPE": "ipk",
236 "DISTRO": "mydistro",
237 "MACHINE": "qemux86",
238 "STARTTIME": 1672527600,
239 "OESELFTEST_METADATA": {"run_all_tests": True,
240 "run_tests": None,
241 "skips": None,
242 "machine": None,
243 "select_tags": ["machine"],
244 "exclude_tags": None}}, "result": {}}
245 self.assertTrue(regression.can_be_compared(self.logger, base_configuration, target_configuration),
246 msg="incorrect metadata filtering, %s tests should be compared" % base_configuration['configuration']['TEST_TYPE'])
247
248 def test_machine_matches(self):
249 base_configuration = {"configuration": {
250 "TEST_TYPE": "runtime",
251 "MACHINE": "qemux86"}, "result": {}}
252 target_configuration = {"configuration": {
253 "TEST_TYPE": "runtime",
254 "MACHINE": "qemux86"
255 }, "result": {}}
256 self.assertTrue(regression.can_be_compared(self.logger, base_configuration, target_configuration),
257 msg="incorrect machine filtering, identical machine tests should be compared")
258
259 def test_machine_mismatches(self):
260 base_configuration = {"configuration": {
261 "TEST_TYPE": "runtime",
262 "MACHINE": "qemux86"
263 }, "result": {}}
264 target_configuration = {"configuration": {
265 "TEST_TYPE": "runtime",
266 "MACHINE": "qemux86_64"
267 }, "result": {}}
268 self.assertFalse(regression.can_be_compared(self.logger, base_configuration, target_configuration),
269 msg="incorrect machine filtering, mismatching machine tests should not be compared")
270
271 def test_can_not_compare_non_ltp_tests(self):
272 base_configuration = {"configuration": {
273 "TEST_TYPE": "runtime",
274 "MACHINE": "qemux86"
275 }, "result": {
276 "ltpresult_foo": {
Andrew Geisslerfc113ea2023-03-31 09:59:46 -0500[diff] [blame]277 "status": "PASSED"
Andrew Geissler6aa7eec2023-03-03 12:41:14 -0600[diff] [blame]278 }}}
279 target_configuration = {"configuration": {
280 "TEST_TYPE": "runtime",
281 "MACHINE": "qemux86_64"
282 }, "result": {
283 "bar": {
Andrew Geisslerfc113ea2023-03-31 09:59:46 -0500[diff] [blame]284 "status": "PASSED"
Andrew Geissler6aa7eec2023-03-03 12:41:14 -0600[diff] [blame]285 }}}
286 self.assertFalse(regression.can_be_compared(self.logger, base_configuration, target_configuration),
287 msg="incorrect ltpresult filtering, mismatching ltpresult content should not be compared")
288
289 def test_can_compare_ltp_tests(self):
290 base_configuration = {"configuration": {
291 "TEST_TYPE": "runtime",
292 "MACHINE": "qemux86"
293 }, "result": {
294 "ltpresult_foo": {
Andrew Geisslerfc113ea2023-03-31 09:59:46 -0500[diff] [blame]295 "status": "PASSED"
Andrew Geissler6aa7eec2023-03-03 12:41:14 -0600[diff] [blame]296 }}}
297 target_configuration = {"configuration": {
298 "TEST_TYPE": "runtime",
299 "MACHINE": "qemux86"
300 }, "result": {
301 "ltpresult_foo": {
Andrew Geisslerfc113ea2023-03-31 09:59:46 -0500[diff] [blame]302 "status": "PASSED"
Andrew Geissler6aa7eec2023-03-03 12:41:14 -0600[diff] [blame]303 }}}
304 self.assertTrue(regression.can_be_compared(self.logger, base_configuration, target_configuration),
305 msg="incorrect ltpresult filtering, matching ltpresult content should be compared")
306
307 def test_can_match_non_static_ptest_names(self):
Andrew Geisslerfc113ea2023-03-31 09:59:46 -0500[diff] [blame]308 base_configuration = {"a": {
309 "conf_X": {
310 "configuration": {
311 "TEST_TYPE": "runtime",
312 "MACHINE": "qemux86"
313 }, "result": {
314 "ptestresult.lttng-tools.foo_-_bar_-_moo": {
315 "status": "PASSED"
316 },
317 "ptestresult.babeltrace.bar_-_moo_-_foo": {
318 "status": "PASSED"
319 },
320 "ptestresult.babeltrace2.moo_-_foo_-_bar": {
321 "status": "PASSED"
322 },
323 "ptestresult.curl.test_0000__foo_out_of_bar": {
324 "status": "PASSED"
325 },
326 "ptestresult.dbus.test_0000__foo_out_of_bar,_remaining:_00:02,_took_0.032s,_duration:_03:32_": {
327 "status": "PASSED"
328 },
329 "ptestresult.binutils-ld.in testcase /foo/build-st-bar/moo/ctf.exp": {
330 "status": "PASSED"
331 },
332 "ptestresult.gcc-libstdc++-v3.Couldn't create remote directory /tmp/runtest.30975 on target": {
333 "status": "PASSED"
334 },
335 "ptestresult.gcc-libgomp.Couldn't create remote directory /tmp/runtest.3657621 on": {
336 "status": "PASSED"
337 }
338 }}}}
339 target_configuration = {"a": {
340 "conf_Y": {
341 "configuration": {
342 "TEST_TYPE": "runtime",
343 "MACHINE": "qemux86"
344 }, "result": {
345 "ptestresult.lttng-tools.foo_-_yyy_-_zzz": {
346 "status": "PASSED"
347 },
348 "ptestresult.babeltrace.bar_-_zzz_-_xxx": {
349 "status": "PASSED"
350 },
351 "ptestresult.babeltrace2.moo_-_xxx_-_yyy": {
352 "status": "PASSED"
353 },
354 "ptestresult.curl.test_0000__xxx_out_of_yyy": {
355 "status": "PASSED"
356 },
357 "ptestresult.dbus.test_0000__yyy_out_of_zzz,_remaining:_00:03,_took_0.034s,_duration:_03:30_": {
358 "status": "PASSED"
359 },
360 "ptestresult.binutils-ld.in testcase /xxx/build-st-yyy/zzz/ctf.exp": {
361 "status": "PASSED"
362 },
363 "ptestresult.gcc-libstdc++-v3.Couldn't create remote directory /tmp/runtest.45678 on target": {
364 "status": "PASSED"
365 },
366 "ptestresult.gcc-libgomp.Couldn't create remote directory /tmp/runtest.3657621 on": {
367 "status": "PASSED"
368 }
369 }}}}
370 regression.fixup_ptest_names(base_configuration, self.logger)
371 regression.fixup_ptest_names(target_configuration, self.logger)
372 result, resultstring = regression.compare_result(
373 self.logger, "A", "B", base_configuration["a"]["conf_X"], target_configuration["a"]["conf_Y"])
374 self.assertDictEqual(
375 result, {}, msg=f"ptests should be compared: {resultstring}")