blob: 3ef902218878d314dc0f8241ff233b1c31b1dc68 [file] [log] [blame]
Brad Bishopc342db32019-05-15 21:57:59 -04001#
Patrick Williams92b42cb2022-09-03 06:53:57 -05002# Copyright OpenEmbedded Contributors
3#
Brad Bishopc342db32019-05-15 21:57:59 -04004# SPDX-License-Identifier: MIT
5#
6
Andrew Geissler82c905d2020-04-13 13:39:40 -05007import os
Brad Bishopf86d0552018-12-04 14:18:15 -08008import unittest
9import pprint
Andrew Geissler99467da2019-02-25 18:54:23 -060010import datetime
Brad Bishopf86d0552018-12-04 14:18:15 -080011
Brad Bishopd7bf8c12018-02-25 22:55:05 -050012from oeqa.runtime.case import OERuntimeTestCase
13from oeqa.core.decorator.depends import OETestDepends
Brad Bishopd7bf8c12018-02-25 22:55:05 -050014from oeqa.core.decorator.data import skipIfNotFeature
Brad Bishop977dc1a2019-02-06 16:01:43 -050015from oeqa.runtime.decorator.package import OEHasPackage
Andrew Geissler99467da2019-02-25 18:54:23 -060016from oeqa.utils.logparser import PtestParser
Brad Bishopd7bf8c12018-02-25 22:55:05 -050017
18class PtestRunnerTest(OERuntimeTestCase):
19
Brad Bishopd7bf8c12018-02-25 22:55:05 -050020 @skipIfNotFeature('ptest', 'Test requires ptest to be in DISTRO_FEATURES')
Brad Bishopd7bf8c12018-02-25 22:55:05 -050021 @OETestDepends(['ssh.SSHTest.test_ssh'])
Brad Bishop977dc1a2019-02-06 16:01:43 -050022 @OEHasPackage(['ptest-runner'])
Brad Bishopf86d0552018-12-04 14:18:15 -080023 @unittest.expectedFailure
Andrew Geissler82c905d2020-04-13 13:39:40 -050024 def test_ptestrunner_expectfail(self):
25 if not self.td.get('PTEST_EXPECT_FAILURE'):
26 self.skipTest('Cannot run ptests with @expectedFailure as ptests are required to pass')
27 self.do_ptestrunner()
28
29 @skipIfNotFeature('ptest', 'Test requires ptest to be in DISTRO_FEATURES')
30 @OETestDepends(['ssh.SSHTest.test_ssh'])
31 @OEHasPackage(['ptest-runner'])
32 def test_ptestrunner_expectsuccess(self):
33 if self.td.get('PTEST_EXPECT_FAILURE'):
34 self.skipTest('Cannot run ptests without @expectedFailure as ptests are expected to fail')
35 self.do_ptestrunner()
36
37 def do_ptestrunner(self):
Brad Bishop316dfdd2018-06-25 12:45:53 -040038 status, output = self.target.run('which ptest-runner', 0)
39 if status != 0:
40 self.skipTest("No -ptest packages are installed in the image")
41
Brad Bishopd7bf8c12018-02-25 22:55:05 -050042 test_log_dir = self.td.get('TEST_LOG_DIR', '')
43 # The TEST_LOG_DIR maybe NULL when testimage is added after
44 # testdata.json is generated.
45 if not test_log_dir:
46 test_log_dir = os.path.join(self.td.get('WORKDIR', ''), 'testimage')
Andrew Geisslerd25ed322020-06-27 00:28:28 -050047 # Make the test output path absolute, otherwise the output content will be
48 # created relative to current directory
49 if not os.path.isabs(test_log_dir):
50 test_log_dir = os.path.join(self.td.get('TOPDIR', ''), test_log_dir)
Brad Bishopd7bf8c12018-02-25 22:55:05 -050051 # Don't use self.td.get('DATETIME'), it's from testdata.json, not
52 # up-to-date, and may cause "File exists" when re-reun.
Andrew Geissler99467da2019-02-25 18:54:23 -060053 timestamp = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
Brad Bishopd7bf8c12018-02-25 22:55:05 -050054 ptest_log_dir_link = os.path.join(test_log_dir, 'ptest_log')
Andrew Geissler99467da2019-02-25 18:54:23 -060055 ptest_log_dir = '%s.%s' % (ptest_log_dir_link, timestamp)
Brad Bishopd7bf8c12018-02-25 22:55:05 -050056 ptest_runner_log = os.path.join(ptest_log_dir, 'ptest-runner.log')
57
Andrew Geissler4b740dc2020-05-05 08:54:39 -050058 libdir = self.td.get('libdir', '')
59 ptest_dirs = [ '/usr/lib' ]
60 if not libdir in ptest_dirs:
61 ptest_dirs.append(libdir)
Patrick Williams213cb262021-08-07 19:21:33 -050062 status, output = self.target.run('ptest-runner -t 450 -d \"{}\"'.format(' '.join(ptest_dirs)), 0)
Brad Bishopd7bf8c12018-02-25 22:55:05 -050063 os.makedirs(ptest_log_dir)
64 with open(ptest_runner_log, 'w') as f:
65 f.write(output)
66
67 # status != 0 is OK since some ptest tests may fail
68 self.assertTrue(status != 127, msg="Cannot execute ptest-runner!")
69
Brad Bishopf86d0552018-12-04 14:18:15 -080070 if not hasattr(self.tc, "extraresults"):
71 self.tc.extraresults = {}
72 extras = self.tc.extraresults
73 extras['ptestresult.rawlogs'] = {'log': output}
74
Brad Bishopd7bf8c12018-02-25 22:55:05 -050075 # Parse and save results
Andrew Geissler99467da2019-02-25 18:54:23 -060076 parser = PtestParser()
77 results, sections = parser.parse(ptest_runner_log)
78 parser.results_as_files(ptest_log_dir)
Brad Bishopd7bf8c12018-02-25 22:55:05 -050079 if os.path.exists(ptest_log_dir_link):
80 # Remove the old link to create a new one
81 os.remove(ptest_log_dir_link)
82 os.symlink(os.path.basename(ptest_log_dir), ptest_log_dir_link)
Brad Bishop316dfdd2018-06-25 12:45:53 -040083
Andrew Geissler99467da2019-02-25 18:54:23 -060084 extras['ptestresult.sections'] = sections
85
Brad Bishopf86d0552018-12-04 14:18:15 -080086 trans = str.maketrans("()", "__")
Andrew Geissler99467da2019-02-25 18:54:23 -060087 for section in results:
88 for test in results[section]:
89 result = results[section][test]
90 testname = "ptestresult." + (section or "No-section") + "." + "_".join(test.translate(trans).split())
91 extras[testname] = {'status': result}
Brad Bishopf86d0552018-12-04 14:18:15 -080092
Brad Bishop316dfdd2018-06-25 12:45:53 -040093 failed_tests = {}
Andrew Geissler82c905d2020-04-13 13:39:40 -050094
95 for section in sections:
96 if 'exitcode' in sections[section].keys():
97 failed_tests[section] = sections[section]["log"]
98
Andrew Geissler99467da2019-02-25 18:54:23 -060099 for section in results:
Andrew Geissler82c905d2020-04-13 13:39:40 -0500100 failed_testcases = [ "_".join(test.translate(trans).split()) for test in results[section] if results[section][test] == 'FAILED' ]
Brad Bishop316dfdd2018-06-25 12:45:53 -0400101 if failed_testcases:
102 failed_tests[section] = failed_testcases
103
Andrew Geissler99467da2019-02-25 18:54:23 -0600104 failmsg = ""
105 status, output = self.target.run('dmesg | grep "Killed process"', 0)
106 if output:
107 failmsg = "ERROR: Processes were killed by the OOM Killer:\n%s\n" % output
108
Brad Bishopf86d0552018-12-04 14:18:15 -0800109 if failed_tests:
Andrew Geissler99467da2019-02-25 18:54:23 -0600110 failmsg = failmsg + "Failed ptests:\n%s" % pprint.pformat(failed_tests)
111
112 if failmsg:
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600113 self.logger.warning("There were failing ptests.")
Andrew Geissler99467da2019-02-25 18:54:23 -0600114 self.fail(failmsg)