blob: 0800f3c27f2f9d82353435a2f5ca3a5def44d1e9 [file] [log] [blame]
Brad Bishopc342db32019-05-15 21:57:59 -04001#
2# SPDX-License-Identifier: MIT
3#
4
Andrew Geissler82c905d2020-04-13 13:39:40 -05005import os
Brad Bishopf86d0552018-12-04 14:18:15 -08006import unittest
7import pprint
Andrew Geissler99467da2019-02-25 18:54:23 -06008import datetime
Brad Bishopf86d0552018-12-04 14:18:15 -08009
Brad Bishopd7bf8c12018-02-25 22:55:05 -050010from oeqa.runtime.case import OERuntimeTestCase
11from oeqa.core.decorator.depends import OETestDepends
Brad Bishopd7bf8c12018-02-25 22:55:05 -050012from oeqa.core.decorator.data import skipIfNotFeature
Brad Bishop977dc1a2019-02-06 16:01:43 -050013from oeqa.runtime.decorator.package import OEHasPackage
Andrew Geissler99467da2019-02-25 18:54:23 -060014from oeqa.utils.logparser import PtestParser
Brad Bishopd7bf8c12018-02-25 22:55:05 -050015
16class PtestRunnerTest(OERuntimeTestCase):
17
Brad Bishopd7bf8c12018-02-25 22:55:05 -050018 @skipIfNotFeature('ptest', 'Test requires ptest to be in DISTRO_FEATURES')
Brad Bishopd7bf8c12018-02-25 22:55:05 -050019 @OETestDepends(['ssh.SSHTest.test_ssh'])
Brad Bishop977dc1a2019-02-06 16:01:43 -050020 @OEHasPackage(['ptest-runner'])
Brad Bishopf86d0552018-12-04 14:18:15 -080021 @unittest.expectedFailure
Andrew Geissler82c905d2020-04-13 13:39:40 -050022 def test_ptestrunner_expectfail(self):
23 if not self.td.get('PTEST_EXPECT_FAILURE'):
24 self.skipTest('Cannot run ptests with @expectedFailure as ptests are required to pass')
25 self.do_ptestrunner()
26
27 @skipIfNotFeature('ptest', 'Test requires ptest to be in DISTRO_FEATURES')
28 @OETestDepends(['ssh.SSHTest.test_ssh'])
29 @OEHasPackage(['ptest-runner'])
30 def test_ptestrunner_expectsuccess(self):
31 if self.td.get('PTEST_EXPECT_FAILURE'):
32 self.skipTest('Cannot run ptests without @expectedFailure as ptests are expected to fail')
33 self.do_ptestrunner()
34
35 def do_ptestrunner(self):
Brad Bishop316dfdd2018-06-25 12:45:53 -040036 status, output = self.target.run('which ptest-runner', 0)
37 if status != 0:
38 self.skipTest("No -ptest packages are installed in the image")
39
Brad Bishopd7bf8c12018-02-25 22:55:05 -050040 test_log_dir = self.td.get('TEST_LOG_DIR', '')
41 # The TEST_LOG_DIR maybe NULL when testimage is added after
42 # testdata.json is generated.
43 if not test_log_dir:
44 test_log_dir = os.path.join(self.td.get('WORKDIR', ''), 'testimage')
Andrew Geisslerd25ed322020-06-27 00:28:28 -050045 # Make the test output path absolute, otherwise the output content will be
46 # created relative to current directory
47 if not os.path.isabs(test_log_dir):
48 test_log_dir = os.path.join(self.td.get('TOPDIR', ''), test_log_dir)
Brad Bishopd7bf8c12018-02-25 22:55:05 -050049 # Don't use self.td.get('DATETIME'), it's from testdata.json, not
50 # up-to-date, and may cause "File exists" when re-reun.
Andrew Geissler99467da2019-02-25 18:54:23 -060051 timestamp = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
Brad Bishopd7bf8c12018-02-25 22:55:05 -050052 ptest_log_dir_link = os.path.join(test_log_dir, 'ptest_log')
Andrew Geissler99467da2019-02-25 18:54:23 -060053 ptest_log_dir = '%s.%s' % (ptest_log_dir_link, timestamp)
Brad Bishopd7bf8c12018-02-25 22:55:05 -050054 ptest_runner_log = os.path.join(ptest_log_dir, 'ptest-runner.log')
55
Andrew Geissler4b740dc2020-05-05 08:54:39 -050056 libdir = self.td.get('libdir', '')
57 ptest_dirs = [ '/usr/lib' ]
58 if not libdir in ptest_dirs:
59 ptest_dirs.append(libdir)
60 status, output = self.target.run('ptest-runner -d \"{}\"'.format(' '.join(ptest_dirs)), 0)
Brad Bishopd7bf8c12018-02-25 22:55:05 -050061 os.makedirs(ptest_log_dir)
62 with open(ptest_runner_log, 'w') as f:
63 f.write(output)
64
65 # status != 0 is OK since some ptest tests may fail
66 self.assertTrue(status != 127, msg="Cannot execute ptest-runner!")
67
Brad Bishopf86d0552018-12-04 14:18:15 -080068 if not hasattr(self.tc, "extraresults"):
69 self.tc.extraresults = {}
70 extras = self.tc.extraresults
71 extras['ptestresult.rawlogs'] = {'log': output}
72
Brad Bishopd7bf8c12018-02-25 22:55:05 -050073 # Parse and save results
Andrew Geissler99467da2019-02-25 18:54:23 -060074 parser = PtestParser()
75 results, sections = parser.parse(ptest_runner_log)
76 parser.results_as_files(ptest_log_dir)
Brad Bishopd7bf8c12018-02-25 22:55:05 -050077 if os.path.exists(ptest_log_dir_link):
78 # Remove the old link to create a new one
79 os.remove(ptest_log_dir_link)
80 os.symlink(os.path.basename(ptest_log_dir), ptest_log_dir_link)
Brad Bishop316dfdd2018-06-25 12:45:53 -040081
Andrew Geissler99467da2019-02-25 18:54:23 -060082 extras['ptestresult.sections'] = sections
83
Brad Bishopf86d0552018-12-04 14:18:15 -080084 trans = str.maketrans("()", "__")
Andrew Geissler99467da2019-02-25 18:54:23 -060085 for section in results:
86 for test in results[section]:
87 result = results[section][test]
88 testname = "ptestresult." + (section or "No-section") + "." + "_".join(test.translate(trans).split())
89 extras[testname] = {'status': result}
Brad Bishopf86d0552018-12-04 14:18:15 -080090
Brad Bishop316dfdd2018-06-25 12:45:53 -040091 failed_tests = {}
Andrew Geissler82c905d2020-04-13 13:39:40 -050092
93 for section in sections:
94 if 'exitcode' in sections[section].keys():
95 failed_tests[section] = sections[section]["log"]
96
Andrew Geissler99467da2019-02-25 18:54:23 -060097 for section in results:
Andrew Geissler82c905d2020-04-13 13:39:40 -050098 failed_testcases = [ "_".join(test.translate(trans).split()) for test in results[section] if results[section][test] == 'FAILED' ]
Brad Bishop316dfdd2018-06-25 12:45:53 -040099 if failed_testcases:
100 failed_tests[section] = failed_testcases
101
Andrew Geissler99467da2019-02-25 18:54:23 -0600102 failmsg = ""
103 status, output = self.target.run('dmesg | grep "Killed process"', 0)
104 if output:
105 failmsg = "ERROR: Processes were killed by the OOM Killer:\n%s\n" % output
106
Brad Bishopf86d0552018-12-04 14:18:15 -0800107 if failed_tests:
Andrew Geissler99467da2019-02-25 18:54:23 -0600108 failmsg = failmsg + "Failed ptests:\n%s" % pprint.pformat(failed_tests)
109
110 if failmsg:
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600111 self.logger.warning("There were failing ptests.")
Andrew Geissler99467da2019-02-25 18:54:23 -0600112 self.fail(failmsg)