blob: 16599269751ccc1ffbd9b1e96452acb23bb9f5a8 [file] [log] [blame]
Brad Bishopc342db32019-05-15 21:57:59 -04001#
Brad Bishopd7bf8c12018-02-25 22:55:05 -05002# Copyright (C) 2017 Intel Corporation
Brad Bishopc342db32019-05-15 21:57:59 -04003#
4# SPDX-License-Identifier: MIT
5#
Brad Bishopd7bf8c12018-02-25 22:55:05 -05006
7import os
8import time
9import glob
10import sys
Brad Bishopf86d0552018-12-04 14:18:15 -080011import importlib
Andrew Geissler82c905d2020-04-13 13:39:40 -050012import subprocess
Andrew Geissler4ed12e12020-06-05 18:00:41 -050013import unittest
Brad Bishopd7bf8c12018-02-25 22:55:05 -050014from random import choice
15
16import oeqa
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080017import oe
Andrew Geissler82c905d2020-04-13 13:39:40 -050018import bb.utils
Brad Bishopd7bf8c12018-02-25 22:55:05 -050019
20from oeqa.core.context import OETestContext, OETestContextExecutor
21from oeqa.core.exception import OEQAPreRun, OEQATestNotFound
22
23from oeqa.utils.commands import runCmd, get_bb_vars, get_test_layer
24
Andrew Geissler475cb722020-07-10 16:00:51 -050025class NonConcurrentTestSuite(unittest.TestSuite):
26 def __init__(self, suite, processes, setupfunc, removefunc):
27 super().__init__([suite])
28 self.processes = processes
29 self.suite = suite
30 self.setupfunc = setupfunc
31 self.removefunc = removefunc
32
33 def run(self, result):
34 (builddir, newbuilddir) = self.setupfunc("-st", None, self.suite)
35 ret = super().run(result)
36 os.chdir(builddir)
Andrew Geisslerd1e89492021-02-12 15:35:20 -060037 if newbuilddir and ret.wasSuccessful() and self.removefunc:
Andrew Geissler475cb722020-07-10 16:00:51 -050038 self.removefunc(newbuilddir)
39
40def removebuilddir(d):
41 delay = 5
42 while delay and os.path.exists(d + "/bitbake.lock"):
43 time.sleep(1)
44 delay = delay - 1
45 # Deleting these directories takes a lot of time, use autobuilder
46 # clobberdir if its available
47 clobberdir = os.path.expanduser("~/yocto-autobuilder-helper/janitor/clobberdir")
48 if os.path.exists(clobberdir):
49 try:
50 subprocess.check_call([clobberdir, d])
51 return
52 except subprocess.CalledProcessError:
53 pass
54 bb.utils.prunedir(d, ionice=True)
55
Brad Bishopd7bf8c12018-02-25 22:55:05 -050056class OESelftestTestContext(OETestContext):
Andrew Geisslerd1e89492021-02-12 15:35:20 -060057 def __init__(self, td=None, logger=None, machines=None, config_paths=None, newbuilddir=None, keep_builddir=None):
Brad Bishopd7bf8c12018-02-25 22:55:05 -050058 super(OESelftestTestContext, self).__init__(td, logger)
59
60 self.machines = machines
61 self.custommachine = None
62 self.config_paths = config_paths
Andrew Geissler4ed12e12020-06-05 18:00:41 -050063 self.newbuilddir = newbuilddir
Brad Bishopd7bf8c12018-02-25 22:55:05 -050064
Andrew Geisslerd1e89492021-02-12 15:35:20 -060065 if keep_builddir:
66 self.removebuilddir = None
67 else:
68 self.removebuilddir = removebuilddir
69
Andrew Geissler82c905d2020-04-13 13:39:40 -050070 def setup_builddir(self, suffix, selftestdir, suite):
71 builddir = os.environ['BUILDDIR']
72 if not selftestdir:
73 selftestdir = get_test_layer()
Andrew Geissler4ed12e12020-06-05 18:00:41 -050074 if self.newbuilddir:
75 newbuilddir = os.path.join(self.newbuilddir, 'build' + suffix)
76 else:
77 newbuilddir = builddir + suffix
Andrew Geissler82c905d2020-04-13 13:39:40 -050078 newselftestdir = newbuilddir + "/meta-selftest"
79
80 if os.path.exists(newbuilddir):
81 self.logger.error("Build directory %s already exists, aborting" % newbuilddir)
82 sys.exit(1)
83
84 bb.utils.mkdirhier(newbuilddir)
85 oe.path.copytree(builddir + "/conf", newbuilddir + "/conf")
86 oe.path.copytree(builddir + "/cache", newbuilddir + "/cache")
87 oe.path.copytree(selftestdir, newselftestdir)
88
89 for e in os.environ:
Andrew Geisslerc9f78652020-09-18 14:11:35 -050090 if builddir + "/" in os.environ[e]:
91 os.environ[e] = os.environ[e].replace(builddir + "/", newbuilddir + "/")
92 if os.environ[e].endswith(builddir):
Andrew Geissler82c905d2020-04-13 13:39:40 -050093 os.environ[e] = os.environ[e].replace(builddir, newbuilddir)
94
95 subprocess.check_output("git init; git add *; git commit -a -m 'initial'", cwd=newselftestdir, shell=True)
96
97 # Tried to used bitbake-layers add/remove but it requires recipe parsing and hence is too slow
98 subprocess.check_output("sed %s/conf/bblayers.conf -i -e 's#%s#%s#g'" % (newbuilddir, selftestdir, newselftestdir), cwd=newbuilddir, shell=True)
99
100 os.chdir(newbuilddir)
101
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500102 def patch_test(t):
Andrew Geissler82c905d2020-04-13 13:39:40 -0500103 if not hasattr(t, "tc"):
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500104 return
Andrew Geissler82c905d2020-04-13 13:39:40 -0500105 cp = t.tc.config_paths
106 for p in cp:
107 if selftestdir in cp[p] and newselftestdir not in cp[p]:
108 cp[p] = cp[p].replace(selftestdir, newselftestdir)
109 if builddir in cp[p] and newbuilddir not in cp[p]:
110 cp[p] = cp[p].replace(builddir, newbuilddir)
111
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500112 def patch_suite(s):
113 for x in s:
114 if isinstance(x, unittest.TestSuite):
115 patch_suite(x)
116 else:
117 patch_test(x)
118
119 patch_suite(suite)
120
Andrew Geissler82c905d2020-04-13 13:39:40 -0500121 return (builddir, newbuilddir)
122
123 def prepareSuite(self, suites, processes):
124 if processes:
125 from oeqa.core.utils.concurrencytest import ConcurrentTestSuite
126
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600127 return ConcurrentTestSuite(suites, processes, self.setup_builddir, self.removebuilddir)
Andrew Geissler82c905d2020-04-13 13:39:40 -0500128 else:
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600129 return NonConcurrentTestSuite(suites, processes, self.setup_builddir, self.removebuilddir)
Andrew Geissler82c905d2020-04-13 13:39:40 -0500130
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800131 def runTests(self, processes=None, machine=None, skips=[]):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500132 if machine:
133 self.custommachine = machine
134 if machine == 'random':
135 self.custommachine = choice(self.machines)
136 self.logger.info('Run tests with custom MACHINE set to: %s' % \
137 self.custommachine)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800138 return super(OESelftestTestContext, self).runTests(processes, skips)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500139
140 def listTests(self, display_type, machine=None):
141 return super(OESelftestTestContext, self).listTests(display_type)
142
143class OESelftestTestContextExecutor(OETestContextExecutor):
144 _context_class = OESelftestTestContext
145 _script_executor = 'oe-selftest'
146
147 name = 'oe-selftest'
148 help = 'oe-selftest test component'
149 description = 'Executes selftest tests'
150
151 def register_commands(self, logger, parser):
152 group = parser.add_mutually_exclusive_group(required=True)
153
154 group.add_argument('-a', '--run-all-tests', default=False,
155 action="store_true", dest="run_all_tests",
156 help='Run all (unhidden) tests')
157 group.add_argument('-R', '--skip-tests', required=False, action='store',
158 nargs='+', dest="skips", default=None,
159 help='Run all (unhidden) tests except the ones specified. Format should be <module>[.<class>[.<test_method>]]')
160 group.add_argument('-r', '--run-tests', required=False, action='store',
161 nargs='+', dest="run_tests", default=None,
162 help='Select what tests to run (modules, classes or test methods). Format should be: <module>.<class>.<test_method>')
163
164 group.add_argument('-m', '--list-modules', required=False,
165 action="store_true", default=False,
166 help='List all available test modules.')
167 group.add_argument('--list-classes', required=False,
168 action="store_true", default=False,
169 help='List all available test classes.')
170 group.add_argument('-l', '--list-tests', required=False,
171 action="store_true", default=False,
172 help='List all available tests.')
173
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800174 parser.add_argument('-j', '--num-processes', dest='processes', action='store',
175 type=int, help="number of processes to execute in parallel with")
176
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500177 parser.add_argument('--machine', required=False, choices=['random', 'all'],
178 help='Run tests on different machines (random/all).')
Brad Bishop79641f22019-09-10 07:20:22 -0400179
Brad Bishopacc069e2019-09-13 06:48:36 -0400180 parser.add_argument('-t', '--select-tag', dest="select_tags",
181 action='append', default=None,
182 help='Filter all (unhidden) tests to any that match any of the specified tag(s).')
183 parser.add_argument('-T', '--exclude-tag', dest="exclude_tags",
184 action='append', default=None,
185 help='Exclude all (unhidden) tests that match any of the specified tag(s). (exclude applies before select)')
Brad Bishop79641f22019-09-10 07:20:22 -0400186
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600187 parser.add_argument('-K', '--keep-builddir', action='store_true',
188 help='Keep the test build directory even if all tests pass')
189
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500190 parser.add_argument('-B', '--newbuilddir', help='New build directory to use for tests.')
191 parser.add_argument('-v', '--verbose', action='store_true')
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500192 parser.set_defaults(func=self.run)
193
194 def _get_available_machines(self):
195 machines = []
196
197 bbpath = self.tc_kwargs['init']['td']['BBPATH'].split(':')
198
199 for path in bbpath:
200 found_machines = glob.glob(os.path.join(path, 'conf', 'machine', '*.conf'))
201 if found_machines:
202 for i in found_machines:
203 # eg: '/home/<user>/poky/meta-intel/conf/machine/intel-core2-32.conf'
204 machines.append(os.path.splitext(os.path.basename(i))[0])
205
206 return machines
207
208 def _get_cases_paths(self, bbpath):
209 cases_paths = []
210 for layer in bbpath:
211 cases_dir = os.path.join(layer, 'lib', 'oeqa', 'selftest', 'cases')
212 if os.path.isdir(cases_dir):
213 cases_paths.append(cases_dir)
214 return cases_paths
215
216 def _process_args(self, logger, args):
Brad Bishopf86d0552018-12-04 14:18:15 -0800217 args.test_start_time = time.strftime("%Y%m%d%H%M%S")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500218 args.test_data_file = None
219 args.CASES_PATHS = None
220
Brad Bishopf86d0552018-12-04 14:18:15 -0800221 bbvars = get_bb_vars()
222 logdir = os.environ.get("BUILDDIR")
223 if 'LOG_DIR' in bbvars:
224 logdir = bbvars['LOG_DIR']
Brad Bishop19323692019-04-05 15:28:33 -0400225 bb.utils.mkdirhier(logdir)
Brad Bishopf86d0552018-12-04 14:18:15 -0800226 args.output_log = logdir + '/%s-results-%s.log' % (self.name, args.test_start_time)
227
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500228 super(OESelftestTestContextExecutor, self)._process_args(logger, args)
229
230 if args.list_modules:
231 args.list_tests = 'module'
232 elif args.list_classes:
233 args.list_tests = 'class'
234 elif args.list_tests:
235 args.list_tests = 'name'
236
Brad Bishopf86d0552018-12-04 14:18:15 -0800237 self.tc_kwargs['init']['td'] = bbvars
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500238 self.tc_kwargs['init']['machines'] = self._get_available_machines()
239
240 builddir = os.environ.get("BUILDDIR")
241 self.tc_kwargs['init']['config_paths'] = {}
Andrew Geissler82c905d2020-04-13 13:39:40 -0500242 self.tc_kwargs['init']['config_paths']['testlayer_path'] = get_test_layer()
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500243 self.tc_kwargs['init']['config_paths']['builddir'] = builddir
Andrew Geissler82c905d2020-04-13 13:39:40 -0500244 self.tc_kwargs['init']['config_paths']['localconf'] = os.path.join(builddir, "conf/local.conf")
245 self.tc_kwargs['init']['config_paths']['bblayers'] = os.path.join(builddir, "conf/bblayers.conf")
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500246 self.tc_kwargs['init']['newbuilddir'] = args.newbuilddir
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600247 self.tc_kwargs['init']['keep_builddir'] = args.keep_builddir
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500248
Brad Bishop79641f22019-09-10 07:20:22 -0400249 def tag_filter(tags):
250 if args.exclude_tags:
251 if any(tag in args.exclude_tags for tag in tags):
252 return True
253 if args.select_tags:
254 if not tags or not any(tag in args.select_tags for tag in tags):
255 return True
256 return False
257
258 if args.select_tags or args.exclude_tags:
259 self.tc_kwargs['load']['tags_filter'] = tag_filter
260
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500261 self.tc_kwargs['run']['skips'] = args.skips
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800262 self.tc_kwargs['run']['processes'] = args.processes
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500263
264 def _pre_run(self):
265 def _check_required_env_variables(vars):
266 for var in vars:
267 if not os.environ.get(var):
268 self.tc.logger.error("%s is not set. Did you forget to source your build environment setup script?" % var)
269 raise OEQAPreRun
270
271 def _check_presence_meta_selftest():
272 builddir = os.environ.get("BUILDDIR")
273 if os.getcwd() != builddir:
274 self.tc.logger.info("Changing cwd to %s" % builddir)
275 os.chdir(builddir)
276
277 if not "meta-selftest" in self.tc.td["BBLAYERS"]:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800278 self.tc.logger.warning("meta-selftest layer not found in BBLAYERS, adding it")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500279 meta_selftestdir = os.path.join(
280 self.tc.td["BBLAYERS_FETCH_DIR"], 'meta-selftest')
281 if os.path.isdir(meta_selftestdir):
282 runCmd("bitbake-layers add-layer %s" %meta_selftestdir)
283 # reload data is needed because a meta-selftest layer was add
284 self.tc.td = get_bb_vars()
285 self.tc.config_paths['testlayer_path'] = get_test_layer()
286 else:
287 self.tc.logger.error("could not locate meta-selftest in:\n%s" % meta_selftestdir)
288 raise OEQAPreRun
289
290 def _add_layer_libs():
291 bbpath = self.tc.td['BBPATH'].split(':')
292 layer_libdirs = [p for p in (os.path.join(l, 'lib') \
293 for l in bbpath) if os.path.exists(p)]
294 if layer_libdirs:
295 self.tc.logger.info("Adding layer libraries:")
296 for l in layer_libdirs:
297 self.tc.logger.info("\t%s" % l)
298
299 sys.path.extend(layer_libdirs)
Brad Bishopf86d0552018-12-04 14:18:15 -0800300 importlib.reload(oeqa.selftest)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500301
302 _check_required_env_variables(["BUILDDIR"])
303 _check_presence_meta_selftest()
304
305 if "buildhistory.bbclass" in self.tc.td["BBINCLUDED"]:
306 self.tc.logger.error("You have buildhistory enabled already and this isn't recommended for selftest, please disable it first.")
307 raise OEQAPreRun
308
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800309 if "rm_work.bbclass" in self.tc.td["BBINCLUDED"]:
310 self.tc.logger.error("You have rm_work enabled which isn't recommended while running oe-selftest. Please disable it before continuing.")
311 raise OEQAPreRun
312
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500313 if "PRSERV_HOST" in self.tc.td:
314 self.tc.logger.error("Please unset PRSERV_HOST in order to run oe-selftest")
315 raise OEQAPreRun
316
317 if "SANITY_TESTED_DISTROS" in self.tc.td:
318 self.tc.logger.error("Please unset SANITY_TESTED_DISTROS in order to run oe-selftest")
319 raise OEQAPreRun
320
321 _add_layer_libs()
322
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800323 self.tc.logger.info("Running bitbake -e to test the configuration is valid/parsable")
324 runCmd("bitbake -e")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500325
Brad Bishopf86d0552018-12-04 14:18:15 -0800326 def get_json_result_dir(self, args):
327 json_result_dir = os.path.join(self.tc.td["LOG_DIR"], 'oeqa')
328 if "OEQA_JSON_RESULT_DIR" in self.tc.td:
329 json_result_dir = self.tc.td["OEQA_JSON_RESULT_DIR"]
330
331 return json_result_dir
332
333 def get_configuration(self, args):
334 import platform
335 from oeqa.utils.metadata import metadata_from_bb
336 metadata = metadata_from_bb()
337 configuration = {'TEST_TYPE': 'oeselftest',
338 'STARTTIME': args.test_start_time,
339 'MACHINE': self.tc.td["MACHINE"],
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800340 'HOST_DISTRO': oe.lsb.distro_identifier().replace(' ', '-'),
Brad Bishopf86d0552018-12-04 14:18:15 -0800341 'HOST_NAME': metadata['hostname'],
342 'LAYERS': metadata['layers']}
343 return configuration
344
345 def get_result_id(self, configuration):
346 return '%s_%s_%s_%s' % (configuration['TEST_TYPE'], configuration['HOST_DISTRO'], configuration['MACHINE'], configuration['STARTTIME'])
347
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500348 def _internal_run(self, logger, args):
349 self.module_paths = self._get_cases_paths(
350 self.tc_kwargs['init']['td']['BBPATH'].split(':'))
351
352 self.tc = self._context_class(**self.tc_kwargs['init'])
353 try:
354 self.tc.loadTests(self.module_paths, **self.tc_kwargs['load'])
355 except OEQATestNotFound as ex:
356 logger.error(ex)
357 sys.exit(1)
358
359 if args.list_tests:
360 rc = self.tc.listTests(args.list_tests, **self.tc_kwargs['list'])
361 else:
362 self._pre_run()
363 rc = self.tc.runTests(**self.tc_kwargs['run'])
Brad Bishopf86d0552018-12-04 14:18:15 -0800364 configuration = self.get_configuration(args)
365 rc.logDetails(self.get_json_result_dir(args),
366 configuration,
367 self.get_result_id(configuration))
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500368 rc.logSummary(self.name)
369
370 return rc
371
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500372 def run(self, logger, args):
373 self._process_args(logger, args)
374
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500375 rc = None
376 try:
377 if args.machine:
378 logger.info('Custom machine mode enabled. MACHINE set to %s' %
379 args.machine)
380
381 if args.machine == 'all':
382 results = []
383 for m in self.tc_kwargs['init']['machines']:
384 self.tc_kwargs['run']['machine'] = m
385 results.append(self._internal_run(logger, args))
386
387 # XXX: the oe-selftest script only needs to know if one
388 # machine run fails
389 for r in results:
390 rc = r
391 if not r.wasSuccessful():
392 break
393
394 else:
395 self.tc_kwargs['run']['machine'] = args.machine
396 return self._internal_run(logger, args)
397
398 else:
399 self.tc_kwargs['run']['machine'] = args.machine
400 rc = self._internal_run(logger, args)
401 finally:
402 config_paths = self.tc_kwargs['init']['config_paths']
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500403
404 output_link = os.path.join(os.path.dirname(args.output_log),
405 "%s-results.log" % self.name)
Brad Bishopf86d0552018-12-04 14:18:15 -0800406 if os.path.lexists(output_link):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500407 os.remove(output_link)
408 os.symlink(args.output_log, output_link)
409
410 return rc
411
412_executor_class = OESelftestTestContextExecutor