Patrick Williams | 92b42cb | 2022-09-03 06:53:57 -0500 | [diff] [blame] | 1 | # Copyright (C) 2013 Intel Corporation |
| 2 | # |
| 3 | # SPDX-License-Identifier: MIT |
| 4 | |
| 5 | inherit metadata_scm |
| 6 | inherit image-artifact-names |
| 7 | |
| 8 | # testimage.bbclass enables testing of qemu images using python unittests. |
| 9 | # Most of the tests are commands run on target image over ssh. |
| 10 | # To use it add testimage to global inherit and call your target image with -c testimage |
| 11 | # You can try it out like this: |
| 12 | # - first add IMAGE_CLASSES += "testimage" in local.conf |
| 13 | # - build a qemu core-image-sato |
| 14 | # - then bitbake core-image-sato -c testimage. That will run a standard suite of tests. |
| 15 | # |
| 16 | # The tests can be run automatically each time an image is built if you set |
| 17 | # TESTIMAGE_AUTO = "1" |
| 18 | |
| 19 | TESTIMAGE_AUTO ??= "0" |
| 20 | |
Patrick Williams | 520786c | 2023-06-25 16:20:36 -0500 | [diff] [blame] | 21 | # When any test fails, TESTIMAGE_FAILED_QA ARTIFACTS will be parsed and for |
| 22 | # each entry in it, if artifact pointed by path description exists on target, |
| 23 | # it will be retrieved onto host |
| 24 | |
| 25 | TESTIMAGE_FAILED_QA_ARTIFACTS ??= "\ |
| 26 | ${localstatedir}/log \ |
| 27 | ${sysconfdir}/version \ |
| 28 | ${sysconfdir}/os-release" |
| 29 | |
Patrick Williams | 92b42cb | 2022-09-03 06:53:57 -0500 | [diff] [blame] | 30 | # You can set (or append to) TEST_SUITES in local.conf to select the tests |
| 31 | # which you want to run for your target. |
| 32 | # The test names are the module names in meta/lib/oeqa/runtime/cases. |
| 33 | # Each name in TEST_SUITES represents a required test for the image. (no skipping allowed) |
| 34 | # Appending "auto" means that it will try to run all tests that are suitable for the image (each test decides that on it's own). |
| 35 | # Note that order in TEST_SUITES is relevant: tests are run in an order such that |
| 36 | # tests mentioned in @skipUnlessPassed run before the tests that depend on them, |
| 37 | # but without such dependencies, tests run in the order in which they are listed |
| 38 | # in TEST_SUITES. |
| 39 | # |
| 40 | # A layer can add its own tests in lib/oeqa/runtime, provided it extends BBPATH as normal in its layer.conf. |
| 41 | |
| 42 | # TEST_LOG_DIR contains a command ssh log and may contain infromation about what command is running, output and return codes and for qemu a boot log till login. |
| 43 | # Booting is handled by this class, and it's not a test in itself. |
| 44 | # TEST_QEMUBOOT_TIMEOUT can be used to set the maximum time in seconds the launch code will wait for the login prompt. |
| 45 | # TEST_OVERALL_TIMEOUT can be used to set the maximum time in seconds the tests will be allowed to run (defaults to no limit). |
| 46 | # TEST_QEMUPARAMS can be used to pass extra parameters to qemu, e.g. "-m 1024" for setting the amount of ram to 1 GB. |
| 47 | # TEST_RUNQEMUPARAMS can be used to pass extra parameters to runqemu, e.g. "gl" to enable OpenGL acceleration. |
| 48 | # QEMU_USE_KVM can be set to "" to disable the use of kvm (by default it is enabled if target_arch == build_arch or both of them are x86 archs) |
| 49 | |
| 50 | # TESTIMAGE_BOOT_PATTERNS can be used to override certain patterns used to communicate with the target when booting, |
| 51 | # if a pattern is not specifically present on this variable a default will be used when booting the target. |
| 52 | # TESTIMAGE_BOOT_PATTERNS[<flag>] overrides the pattern used for that specific flag, where flag comes from a list of accepted flags |
| 53 | # e.g. normally the system boots and waits for a login prompt (login:), after that it sends the command: "root\n" to log as the root user |
| 54 | # if we wanted to log in as the hypothetical "webserver" user for example we could set the following: |
| 55 | # TESTIMAGE_BOOT_PATTERNS = "send_login_user search_login_succeeded" |
| 56 | # TESTIMAGE_BOOT_PATTERNS[send_login_user] = "webserver\n" |
| 57 | # TESTIMAGE_BOOT_PATTERNS[search_login_succeeded] = "webserver@[a-zA-Z0-9\-]+:~#" |
| 58 | # The accepted flags are the following: search_reached_prompt, send_login_user, search_login_succeeded, search_cmd_finished. |
| 59 | # They are prefixed with either search/send, to differentiate if the pattern is meant to be sent or searched to/from the target terminal |
| 60 | |
| 61 | TEST_LOG_DIR ?= "${WORKDIR}/testimage" |
| 62 | |
| 63 | TEST_EXPORT_DIR ?= "${TMPDIR}/testimage/${PN}" |
| 64 | TEST_INSTALL_TMP_DIR ?= "${WORKDIR}/testimage/install_tmp" |
| 65 | TEST_NEEDED_PACKAGES_DIR ?= "${WORKDIR}/testimage/packages" |
| 66 | TEST_EXTRACTED_DIR ?= "${TEST_NEEDED_PACKAGES_DIR}/extracted" |
| 67 | TEST_PACKAGED_DIR ?= "${TEST_NEEDED_PACKAGES_DIR}/packaged" |
| 68 | |
| 69 | BASICTESTSUITE = "\ |
| 70 | ping date df ssh scp python perl gi ptest parselogs \ |
| 71 | logrotate connman systemd oe_syslog pam stap ldd xorg \ |
| 72 | kernelmodule gcc buildcpio buildlzip buildgalculator \ |
| 73 | dnf rpm opkg apt weston go rust" |
| 74 | |
| 75 | DEFAULT_TEST_SUITES = "${BASICTESTSUITE}" |
| 76 | |
| 77 | # musl doesn't support systemtap |
| 78 | DEFAULT_TEST_SUITES:remove:libc-musl = "stap" |
| 79 | |
| 80 | # qemumips is quite slow and has reached the timeout limit several times on the YP build cluster, |
| 81 | # mitigate this by removing build tests for qemumips machines. |
| 82 | MIPSREMOVE ??= "buildcpio buildlzip buildgalculator" |
| 83 | DEFAULT_TEST_SUITES:remove:qemumips = "${MIPSREMOVE}" |
| 84 | DEFAULT_TEST_SUITES:remove:qemumips64 = "${MIPSREMOVE}" |
| 85 | |
| 86 | TEST_SUITES ?= "${DEFAULT_TEST_SUITES}" |
| 87 | |
| 88 | QEMU_USE_KVM ?= "1" |
| 89 | TEST_QEMUBOOT_TIMEOUT ?= "1000" |
| 90 | TEST_OVERALL_TIMEOUT ?= "" |
| 91 | TEST_TARGET ?= "qemu" |
| 92 | TEST_QEMUPARAMS ?= "" |
| 93 | TEST_RUNQEMUPARAMS ?= "" |
| 94 | |
| 95 | TESTIMAGE_BOOT_PATTERNS ?= "" |
| 96 | |
| 97 | TESTIMAGEDEPENDS = "" |
| 98 | TESTIMAGEDEPENDS:append:qemuall = " qemu-native:do_populate_sysroot qemu-helper-native:do_populate_sysroot qemu-helper-native:do_addto_recipe_sysroot" |
| 99 | TESTIMAGEDEPENDS += "${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'cpio-native:do_populate_sysroot', '', d)}" |
| 100 | TESTIMAGEDEPENDS += "${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'dnf-native:do_populate_sysroot', '', d)}" |
| 101 | TESTIMAGEDEPENDS += "${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'createrepo-c-native:do_populate_sysroot', '', d)}" |
| 102 | TESTIMAGEDEPENDS += "${@bb.utils.contains('IMAGE_PKGTYPE', 'ipk', 'opkg-utils-native:do_populate_sysroot package-index:do_package_index', '', d)}" |
| 103 | TESTIMAGEDEPENDS += "${@bb.utils.contains('IMAGE_PKGTYPE', 'deb', 'apt-native:do_populate_sysroot package-index:do_package_index', '', d)}" |
| 104 | |
| 105 | TESTIMAGELOCK = "${TMPDIR}/testimage.lock" |
| 106 | TESTIMAGELOCK:qemuall = "" |
| 107 | |
| 108 | TESTIMAGE_DUMP_DIR ?= "${LOG_DIR}/runtime-hostdump/" |
| 109 | |
Patrick Williams | 8e7b46e | 2023-05-01 14:19:06 -0500 | [diff] [blame] | 110 | TESTIMAGE_UPDATE_VARS ?= "DL_DIR WORKDIR DEPLOY_DIR_IMAGE IMAGE_LINK_NAME" |
Patrick Williams | 92b42cb | 2022-09-03 06:53:57 -0500 | [diff] [blame] | 111 | |
| 112 | testimage_dump_target () { |
| 113 | top -bn1 |
| 114 | ps |
| 115 | free |
| 116 | df |
| 117 | # The next command will export the default gateway IP |
| 118 | export DEFAULT_GATEWAY=$(ip route | awk '/default/ { print $3}') |
| 119 | ping -c3 $DEFAULT_GATEWAY |
| 120 | dmesg |
| 121 | netstat -an |
| 122 | ip address |
| 123 | # Next command will dump logs from /var/log/ |
| 124 | find /var/log/ -type f 2>/dev/null -exec echo "====================" \; -exec echo {} \; -exec echo "====================" \; -exec cat {} \; -exec echo "" \; |
| 125 | } |
| 126 | |
Patrick Williams | 92b42cb | 2022-09-03 06:53:57 -0500 | [diff] [blame] | 127 | testimage_dump_monitor () { |
| 128 | query-status |
| 129 | query-block |
| 130 | dump-guest-memory {"paging":false,"protocol":"file:%s.img"} |
| 131 | } |
| 132 | |
| 133 | python do_testimage() { |
| 134 | testimage_main(d) |
| 135 | } |
| 136 | |
| 137 | addtask testimage |
| 138 | do_testimage[nostamp] = "1" |
| 139 | do_testimage[network] = "1" |
| 140 | do_testimage[depends] += "${TESTIMAGEDEPENDS}" |
| 141 | do_testimage[lockfiles] += "${TESTIMAGELOCK}" |
| 142 | |
| 143 | def testimage_sanity(d): |
| 144 | if (d.getVar('TEST_TARGET') == 'simpleremote' |
| 145 | and (not d.getVar('TEST_TARGET_IP') |
| 146 | or not d.getVar('TEST_SERVER_IP'))): |
| 147 | bb.fatal('When TEST_TARGET is set to "simpleremote" ' |
| 148 | 'TEST_TARGET_IP and TEST_SERVER_IP are needed too.') |
| 149 | |
| 150 | def get_testimage_configuration(d, test_type, machine): |
| 151 | import platform |
| 152 | from oeqa.utils.metadata import get_layers |
| 153 | configuration = {'TEST_TYPE': test_type, |
| 154 | 'MACHINE': machine, |
| 155 | 'DISTRO': d.getVar("DISTRO"), |
| 156 | 'IMAGE_BASENAME': d.getVar("IMAGE_BASENAME"), |
| 157 | 'IMAGE_PKGTYPE': d.getVar("IMAGE_PKGTYPE"), |
| 158 | 'STARTTIME': d.getVar("DATETIME"), |
| 159 | 'HOST_DISTRO': oe.lsb.distro_identifier().replace(' ', '-'), |
| 160 | 'LAYERS': get_layers(d.getVar("BBLAYERS"))} |
| 161 | return configuration |
| 162 | get_testimage_configuration[vardepsexclude] = "DATETIME" |
| 163 | |
| 164 | def get_testimage_json_result_dir(d): |
| 165 | json_result_dir = os.path.join(d.getVar("LOG_DIR"), 'oeqa') |
| 166 | custom_json_result_dir = d.getVar("OEQA_JSON_RESULT_DIR") |
| 167 | if custom_json_result_dir: |
| 168 | json_result_dir = custom_json_result_dir |
| 169 | return json_result_dir |
| 170 | |
| 171 | def get_testimage_result_id(configuration): |
| 172 | return '%s_%s_%s_%s' % (configuration['TEST_TYPE'], configuration['IMAGE_BASENAME'], configuration['MACHINE'], configuration['STARTTIME']) |
| 173 | |
| 174 | def get_testimage_boot_patterns(d): |
| 175 | from collections import defaultdict |
| 176 | boot_patterns = defaultdict(str) |
| 177 | # Only accept certain values |
| 178 | accepted_patterns = ['search_reached_prompt', 'send_login_user', 'search_login_succeeded', 'search_cmd_finished'] |
| 179 | # Not all patterns need to be overriden, e.g. perhaps we only want to change the user |
| 180 | boot_patterns_flags = d.getVarFlags('TESTIMAGE_BOOT_PATTERNS') or {} |
| 181 | if boot_patterns_flags: |
| 182 | patterns_set = [p for p in boot_patterns_flags.items() if p[0] in d.getVar('TESTIMAGE_BOOT_PATTERNS').split()] |
| 183 | for flag, flagval in patterns_set: |
| 184 | if flag not in accepted_patterns: |
| 185 | bb.fatal('Testimage: The only accepted boot patterns are: search_reached_prompt,send_login_user, \ |
| 186 | search_login_succeeded,search_cmd_finished\n Make sure your TESTIMAGE_BOOT_PATTERNS=%s \ |
| 187 | contains an accepted flag.' % d.getVar('TESTIMAGE_BOOT_PATTERNS')) |
| 188 | return |
Andrew Geissler | 87f5cff | 2022-09-30 13:13:31 -0500 | [diff] [blame] | 189 | boot_patterns[flag] = flagval.encode().decode('unicode-escape') |
Patrick Williams | 92b42cb | 2022-09-03 06:53:57 -0500 | [diff] [blame] | 190 | return boot_patterns |
| 191 | |
Patrick Williams | 520786c | 2023-06-25 16:20:36 -0500 | [diff] [blame] | 192 | def get_artifacts_list(target, raw_list): |
| 193 | result = [] |
| 194 | # Passed list may contains patterns in paths, expand them directly on target |
| 195 | for raw_path in raw_list.split(): |
| 196 | cmd = f"for p in {raw_path}; do if [ -e $p ]; then echo $p; fi; done" |
| 197 | try: |
| 198 | status, output = target.run(cmd) |
| 199 | if status != 0 or not output: |
| 200 | raise Exception() |
| 201 | result += output.split() |
| 202 | except: |
| 203 | bb.note(f"No file/directory matching path {raw_path}") |
| 204 | |
| 205 | return result |
| 206 | |
| 207 | def retrieve_test_artifacts(target, artifacts_list, target_dir): |
| 208 | import shutil |
| 209 | |
| 210 | local_artifacts_dir = os.path.join(target_dir, "artifacts") |
| 211 | if os.path.isdir(local_artifacts_dir): |
| 212 | shutil.rmtree(local_artifacts_dir) |
| 213 | |
| 214 | os.makedirs(local_artifacts_dir) |
| 215 | for artifact_path in artifacts_list: |
| 216 | if not os.path.isabs(artifact_path): |
| 217 | bb.warn(f"{artifact_path} is not an absolute path") |
| 218 | continue |
| 219 | try: |
| 220 | dest_dir = os.path.join(local_artifacts_dir, os.path.dirname(artifact_path[1:])) |
| 221 | os.makedirs(dest_dir, exist_ok=True) |
| 222 | target.copyFrom(artifact_path, dest_dir) |
| 223 | except: |
| 224 | bb.warn(f"Can not retrieve {artifact_path} from test target") |
Patrick Williams | 92b42cb | 2022-09-03 06:53:57 -0500 | [diff] [blame] | 225 | |
| 226 | def testimage_main(d): |
| 227 | import os |
| 228 | import json |
| 229 | import signal |
| 230 | import logging |
| 231 | import shutil |
| 232 | |
| 233 | from bb.utils import export_proxies |
| 234 | from oeqa.runtime.context import OERuntimeTestContext |
| 235 | from oeqa.runtime.context import OERuntimeTestContextExecutor |
| 236 | from oeqa.core.target.qemu import supported_fstypes |
| 237 | from oeqa.core.utils.test import getSuiteCases |
| 238 | from oeqa.utils import make_logger_bitbake_compatible |
| 239 | |
| 240 | def sigterm_exception(signum, stackframe): |
| 241 | """ |
| 242 | Catch SIGTERM from worker in order to stop qemu. |
| 243 | """ |
| 244 | os.kill(os.getpid(), signal.SIGINT) |
| 245 | |
| 246 | def handle_test_timeout(timeout): |
| 247 | bb.warn("Global test timeout reached (%s seconds), stopping the tests." %(timeout)) |
| 248 | os.kill(os.getpid(), signal.SIGINT) |
| 249 | |
| 250 | testimage_sanity(d) |
| 251 | |
| 252 | if (d.getVar('IMAGE_PKGTYPE') == 'rpm' |
| 253 | and ('dnf' in d.getVar('TEST_SUITES') or 'auto' in d.getVar('TEST_SUITES'))): |
| 254 | create_rpm_index(d) |
| 255 | |
| 256 | logger = make_logger_bitbake_compatible(logging.getLogger("BitBake")) |
| 257 | pn = d.getVar("PN") |
| 258 | |
| 259 | bb.utils.mkdirhier(d.getVar("TEST_LOG_DIR")) |
| 260 | |
| 261 | image_name = ("%s/%s" % (d.getVar('DEPLOY_DIR_IMAGE'), |
| 262 | d.getVar('IMAGE_LINK_NAME'))) |
| 263 | |
| 264 | tdname = "%s.testdata.json" % image_name |
| 265 | try: |
| 266 | with open(tdname, "r") as f: |
| 267 | td = json.load(f) |
| 268 | except FileNotFoundError as err: |
Patrick Williams | 864cc43 | 2023-02-09 14:54:44 -0600 | [diff] [blame] | 269 | bb.fatal('File %s not found (%s).\nHave you built the image with IMAGE_CLASSES += "testimage" in the conf/local.conf?' % (tdname, err)) |
Patrick Williams | 92b42cb | 2022-09-03 06:53:57 -0500 | [diff] [blame] | 270 | |
| 271 | # Some variables need to be updates (mostly paths) with the |
| 272 | # ones of the current environment because some tests require them. |
| 273 | for var in d.getVar('TESTIMAGE_UPDATE_VARS').split(): |
| 274 | td[var] = d.getVar(var) |
| 275 | |
| 276 | image_manifest = "%s.manifest" % image_name |
| 277 | image_packages = OERuntimeTestContextExecutor.readPackagesManifest(image_manifest) |
| 278 | |
| 279 | extract_dir = d.getVar("TEST_EXTRACTED_DIR") |
| 280 | |
| 281 | # Get machine |
| 282 | machine = d.getVar("MACHINE") |
| 283 | |
| 284 | # Get rootfs |
| 285 | fstypes = d.getVar('IMAGE_FSTYPES').split() |
| 286 | if d.getVar("TEST_TARGET") == "qemu": |
| 287 | fstypes = [fs for fs in fstypes if fs in supported_fstypes] |
| 288 | if not fstypes: |
| 289 | bb.fatal('Unsupported image type built. Add a compatible image to ' |
| 290 | 'IMAGE_FSTYPES. Supported types: %s' % |
| 291 | ', '.join(supported_fstypes)) |
| 292 | qfstype = fstypes[0] |
| 293 | qdeffstype = d.getVar("QB_DEFAULT_FSTYPE") |
| 294 | if qdeffstype: |
| 295 | qfstype = qdeffstype |
| 296 | rootfs = '%s.%s' % (image_name, qfstype) |
| 297 | |
| 298 | # Get tmpdir (not really used, just for compatibility) |
| 299 | tmpdir = d.getVar("TMPDIR") |
| 300 | |
| 301 | # Get deploy_dir_image (not really used, just for compatibility) |
| 302 | dir_image = d.getVar("DEPLOY_DIR_IMAGE") |
| 303 | |
| 304 | # Get bootlog |
| 305 | bootlog = os.path.join(d.getVar("TEST_LOG_DIR"), |
| 306 | 'qemu_boot_log.%s' % d.getVar('DATETIME')) |
| 307 | |
| 308 | # Get display |
| 309 | display = d.getVar("BB_ORIGENV").getVar("DISPLAY") |
| 310 | |
| 311 | # Get kernel |
| 312 | kernel_name = ('%s-%s.bin' % (d.getVar("KERNEL_IMAGETYPE"), machine)) |
| 313 | kernel = os.path.join(d.getVar("DEPLOY_DIR_IMAGE"), kernel_name) |
| 314 | |
| 315 | # Get boottime |
| 316 | boottime = int(d.getVar("TEST_QEMUBOOT_TIMEOUT")) |
| 317 | |
| 318 | # Get use_kvm |
| 319 | kvm = oe.types.qemu_use_kvm(d.getVar('QEMU_USE_KVM'), d.getVar('TARGET_ARCH')) |
| 320 | |
| 321 | # Get OVMF |
| 322 | ovmf = d.getVar("QEMU_USE_OVMF") |
| 323 | |
| 324 | slirp = False |
Andrew Geissler | 220dafd | 2023-10-04 10:18:08 -0500 | [diff] [blame] | 325 | if bb.utils.contains('TEST_RUNQEMUPARAMS', 'slirp', True, False, d): |
Patrick Williams | 92b42cb | 2022-09-03 06:53:57 -0500 | [diff] [blame] | 326 | slirp = True |
| 327 | |
| 328 | # TODO: We use the current implementation of qemu runner because of |
| 329 | # time constrains, qemu runner really needs a refactor too. |
| 330 | target_kwargs = { 'machine' : machine, |
| 331 | 'rootfs' : rootfs, |
| 332 | 'tmpdir' : tmpdir, |
| 333 | 'dir_image' : dir_image, |
| 334 | 'display' : display, |
| 335 | 'kernel' : kernel, |
| 336 | 'boottime' : boottime, |
| 337 | 'bootlog' : bootlog, |
| 338 | 'kvm' : kvm, |
| 339 | 'slirp' : slirp, |
| 340 | 'dump_dir' : d.getVar("TESTIMAGE_DUMP_DIR"), |
| 341 | 'serial_ports': len(d.getVar("SERIAL_CONSOLES").split()), |
| 342 | 'ovmf' : ovmf, |
| 343 | 'tmpfsdir' : d.getVar("RUNQEMU_TMPFS_DIR"), |
| 344 | } |
| 345 | |
| 346 | if d.getVar("TESTIMAGE_BOOT_PATTERNS"): |
| 347 | target_kwargs['boot_patterns'] = get_testimage_boot_patterns(d) |
| 348 | |
| 349 | # hardware controlled targets might need further access |
| 350 | target_kwargs['powercontrol_cmd'] = d.getVar("TEST_POWERCONTROL_CMD") or None |
| 351 | target_kwargs['powercontrol_extra_args'] = d.getVar("TEST_POWERCONTROL_EXTRA_ARGS") or "" |
| 352 | target_kwargs['serialcontrol_cmd'] = d.getVar("TEST_SERIALCONTROL_CMD") or None |
| 353 | target_kwargs['serialcontrol_extra_args'] = d.getVar("TEST_SERIALCONTROL_EXTRA_ARGS") or "" |
| 354 | target_kwargs['testimage_dump_monitor'] = d.getVar("testimage_dump_monitor") or "" |
| 355 | target_kwargs['testimage_dump_target'] = d.getVar("testimage_dump_target") or "" |
| 356 | |
| 357 | def export_ssh_agent(d): |
| 358 | import os |
| 359 | |
| 360 | variables = ['SSH_AGENT_PID', 'SSH_AUTH_SOCK'] |
| 361 | for v in variables: |
| 362 | if v not in os.environ.keys(): |
| 363 | val = d.getVar(v) |
| 364 | if val is not None: |
| 365 | os.environ[v] = val |
| 366 | |
| 367 | export_ssh_agent(d) |
| 368 | |
| 369 | # runtime use network for download projects for build |
| 370 | export_proxies(d) |
| 371 | |
Andrew Geissler | 220dafd | 2023-10-04 10:18:08 -0500 | [diff] [blame] | 372 | if slirp: |
| 373 | # Default to 127.0.0.1 and let the runner identify the port forwarding |
| 374 | # (as OEQemuTarget does), but allow overriding. |
| 375 | target_ip = d.getVar("TEST_TARGET_IP") or "127.0.0.1" |
| 376 | # Default to 10.0.2.2 as this is the IP that the guest has with the |
| 377 | # default qemu slirp networking configuration, but allow overriding. |
| 378 | server_ip = d.getVar("TEST_SERVER_IP") or "10.0.2.2" |
| 379 | else: |
| 380 | target_ip = d.getVar("TEST_TARGET_IP") |
| 381 | server_ip = d.getVar("TEST_SERVER_IP") |
| 382 | |
Patrick Williams | 92b42cb | 2022-09-03 06:53:57 -0500 | [diff] [blame] | 383 | # the robot dance |
| 384 | target = OERuntimeTestContextExecutor.getTarget( |
Andrew Geissler | 220dafd | 2023-10-04 10:18:08 -0500 | [diff] [blame] | 385 | d.getVar("TEST_TARGET"), logger, target_ip, |
| 386 | server_ip, **target_kwargs) |
Patrick Williams | 92b42cb | 2022-09-03 06:53:57 -0500 | [diff] [blame] | 387 | |
| 388 | # test context |
Andrew Geissler | 8f84068 | 2023-07-21 09:09:43 -0500 | [diff] [blame] | 389 | tc = OERuntimeTestContext(td, logger, target, image_packages, extract_dir) |
Patrick Williams | 92b42cb | 2022-09-03 06:53:57 -0500 | [diff] [blame] | 390 | |
| 391 | # Load tests before starting the target |
| 392 | test_paths = get_runtime_paths(d) |
| 393 | test_modules = d.getVar('TEST_SUITES').split() |
| 394 | if not test_modules: |
| 395 | bb.fatal('Empty test suite, please verify TEST_SUITES variable') |
| 396 | |
| 397 | tc.loadTests(test_paths, modules=test_modules) |
| 398 | |
| 399 | suitecases = getSuiteCases(tc.suites) |
| 400 | if not suitecases: |
| 401 | bb.fatal('Empty test suite, please verify TEST_SUITES variable') |
| 402 | else: |
| 403 | bb.debug(2, 'test suites:\n\t%s' % '\n\t'.join([str(c) for c in suitecases])) |
| 404 | |
| 405 | package_extraction(d, tc.suites) |
| 406 | |
| 407 | results = None |
| 408 | complete = False |
| 409 | orig_sigterm_handler = signal.signal(signal.SIGTERM, sigterm_exception) |
| 410 | try: |
| 411 | # We need to check if runqemu ends unexpectedly |
| 412 | # or if the worker send us a SIGTERM |
| 413 | tc.target.start(params=d.getVar("TEST_QEMUPARAMS"), runqemuparams=d.getVar("TEST_RUNQEMUPARAMS")) |
| 414 | import threading |
| 415 | try: |
| 416 | threading.Timer(int(d.getVar("TEST_OVERALL_TIMEOUT")), handle_test_timeout, (int(d.getVar("TEST_OVERALL_TIMEOUT")),)).start() |
| 417 | except ValueError: |
| 418 | pass |
| 419 | results = tc.runTests() |
| 420 | complete = True |
Patrick Williams | 520786c | 2023-06-25 16:20:36 -0500 | [diff] [blame] | 421 | if results.hasAnyFailingTest(): |
| 422 | artifacts_list = get_artifacts_list(tc.target, d.getVar("TESTIMAGE_FAILED_QA_ARTIFACTS")) |
| 423 | if not artifacts_list: |
| 424 | bb.warn("Could not load artifacts list, skip artifacts retrieval") |
| 425 | else: |
| 426 | retrieve_test_artifacts(tc.target, artifacts_list, get_testimage_json_result_dir(d)) |
Patrick Williams | 92b42cb | 2022-09-03 06:53:57 -0500 | [diff] [blame] | 427 | except (KeyboardInterrupt, BlockingIOError) as err: |
| 428 | if isinstance(err, KeyboardInterrupt): |
| 429 | bb.error('testimage interrupted, shutting down...') |
| 430 | else: |
| 431 | bb.error('runqemu failed, shutting down...') |
| 432 | if results: |
| 433 | results.stop() |
| 434 | results = tc.results |
| 435 | finally: |
| 436 | signal.signal(signal.SIGTERM, orig_sigterm_handler) |
| 437 | tc.target.stop() |
| 438 | |
| 439 | # Show results (if we have them) |
| 440 | if results: |
| 441 | configuration = get_testimage_configuration(d, 'runtime', machine) |
| 442 | results.logDetails(get_testimage_json_result_dir(d), |
| 443 | configuration, |
| 444 | get_testimage_result_id(configuration), |
| 445 | dump_streams=d.getVar('TESTREPORT_FULLLOGS')) |
| 446 | results.logSummary(pn) |
| 447 | |
| 448 | # Copy additional logs to tmp/log/oeqa so it's easier to find them |
| 449 | targetdir = os.path.join(get_testimage_json_result_dir(d), d.getVar("PN")) |
| 450 | os.makedirs(targetdir, exist_ok=True) |
| 451 | os.symlink(bootlog, os.path.join(targetdir, os.path.basename(bootlog))) |
| 452 | os.symlink(d.getVar("BB_LOGFILE"), os.path.join(targetdir, os.path.basename(d.getVar("BB_LOGFILE") + "." + d.getVar('DATETIME')))) |
| 453 | |
| 454 | if not results or not complete: |
| 455 | bb.fatal('%s - FAILED - tests were interrupted during execution, check the logs in %s' % (pn, d.getVar("LOG_DIR")), forcelog=True) |
| 456 | if not results.wasSuccessful(): |
| 457 | bb.fatal('%s - FAILED - also check the logs in %s' % (pn, d.getVar("LOG_DIR")), forcelog=True) |
| 458 | |
| 459 | def get_runtime_paths(d): |
| 460 | """ |
| 461 | Returns a list of paths where runtime test must reside. |
| 462 | |
| 463 | Runtime tests are expected in <LAYER_DIR>/lib/oeqa/runtime/cases/ |
| 464 | """ |
| 465 | paths = [] |
| 466 | |
| 467 | for layer in d.getVar('BBLAYERS').split(): |
| 468 | path = os.path.join(layer, 'lib/oeqa/runtime/cases') |
| 469 | if os.path.isdir(path): |
| 470 | paths.append(path) |
| 471 | return paths |
| 472 | |
| 473 | def create_index(arg): |
| 474 | import subprocess |
| 475 | |
| 476 | index_cmd = arg |
| 477 | try: |
| 478 | bb.note("Executing '%s' ..." % index_cmd) |
| 479 | result = subprocess.check_output(index_cmd, |
| 480 | stderr=subprocess.STDOUT, |
| 481 | shell=True) |
| 482 | result = result.decode('utf-8') |
| 483 | except subprocess.CalledProcessError as e: |
| 484 | return("Index creation command '%s' failed with return code " |
| 485 | '%d:\n%s' % (e.cmd, e.returncode, e.output.decode("utf-8"))) |
| 486 | if result: |
| 487 | bb.note(result) |
| 488 | return None |
| 489 | |
| 490 | def create_rpm_index(d): |
| 491 | import glob |
| 492 | # Index RPMs |
| 493 | rpm_createrepo = bb.utils.which(os.getenv('PATH'), "createrepo_c") |
| 494 | index_cmds = [] |
| 495 | archs = (d.getVar('ALL_MULTILIB_PACKAGE_ARCHS') or '').replace('-', '_') |
| 496 | |
| 497 | for arch in archs.split(): |
| 498 | rpm_dir = os.path.join(d.getVar('DEPLOY_DIR_RPM'), arch) |
| 499 | idx_path = os.path.join(d.getVar('WORKDIR'), 'oe-testimage-repo', arch) |
| 500 | |
| 501 | if not os.path.isdir(rpm_dir): |
| 502 | continue |
| 503 | |
| 504 | lockfilename = os.path.join(d.getVar('DEPLOY_DIR_RPM'), 'rpm.lock') |
| 505 | lf = bb.utils.lockfile(lockfilename, False) |
| 506 | oe.path.copyhardlinktree(rpm_dir, idx_path) |
| 507 | # Full indexes overload a 256MB image so reduce the number of rpms |
| 508 | # in the feed by filtering to specific packages needed by the tests. |
| 509 | package_list = glob.glob(idx_path + "*/*.rpm") |
| 510 | |
| 511 | for pkg in package_list: |
Andrew Geissler | 87f5cff | 2022-09-30 13:13:31 -0500 | [diff] [blame] | 512 | if not os.path.basename(pkg).startswith(("dnf-test-", "busybox", "update-alternatives", "libc6", "musl")): |
Patrick Williams | 92b42cb | 2022-09-03 06:53:57 -0500 | [diff] [blame] | 513 | bb.utils.remove(pkg) |
| 514 | |
| 515 | bb.utils.unlockfile(lf) |
| 516 | cmd = '%s --update -q %s' % (rpm_createrepo, idx_path) |
| 517 | |
| 518 | # Create repodata |
| 519 | result = create_index(cmd) |
| 520 | if result: |
| 521 | bb.fatal('%s' % ('\n'.join(result))) |
| 522 | |
| 523 | def package_extraction(d, test_suites): |
| 524 | from oeqa.utils.package_manager import find_packages_to_extract |
| 525 | from oeqa.utils.package_manager import extract_packages |
| 526 | |
| 527 | bb.utils.remove(d.getVar("TEST_NEEDED_PACKAGES_DIR"), recurse=True) |
| 528 | packages = find_packages_to_extract(test_suites) |
| 529 | if packages: |
| 530 | bb.utils.mkdirhier(d.getVar("TEST_INSTALL_TMP_DIR")) |
| 531 | bb.utils.mkdirhier(d.getVar("TEST_PACKAGED_DIR")) |
| 532 | bb.utils.mkdirhier(d.getVar("TEST_EXTRACTED_DIR")) |
| 533 | extract_packages(d, packages) |
| 534 | |
| 535 | testimage_main[vardepsexclude] += "BB_ORIGENV DATETIME" |
| 536 | |
| 537 | python () { |
| 538 | if oe.types.boolean(d.getVar("TESTIMAGE_AUTO") or "False"): |
| 539 | bb.build.addtask("testimage", "do_build", "do_image_complete", d) |
| 540 | } |
| 541 | |
| 542 | inherit testsdk |