blob: 6ff7c8e498c330738f894f1ca651450281a43e4a [file] [log] [blame]
Patrick Williams02871c92021-02-01 20:57:19 -06001#!/usr/bin/env python3
2#
3# Build the required docker image to run package unit tests
4#
5# Script Variables:
6# DOCKER_IMG_NAME: <optional, the name of the docker image to generate>
7# default is openbmc/ubuntu-unit-test
8# DISTRO: <optional, the distro to build a docker image against>
Patrick Williams50837432021-02-06 12:24:05 -06009# FORCE_DOCKER_BUILD: <optional, a non-zero value with force all Docker
10# images to be rebuilt rather than reusing caches.>
11# BUILD_URL: <optional, used to detect running under CI context
12# (ex. Jenkins)>
Patrick Williams02871c92021-02-01 20:57:19 -060013# BRANCH: <optional, branch to build from each of the openbmc/
14# repositories>
15# default is master, which will be used if input branch not
16# provided or not found
17# UBUNTU_MIRROR: <optional, the URL of a mirror of Ubuntu to override the
18# default ones in /etc/apt/sources.list>
19# default is empty, and no mirror is used.
Andrew Geisslerfe2768c2024-10-02 10:10:29 -040020# DOCKER_REG: <optional, the URL of a docker registry to utilize
Andrew Geissler23ec3322024-10-02 10:45:32 -040021# instead of our default (public.ecr.aws/ubuntu)
22# (ex. docker.io)
Patrick Williams02871c92021-02-01 20:57:19 -060023# http_proxy The HTTP address of the proxy server to connect to.
24# Default: "", proxy is not setup if this is not set
25
Patrick Williams276bd0e2024-10-02 10:34:32 -040026import json
Patrick Williams02871c92021-02-01 20:57:19 -060027import os
Andrew Geisslerf3d27e62024-04-09 15:24:49 -050028import re
Patrick Williams02871c92021-02-01 20:57:19 -060029import sys
Patrick Williamsb16f3e22021-02-06 08:16:47 -060030import threading
Patrick Williams276bd0e2024-10-02 10:34:32 -040031import urllib.request
Patrick Williamsa18d9c52021-02-05 09:52:26 -060032from datetime import date
33from hashlib import sha256
Patrick Williamse08ffba2022-12-05 10:33:46 -060034
35# typing.Dict is used for type-hints.
36from typing import Any, Callable, Dict, Iterable, Optional # noqa: F401
Patrick Williams02871c92021-02-01 20:57:19 -060037
Andrew Geissler8f7146f2024-12-11 14:20:47 -060038from sh import git, nproc # type: ignore
39
40try:
41 # System may have docker or it may have podman, try docker first
42 from sh import docker
43
44 container = docker
45except ImportError:
46 try:
47 from sh import podman
48
49 container = podman
50 except Exception:
51 print("No docker or podman found on system")
52 exit(1)
Patrick Williams41d86212022-11-25 18:28:43 -060053
Patrick Williamsee3c9ee2021-02-12 20:56:01 -060054try:
55 # Python before 3.8 doesn't have TypedDict, so reroute to standard 'dict'.
56 from typing import TypedDict
Patrick Williams41d86212022-11-25 18:28:43 -060057except Exception:
Patrick Williamsee3c9ee2021-02-12 20:56:01 -060058
59 class TypedDict(dict): # type: ignore
60 # We need to do this to eat the 'total' argument.
Patrick Williams41d86212022-11-25 18:28:43 -060061 def __init_subclass__(cls, **kwargs: Any) -> None:
Patrick Williamsee3c9ee2021-02-12 20:56:01 -060062 super().__init_subclass__()
63
64
65# Declare some variables used in package definitions.
Patrick Williamsaae36d12021-02-04 16:30:04 -060066prefix = "/usr/local"
Patrick Williams02871c92021-02-01 20:57:19 -060067proc_count = nproc().strip()
Patrick Williams02871c92021-02-01 20:57:19 -060068
Patrick Williamsee3c9ee2021-02-12 20:56:01 -060069
70class PackageDef(TypedDict, total=False):
Patrick Williams05fb2a02022-10-11 17:22:33 -050071 """Package Definition for packages dictionary."""
Patrick Williamsee3c9ee2021-02-12 20:56:01 -060072
73 # rev [optional]: Revision of package to use.
74 rev: str
75 # url [optional]: lambda function to create URL: (package, rev) -> url.
76 url: Callable[[str, str], str]
77 # depends [optional]: List of package dependencies.
78 depends: Iterable[str]
79 # build_type [required]: Build type used for package.
80 # Currently supported: autoconf, cmake, custom, make, meson
81 build_type: str
82 # build_steps [optional]: Steps to run for 'custom' build_type.
83 build_steps: Iterable[str]
84 # config_flags [optional]: List of options to pass configuration tool.
85 config_flags: Iterable[str]
86 # config_env [optional]: List of environment variables to set for config.
87 config_env: Iterable[str]
88 # custom_post_dl [optional]: List of steps to run after download, but
89 # before config / build / install.
90 custom_post_dl: Iterable[str]
Patrick Williams6bce2ca2021-02-12 21:13:37 -060091 # custom_post_install [optional]: List of steps to run after install.
92 custom_post_install: Iterable[str]
Patrick Williamsee3c9ee2021-02-12 20:56:01 -060093
94 # __tag [private]: Generated Docker tag name for package stage.
95 __tag: str
96 # __package [private]: Package object associated with this package.
97 __package: Any # Type is Package, but not defined yet.
98
Patrick Williams02871c92021-02-01 20:57:19 -060099
Patrick Williams72043242021-02-02 10:31:45 -0600100# Packages to include in image.
101packages = {
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600102 "boost": PackageDef(
Patrick Williamse4b761f2025-07-23 11:34:07 -0400103 rev="1.88.0",
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600104 url=(
Jayanth Othayoth96982152024-12-12 05:13:54 -0600105 lambda pkg, rev: f"https://github.com/boostorg/{pkg}/releases/download/{pkg}-{rev}/{pkg}-{rev}-cmake.tar.gz"
Patrick Williams2abc4a42021-02-03 06:11:40 -0600106 ),
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600107 build_type="custom",
108 build_steps=[
Patrick Williamse08ffba2022-12-05 10:33:46 -0600109 (
Andrew Geissler38b46872024-01-07 07:20:27 -0600110 "./bootstrap.sh"
Jayanth Othayoth96982152024-12-12 05:13:54 -0600111 f" --prefix={prefix} --with-libraries=atomic,context,coroutine,filesystem,process,url"
Patrick Williamse08ffba2022-12-05 10:33:46 -0600112 ),
Patrick Williamsaae36d12021-02-04 16:30:04 -0600113 "./b2",
Michal Orzel04770cc2024-06-18 10:38:22 +0200114 f"./b2 install --prefix={prefix} valgrind=on",
Patrick Williamsaae36d12021-02-04 16:30:04 -0600115 ],
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600116 ),
117 "USCiLab/cereal": PackageDef(
Patrick Williamsc1977832022-09-27 16:54:34 -0500118 rev="v1.3.2",
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600119 build_type="custom",
120 build_steps=[f"cp -a include/cereal/ {prefix}/include/"],
121 ),
Ed Tanousc7198552022-07-01 08:15:50 -0700122 "danmar/cppcheck": PackageDef(
Patrick Williams51021782023-12-05 19:10:44 -0600123 rev="2.12.1",
Ed Tanousc7198552022-07-01 08:15:50 -0700124 build_type="cmake",
125 ),
Ratan Gupta3dc37e62025-07-17 16:47:13 +0530126 "DMTF/libspdm": PackageDef(
127 rev="3.7.0",
128 url=lambda pkg, rev: f"https://github.com/DMTF/libspdm/archive/{rev}.tar.gz",
129 build_type="cmake",
130 config_flags=(
131 lambda: (
132 lambda arch_mapping={
133 "x86_64": "x64",
134 "i586": "ia32",
135 "i686": "ia32",
136 "arm": "arm",
137 "aarch64": "aarch64",
138 "arm64": "aarch64",
139 "riscv32": "riscv32",
140 "riscv64": "riscv64",
141 "ppc64le": "ppc64le",
142 }: [
143 f"-DARCH={arch_mapping.get(__import__('platform').machine(), 'x64')}",
144 "-DTOOLCHAIN=GCC",
145 "-DTARGET=Release",
146 "-DCRYPTO=openssl",
147 "-DBUILD_LINUX_SHARED_LIB=ON",
148 "-DENABLE_BINARY_BUILD=1",
149 "-DDISABLE_TESTS=1",
150 f"-DCOMPILED_LIBCRYPTO_PATH={prefix}/lib",
151 f"-DCOMPILED_LIBSSL_PATH={prefix}/lib",
152 ]
153 )()
154 )(),
155 ),
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600156 "CLIUtils/CLI11": PackageDef(
Patrick Williamsfc397332023-07-17 11:35:43 -0500157 rev="v2.3.2",
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600158 build_type="cmake",
159 config_flags=[
Patrick Williamsaae36d12021-02-04 16:30:04 -0600160 "-DBUILD_TESTING=OFF",
161 "-DCLI11_BUILD_DOCS=OFF",
162 "-DCLI11_BUILD_EXAMPLES=OFF",
163 ],
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600164 ),
165 "fmtlib/fmt": PackageDef(
Patrick Williamse4b761f2025-07-23 11:34:07 -0400166 rev="11.2.0",
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600167 build_type="cmake",
168 config_flags=[
Patrick Williamsaae36d12021-02-04 16:30:04 -0600169 "-DFMT_DOC=OFF",
170 "-DFMT_TEST=OFF",
171 ],
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600172 ),
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600173 "Naios/function2": PackageDef(
Patrick Williamscb099742023-12-05 19:12:09 -0600174 rev="4.2.4",
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600175 build_type="custom",
176 build_steps=[
Patrick Williamsaae36d12021-02-04 16:30:04 -0600177 f"mkdir {prefix}/include/function2",
178 f"cp include/function2/function2.hpp {prefix}/include/function2/",
179 ],
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600180 ),
181 "google/googletest": PackageDef(
Patrick Williamse4b761f2025-07-23 11:34:07 -0400182 rev="v1.16.0",
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600183 build_type="cmake",
William A. Kennington III4dd32c02021-05-28 01:58:13 -0700184 config_env=["CXXFLAGS=-std=c++20"],
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600185 config_flags=["-DTHREADS_PREFER_PTHREAD_FLAG=ON"],
186 ),
Ed Tanous178b4b22023-06-15 09:03:11 -0700187 "nghttp2/nghttp2": PackageDef(
Patrick Williamse4b761f2025-07-23 11:34:07 -0400188 rev="v1.65.0",
Ed Tanous178b4b22023-06-15 09:03:11 -0700189 build_type="cmake",
190 config_env=["CXXFLAGS=-std=c++20"],
191 config_flags=[
192 "-DENABLE_LIB_ONLY=ON",
193 "-DENABLE_STATIC_LIB=ON",
194 ],
195 ),
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600196 "nlohmann/json": PackageDef(
Patrick Williamse4b761f2025-07-23 11:34:07 -0400197 rev="v3.12.0",
Patrick Williams6bce2ca2021-02-12 21:13:37 -0600198 build_type="cmake",
199 config_flags=["-DJSON_BuildTests=OFF"],
200 custom_post_install=[
Patrick Williamse08ffba2022-12-05 10:33:46 -0600201 (
202 f"ln -s {prefix}/include/nlohmann/json.hpp"
203 f" {prefix}/include/json.hpp"
204 ),
Patrick Williamsaae36d12021-02-04 16:30:04 -0600205 ],
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600206 ),
Przemyslaw Czarnowski058e3a32022-12-21 14:13:23 +0100207 "json-c/json-c": PackageDef(
Patrick Williamse4b761f2025-07-23 11:34:07 -0400208 rev="json-c-0.18-20240915",
Przemyslaw Czarnowski058e3a32022-12-21 14:13:23 +0100209 build_type="cmake",
210 ),
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600211 "LibVNC/libvncserver": PackageDef(
Patrick Williamsc0421322023-12-05 19:18:57 -0600212 rev="LibVNCServer-0.9.14",
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600213 build_type="cmake",
214 ),
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600215 "leethomason/tinyxml2": PackageDef(
Patrick Williamse4b761f2025-07-23 11:34:07 -0400216 rev="11.0.0",
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600217 build_type="cmake",
218 ),
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600219 "tristanpenman/valijson": PackageDef(
Patrick Williamse4b761f2025-07-23 11:34:07 -0400220 rev="v1.0.5",
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600221 build_type="cmake",
222 config_flags=[
Patrick Williams0eedeed2021-02-06 19:06:09 -0600223 "-Dvalijson_BUILD_TESTS=0",
224 "-Dvalijson_INSTALL_HEADERS=1",
Patrick Williamsaae36d12021-02-04 16:30:04 -0600225 ],
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600226 ),
Patrick Williamsc7e719f2025-07-24 16:51:57 -0400227 "libgpiod": PackageDef(
228 rev="1.6.5",
229 url=(
230 lambda pkg, rev: f"https://git.kernel.org/pub/scm/libs/{pkg}/{pkg}.git/snapshot/{pkg}-{rev}.tar.gz"
231 ),
232 build_type="autogen",
233 config_flags=["--enable-bindings-cxx"],
234 ),
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600235 "open-power/pdbg": PackageDef(build_type="autoconf"),
236 "openbmc/gpioplus": PackageDef(
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600237 build_type="meson",
238 config_flags=[
Patrick Williamsaae36d12021-02-04 16:30:04 -0600239 "-Dexamples=false",
240 "-Dtests=disabled",
241 ],
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600242 ),
243 "openbmc/phosphor-dbus-interfaces": PackageDef(
244 depends=["openbmc/sdbusplus"],
245 build_type="meson",
William A. Kennington III4fe87772022-02-11 15:44:29 -0800246 config_flags=["-Dgenerate_md=false"],
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600247 ),
248 "openbmc/phosphor-logging": PackageDef(
249 depends=[
Patrick Williams83394612021-02-03 07:12:50 -0600250 "USCiLab/cereal",
Patrick Williams83394612021-02-03 07:12:50 -0600251 "openbmc/phosphor-dbus-interfaces",
252 "openbmc/sdbusplus",
253 "openbmc/sdeventplus",
Patrick Williamsaae36d12021-02-04 16:30:04 -0600254 ],
Patrick Williamsf79ce4c2021-04-30 16:00:49 -0500255 build_type="meson",
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600256 config_flags=[
William A. Kennington III6c98f282022-10-05 13:37:04 -0700257 "-Dlibonly=true",
258 "-Dtests=disabled",
Patrick Williamsaae36d12021-02-04 16:30:04 -0600259 ],
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600260 ),
261 "openbmc/phosphor-objmgr": PackageDef(
262 depends=[
Brad Bishop11e57622022-09-14 16:10:25 -0400263 "CLIUtils/CLI11",
Patrick Williams70af95c2022-09-27 16:55:41 -0500264 "boost",
Patrick Williams83394612021-02-03 07:12:50 -0600265 "leethomason/tinyxml2",
Patrick Williams70af95c2022-09-27 16:55:41 -0500266 "openbmc/phosphor-dbus-interfaces",
Patrick Williams83394612021-02-03 07:12:50 -0600267 "openbmc/phosphor-logging",
268 "openbmc/sdbusplus",
Patrick Williamsaae36d12021-02-04 16:30:04 -0600269 ],
Brad Bishop1197e352021-08-03 19:25:46 -0400270 build_type="meson",
271 config_flags=[
272 "-Dtests=disabled",
273 ],
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600274 ),
Jason M. Billsc02ff272023-08-02 10:55:22 -0700275 "openbmc/libpeci": PackageDef(
276 build_type="meson",
277 config_flags=[
278 "-Draw-peci=disabled",
279 ],
280 ),
Manojkiran Eda1c19e452022-10-03 11:01:59 +0530281 "openbmc/libpldm": PackageDef(
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600282 build_type="meson",
Andrew Jeffery29163972024-12-06 17:01:56 +1030283 config_flags=[
284 "-Dabi=deprecated,stable",
285 "-Dtests=false",
286 "-Dabi-compliance-check=false",
287 ],
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600288 ),
289 "openbmc/sdbusplus": PackageDef(
Patrick Williams54d01da2024-09-25 06:40:25 -0400290 depends=[
291 "nlohmann/json",
292 ],
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600293 build_type="meson",
294 custom_post_dl=[
Patrick Williamsaae36d12021-02-04 16:30:04 -0600295 "cd tools",
Patrick Williams3e9c0072025-08-27 10:45:22 -0400296 "python3 -m pip install --break-system-packages --root-user-action ignore .",
Patrick Williamsaae36d12021-02-04 16:30:04 -0600297 "cd ..",
298 ],
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600299 config_flags=[
Patrick Williamsaae36d12021-02-04 16:30:04 -0600300 "-Dexamples=disabled",
301 "-Dtests=disabled",
302 ],
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600303 ),
304 "openbmc/sdeventplus": PackageDef(
Patrick Williams70af95c2022-09-27 16:55:41 -0500305 depends=[
Patrick Williams70af95c2022-09-27 16:55:41 -0500306 "openbmc/stdplus",
307 ],
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600308 build_type="meson",
309 config_flags=[
Patrick Williamsaae36d12021-02-04 16:30:04 -0600310 "-Dexamples=false",
311 "-Dtests=disabled",
312 ],
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600313 ),
314 "openbmc/stdplus": PackageDef(
Patrick Williams70af95c2022-09-27 16:55:41 -0500315 depends=[
Patrick Williams70af95c2022-09-27 16:55:41 -0500316 "fmtlib/fmt",
William A. Kennington IIIca1bf0c2022-10-05 02:23:30 -0700317 "google/googletest",
318 "Naios/function2",
Patrick Williams70af95c2022-09-27 16:55:41 -0500319 ],
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600320 build_type="meson",
321 config_flags=[
Patrick Williamsaae36d12021-02-04 16:30:04 -0600322 "-Dexamples=false",
323 "-Dtests=disabled",
William A. Kennington IIIca1bf0c2022-10-05 02:23:30 -0700324 "-Dgtest=enabled",
Patrick Williamsaae36d12021-02-04 16:30:04 -0600325 ],
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600326 ),
327} # type: Dict[str, PackageDef]
Patrick Williams02871c92021-02-01 20:57:19 -0600328
329# Define common flags used for builds
Patrick Williams02871c92021-02-01 20:57:19 -0600330configure_flags = " ".join(
331 [
332 f"--prefix={prefix}",
333 ]
334)
335cmake_flags = " ".join(
336 [
Patrick Williams02871c92021-02-01 20:57:19 -0600337 "-DBUILD_SHARED_LIBS=ON",
Patrick Williams0f2086b2021-02-05 06:49:49 -0600338 "-DCMAKE_BUILD_TYPE=RelWithDebInfo",
Patrick Williams02871c92021-02-01 20:57:19 -0600339 f"-DCMAKE_INSTALL_PREFIX:PATH={prefix}",
Patrick Williams0f2086b2021-02-05 06:49:49 -0600340 "-GNinja",
341 "-DCMAKE_MAKE_PROGRAM=ninja",
Patrick Williams02871c92021-02-01 20:57:19 -0600342 ]
343)
344meson_flags = " ".join(
345 [
346 "--wrap-mode=nodownload",
347 f"-Dprefix={prefix}",
348 ]
349)
350
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600351
352class Package(threading.Thread):
353 """Class used to build the Docker stages for each package.
354
355 Generally, this class should not be instantiated directly but through
356 Package.generate_all().
357 """
358
359 # Copy the packages dictionary.
360 packages = packages.copy()
361
362 # Lock used for thread-safety.
363 lock = threading.Lock()
364
365 def __init__(self, pkg: str):
Patrick Williams05fb2a02022-10-11 17:22:33 -0500366 """pkg - The name of this package (ex. foo/bar )"""
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600367 super(Package, self).__init__()
368
369 self.package = pkg
370 self.exception = None # type: Optional[Exception]
371
372 # Reference to this package's
373 self.pkg_def = Package.packages[pkg]
374 self.pkg_def["__package"] = self
375
376 def run(self) -> None:
Patrick Williams05fb2a02022-10-11 17:22:33 -0500377 """Thread 'run' function. Builds the Docker stage."""
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600378
379 # In case this package has no rev, fetch it from Github.
380 self._update_rev()
381
382 # Find all the Package objects that this package depends on.
383 # This section is locked because we are looking into another
384 # package's PackageDef dict, which could be being modified.
385 Package.lock.acquire()
386 deps: Iterable[Package] = [
387 Package.packages[deppkg]["__package"]
388 for deppkg in self.pkg_def.get("depends", [])
389 ]
390 Package.lock.release()
391
392 # Wait until all the depends finish building. We need them complete
393 # for the "COPY" commands.
394 for deppkg in deps:
395 deppkg.join()
396
397 # Generate this package's Dockerfile.
398 dockerfile = f"""
399FROM {docker_base_img_name}
400{self._df_copycmds()}
401{self._df_build()}
402"""
403
404 # Generate the resulting tag name and save it to the PackageDef.
405 # This section is locked because we are modifying the PackageDef,
406 # which can be accessed by other threads.
407 Package.lock.acquire()
408 tag = Docker.tagname(self._stagename(), dockerfile)
409 self.pkg_def["__tag"] = tag
410 Package.lock.release()
411
412 # Do the build / save any exceptions.
413 try:
414 Docker.build(self.package, tag, dockerfile)
415 except Exception as e:
416 self.exception = e
417
418 @classmethod
419 def generate_all(cls) -> None:
420 """Ensure a Docker stage is created for all defined packages.
421
422 These are done in parallel but with appropriate blocking per
423 package 'depends' specifications.
424 """
425
426 # Create a Package for each defined package.
427 pkg_threads = [Package(p) for p in cls.packages.keys()]
428
429 # Start building them all.
Patrick Williams6dbd7802021-02-20 08:34:10 -0600430 # This section is locked because threads depend on each other,
431 # based on the packages, and they cannot 'join' on a thread
432 # which is not yet started. Adding a lock here allows all the
433 # threads to start before they 'join' their dependencies.
434 Package.lock.acquire()
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600435 for t in pkg_threads:
436 t.start()
Patrick Williams6dbd7802021-02-20 08:34:10 -0600437 Package.lock.release()
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600438
439 # Wait for completion.
440 for t in pkg_threads:
441 t.join()
442 # Check if the thread saved off its own exception.
443 if t.exception:
444 print(f"Package {t.package} failed!", file=sys.stderr)
445 raise t.exception
446
447 @staticmethod
448 def df_all_copycmds() -> str:
449 """Formulate the Dockerfile snippet necessary to copy all packages
450 into the final image.
451 """
452 return Package.df_copycmds_set(Package.packages.keys())
453
454 @classmethod
455 def depcache(cls) -> str:
456 """Create the contents of the '/tmp/depcache'.
457 This file is a comma-separated list of "<pkg>:<rev>".
458 """
459
460 # This needs to be sorted for consistency.
461 depcache = ""
462 for pkg in sorted(cls.packages.keys()):
463 depcache += "%s:%s," % (pkg, cls.packages[pkg]["rev"])
464 return depcache
465
Patrick Williams276bd0e2024-10-02 10:34:32 -0400466 def _check_gerrit_topic(self) -> bool:
467 if not gerrit_topic:
468 return False
469 if not self.package.startswith("openbmc/"):
470 return False
471 if gerrit_project == self.package and gerrit_rev:
472 return False
473
Patrick Williams1c847972025-09-12 17:10:31 -0400474 # URL escape any spaces. Gerrit uses pluses.
475 gerrit_topic_escape = urllib.parse.quote_plus(gerrit_topic)
476
Patrick Williams276bd0e2024-10-02 10:34:32 -0400477 try:
478 commits = json.loads(
479 urllib.request.urlopen(
Patrick Williams1c847972025-09-12 17:10:31 -0400480 f'https://gerrit.openbmc.org/changes/?q=status:open+project:{self.package}+topic:"{gerrit_topic_escape}"'
Patrick Williams276bd0e2024-10-02 10:34:32 -0400481 )
482 .read()
483 .splitlines()[-1]
484 )
485
486 if len(commits) == 0:
487 return False
488 if len(commits) > 1:
489 print(
490 f"{self.package} has more than 1 commit under {gerrit_topic}; using lastest upstream: {len(commits)}",
491 file=sys.stderr,
492 )
493 return False
494
495 change_id = commits[0]["id"]
496
497 commit = json.loads(
498 urllib.request.urlopen(
499 f"https://gerrit.openbmc.org/changes/{change_id}/revisions/current/commit"
500 )
501 .read()
502 .splitlines()[-1]
503 )["commit"]
504
505 print(
506 f"Using {commit} from {gerrit_topic} for {self.package}",
507 file=sys.stderr,
508 )
509 self.pkg_def["rev"] = commit
510 return True
511
512 except urllib.error.HTTPError as e:
513 print(
514 f"Error loading topic {gerrit_topic} for {self.package}: ",
515 e,
516 file=sys.stderr,
517 )
518 return False
519
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600520 def _update_rev(self) -> None:
Patrick Williams05fb2a02022-10-11 17:22:33 -0500521 """Look up the HEAD for missing a static rev."""
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600522
523 if "rev" in self.pkg_def:
524 return
525
Patrick Williams276bd0e2024-10-02 10:34:32 -0400526 if self._check_gerrit_topic():
527 return
528
Patrick Williams65b21fb2021-02-12 21:21:14 -0600529 # Check if Jenkins/Gerrit gave us a revision and use it.
530 if gerrit_project == self.package and gerrit_rev:
531 print(
532 f"Found Gerrit revision for {self.package}: {gerrit_rev}",
533 file=sys.stderr,
534 )
535 self.pkg_def["rev"] = gerrit_rev
536 return
537
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600538 # Ask Github for all the branches.
Patrick Williams05fb2a02022-10-11 17:22:33 -0500539 lookup = git(
540 "ls-remote", "--heads", f"https://github.com/{self.package}"
541 )
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600542
543 # Find the branch matching {branch} (or fallback to master).
544 # This section is locked because we are modifying the PackageDef.
545 Package.lock.acquire()
546 for line in lookup.split("\n"):
Andrew Geisslerf3d27e62024-04-09 15:24:49 -0500547 if re.fullmatch(f".*{branch}$", line.strip()):
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600548 self.pkg_def["rev"] = line.split()[0]
Andrew Geisslerf3d27e62024-04-09 15:24:49 -0500549 break
Patrick Williamsc7d73642022-10-11 17:22:06 -0500550 elif (
551 "refs/heads/master" in line or "refs/heads/main" in line
552 ) and "rev" not in self.pkg_def:
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600553 self.pkg_def["rev"] = line.split()[0]
554 Package.lock.release()
555
556 def _stagename(self) -> str:
Patrick Williams05fb2a02022-10-11 17:22:33 -0500557 """Create a name for the Docker stage associated with this pkg."""
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600558 return self.package.replace("/", "-").lower()
559
560 def _url(self) -> str:
Patrick Williams05fb2a02022-10-11 17:22:33 -0500561 """Get the URL for this package."""
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600562 rev = self.pkg_def["rev"]
563
564 # If the lambda exists, call it.
565 if "url" in self.pkg_def:
566 return self.pkg_def["url"](self.package, rev)
567
568 # Default to the github archive URL.
569 return f"https://github.com/{self.package}/archive/{rev}.tar.gz"
570
571 def _cmd_download(self) -> str:
572 """Formulate the command necessary to download and unpack to source."""
573
574 url = self._url()
575 if ".tar." not in url:
576 raise NotImplementedError(
577 f"Unhandled download type for {self.package}: {url}"
578 )
579
580 cmd = f"curl -L {url} | tar -x"
581
582 if url.endswith(".bz2"):
583 cmd += "j"
584 elif url.endswith(".gz"):
585 cmd += "z"
586 else:
587 raise NotImplementedError(
588 f"Unknown tar flags needed for {self.package}: {url}"
589 )
590
591 return cmd
592
593 def _cmd_cd_srcdir(self) -> str:
Patrick Williams05fb2a02022-10-11 17:22:33 -0500594 """Formulate the command necessary to 'cd' into the source dir."""
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600595 return f"cd {self.package.split('/')[-1]}*"
596
597 def _df_copycmds(self) -> str:
Patrick Williams05fb2a02022-10-11 17:22:33 -0500598 """Formulate the dockerfile snippet necessary to COPY all depends."""
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600599
600 if "depends" not in self.pkg_def:
601 return ""
602 return Package.df_copycmds_set(self.pkg_def["depends"])
603
604 @staticmethod
605 def df_copycmds_set(pkgs: Iterable[str]) -> str:
606 """Formulate the Dockerfile snippet necessary to COPY a set of
607 packages into a Docker stage.
608 """
609
610 copy_cmds = ""
611
612 # Sort the packages for consistency.
613 for p in sorted(pkgs):
614 tag = Package.packages[p]["__tag"]
615 copy_cmds += f"COPY --from={tag} {prefix} {prefix}\n"
616 # Workaround for upstream docker bug and multiple COPY cmds
617 # https://github.com/moby/moby/issues/37965
618 copy_cmds += "RUN true\n"
619
620 return copy_cmds
621
622 def _df_build(self) -> str:
623 """Formulate the Dockerfile snippet necessary to download, build, and
624 install a package into a Docker stage.
625 """
626
627 # Download and extract source.
628 result = f"RUN {self._cmd_download()} && {self._cmd_cd_srcdir()} && "
629
630 # Handle 'custom_post_dl' commands.
631 custom_post_dl = self.pkg_def.get("custom_post_dl")
632 if custom_post_dl:
633 result += " && ".join(custom_post_dl) + " && "
634
635 # Build and install package based on 'build_type'.
636 build_type = self.pkg_def["build_type"]
637 if build_type == "autoconf":
638 result += self._cmd_build_autoconf()
Patrick Williamsc7e719f2025-07-24 16:51:57 -0400639 elif build_type == "autogen":
640 result += self._cmd_build_autogen()
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600641 elif build_type == "cmake":
642 result += self._cmd_build_cmake()
643 elif build_type == "custom":
644 result += self._cmd_build_custom()
645 elif build_type == "make":
646 result += self._cmd_build_make()
647 elif build_type == "meson":
648 result += self._cmd_build_meson()
649 else:
650 raise NotImplementedError(
651 f"Unhandled build type for {self.package}: {build_type}"
652 )
653
Patrick Williams6bce2ca2021-02-12 21:13:37 -0600654 # Handle 'custom_post_install' commands.
655 custom_post_install = self.pkg_def.get("custom_post_install")
656 if custom_post_install:
657 result += " && " + " && ".join(custom_post_install)
658
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600659 return result
660
661 def _cmd_build_autoconf(self) -> str:
662 options = " ".join(self.pkg_def.get("config_flags", []))
663 env = " ".join(self.pkg_def.get("config_env", []))
664 result = "./bootstrap.sh && "
665 result += f"{env} ./configure {configure_flags} {options} && "
666 result += f"make -j{proc_count} && make install"
667 return result
668
Patrick Williamsc7e719f2025-07-24 16:51:57 -0400669 def _cmd_build_autogen(self) -> str:
670 options = " ".join(self.pkg_def.get("config_flags", []))
671 env = " ".join(self.pkg_def.get("config_env", []))
672 result = f"{env} ./autogen.sh {configure_flags} {options} && "
673 result += "make && make install"
674 return result
675
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600676 def _cmd_build_cmake(self) -> str:
677 options = " ".join(self.pkg_def.get("config_flags", []))
678 env = " ".join(self.pkg_def.get("config_env", []))
679 result = "mkdir builddir && cd builddir && "
680 result += f"{env} cmake {cmake_flags} {options} .. && "
681 result += "cmake --build . --target all && "
682 result += "cmake --build . --target install && "
683 result += "cd .."
684 return result
685
686 def _cmd_build_custom(self) -> str:
687 return " && ".join(self.pkg_def.get("build_steps", []))
688
689 def _cmd_build_make(self) -> str:
690 return f"make -j{proc_count} && make install"
691
692 def _cmd_build_meson(self) -> str:
693 options = " ".join(self.pkg_def.get("config_flags", []))
694 env = " ".join(self.pkg_def.get("config_env", []))
Andrew Jefferye2da11a2023-06-15 10:16:37 +0930695 result = f"{env} meson setup builddir {meson_flags} {options} && "
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600696 result += "ninja -C builddir && ninja -C builddir install"
697 return result
698
699
700class Docker:
701 """Class to assist with Docker interactions. All methods are static."""
702
703 @staticmethod
704 def timestamp() -> str:
Patrick Williams05fb2a02022-10-11 17:22:33 -0500705 """Generate a timestamp for today using the ISO week."""
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600706 today = date.today().isocalendar()
707 return f"{today[0]}-W{today[1]:02}"
708
709 @staticmethod
Patrick Williams41d86212022-11-25 18:28:43 -0600710 def tagname(pkgname: Optional[str], dockerfile: str) -> str:
Patrick Williams05fb2a02022-10-11 17:22:33 -0500711 """Generate a tag name for a package using a hash of the Dockerfile."""
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600712 result = docker_image_name
713 if pkgname:
714 result += "-" + pkgname
715
716 result += ":" + Docker.timestamp()
717 result += "-" + sha256(dockerfile.encode()).hexdigest()[0:16]
718
719 return result
720
721 @staticmethod
722 def build(pkg: str, tag: str, dockerfile: str) -> None:
Andrew Geissler22e61102023-02-14 14:44:00 -0600723 """Build a docker image using the Dockerfile and tagging it with 'tag'."""
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600724
725 # If we're not forcing builds, check if it already exists and skip.
726 if not force_build:
Andrew Geissler8f7146f2024-12-11 14:20:47 -0600727 if container.image.ls(
728 tag, "--format", '"{{.Repository}}:{{.Tag}}"'
729 ):
Patrick Williams05fb2a02022-10-11 17:22:33 -0500730 print(
731 f"Image {tag} already exists. Skipping.", file=sys.stderr
732 )
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600733 return
734
735 # Build it.
736 # Capture the output of the 'docker build' command and send it to
737 # stderr (prefixed with the package name). This allows us to see
Manojkiran Edaa6ebc6e2024-06-17 11:51:45 +0530738 # progress but not pollute stdout. Later on we output the final
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600739 # docker tag to stdout and we want to keep that pristine.
740 #
741 # Other unusual flags:
742 # --no-cache: Bypass the Docker cache if 'force_build'.
743 # --force-rm: Clean up Docker processes if they fail.
Andrew Geissler8f7146f2024-12-11 14:20:47 -0600744 container.build(
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600745 proxy_args,
746 "--network=host",
747 "--force-rm",
748 "--no-cache=true" if force_build else "--no-cache=false",
749 "-t",
750 tag,
751 "-",
752 _in=dockerfile,
753 _out=(
754 lambda line: print(
755 pkg + ":", line, end="", file=sys.stderr, flush=True
756 )
757 ),
Jonathan Doman88dd7922024-05-02 10:34:21 -0700758 _err_to_out=True,
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600759 )
760
761
762# Read a bunch of environment variables.
Patrick Williams05fb2a02022-10-11 17:22:33 -0500763docker_image_name = os.environ.get(
764 "DOCKER_IMAGE_NAME", "openbmc/ubuntu-unit-test"
765)
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600766force_build = os.environ.get("FORCE_DOCKER_BUILD")
767is_automated_ci_build = os.environ.get("BUILD_URL", False)
Patrick Williams6b141902025-07-23 11:24:11 -0400768distro = os.environ.get("DISTRO", "ubuntu:plucky")
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600769branch = os.environ.get("BRANCH", "master")
770ubuntu_mirror = os.environ.get("UBUNTU_MIRROR")
Andrew Geissler23ec3322024-10-02 10:45:32 -0400771docker_reg = os.environ.get("DOCKER_REG", "public.ecr.aws/ubuntu")
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600772http_proxy = os.environ.get("http_proxy")
773
Patrick Williams65b21fb2021-02-12 21:21:14 -0600774gerrit_project = os.environ.get("GERRIT_PROJECT")
775gerrit_rev = os.environ.get("GERRIT_PATCHSET_REVISION")
Patrick Williams276bd0e2024-10-02 10:34:32 -0400776gerrit_topic = os.environ.get("GERRIT_TOPIC")
Patrick Williams65b21fb2021-02-12 21:21:14 -0600777
Andrew Geisslerd0dabc32023-04-04 08:09:21 -0600778# Ensure appropriate docker build output to see progress and identify
779# any issues
780os.environ["BUILDKIT_PROGRESS"] = "plain"
781
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600782# Set up some common variables.
783username = os.environ.get("USER", "root")
784homedir = os.environ.get("HOME", "/root")
785gid = os.getgid()
786uid = os.getuid()
787
Josh Lehan6825a012022-03-17 18:31:39 -0700788# Use well-known constants if user is root
789if username == "root":
790 homedir = "/root"
791 gid = 0
792 uid = 0
793
Patrick Williams02871c92021-02-01 20:57:19 -0600794# Special flags if setting up a deb mirror.
795mirror = ""
796if "ubuntu" in distro and ubuntu_mirror:
797 mirror = f"""
Patrick Williamse08ffba2022-12-05 10:33:46 -0600798RUN echo "deb {ubuntu_mirror} \
799 $(. /etc/os-release && echo $VERSION_CODENAME) \
800 main restricted universe multiverse" > /etc/apt/sources.list && \\
801 echo "deb {ubuntu_mirror} \
802 $(. /etc/os-release && echo $VERSION_CODENAME)-updates \
803 main restricted universe multiverse" >> /etc/apt/sources.list && \\
804 echo "deb {ubuntu_mirror} \
805 $(. /etc/os-release && echo $VERSION_CODENAME)-security \
806 main restricted universe multiverse" >> /etc/apt/sources.list && \\
807 echo "deb {ubuntu_mirror} \
808 $(. /etc/os-release && echo $VERSION_CODENAME)-proposed \
809 main restricted universe multiverse" >> /etc/apt/sources.list && \\
810 echo "deb {ubuntu_mirror} \
811 $(. /etc/os-release && echo $VERSION_CODENAME)-backports \
812 main restricted universe multiverse" >> /etc/apt/sources.list
Patrick Williams02871c92021-02-01 20:57:19 -0600813"""
814
815# Special flags for proxying.
816proxy_cmd = ""
Adrian Ambrożewicz34ec77e2021-06-02 10:23:38 +0200817proxy_keyserver = ""
Patrick Williams02871c92021-02-01 20:57:19 -0600818proxy_args = []
819if http_proxy:
820 proxy_cmd = f"""
821RUN echo "[http]" >> {homedir}/.gitconfig && \
822 echo "proxy = {http_proxy}" >> {homedir}/.gitconfig
Tan Siewert3aa71c82025-01-24 15:26:55 +0100823COPY <<EOF_WGETRC {homedir}/.wgetrc
824https_proxy = {http_proxy}
825http_proxy = {http_proxy}
826use_proxy = on
Lei YUf7e52612025-01-08 11:06:59 +0000827EOF_WGETRC
Patrick Williams02871c92021-02-01 20:57:19 -0600828"""
Adrian Ambrożewicz34ec77e2021-06-02 10:23:38 +0200829 proxy_keyserver = f"--keyserver-options http-proxy={http_proxy}"
830
Patrick Williams02871c92021-02-01 20:57:19 -0600831 proxy_args.extend(
832 [
833 "--build-arg",
834 f"http_proxy={http_proxy}",
835 "--build-arg",
Lei YUd461cd62021-02-18 14:25:49 +0800836 f"https_proxy={http_proxy}",
Patrick Williams02871c92021-02-01 20:57:19 -0600837 ]
838 )
839
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600840# Create base Dockerfile.
Patrick Williamsa18d9c52021-02-05 09:52:26 -0600841dockerfile_base = f"""
Andrew Geisslerfe2768c2024-10-02 10:10:29 -0400842FROM {docker_reg}/{distro}
Patrick Williams02871c92021-02-01 20:57:19 -0600843
844{mirror}
845
846ENV DEBIAN_FRONTEND noninteractive
847
Patrick Williams8949d3c2022-04-27 16:41:27 -0500848ENV PYTHONPATH "/usr/local/lib/python3.10/site-packages/"
Patrick Williams02871c92021-02-01 20:57:19 -0600849
Patrick Williamsbb16ac12021-04-12 12:23:51 -0500850# Sometimes the ubuntu key expires and we need a way to force an execution
851# of the apt-get commands for the dbgsym-keyring. When this happens we see
852# an error like: "Release: The following signatures were invalid:"
853# Insert a bogus echo that we can change here when we get this error to force
854# the update.
Patrick Williamsa1cbd402025-06-25 10:23:50 -0400855RUN echo "ubuntu keyserver rev as of 2025-06-25"
Patrick Williamsbb16ac12021-04-12 12:23:51 -0500856
Patrick Williams02871c92021-02-01 20:57:19 -0600857# We need the keys to be imported for dbgsym repos
858# New releases have a package, older ones fall back to manual fetching
859# https://wiki.ubuntu.com/Debug%20Symbol%20Packages
Jagpal Singh Gill575b5e42023-04-14 15:52:10 -0700860# Known issue with gpg to get keys via proxy -
861# https://bugs.launchpad.net/ubuntu/+source/gnupg2/+bug/1788190, hence using
862# curl to get keys.
Patrick Williams50837432021-02-06 12:24:05 -0600863RUN apt-get update && apt-get dist-upgrade -yy && \
Jian Zhang938d3032023-07-05 13:35:35 +0800864 ( apt-get install -yy gpgv ubuntu-dbgsym-keyring || \
Jagpal Singh Gill575b5e42023-04-14 15:52:10 -0700865 ( apt-get install -yy dirmngr curl && \
866 curl -sSL \
867 'https://keyserver.ubuntu.com/pks/lookup?op=get&search=0xF2EDC64DC5AEE1F6B9C621F0C8CAB6595FDFF622' \
868 | apt-key add - ))
Patrick Williams02871c92021-02-01 20:57:19 -0600869
870# Parse the current repo list into a debug repo list
Patrick Williamse08ffba2022-12-05 10:33:46 -0600871RUN sed -n '/^deb /s,^deb [^ ]* ,deb http://ddebs.ubuntu.com ,p' \
872 /etc/apt/sources.list >/etc/apt/sources.list.d/debug.list
Patrick Williams02871c92021-02-01 20:57:19 -0600873
874# Remove non-existent debug repos
Patrick Williams41d86212022-11-25 18:28:43 -0600875RUN sed -i '/-\\(backports\\|security\\) /d' /etc/apt/sources.list.d/debug.list
Patrick Williams02871c92021-02-01 20:57:19 -0600876
877RUN cat /etc/apt/sources.list.d/debug.list
878
879RUN apt-get update && apt-get dist-upgrade -yy && apt-get install -yy \
Andrew Jeffery58f19152023-05-22 16:41:32 +0930880 abi-compliance-checker \
Andrew Jeffery8b112062023-05-22 20:49:11 +0930881 abi-dumper \
Patrick Williams02871c92021-02-01 20:57:19 -0600882 autoconf \
883 autoconf-archive \
Andrew Geissleraf49ed52022-12-14 11:41:35 -0600884 bison \
Andrew Geissleraf49ed52022-12-14 11:41:35 -0600885 cmake \
886 curl \
887 dbus \
888 device-tree-compiler \
Andrew Jeffery1c28d962025-05-09 14:17:29 +0930889 doxygen \
Andrew Geissleraf49ed52022-12-14 11:41:35 -0600890 flex \
Patrick Williamsea1bfb22025-07-23 10:46:34 -0400891 g++-15 \
892 gcc-15 \
Andrew Geissleraf49ed52022-12-14 11:41:35 -0600893 git \
Patrick Williamsb4eec872024-10-04 10:49:50 -0400894 glib-2.0 \
Patrick Williams6968e832024-08-16 17:43:24 -0400895 gnupg \
Patrick Williams02871c92021-02-01 20:57:19 -0600896 iproute2 \
Andrew Geissleraf49ed52022-12-14 11:41:35 -0600897 iputils-ping \
Manojkiran Eda524a3312023-04-05 15:37:47 +0530898 libaudit-dev \
Andrew Geissleraf49ed52022-12-14 11:41:35 -0600899 libc6-dbg \
900 libc6-dev \
Patrick Williamsc7bc4d12024-10-04 11:22:02 -0400901 libcjson-dev \
Andrew Geissleraf49ed52022-12-14 11:41:35 -0600902 libconfig++-dev \
903 libcryptsetup-dev \
Anirban Banerjeea7a30552024-12-20 19:12:42 -0800904 libcurl4-openssl-dev \
Andrew Geissleraf49ed52022-12-14 11:41:35 -0600905 libdbus-1-dev \
906 libevdev-dev \
Andrew Geissleraf49ed52022-12-14 11:41:35 -0600907 libi2c-dev \
908 libjpeg-dev \
909 libjson-perl \
910 libldap2-dev \
911 libmimetic-dev \
Ewelina Walkusz3ee62fb2025-02-25 16:04:52 +0100912 libmpfr-dev \
Patrick Williams02871c92021-02-01 20:57:19 -0600913 libnl-3-dev \
914 libnl-genl-3-dev \
Patrick Williams02871c92021-02-01 20:57:19 -0600915 libpam0g-dev \
Patrick Williams02871c92021-02-01 20:57:19 -0600916 libpciaccess-dev \
Andrew Geissleraf49ed52022-12-14 11:41:35 -0600917 libperlio-gzip-perl \
918 libpng-dev \
919 libprotobuf-dev \
920 libsnmp-dev \
921 libssl-dev \
922 libsystemd-dev \
923 libtool \
Andrew Geissleraf49ed52022-12-14 11:41:35 -0600924 liburing-dev \
Patrick Williams02871c92021-02-01 20:57:19 -0600925 libxml2-utils \
Patrick Williams0eedeed2021-02-06 19:06:09 -0600926 libxml-simple-perl \
Patrick Williams6968e832024-08-16 17:43:24 -0400927 lsb-release \
Andrew Geissleraf49ed52022-12-14 11:41:35 -0600928 ninja-build \
929 npm \
930 pkg-config \
931 protobuf-compiler \
932 python3 \
933 python3-dev\
934 python3-git \
935 python3-mako \
936 python3-pip \
William A. Kennington III25ba1e22024-03-24 15:47:51 -0700937 python3-protobuf \
Andrew Geissleraf49ed52022-12-14 11:41:35 -0600938 python3-setuptools \
939 python3-socks \
940 python3-yaml \
John Wedig9adf68d2021-11-16 14:00:39 -0800941 rsync \
Andrew Geissleraf49ed52022-12-14 11:41:35 -0600942 shellcheck \
Ewelina Walkusz8dd1bfe2024-05-27 09:34:50 +0200943 socat \
Patrick Williams6968e832024-08-16 17:43:24 -0400944 software-properties-common \
Andrew Geissleraf49ed52022-12-14 11:41:35 -0600945 sudo \
946 systemd \
Patrick Williams917b1772024-12-11 15:15:44 -0500947 systemd-dev \
Andrew Geissleraf49ed52022-12-14 11:41:35 -0600948 valgrind \
Andrew Geisslerb565f822022-12-14 11:43:25 -0600949 vim \
Andrew Geissleraf49ed52022-12-14 11:41:35 -0600950 wget \
951 xxd
Patrick Williams02871c92021-02-01 20:57:19 -0600952
Patrick Williamsea1bfb22025-07-23 10:46:34 -0400953RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-15 15 \
954 --slave /usr/bin/g++ g++ /usr/bin/g++-15 \
955 --slave /usr/bin/gcov gcov /usr/bin/gcov-15 \
956 --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-15 \
957 --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-15
Patrick Williams961f1482023-05-30 09:24:16 -0500958RUN update-alternatives --remove cpp /usr/bin/cpp && \
Patrick Williamsea1bfb22025-07-23 10:46:34 -0400959 update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-15 15
Patrick Williams02871c92021-02-01 20:57:19 -0600960
Patrick Williams6968e832024-08-16 17:43:24 -0400961# Set up LLVM apt repository.
Patrick Williamse31ec4e2025-09-03 15:30:22 -0400962RUN bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" -- 21
Patrick Williams6968e832024-08-16 17:43:24 -0400963
964# Install extra clang tools
Patrick Williamsed8aeca2024-12-18 11:08:29 -0500965RUN apt-get install -y \
Patrick Williamse31ec4e2025-09-03 15:30:22 -0400966 clang-21 \
967 clang-format-21 \
968 clang-tidy-21 \
969 lld-21
Patrick Williams6968e832024-08-16 17:43:24 -0400970
Patrick Williamse31ec4e2025-09-03 15:30:22 -0400971RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-21 1000 \
972 --slave /usr/bin/clang++ clang++ /usr/bin/clang++-21 \
973 --slave /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-21 \
Patrick Williamsc5f92c92025-09-26 07:58:40 -0400974 --slave /usr/bin/clang-apply-replacements clang-apply-replacements \
975 /usr/bin/clang-apply-replacements-21 \
Patrick Williamse31ec4e2025-09-03 15:30:22 -0400976 --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-21 \
Patrick Williamse08ffba2022-12-05 10:33:46 -0600977 --slave /usr/bin/run-clang-tidy run-clang-tidy.py \
Patrick Williamse31ec4e2025-09-03 15:30:22 -0400978 /usr/bin/run-clang-tidy-21 \
979 --slave /usr/bin/scan-build scan-build /usr/bin/scan-build-21 \
980 --slave /usr/bin/lld lld /usr/bin/lld-21
Patrick Williams02871c92021-02-01 20:57:19 -0600981
Patrick Williams50837432021-02-06 12:24:05 -0600982"""
983
984if is_automated_ci_build:
985 dockerfile_base += f"""
Manojkiran Edaa6ebc6e2024-06-17 11:51:45 +0530986# Run an arbitrary command to pollute the docker cache regularly force us
Patrick Williams50837432021-02-06 12:24:05 -0600987# to re-run `apt-get update` daily.
Patrick Williamsee3c9ee2021-02-12 20:56:01 -0600988RUN echo {Docker.timestamp()}
Patrick Williams50837432021-02-06 12:24:05 -0600989RUN apt-get update && apt-get dist-upgrade -yy
990
991"""
992
Patrick Williams41d86212022-11-25 18:28:43 -0600993dockerfile_base += """
Patrick Williams5e4d8402023-04-11 22:19:30 -0500994RUN pip3 install --break-system-packages \
Patrick Williams818023d2023-04-10 13:07:15 -0500995 beautysh \
996 black \
997 codespell \
998 flake8 \
Ewelina Walkusz2d8c5512024-07-02 10:49:38 +0200999 gcovr \
Patrick Williams818023d2023-04-10 13:07:15 -05001000 gitlint \
1001 inflection \
Arya K Padmanf7381ad2024-10-14 02:29:53 -05001002 isoduration \
Patrick Williams818023d2023-04-10 13:07:15 -05001003 isort \
1004 jsonschema \
Ed Tanous82425ec2025-08-25 17:57:30 -07001005 meson==1.9.0 \
Patrick Williams9fdba2d2025-05-14 12:36:55 -04001006 referencing \
Patrick Williams818023d2023-04-10 13:07:15 -05001007 requests
Patrick Williamsb08ddf72022-12-06 08:56:31 -06001008
1009RUN npm install -g \
Xinnan Xied0757de2024-05-27 14:22:58 +08001010 eslint@v8.56.0 eslint-plugin-json@v3.1.0 \
Patrick Williams7d41f6d2022-12-06 10:19:43 -06001011 markdownlint-cli@latest \
Patrick Williamsb08ddf72022-12-06 08:56:31 -06001012 prettier@latest
Ed Tanousfb9948a2022-06-21 09:10:24 -07001013"""
1014
Patrick Williamsee3c9ee2021-02-12 20:56:01 -06001015# Build the base and stage docker images.
1016docker_base_img_name = Docker.tagname("base", dockerfile_base)
1017Docker.build("base", docker_base_img_name, dockerfile_base)
1018Package.generate_all()
Patrick Williams02871c92021-02-01 20:57:19 -06001019
Patrick Williamsee3c9ee2021-02-12 20:56:01 -06001020# Create the final Dockerfile.
Patrick Williamsa18d9c52021-02-05 09:52:26 -06001021dockerfile = f"""
Patrick Williams02871c92021-02-01 20:57:19 -06001022# Build the final output image
Patrick Williamsa18d9c52021-02-05 09:52:26 -06001023FROM {docker_base_img_name}
Patrick Williamsee3c9ee2021-02-12 20:56:01 -06001024{Package.df_all_copycmds()}
Patrick Williams02871c92021-02-01 20:57:19 -06001025
1026# Some of our infrastructure still relies on the presence of this file
1027# even though it is no longer needed to rebuild the docker environment
1028# NOTE: The file is sorted to ensure the ordering is stable.
Patrick Williamsee3c9ee2021-02-12 20:56:01 -06001029RUN echo '{Package.depcache()}' > /tmp/depcache
Patrick Williams02871c92021-02-01 20:57:19 -06001030
Patrick Williams67cc0612023-04-11 22:16:46 -05001031# Ensure the group, user, and home directory are created (or rename them if
1032# they already exist).
1033RUN if grep -q ":{gid}:" /etc/group ; then \
1034 groupmod -n {username} $(awk -F : '{{ if ($3 == {gid}) {{ print $1 }} }}' /etc/group) ; \
1035 else \
1036 groupadd -f -g {gid} {username} ; \
1037 fi
Patrick Williams02871c92021-02-01 20:57:19 -06001038RUN mkdir -p "{os.path.dirname(homedir)}"
Patrick Williams67cc0612023-04-11 22:16:46 -05001039RUN if grep -q ":{uid}:" /etc/passwd ; then \
Patrick Williams73b3ee92023-04-24 10:11:01 -05001040 usermod -l {username} -d {homedir} -m $(awk -F : '{{ if ($3 == {uid}) {{ print $1 }} }}' /etc/passwd) ; \
Patrick Williams67cc0612023-04-11 22:16:46 -05001041 else \
1042 useradd -d {homedir} -m -u {uid} -g {gid} {username} ; \
1043 fi
Patrick Williams02871c92021-02-01 20:57:19 -06001044RUN sed -i '1iDefaults umask=000' /etc/sudoers
1045RUN echo "{username} ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers
1046
Andrew Geissler305a9a52021-04-07 11:08:40 -05001047# Ensure user has ability to write to /usr/local for different tool
1048# and data installs
Andrew Geissler7bb00b12021-05-10 15:12:08 -05001049RUN chown -R {username}:{username} /usr/local/share
Andrew Geissler305a9a52021-04-07 11:08:40 -05001050
Jonathan Domanab4fee82024-01-31 15:39:20 -08001051# Update library cache
1052RUN ldconfig
1053
Patrick Williams02871c92021-02-01 20:57:19 -06001054{proxy_cmd}
1055
1056RUN /bin/bash
1057"""
1058
Patrick Williamsa18d9c52021-02-05 09:52:26 -06001059# Do the final docker build
Patrick Williamsee3c9ee2021-02-12 20:56:01 -06001060docker_final_img_name = Docker.tagname(None, dockerfile)
1061Docker.build("final", docker_final_img_name, dockerfile)
1062
Patrick Williams00536fb2021-02-11 14:28:49 -06001063# Print the tag of the final image.
1064print(docker_final_img_name)