build-unit-test-docker: convert to Python
Do a near identical conversion from bash to Python for the script which
builds the unit-test docker containers. I plan to do additional
enhancements to build the sub-containers in parallel, so I needed to
first do this conversion to Python in order to make that implementation
sane.
Signed-off-by: Patrick Williams <patrick@stwcx.xyz>
Change-Id: I6df6f294b41f3a01f95fbe23a6dcd1ea21a315a9
diff --git a/scripts/build-unit-test-docker b/scripts/build-unit-test-docker
new file mode 100755
index 0000000..94baa67
--- /dev/null
+++ b/scripts/build-unit-test-docker
@@ -0,0 +1,520 @@
+#!/usr/bin/env python3
+#
+# Build the required docker image to run package unit tests
+#
+# Script Variables:
+# DOCKER_IMG_NAME: <optional, the name of the docker image to generate>
+# default is openbmc/ubuntu-unit-test
+# DISTRO: <optional, the distro to build a docker image against>
+# default is ubuntu:eoan
+# BRANCH: <optional, branch to build from each of the openbmc/
+# repositories>
+# default is master, which will be used if input branch not
+# provided or not found
+# UBUNTU_MIRROR: <optional, the URL of a mirror of Ubuntu to override the
+# default ones in /etc/apt/sources.list>
+# default is empty, and no mirror is used.
+# http_proxy The HTTP address of the proxy server to connect to.
+# Default: "", proxy is not setup if this is not set
+
+import os
+import sys
+from sh import docker, git, nproc, uname
+
+# Read a bunch of environment variables.
+docker_image_name = os.environ.get("DOCKER_IMAGE_NAME", "openbmc/ubuntu-unit-test")
+distro = os.environ.get("DISTRO", "ubuntu:focal")
+branch = os.environ.get("BRANCH", "master")
+ubuntu_mirror = os.environ.get("UBUNTU_MIRROR")
+http_proxy = os.environ.get("http_proxy")
+
+# Set up some common variables.
+proc_count = nproc().strip()
+username = os.environ.get("USER")
+homedir = os.environ.get("HOME")
+gid = os.getgid()
+uid = os.getuid()
+
+# Determine the architecture for Docker.
+arch = uname("-m").strip()
+if arch == "ppc64le":
+ docker_base = "ppc64le/"
+elif arch == "x86_64":
+ docker_base = ""
+else:
+ print(f"Unsupported system architecture({arch}) found for docker image")
+ sys.exit(1)
+
+# These packages we use 'HEAD' for.
+head_pkgs = [
+ "openbmc/phosphor-objmgr",
+ "openbmc/sdbusplus",
+ "openbmc/sdeventplus",
+ "openbmc/stdplus",
+ "openbmc/gpioplus",
+ "openbmc/phosphor-logging",
+ "openbmc/phosphor-dbus-interfaces",
+ "open-power/pdbg",
+ "openbmc/pldm",
+]
+
+# Packages with fixed revisions.
+pkg_rev = {
+ "boost": "1.74.0",
+ "cereal": "v1.3.0",
+ "catch2": "v2.12.2",
+ "CLI11": "v1.9.0",
+ "fmt": "6.2.1",
+ # Snapshot from 2020-01-03
+ "function2": "3a0746bf5f601dfed05330aefcb6854354fce07d",
+ # Snapshot from 2020-02-13
+ "googletest": "23b2a3b1cf803999fb38175f6e9e038a4495c8a5",
+ # Release 2020-08-06
+ "json": "v3.9.1",
+ # Snapshot from 2019-05-24
+ "lcov": "75fbae1cfc5027f818a0bb865bf6f96fab3202da",
+ # dev-5.0 2019-05-03
+ "linux-headers": "8bf6567e77f7aa68975b7c9c6d044bba690bf327",
+ # Snapshot from 2019-09-03
+ "libvncserver": "1354f7f1bb6962dab209eddb9d6aac1f03408110",
+ "span-lite": "v0.7.0",
+ # version from meta-openembedded/meta-oe/recipes-support/libtinyxml2/libtinyxml2_5.0.1.bb
+ "tinyxml2": "37bc3aca429f0164adf68c23444540b4a24b5778",
+ # version from /meta-openembedded/meta-oe/recipes-devtools/boost-url/boost-url_git.bb
+ "boost-url": "a56ae0df6d3078319755fbaa67822b4fa7fd352b",
+ # version from meta-openembedded/meta-oe/recipes-devtools/valijson/valijson_git.bb
+ "valijson": "c2f22fddf599d04dc33fcd7ed257c698a05345d9",
+ # version from meta-openembedded/meta-oe/recipes-devtools/nlohmann-fifo/nlohmann-fifo_git.bb
+ "fifo_map": "0dfbf5dacbb15a32c43f912a7e66a54aae39d0f9",
+}
+
+# Look up the HEAD for 'head_pkgs' and insert them into 'pkg_rev'.
+pkg_lookups = {}
+for pkg in head_pkgs:
+ pkg_lookups[pkg] = git(
+ "ls-remote", "--heads", f"https://github.com/{pkg}", _bg=True
+ )
+for pkg, result in pkg_lookups.items():
+ for line in result.stdout.decode().split("\n"):
+ if f"refs/heads/{branch}" in line:
+ pkg_rev[pkg] = line.strip().split()[0]
+ elif "refs/heads/master" in line and p not in pkg_rev:
+ pkg_rev[pkg] = line.strip().split()[0]
+
+# Create the contents of the '/tmp/depcache'.
+# This needs to be sorted for consistency.
+depcache = ""
+for pkg in sorted(head_pkgs):
+ if pkg in pkg_rev:
+ depcache += "%s:%s," % (pkg, pkg_rev[pkg])
+
+# Define common flags used for builds
+prefix = "/usr/local"
+configure_flags = " ".join(
+ [
+ f"--prefix={prefix}",
+ ]
+)
+cmake_flags = " ".join(
+ [
+ "-DCMAKE_BUILD_TYPE=RelWithDebInfo",
+ "-DBUILD_SHARED_LIBS=ON",
+ f"-DCMAKE_INSTALL_PREFIX:PATH={prefix}",
+ ]
+)
+meson_flags = " ".join(
+ [
+ "--wrap-mode=nodownload",
+ f"-Dprefix={prefix}",
+ ]
+)
+
+
+def stagename(name):
+ if not name.startswith("openbmc/"):
+ name = "openbmc/" + name
+ return name.replace("/", "-")
+
+
+# Build the commands needed to compose our final image
+# We must sort the packages, otherwise we might produce an unstable
+# docker file and rebuild the image unnecessarily
+copy_cmds = ""
+for pkg in sorted(pkg_rev.keys()):
+ copy_cmds += f"COPY --from={stagename(pkg)} {prefix} {prefix}\n"
+ # Workaround for upstream docker bug and multiple COPY cmds
+ # https://github.com/moby/moby/issues/37965
+ copy_cmds += "RUN true\n"
+
+# Special flags if setting up a deb mirror.
+mirror = ""
+if "ubuntu" in distro and ubuntu_mirror:
+ mirror = f"""
+RUN echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME) main restricted universe multiverse" > /etc/apt/sources.list && \\
+ echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-updates main restricted universe multiverse" >> /etc/apt/sources.list && \\
+ echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-security main restricted universe multiverse" >> /etc/apt/sources.list && \\
+ echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-proposed main restricted universe multiverse" >> /etc/apt/sources.list && \\
+ echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-backports main restricted universe multiverse" >> /etc/apt/sources.list
+"""
+
+# Special flags for proxying.
+proxy_cmd = ""
+proxy_args = []
+if http_proxy:
+ proxy_cmd = f"""
+RUN echo "[http]" >> {homedir}/.gitconfig && \
+ echo "proxy = {http_proxy}" >> {homedir}/.gitconfig
+"""
+ proxy_args.extend(
+ [
+ "--build-arg",
+ f"http_proxy={http_proxy}",
+ "--build-arg",
+ "https_proxy={https_proxy}",
+ ]
+ )
+
+# Create docker image that can run package unit tests
+dockerfile = f"""
+FROM {docker_base}{distro} as openbmc-base
+
+{mirror}
+
+ENV DEBIAN_FRONTEND noninteractive
+
+ENV PYTHONPATH "/usr/local/lib/python3.8/site-packages/"
+
+# We need the keys to be imported for dbgsym repos
+# New releases have a package, older ones fall back to manual fetching
+# https://wiki.ubuntu.com/Debug%20Symbol%20Packages
+RUN apt-get update && ( apt-get install ubuntu-dbgsym-keyring || ( apt-get install -yy dirmngr && \
+ apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F2EDC64DC5AEE1F6B9C621F0C8CAB6595FDFF622 ) )
+
+# Parse the current repo list into a debug repo list
+RUN sed -n '/^deb /s,^deb [^ ]* ,deb http://ddebs.ubuntu.com ,p' /etc/apt/sources.list >/etc/apt/sources.list.d/debug.list
+
+# Remove non-existent debug repos
+RUN sed -i '/-\(backports\|security\) /d' /etc/apt/sources.list.d/debug.list
+
+RUN cat /etc/apt/sources.list.d/debug.list
+
+RUN apt-get update && apt-get dist-upgrade -yy && apt-get install -yy \
+ gcc-10 \
+ g++-10 \
+ libc6-dbg \
+ libc6-dev \
+ libtool \
+ bison \
+ libdbus-1-dev \
+ flex \
+ cmake \
+ python3 \
+ python3-dev\
+ python3-yaml \
+ python3-mako \
+ python3-pip \
+ python3-setuptools \
+ python3-git \
+ python3-socks \
+ pkg-config \
+ autoconf \
+ autoconf-archive \
+ libsystemd-dev \
+ systemd \
+ libssl-dev \
+ libevdev-dev \
+ libevdev2-dbgsym \
+ libjpeg-dev \
+ libpng-dev \
+ ninja-build \
+ sudo \
+ curl \
+ git \
+ dbus \
+ iputils-ping \
+ clang-10 \
+ clang-format-10 \
+ clang-tidy-10 \
+ clang-tools-10 \
+ shellcheck \
+ npm \
+ iproute2 \
+ libnl-3-dev \
+ libnl-genl-3-dev \
+ libconfig++-dev \
+ libsnmp-dev \
+ valgrind \
+ valgrind-dbg \
+ libpam0g-dev \
+ xxd \
+ libi2c-dev \
+ wget \
+ libldap2-dev \
+ libprotobuf-dev \
+ libperlio-gzip-perl \
+ libjson-perl \
+ protobuf-compiler \
+ libgpiod-dev \
+ device-tree-compiler \
+ cppcheck \
+ libpciaccess-dev \
+ libmimetic-dev \
+ libxml2-utils \
+ libxml-simple-perl
+
+RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 1000 \
+ --slave /usr/bin/g++ g++ /usr/bin/g++-10 \
+ --slave /usr/bin/gcov gcov /usr/bin/gcov-10 \
+ --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-10 \
+ --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-10
+
+
+RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-10 1000 \
+ --slave /usr/bin/clang++ clang++ /usr/bin/clang++-10 \
+ --slave /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-10 \
+ --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-10 \
+ --slave /usr/bin/run-clang-tidy.py run-clang-tidy.py /usr/bin/run-clang-tidy-10.py
+
+RUN pip3 install inflection
+RUN pip3 install pycodestyle
+RUN pip3 install jsonschema
+RUN pip3 install meson==0.54.3
+RUN pip3 install protobuf
+
+FROM openbmc-base as openbmc-lcov
+RUN curl -L https://github.com/linux-test-project/lcov/archive/{pkg_rev['lcov']}.tar.gz | tar -xz && \
+cd lcov-* && \
+make -j{proc_count} && \
+make install
+
+FROM openbmc-base as openbmc-function2
+RUN mkdir {prefix}/include/function2 && \
+curl -L -o {prefix}/include/function2/function2.hpp https://raw.githubusercontent.com/Naios/function2/{pkg_rev['function2']}/include/function2/function2.hpp
+
+FROM openbmc-base as openbmc-googletest
+RUN curl -L https://github.com/google/googletest/archive/{pkg_rev['googletest']}.tar.gz | tar -xz && \
+cd googletest-* && \
+mkdir build && \
+cd build && \
+CXXFLAGS=-std=c++17 cmake {cmake_flags} -DTHREADS_PREFER_PTHREAD_FLAG=ON .. && \
+make -j{proc_count} && \
+make install
+
+FROM openbmc-base as openbmc-catch2
+RUN curl -L https://github.com/catchorg/Catch2/archive/{pkg_rev['catch2']}.tar.gz | tar -xz && \
+cd Catch2-* && \
+mkdir build && \
+cd build && \
+cmake {cmake_flags} -DBUILD_TESTING=OFF -DCATCH_INSTALL_DOCS=OFF .. && \
+make -j{proc_count} && \
+make install
+
+FROM openbmc-base as openbmc-cereal
+RUN curl -L https://github.com/USCiLab/cereal/archive/{pkg_rev['cereal']}.tar.gz | tar -xz && \
+cp -a cereal-*/include/cereal/ {prefix}/include/
+
+FROM openbmc-base as openbmc-CLI11
+RUN curl -L https://github.com/CLIUtils/CLI11/archive/{pkg_rev['CLI11']}.tar.gz | tar -xz && \
+cd CLI11-* && \
+mkdir build && \
+cd build && \
+cmake {cmake_flags} -DCLI11_BUILD_DOCS=OFF -DBUILD_TESTING=OFF -DCLI11_BUILD_EXAMPLES=OFF .. && \
+make -j{proc_count} && \
+make install
+
+FROM openbmc-base as openbmc-fmt
+RUN curl -L https://github.com/fmtlib/fmt/archive/{pkg_rev['fmt']}.tar.gz | tar -xz && \
+cd fmt-* && \
+mkdir build && \
+cd build && \
+cmake {cmake_flags} -DFMT_DOC=OFF -DFMT_TEST=OFF .. && \
+make -j{proc_count} && \
+make install
+
+FROM openbmc-base as openbmc-json
+RUN mkdir {prefix}/include/nlohmann/ && \
+curl -L -o {prefix}/include/nlohmann/json.hpp https://github.com/nlohmann/json/releases/download/{pkg_rev['json']}/json.hpp && \
+ln -s nlohmann/json.hpp {prefix}/include/json.hpp
+
+FROM openbmc-base as openbmc-fifo_map
+RUN curl -L https://github.com/nlohmann/fifo_map/archive/{pkg_rev['fifo_map']}.tar.gz | tar -xz && \
+cd fifo_map-*/src && cp fifo_map.hpp {prefix}/include/
+
+FROM openbmc-base as openbmc-span-lite
+RUN curl -L https://github.com/martinmoene/span-lite/archive/{pkg_rev['span-lite']}.tar.gz | tar -xz && \
+cd span-lite-* && \
+mkdir build && \
+cd build && \
+cmake {cmake_flags} -DSPAN_LITE_OPT_BUILD_TESTS=OFF .. && \
+make -j{proc_count} && \
+make install
+
+FROM openbmc-base as openbmc-linux-headers
+RUN curl -L https://github.com/openbmc/linux/archive/{pkg_rev['linux-headers']}.tar.gz | tar -xz && \
+cd linux-* && \
+make -j{proc_count} defconfig && \
+make INSTALL_HDR_PATH=/usr/local headers_install
+
+FROM openbmc-base as openbmc-boost
+RUN curl -L https://dl.bintray.com/boostorg/release/{pkg_rev['boost']}/source/boost_$(echo "{pkg_rev['boost']}" | tr '.' '_').tar.bz2 | tar -xj && \
+cd boost_*/ && \
+./bootstrap.sh --prefix={prefix} --with-libraries=context,coroutine && \
+./b2 && ./b2 install --prefix={prefix}
+
+FROM openbmc-base as openbmc-tinyxml2
+RUN curl -L https://github.com/leethomason/tinyxml2/archive/{pkg_rev['tinyxml2']}.tar.gz | tar -xz && \
+cd tinyxml2-* && \
+mkdir build && \
+cd build && \
+cmake {cmake_flags} .. && \
+make -j{proc_count} && \
+make install
+
+FROM openbmc-base as openbmc-boost-url
+RUN curl -L https://github.com/CPPAlliance/url/archive/{pkg_rev['boost-url']}.tar.gz | tar -xz && \
+cd url-* && \
+mkdir buildir && \
+cd buildir && \
+cmake {cmake_flags} -DBOOST_URL_STANDALONE=ON -DBOOST_URL_BUILD_TESTS=OFF -DBOOST_URL_BUILD_EXAMPLES=OFF .. && \
+make -j{proc_count} && \
+make install
+
+FROM openbmc-base as openbmc-valijson
+RUN curl -L https://github.com/tristanpenman/valijson/archive/{pkg_rev['valijson']}.tar.gz | tar -xz && \
+cd valijson-* && \
+mkdir build && \
+cd build && \
+cmake {cmake_flags} -DINSTALL_HEADERS=1 -DBUILD_TESTS=0 .. && \
+make -j{proc_count} && \
+make install
+
+FROM openbmc-base as openbmc-libvncserver
+RUN curl -L https://github.com/LibVNC/libvncserver/archive/{pkg_rev['libvncserver']}.tar.gz | tar -xz && \
+cd libvncserver-* && \
+mkdir build && \
+cd build && \
+cmake {cmake_flags} .. && \
+make -j{proc_count} && \
+make install
+
+FROM openbmc-base as openbmc-stdplus
+COPY --from=openbmc-fmt {prefix} {prefix}
+COPY --from=openbmc-span-lite {prefix} {prefix}
+RUN curl -L https://github.com/openbmc/stdplus/archive/{pkg_rev['openbmc/stdplus']}.tar.gz | tar -xz && \
+cd stdplus-* && \
+meson build {meson_flags} -Dtests=disabled -Dexamples=false && \
+ninja -C build && \
+ninja -C build install
+
+FROM openbmc-base as openbmc-sdbusplus
+RUN curl -L https://github.com/openbmc/sdbusplus/archive/{pkg_rev['openbmc/sdbusplus']}.tar.gz | tar -xz && \
+cd sdbusplus-* && \
+cd tools && ./setup.py install --root=/ --prefix={prefix} && \
+cd .. && meson build {meson_flags} -Dtests=disabled -Dexamples=disabled && \
+ninja -C build && \
+ninja -C build install
+
+FROM openbmc-base as openbmc-sdeventplus
+COPY --from=openbmc-function2 {prefix} {prefix}
+COPY --from=openbmc-stdplus {prefix} {prefix}
+RUN curl -L https://github.com/openbmc/sdeventplus/archive/{pkg_rev['openbmc/sdeventplus']}.tar.gz | tar -xz && \
+cd sdeventplus-* && \
+meson build {meson_flags} -Dtests=disabled -Dexamples=false && \
+ninja -C build && \
+ninja -C build install
+
+FROM openbmc-base as openbmc-gpioplus
+COPY --from=openbmc-stdplus {prefix} {prefix}
+RUN curl -L https://github.com/openbmc/gpioplus/archive/{pkg_rev['openbmc/gpioplus']}.tar.gz | tar -xz && \
+cd gpioplus-* && \
+meson build {meson_flags} -Dtests=disabled -Dexamples=false && \
+ninja -C build && \
+ninja -C build install
+
+FROM openbmc-base as openbmc-phosphor-dbus-interfaces
+COPY --from=openbmc-sdbusplus {prefix} {prefix}
+RUN curl -L https://github.com/openbmc/phosphor-dbus-interfaces/archive/{pkg_rev['openbmc/phosphor-dbus-interfaces']}.tar.gz | tar -xz && \
+cd phosphor-dbus-interfaces-* && \
+meson build {meson_flags} -Ddata_org_open_power=true -Ddata_com_ibm=true && \
+ninja -C build && \
+ninja -C build install
+
+FROM openbmc-base as openbmc-phosphor-logging
+COPY --from=openbmc-cereal {prefix} {prefix}
+COPY --from=openbmc-sdbusplus {prefix} {prefix}
+COPY --from=openbmc-sdeventplus {prefix} {prefix}
+COPY --from=openbmc-phosphor-dbus-interfaces {prefix} {prefix}
+COPY --from=openbmc-fifo_map {prefix} {prefix}
+RUN curl -L https://github.com/openbmc/phosphor-logging/archive/{pkg_rev['openbmc/phosphor-logging']}.tar.gz | tar -xz && \
+cd phosphor-logging-* && \
+./bootstrap.sh && \
+./configure {configure_flags} --enable-metadata-processing YAML_DIR={prefix}/share/phosphor-dbus-yaml/yaml && \
+make -j{proc_count} && \
+make install
+
+FROM openbmc-base as openbmc-phosphor-objmgr
+COPY --from=openbmc-boost {prefix} {prefix}
+COPY --from=openbmc-sdbusplus {prefix} {prefix}
+COPY --from=openbmc-tinyxml2 {prefix} {prefix}
+COPY --from=openbmc-phosphor-logging {prefix} {prefix}
+RUN curl -L https://github.com/openbmc/phosphor-objmgr/archive/{pkg_rev['openbmc/phosphor-objmgr']}.tar.gz | tar -xz && \
+cd phosphor-objmgr-* && \
+./bootstrap.sh && \
+./configure {configure_flags} && \
+make -j{proc_count} && \
+make install
+
+FROM openbmc-base as openbmc-open-power-pdbg
+RUN curl -L https://github.com/open-power/pdbg/archive/{pkg_rev['open-power/pdbg']}.tar.gz | tar -xz && \
+cd pdbg-* && \
+./bootstrap.sh && \
+./configure {configure_flags} && \
+make -j{proc_count} && \
+make install
+
+FROM openbmc-base as openbmc-pldm
+COPY --from=openbmc-sdbusplus {prefix} {prefix}
+COPY --from=openbmc-sdeventplus {prefix} {prefix}
+COPY --from=openbmc-boost {prefix} {prefix}
+COPY --from=openbmc-phosphor-dbus-interfaces {prefix} {prefix}
+COPY --from=openbmc-phosphor-logging {prefix} {prefix}
+COPY --from=openbmc-json {prefix} {prefix}
+COPY --from=openbmc-CLI11 {prefix} {prefix}
+RUN curl -L https://github.com/openbmc/pldm/archive/{pkg_rev['openbmc/pldm']}.tar.gz | tar -xz && \
+cd pldm-* && \
+meson build {meson_flags} -Dlibpldm-only=enabled -Doem-ibm=enabled -Dtests=disabled && \
+ninja -C build && \
+ninja -C build install
+
+# Build the final output image
+FROM openbmc-base
+{copy_cmds}
+
+# Some of our infrastructure still relies on the presence of this file
+# even though it is no longer needed to rebuild the docker environment
+# NOTE: The file is sorted to ensure the ordering is stable.
+RUN echo '{depcache}' > /tmp/depcache
+
+# Final configuration for the workspace
+RUN grep -q {gid} /etc/group || groupadd -g {gid} {username}
+RUN mkdir -p "{os.path.dirname(homedir)}"
+RUN grep -q {uid} /etc/passwd || useradd -d {homedir} -m -u {uid} -g {gid} {username}
+RUN sed -i '1iDefaults umask=000' /etc/sudoers
+RUN echo "{username} ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers
+
+{proxy_cmd}
+
+RUN /bin/bash
+"""
+
+# Do the docker build.
+for line in docker.build(
+ proxy_args,
+ "--network=host",
+ "-t",
+ docker_image_name,
+ "-",
+ _in=dockerfile,
+ _iter=True,
+):
+ print(line, end="", flush=True)