scripts/build-unit-test-docker

#!/usr/bin/env python3
#
# Build the required docker image to run package unit tests
#
# Script Variables:
#   DOCKER_IMG_NAME:  <optional, the name of the docker image to generate>
#                     default is openbmc/ubuntu-unit-test
#   DISTRO:           <optional, the distro to build a docker image against>
#                     default is ubuntu:eoan
#   BRANCH:           <optional, branch to build from each of the openbmc/
#                     repositories>
#                     default is master, which will be used if input branch not
#                     provided or not found
#   UBUNTU_MIRROR:    <optional, the URL of a mirror of Ubuntu to override the
#                     default ones in /etc/apt/sources.list>
#                     default is empty, and no mirror is used.
#   http_proxy        The HTTP address of the proxy server to connect to.
#                     Default: "", proxy is not setup if this is not set

import os
import sys
from sh import docker, git, nproc, uname

# Read a bunch of environment variables.
docker_image_name = os.environ.get("DOCKER_IMAGE_NAME", "openbmc/ubuntu-unit-test")
distro = os.environ.get("DISTRO", "ubuntu:focal")
branch = os.environ.get("BRANCH", "master")
ubuntu_mirror = os.environ.get("UBUNTU_MIRROR")
http_proxy = os.environ.get("http_proxy")

# Set up some common variables.
proc_count = nproc().strip()
username = os.environ.get("USER")
homedir = os.environ.get("HOME")
gid = os.getgid()
uid = os.getuid()

# Determine the architecture for Docker.
arch = uname("-m").strip()
if arch == "ppc64le":
    docker_base = "ppc64le/"
elif arch == "x86_64":
    docker_base = ""
else:
    print(f"Unsupported system architecture({arch}) found for docker image")
    sys.exit(1)

# Packages to include in image.
packages = {
    "boost": {
        "rev": "1.74.0",
        "url": (
            lambda pkg, rev: f"https://dl.bintray.com/boostorg/release/{rev}/source/{pkg}_{rev.replace('.', '_')}.tar.bz2"
        ),
    },
    "USCiLab/cereal": {"rev": "v1.3.0"},
    "catchorg/Catch2": {"rev": "v2.12.2"},
    "CLIUtils/CLI11": {"rev": "v1.9.0"},
    "fmtlib/fmt": {"rev": "6.2.1"},
    # Snapshot from 2020-01-03
    "Naios/function2": {"rev": "3a0746bf5f601dfed05330aefcb6854354fce07d"},
    # Snapshot from 2020-02-13
    "google/googletest": {"rev": "23b2a3b1cf803999fb38175f6e9e038a4495c8a5"},
    # Release 2020-08-06
    "nlohmann/json": {"rev": "v3.9.1"},
    # Snapshot from 2019-05-24
    "linux-test-project/lcov": {"rev": "75fbae1cfc5027f818a0bb865bf6f96fab3202da"},
    # dev-5.0 2019-05-03
    "openbmc/linux": {"rev": "8bf6567e77f7aa68975b7c9c6d044bba690bf327"},
    # Snapshot from 2019-09-03
    "LibVNC/libvncserver": {"rev": "1354f7f1bb6962dab209eddb9d6aac1f03408110"},
    "martinmoene/span-lite": {"rev": "v0.7.0"},
    # version from meta-openembedded/meta-oe/recipes-support/libtinyxml2/libtinyxml2_5.0.1.bb
    "leethomason/tinyxml2": {"rev": "37bc3aca429f0164adf68c23444540b4a24b5778"},
    # version from /meta-openembedded/meta-oe/recipes-devtools/boost-url/boost-url_git.bb
    "CPPAlliance/url": {"rev": "a56ae0df6d3078319755fbaa67822b4fa7fd352b"},
    # version from meta-openembedded/meta-oe/recipes-devtools/valijson/valijson_git.bb
    "tristanpenman/valijson": {"rev": "c2f22fddf599d04dc33fcd7ed257c698a05345d9"},
    # version from meta-openembedded/meta-oe/recipes-devtools/nlohmann-fifo/nlohmann-fifo_git.bb
    "nlohmann/fifo_map": {"rev": "0dfbf5dacbb15a32c43f912a7e66a54aae39d0f9"},
    "open-power/pdbg": {},
    "openbmc/gpioplus": {},
    "openbmc/phosphor-dbus-interfaces": {},
    "openbmc/phosphor-logging": {},
    "openbmc/phosphor-objmgr": {},
    "openbmc/pldm": {},
    "openbmc/sdbusplus": {},
    "openbmc/sdeventplus": {},
    "openbmc/stdplus": {},
}


def pkg_rev(pkg):
    return packages[pkg]["rev"]


def pkg_stagename(pkg):
    if not pkg.startswith("openbmc/"):
        pkg = "openbmc/" + pkg
    return pkg.replace("/", "-")


def pkg_url(pkg):
    if "url" in packages[pkg]:
        return packages[pkg]["url"](pkg, pkg_rev(pkg))
    return f"https://github.com/{pkg}/archive/{pkg_rev(pkg)}.tar.gz"


# Look up the HEAD for missing a static rev.
pkg_lookups = {}
for pkg in packages.keys():
    if "rev" in packages[pkg]:
        continue
    pkg_lookups[pkg] = git(
        "ls-remote", "--heads", f"https://github.com/{pkg}", _bg=True
    )
for pkg, result in pkg_lookups.items():
    for line in result.stdout.decode().split("\n"):
        if f"refs/heads/{branch}" in line:
            packages[pkg]["rev"] = line.strip().split()[0]
        elif "refs/heads/master" in line and p not in packages:
            packages[pkg]["rev"] = line.strip().split()[0]

# Create the contents of the '/tmp/depcache'.
# This needs to be sorted for consistency.
depcache = ""
for pkg in sorted(packages.keys()):
    depcache += "%s:%s," % (pkg, pkg_rev(pkg))

# Define common flags used for builds
prefix = "/usr/local"
configure_flags = " ".join(
    [
        f"--prefix={prefix}",
    ]
)
cmake_flags = " ".join(
    [
        "-DCMAKE_BUILD_TYPE=RelWithDebInfo",
        "-DBUILD_SHARED_LIBS=ON",
        f"-DCMAKE_INSTALL_PREFIX:PATH={prefix}",
    ]
)
meson_flags = " ".join(
    [
        "--wrap-mode=nodownload",
        f"-Dprefix={prefix}",
    ]
)

# Build the commands needed to compose our final image
# We must sort the packages, otherwise we might produce an unstable
# docker file and rebuild the image unnecessarily
copy_cmds = ""
for pkg in sorted(packages.keys()):
    copy_cmds += f"COPY --from={pkg_stagename(pkg)} {prefix} {prefix}\n"
    # Workaround for upstream docker bug and multiple COPY cmds
    # https://github.com/moby/moby/issues/37965
    copy_cmds += "RUN true\n"

# Special flags if setting up a deb mirror.
mirror = ""
if "ubuntu" in distro and ubuntu_mirror:
    mirror = f"""
RUN echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME) main restricted universe multiverse" > /etc/apt/sources.list && \\
    echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-updates main restricted universe multiverse" >> /etc/apt/sources.list && \\
    echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-security main restricted universe multiverse" >> /etc/apt/sources.list && \\
    echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-proposed main restricted universe multiverse" >> /etc/apt/sources.list && \\
    echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-backports main restricted universe multiverse" >> /etc/apt/sources.list
"""

# Special flags for proxying.
proxy_cmd = ""
proxy_args = []
if http_proxy:
    proxy_cmd = f"""
RUN echo "[http]" >> {homedir}/.gitconfig && \
    echo "proxy = {http_proxy}" >> {homedir}/.gitconfig
"""
    proxy_args.extend(
        [
            "--build-arg",
            f"http_proxy={http_proxy}",
            "--build-arg",
            "https_proxy={https_proxy}",
        ]
    )

# Create docker image that can run package unit tests
dockerfile = f"""
FROM {docker_base}{distro} as openbmc-base

{mirror}

ENV DEBIAN_FRONTEND noninteractive

ENV PYTHONPATH "/usr/local/lib/python3.8/site-packages/"

# We need the keys to be imported for dbgsym repos
# New releases have a package, older ones fall back to manual fetching
# https://wiki.ubuntu.com/Debug%20Symbol%20Packages
RUN apt-get update && ( apt-get install ubuntu-dbgsym-keyring || ( apt-get install -yy dirmngr && \
    apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F2EDC64DC5AEE1F6B9C621F0C8CAB6595FDFF622 ) )

# Parse the current repo list into a debug repo list
RUN sed -n '/^deb /s,^deb [^ ]* ,deb http://ddebs.ubuntu.com ,p' /etc/apt/sources.list >/etc/apt/sources.list.d/debug.list

# Remove non-existent debug repos
RUN sed -i '/-\(backports\|security\) /d' /etc/apt/sources.list.d/debug.list

RUN cat /etc/apt/sources.list.d/debug.list

RUN apt-get update && apt-get dist-upgrade -yy && apt-get install -yy \
    gcc-10 \
    g++-10 \
    libc6-dbg \
    libc6-dev \
    libtool \
    bison \
    libdbus-1-dev \
    flex \
    cmake \
    python3 \
    python3-dev\
    python3-yaml \
    python3-mako \
    python3-pip \
    python3-setuptools \
    python3-git \
    python3-socks \
    pkg-config \
    autoconf \
    autoconf-archive \
    libsystemd-dev \
    systemd \
    libssl-dev \
    libevdev-dev \
    libevdev2-dbgsym \
    libjpeg-dev \
    libpng-dev \
    ninja-build \
    sudo \
    curl \
    git \
    dbus \
    iputils-ping \
    clang-10 \
    clang-format-10 \
    clang-tidy-10 \
    clang-tools-10 \
    shellcheck \
    npm \
    iproute2 \
    libnl-3-dev \
    libnl-genl-3-dev \
    libconfig++-dev \
    libsnmp-dev \
    valgrind \
    valgrind-dbg \
    libpam0g-dev \
    xxd \
    libi2c-dev \
    wget \
    libldap2-dev \
    libprotobuf-dev \
    libperlio-gzip-perl \
    libjson-perl \
    protobuf-compiler \
    libgpiod-dev \
    device-tree-compiler \
    cppcheck \
    libpciaccess-dev \
    libmimetic-dev \
    libxml2-utils \
    libxml-simple-perl

RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 1000 \
  --slave /usr/bin/g++ g++ /usr/bin/g++-10 \
  --slave /usr/bin/gcov gcov /usr/bin/gcov-10 \
  --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-10 \
  --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-10


RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-10 1000 \
  --slave /usr/bin/clang++ clang++ /usr/bin/clang++-10 \
  --slave /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-10 \
  --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-10 \
  --slave /usr/bin/run-clang-tidy.py run-clang-tidy.py /usr/bin/run-clang-tidy-10.py

RUN pip3 install inflection
RUN pip3 install pycodestyle
RUN pip3 install jsonschema
RUN pip3 install meson==0.54.3
RUN pip3 install protobuf

FROM openbmc-base as {pkg_stagename('linux-test-project/lcov')}
RUN curl -L {pkg_url('linux-test-project/lcov')} | tar -xz && \
cd lcov-* && \
make -j{proc_count} && \
make install

FROM openbmc-base as {pkg_stagename('Naios/function2')}
RUN curl -L {pkg_url('Naios/function2')} | tar -xz && \
cd function2-* && \
mkdir {prefix}/include/function2 && \
cp include/function2/function2.hpp {prefix}/include/function2/

FROM openbmc-base as {pkg_stagename('google/googletest')}
RUN curl -L {pkg_url('google/googletest')} | tar -xz && \
cd googletest-* && \
mkdir build && \
cd build && \
CXXFLAGS=-std=c++17 cmake {cmake_flags} -DTHREADS_PREFER_PTHREAD_FLAG=ON .. && \
make -j{proc_count} && \
make install

FROM openbmc-base as {pkg_stagename('catchorg/Catch2')}
RUN curl -L {pkg_url('catchorg/Catch2')} | tar -xz && \
cd Catch2-* && \
mkdir build && \
cd build && \
cmake {cmake_flags} -DBUILD_TESTING=OFF -DCATCH_INSTALL_DOCS=OFF .. && \
make -j{proc_count} && \
make install

FROM openbmc-base as {pkg_stagename('USCiLab/cereal')}
RUN curl -L {pkg_url('USCiLab/cereal')} | tar -xz && \
cp -a cereal-*/include/cereal/ {prefix}/include/

FROM openbmc-base as {pkg_stagename('CLIUtils/CLI11')}
RUN curl -L {pkg_url('CLIUtils/CLI11')} | tar -xz && \
cd CLI11-* && \
mkdir build && \
cd build && \
cmake {cmake_flags} -DCLI11_BUILD_DOCS=OFF -DBUILD_TESTING=OFF -DCLI11_BUILD_EXAMPLES=OFF .. && \
make -j{proc_count} && \
make install

FROM openbmc-base as {pkg_stagename('fmtlib/fmt')}
RUN curl -L {pkg_url('fmtlib/fmt')} | tar -xz && \
cd fmt-* && \
mkdir build && \
cd build && \
cmake {cmake_flags} -DFMT_DOC=OFF -DFMT_TEST=OFF .. && \
make -j{proc_count} && \
make install

FROM openbmc-base as {pkg_stagename('nlohmann/json')}
RUN curl -L {pkg_url('nlohmann/json')} | tar -xz && \
cd json-* && \
mkdir {prefix}/include/nlohmann && \
cp include/nlohmann/json.hpp {prefix}/include/nlohmann && \
ln -s {prefix}/include/nlohmann/json.hpp {prefix}/include/json.hpp

FROM openbmc-base as {pkg_stagename('nlohmann/fifo_map')}
RUN curl -L {pkg_url('nlohmann/fifo_map')} | tar -xz && \
cd fifo_map-*/src && cp fifo_map.hpp {prefix}/include/

FROM openbmc-base as {pkg_stagename('martinmoene/span-lite')}
RUN curl -L {pkg_url('martinmoene/span-lite')} | tar -xz && \
cd span-lite-* && \
mkdir build && \
cd build && \
cmake {cmake_flags} -DSPAN_LITE_OPT_BUILD_TESTS=OFF .. && \
make -j{proc_count} && \
make install

FROM openbmc-base as {pkg_stagename('openbmc/linux')}
RUN curl -L {pkg_url('openbmc/linux')} | tar -xz && \
cd linux-* && \
make -j{proc_count} defconfig && \
make INSTALL_HDR_PATH=/usr/local headers_install

FROM openbmc-base as {pkg_stagename('boost')}
RUN curl -L {pkg_url('boost')} | tar -xj && \
cd boost_*/ && \
./bootstrap.sh --prefix={prefix} --with-libraries=context,coroutine && \
./b2 && ./b2 install --prefix={prefix}

FROM openbmc-base as {pkg_stagename('leethomason/tinyxml2')}
RUN curl -L {pkg_url('leethomason/tinyxml2')} | tar -xz && \
cd tinyxml2-* && \
mkdir build && \
cd build && \
cmake {cmake_flags} .. && \
make -j{proc_count} && \
make install

FROM openbmc-base as {pkg_stagename('CPPAlliance/url')}
RUN curl -L {pkg_url('CPPAlliance/url')} | tar -xz && \
cd url-* && \
mkdir buildir && \
cd buildir && \
cmake {cmake_flags} -DBOOST_URL_STANDALONE=ON -DBOOST_URL_BUILD_TESTS=OFF -DBOOST_URL_BUILD_EXAMPLES=OFF .. && \
make -j{proc_count} && \
make install

FROM openbmc-base as {pkg_stagename('tristanpenman/valijson')}
RUN curl -L {pkg_url('tristanpenman/valijson')} | tar -xz && \
cd valijson-* && \
mkdir build && \
cd build && \
cmake {cmake_flags} -DINSTALL_HEADERS=1 -DBUILD_TESTS=0 .. && \
make -j{proc_count} && \
make install

FROM openbmc-base as {pkg_stagename('LibVNC/libvncserver')}
RUN curl -L {pkg_url('LibVNC/libvncserver')} | tar -xz && \
cd libvncserver-* && \
mkdir build && \
cd build && \
cmake {cmake_flags} .. && \
make -j{proc_count} && \
make install

FROM openbmc-base as {pkg_stagename('openbmc/stdplus')}
COPY --from={pkg_stagename('fmtlib/fmt')} {prefix} {prefix}
COPY --from={pkg_stagename('martinmoene/span-lite')} {prefix} {prefix}
RUN curl -L {pkg_url('openbmc/stdplus')} | tar -xz && \
cd stdplus-* && \
meson build {meson_flags} -Dtests=disabled -Dexamples=false && \
ninja -C build && \
ninja -C build install

FROM openbmc-base as {pkg_stagename('openbmc/sdbusplus')}
RUN curl -L {pkg_url('openbmc/sdbusplus')} | tar -xz && \
cd sdbusplus-* && \
cd tools && ./setup.py install --root=/ --prefix={prefix} && \
cd .. && meson build {meson_flags} -Dtests=disabled -Dexamples=disabled && \
ninja -C build && \
ninja -C build install

FROM openbmc-base as {pkg_stagename('openbmc/sdeventplus')}
COPY --from={pkg_stagename('Naios/function2')} {prefix} {prefix}
COPY --from={pkg_stagename('openbmc/stdplus')} {prefix} {prefix}
RUN curl -L {pkg_url('openbmc/sdeventplus')} | tar -xz && \
cd sdeventplus-* && \
meson build {meson_flags} -Dtests=disabled -Dexamples=false && \
ninja -C build && \
ninja -C build install

FROM openbmc-base as {pkg_stagename('openbmc/gpioplus')}
COPY --from={pkg_stagename('openbmc/stdplus')} {prefix} {prefix}
RUN curl -L {pkg_url('openbmc/gpioplus')} | tar -xz && \
cd gpioplus-* && \
meson build {meson_flags} -Dtests=disabled -Dexamples=false && \
ninja -C build && \
ninja -C build install

FROM openbmc-base as {pkg_stagename('openbmc/phosphor-dbus-interfaces')}
COPY --from={pkg_stagename('openbmc/sdbusplus')} {prefix} {prefix}
RUN curl -L {pkg_url('openbmc/phosphor-dbus-interfaces')} | tar -xz && \
cd phosphor-dbus-interfaces-* && \
meson build {meson_flags} -Ddata_org_open_power=true -Ddata_com_ibm=true && \
ninja -C build && \
ninja -C build install

FROM openbmc-base as {pkg_stagename('openbmc/phosphor-logging')}
COPY --from={pkg_stagename('USCiLab/cereal')} {prefix} {prefix}
COPY --from={pkg_stagename('openbmc/sdbusplus')} {prefix} {prefix}
COPY --from={pkg_stagename('openbmc/sdeventplus')} {prefix} {prefix}
COPY --from={pkg_stagename('openbmc/phosphor-dbus-interfaces')} {prefix} {prefix}
COPY --from={pkg_stagename('nlohmann/fifo_map')} {prefix} {prefix}
RUN curl -L {pkg_url('openbmc/phosphor-logging')} | tar -xz && \
cd phosphor-logging-* && \
./bootstrap.sh && \
./configure {configure_flags} --enable-metadata-processing YAML_DIR={prefix}/share/phosphor-dbus-yaml/yaml && \
make -j{proc_count} && \
make install

FROM openbmc-base as {pkg_stagename('openbmc/phosphor-objmgr')}
COPY --from={pkg_stagename('boost')} {prefix} {prefix}
COPY --from={pkg_stagename('openbmc/sdbusplus')} {prefix} {prefix}
COPY --from={pkg_stagename('leethomason/tinyxml2')} {prefix} {prefix}
COPY --from={pkg_stagename('openbmc/phosphor-logging')} {prefix} {prefix}
RUN curl -L {pkg_url('openbmc/phosphor-objmgr')} | tar -xz && \
cd phosphor-objmgr-* && \
./bootstrap.sh && \
./configure {configure_flags} && \
make -j{proc_count} && \
make install

FROM openbmc-base as {pkg_stagename('open-power/pdbg')}
RUN curl -L {pkg_url('open-power/pdbg')} | tar -xz && \
cd pdbg-* && \
./bootstrap.sh && \
./configure {configure_flags} && \
make -j{proc_count} && \
make install

FROM openbmc-base as {pkg_stagename('openbmc/pldm')}
COPY --from={pkg_stagename('openbmc/sdbusplus')} {prefix} {prefix}
COPY --from={pkg_stagename('openbmc/sdeventplus')} {prefix} {prefix}
COPY --from={pkg_stagename('boost')} {prefix} {prefix}
COPY --from={pkg_stagename('openbmc/phosphor-dbus-interfaces')} {prefix} {prefix}
COPY --from={pkg_stagename('openbmc/phosphor-logging')} {prefix} {prefix}
COPY --from={pkg_stagename('nlohmann/json')} {prefix} {prefix}
COPY --from={pkg_stagename('CLIUtils/CLI11')} {prefix} {prefix}
RUN curl -L {pkg_url('openbmc/pldm')} | tar -xz && \
cd pldm-* && \
meson build {meson_flags} -Dlibpldm-only=enabled -Doem-ibm=enabled -Dtests=disabled && \
ninja -C build && \
ninja -C build install

# Build the final output image
FROM openbmc-base
{copy_cmds}

# Some of our infrastructure still relies on the presence of this file
# even though it is no longer needed to rebuild the docker environment
# NOTE: The file is sorted to ensure the ordering is stable.
RUN echo '{depcache}' > /tmp/depcache

# Final configuration for the workspace
RUN grep -q {gid} /etc/group || groupadd -g {gid} {username}
RUN mkdir -p "{os.path.dirname(homedir)}"
RUN grep -q {uid} /etc/passwd || useradd -d {homedir} -m -u {uid} -g {gid} {username}
RUN sed -i '1iDefaults umask=000' /etc/sudoers
RUN echo "{username} ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers

{proxy_cmd}

RUN /bin/bash
"""

# Do the docker build.
for line in docker.build(
    proxy_args,
    "--network=host",
    "-t",
    docker_image_name,
    "-",
    _in=dockerfile,
    _iter=True,
):
    print(line, end="", flush=True)