scripts/build-unit-test-docker

#!/usr/bin/env python3
#
# Build the required docker image to run package unit tests
#
# Script Variables:
#   DOCKER_IMG_NAME:  <optional, the name of the docker image to generate>
#                     default is openbmc/ubuntu-unit-test
#   DISTRO:           <optional, the distro to build a docker image against>
#                     default is ubuntu:eoan
#   BRANCH:           <optional, branch to build from each of the openbmc/
#                     repositories>
#                     default is master, which will be used if input branch not
#                     provided or not found
#   UBUNTU_MIRROR:    <optional, the URL of a mirror of Ubuntu to override the
#                     default ones in /etc/apt/sources.list>
#                     default is empty, and no mirror is used.
#   http_proxy        The HTTP address of the proxy server to connect to.
#                     Default: "", proxy is not setup if this is not set

import os
import sys
from sh import docker, git, nproc, uname

# Read a bunch of environment variables.
docker_image_name = os.environ.get("DOCKER_IMAGE_NAME", "openbmc/ubuntu-unit-test")
distro = os.environ.get("DISTRO", "ubuntu:focal")
branch = os.environ.get("BRANCH", "master")
ubuntu_mirror = os.environ.get("UBUNTU_MIRROR")
http_proxy = os.environ.get("http_proxy")

# Set up some common variables.
proc_count = nproc().strip()
username = os.environ.get("USER")
homedir = os.environ.get("HOME")
gid = os.getgid()
uid = os.getuid()

# Determine the architecture for Docker.
arch = uname("-m").strip()
if arch == "ppc64le":
    docker_base = "ppc64le/"
elif arch == "x86_64":
    docker_base = ""
else:
    print(f"Unsupported system architecture({arch}) found for docker image")
    sys.exit(1)

# These packages we use 'HEAD' for.
head_pkgs = [
    "openbmc/phosphor-objmgr",
    "openbmc/sdbusplus",
    "openbmc/sdeventplus",
    "openbmc/stdplus",
    "openbmc/gpioplus",
    "openbmc/phosphor-logging",
    "openbmc/phosphor-dbus-interfaces",
    "open-power/pdbg",
    "openbmc/pldm",
]

# Packages with fixed revisions.
pkg_rev = {
    "boost": "1.74.0",
    "cereal": "v1.3.0",
    "catch2": "v2.12.2",
    "CLI11": "v1.9.0",
    "fmt": "6.2.1",
    # Snapshot from 2020-01-03
    "function2": "3a0746bf5f601dfed05330aefcb6854354fce07d",
    # Snapshot from 2020-02-13
    "googletest": "23b2a3b1cf803999fb38175f6e9e038a4495c8a5",
    # Release 2020-08-06
    "json": "v3.9.1",
    # Snapshot from 2019-05-24
    "lcov": "75fbae1cfc5027f818a0bb865bf6f96fab3202da",
    # dev-5.0 2019-05-03
    "linux-headers": "8bf6567e77f7aa68975b7c9c6d044bba690bf327",
    # Snapshot from 2019-09-03
    "libvncserver": "1354f7f1bb6962dab209eddb9d6aac1f03408110",
    "span-lite": "v0.7.0",
    # version from meta-openembedded/meta-oe/recipes-support/libtinyxml2/libtinyxml2_5.0.1.bb
    "tinyxml2": "37bc3aca429f0164adf68c23444540b4a24b5778",
    # version from /meta-openembedded/meta-oe/recipes-devtools/boost-url/boost-url_git.bb
    "boost-url": "a56ae0df6d3078319755fbaa67822b4fa7fd352b",
    # version from meta-openembedded/meta-oe/recipes-devtools/valijson/valijson_git.bb
    "valijson": "c2f22fddf599d04dc33fcd7ed257c698a05345d9",
    # version from meta-openembedded/meta-oe/recipes-devtools/nlohmann-fifo/nlohmann-fifo_git.bb
    "fifo_map": "0dfbf5dacbb15a32c43f912a7e66a54aae39d0f9",
}

# Look up the HEAD for 'head_pkgs' and insert them into 'pkg_rev'.
pkg_lookups = {}
for pkg in head_pkgs:
    pkg_lookups[pkg] = git(
        "ls-remote", "--heads", f"https://github.com/{pkg}", _bg=True
    )
for pkg, result in pkg_lookups.items():
    for line in result.stdout.decode().split("\n"):
        if f"refs/heads/{branch}" in line:
            pkg_rev[pkg] = line.strip().split()[0]
        elif "refs/heads/master" in line and p not in pkg_rev:
            pkg_rev[pkg] = line.strip().split()[0]

# Create the contents of the '/tmp/depcache'.
# This needs to be sorted for consistency.
depcache = ""
for pkg in sorted(head_pkgs):
    if pkg in pkg_rev:
        depcache += "%s:%s," % (pkg, pkg_rev[pkg])

# Define common flags used for builds
prefix = "/usr/local"
configure_flags = " ".join(
    [
        f"--prefix={prefix}",
    ]
)
cmake_flags = " ".join(
    [
        "-DCMAKE_BUILD_TYPE=RelWithDebInfo",
        "-DBUILD_SHARED_LIBS=ON",
        f"-DCMAKE_INSTALL_PREFIX:PATH={prefix}",
    ]
)
meson_flags = " ".join(
    [
        "--wrap-mode=nodownload",
        f"-Dprefix={prefix}",
    ]
)


def stagename(name):
    if not name.startswith("openbmc/"):
        name = "openbmc/" + name
    return name.replace("/", "-")


# Build the commands needed to compose our final image
# We must sort the packages, otherwise we might produce an unstable
# docker file and rebuild the image unnecessarily
copy_cmds = ""
for pkg in sorted(pkg_rev.keys()):
    copy_cmds += f"COPY --from={stagename(pkg)} {prefix} {prefix}\n"
    # Workaround for upstream docker bug and multiple COPY cmds
    # https://github.com/moby/moby/issues/37965
    copy_cmds += "RUN true\n"

# Special flags if setting up a deb mirror.
mirror = ""
if "ubuntu" in distro and ubuntu_mirror:
    mirror = f"""
RUN echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME) main restricted universe multiverse" > /etc/apt/sources.list && \\
    echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-updates main restricted universe multiverse" >> /etc/apt/sources.list && \\
    echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-security main restricted universe multiverse" >> /etc/apt/sources.list && \\
    echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-proposed main restricted universe multiverse" >> /etc/apt/sources.list && \\
    echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-backports main restricted universe multiverse" >> /etc/apt/sources.list
"""

# Special flags for proxying.
proxy_cmd = ""
proxy_args = []
if http_proxy:
    proxy_cmd = f"""
RUN echo "[http]" >> {homedir}/.gitconfig && \
    echo "proxy = {http_proxy}" >> {homedir}/.gitconfig
"""
    proxy_args.extend(
        [
            "--build-arg",
            f"http_proxy={http_proxy}",
            "--build-arg",
            "https_proxy={https_proxy}",
        ]
    )

# Create docker image that can run package unit tests
dockerfile = f"""
FROM {docker_base}{distro} as openbmc-base

{mirror}

ENV DEBIAN_FRONTEND noninteractive

ENV PYTHONPATH "/usr/local/lib/python3.8/site-packages/"

# We need the keys to be imported for dbgsym repos
# New releases have a package, older ones fall back to manual fetching
# https://wiki.ubuntu.com/Debug%20Symbol%20Packages
RUN apt-get update && ( apt-get install ubuntu-dbgsym-keyring || ( apt-get install -yy dirmngr && \
    apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F2EDC64DC5AEE1F6B9C621F0C8CAB6595FDFF622 ) )

# Parse the current repo list into a debug repo list
RUN sed -n '/^deb /s,^deb [^ ]* ,deb http://ddebs.ubuntu.com ,p' /etc/apt/sources.list >/etc/apt/sources.list.d/debug.list

# Remove non-existent debug repos
RUN sed -i '/-\(backports\|security\) /d' /etc/apt/sources.list.d/debug.list

RUN cat /etc/apt/sources.list.d/debug.list

RUN apt-get update && apt-get dist-upgrade -yy && apt-get install -yy \
    gcc-10 \
    g++-10 \
    libc6-dbg \
    libc6-dev \
    libtool \
    bison \
    libdbus-1-dev \
    flex \
    cmake \
    python3 \
    python3-dev\
    python3-yaml \
    python3-mako \
    python3-pip \
    python3-setuptools \
    python3-git \
    python3-socks \
    pkg-config \
    autoconf \
    autoconf-archive \
    libsystemd-dev \
    systemd \
    libssl-dev \
    libevdev-dev \
    libevdev2-dbgsym \
    libjpeg-dev \
    libpng-dev \
    ninja-build \
    sudo \
    curl \
    git \
    dbus \
    iputils-ping \
    clang-10 \
    clang-format-10 \
    clang-tidy-10 \
    clang-tools-10 \
    shellcheck \
    npm \
    iproute2 \
    libnl-3-dev \
    libnl-genl-3-dev \
    libconfig++-dev \
    libsnmp-dev \
    valgrind \
    valgrind-dbg \
    libpam0g-dev \
    xxd \
    libi2c-dev \
    wget \
    libldap2-dev \
    libprotobuf-dev \
    libperlio-gzip-perl \
    libjson-perl \
    protobuf-compiler \
    libgpiod-dev \
    device-tree-compiler \
    cppcheck \
    libpciaccess-dev \
    libmimetic-dev \
    libxml2-utils \
    libxml-simple-perl

RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 1000 \
  --slave /usr/bin/g++ g++ /usr/bin/g++-10 \
  --slave /usr/bin/gcov gcov /usr/bin/gcov-10 \
  --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-10 \
  --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-10


RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-10 1000 \
  --slave /usr/bin/clang++ clang++ /usr/bin/clang++-10 \
  --slave /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-10 \
  --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-10 \
  --slave /usr/bin/run-clang-tidy.py run-clang-tidy.py /usr/bin/run-clang-tidy-10.py

RUN pip3 install inflection
RUN pip3 install pycodestyle
RUN pip3 install jsonschema
RUN pip3 install meson==0.54.3
RUN pip3 install protobuf

FROM openbmc-base as openbmc-lcov
RUN curl -L https://github.com/linux-test-project/lcov/archive/{pkg_rev['lcov']}.tar.gz | tar -xz && \
cd lcov-* && \
make -j{proc_count} && \
make install

FROM openbmc-base as openbmc-function2
RUN mkdir {prefix}/include/function2 && \
curl -L -o {prefix}/include/function2/function2.hpp https://raw.githubusercontent.com/Naios/function2/{pkg_rev['function2']}/include/function2/function2.hpp

FROM openbmc-base as openbmc-googletest
RUN curl -L https://github.com/google/googletest/archive/{pkg_rev['googletest']}.tar.gz | tar -xz && \
cd googletest-* && \
mkdir build && \
cd build && \
CXXFLAGS=-std=c++17 cmake {cmake_flags} -DTHREADS_PREFER_PTHREAD_FLAG=ON .. && \
make -j{proc_count} && \
make install

FROM openbmc-base as openbmc-catch2
RUN curl -L https://github.com/catchorg/Catch2/archive/{pkg_rev['catch2']}.tar.gz | tar -xz && \
cd Catch2-* && \
mkdir build && \
cd build && \
cmake {cmake_flags} -DBUILD_TESTING=OFF -DCATCH_INSTALL_DOCS=OFF .. && \
make -j{proc_count} && \
make install

FROM openbmc-base as openbmc-cereal
RUN curl -L https://github.com/USCiLab/cereal/archive/{pkg_rev['cereal']}.tar.gz | tar -xz && \
cp -a cereal-*/include/cereal/ {prefix}/include/

FROM openbmc-base as openbmc-CLI11
RUN curl -L https://github.com/CLIUtils/CLI11/archive/{pkg_rev['CLI11']}.tar.gz | tar -xz && \
cd CLI11-* && \
mkdir build && \
cd build && \
cmake {cmake_flags} -DCLI11_BUILD_DOCS=OFF -DBUILD_TESTING=OFF -DCLI11_BUILD_EXAMPLES=OFF .. && \
make -j{proc_count} && \
make install

FROM openbmc-base as openbmc-fmt
RUN curl -L https://github.com/fmtlib/fmt/archive/{pkg_rev['fmt']}.tar.gz | tar -xz && \
cd fmt-* && \
mkdir build && \
cd build && \
cmake {cmake_flags} -DFMT_DOC=OFF -DFMT_TEST=OFF .. && \
make -j{proc_count} && \
make install

FROM openbmc-base as openbmc-json
RUN mkdir {prefix}/include/nlohmann/ && \
curl -L -o {prefix}/include/nlohmann/json.hpp https://github.com/nlohmann/json/releases/download/{pkg_rev['json']}/json.hpp && \
ln -s nlohmann/json.hpp {prefix}/include/json.hpp

FROM openbmc-base as openbmc-fifo_map
RUN curl -L https://github.com/nlohmann/fifo_map/archive/{pkg_rev['fifo_map']}.tar.gz | tar -xz && \
cd fifo_map-*/src && cp fifo_map.hpp {prefix}/include/

FROM openbmc-base as openbmc-span-lite
RUN curl -L https://github.com/martinmoene/span-lite/archive/{pkg_rev['span-lite']}.tar.gz | tar -xz && \
cd span-lite-* && \
mkdir build && \
cd build && \
cmake {cmake_flags} -DSPAN_LITE_OPT_BUILD_TESTS=OFF .. && \
make -j{proc_count} && \
make install

FROM openbmc-base as openbmc-linux-headers
RUN curl -L https://github.com/openbmc/linux/archive/{pkg_rev['linux-headers']}.tar.gz | tar -xz && \
cd linux-* && \
make -j{proc_count} defconfig && \
make INSTALL_HDR_PATH=/usr/local headers_install

FROM openbmc-base as openbmc-boost
RUN curl -L https://dl.bintray.com/boostorg/release/{pkg_rev['boost']}/source/boost_$(echo "{pkg_rev['boost']}" | tr '.' '_').tar.bz2 | tar -xj && \
cd boost_*/ && \
./bootstrap.sh --prefix={prefix} --with-libraries=context,coroutine && \
./b2 && ./b2 install --prefix={prefix}

FROM openbmc-base as openbmc-tinyxml2
RUN curl -L https://github.com/leethomason/tinyxml2/archive/{pkg_rev['tinyxml2']}.tar.gz | tar -xz && \
cd tinyxml2-* && \
mkdir build && \
cd build && \
cmake {cmake_flags} .. && \
make -j{proc_count} && \
make install

FROM openbmc-base as openbmc-boost-url
RUN curl -L https://github.com/CPPAlliance/url/archive/{pkg_rev['boost-url']}.tar.gz | tar -xz && \
cd url-* && \
mkdir buildir && \
cd buildir && \
cmake {cmake_flags} -DBOOST_URL_STANDALONE=ON -DBOOST_URL_BUILD_TESTS=OFF -DBOOST_URL_BUILD_EXAMPLES=OFF .. && \
make -j{proc_count} && \
make install

FROM openbmc-base as openbmc-valijson
RUN curl -L https://github.com/tristanpenman/valijson/archive/{pkg_rev['valijson']}.tar.gz | tar -xz && \
cd valijson-* && \
mkdir build && \
cd build && \
cmake {cmake_flags} -DINSTALL_HEADERS=1 -DBUILD_TESTS=0 .. && \
make -j{proc_count} && \
make install

FROM openbmc-base as openbmc-libvncserver
RUN curl -L https://github.com/LibVNC/libvncserver/archive/{pkg_rev['libvncserver']}.tar.gz | tar -xz && \
cd libvncserver-* && \
mkdir build && \
cd build && \
cmake {cmake_flags} .. && \
make -j{proc_count} && \
make install

FROM openbmc-base as openbmc-stdplus
COPY --from=openbmc-fmt {prefix} {prefix}
COPY --from=openbmc-span-lite {prefix} {prefix}
RUN curl -L https://github.com/openbmc/stdplus/archive/{pkg_rev['openbmc/stdplus']}.tar.gz | tar -xz && \
cd stdplus-* && \
meson build {meson_flags} -Dtests=disabled -Dexamples=false && \
ninja -C build && \
ninja -C build install

FROM openbmc-base as openbmc-sdbusplus
RUN curl -L https://github.com/openbmc/sdbusplus/archive/{pkg_rev['openbmc/sdbusplus']}.tar.gz | tar -xz && \
cd sdbusplus-* && \
cd tools && ./setup.py install --root=/ --prefix={prefix} && \
cd .. && meson build {meson_flags} -Dtests=disabled -Dexamples=disabled && \
ninja -C build && \
ninja -C build install

FROM openbmc-base as openbmc-sdeventplus
COPY --from=openbmc-function2 {prefix} {prefix}
COPY --from=openbmc-stdplus {prefix} {prefix}
RUN curl -L https://github.com/openbmc/sdeventplus/archive/{pkg_rev['openbmc/sdeventplus']}.tar.gz | tar -xz && \
cd sdeventplus-* && \
meson build {meson_flags} -Dtests=disabled -Dexamples=false && \
ninja -C build && \
ninja -C build install

FROM openbmc-base as openbmc-gpioplus
COPY --from=openbmc-stdplus {prefix} {prefix}
RUN curl -L https://github.com/openbmc/gpioplus/archive/{pkg_rev['openbmc/gpioplus']}.tar.gz | tar -xz && \
cd gpioplus-* && \
meson build {meson_flags} -Dtests=disabled -Dexamples=false && \
ninja -C build && \
ninja -C build install

FROM openbmc-base as openbmc-phosphor-dbus-interfaces
COPY --from=openbmc-sdbusplus {prefix} {prefix}
RUN curl -L https://github.com/openbmc/phosphor-dbus-interfaces/archive/{pkg_rev['openbmc/phosphor-dbus-interfaces']}.tar.gz | tar -xz && \
cd phosphor-dbus-interfaces-* && \
meson build {meson_flags} -Ddata_org_open_power=true -Ddata_com_ibm=true && \
ninja -C build && \
ninja -C build install

FROM openbmc-base as openbmc-phosphor-logging
COPY --from=openbmc-cereal {prefix} {prefix}
COPY --from=openbmc-sdbusplus {prefix} {prefix}
COPY --from=openbmc-sdeventplus {prefix} {prefix}
COPY --from=openbmc-phosphor-dbus-interfaces {prefix} {prefix}
COPY --from=openbmc-fifo_map {prefix} {prefix}
RUN curl -L https://github.com/openbmc/phosphor-logging/archive/{pkg_rev['openbmc/phosphor-logging']}.tar.gz | tar -xz && \
cd phosphor-logging-* && \
./bootstrap.sh && \
./configure {configure_flags} --enable-metadata-processing YAML_DIR={prefix}/share/phosphor-dbus-yaml/yaml && \
make -j{proc_count} && \
make install

FROM openbmc-base as openbmc-phosphor-objmgr
COPY --from=openbmc-boost {prefix} {prefix}
COPY --from=openbmc-sdbusplus {prefix} {prefix}
COPY --from=openbmc-tinyxml2 {prefix} {prefix}
COPY --from=openbmc-phosphor-logging {prefix} {prefix}
RUN curl -L https://github.com/openbmc/phosphor-objmgr/archive/{pkg_rev['openbmc/phosphor-objmgr']}.tar.gz | tar -xz && \
cd phosphor-objmgr-* && \
./bootstrap.sh && \
./configure {configure_flags} && \
make -j{proc_count} && \
make install

FROM openbmc-base as openbmc-open-power-pdbg
RUN curl -L https://github.com/open-power/pdbg/archive/{pkg_rev['open-power/pdbg']}.tar.gz | tar -xz && \
cd pdbg-* && \
./bootstrap.sh && \
./configure {configure_flags} && \
make -j{proc_count} && \
make install

FROM openbmc-base as openbmc-pldm
COPY --from=openbmc-sdbusplus {prefix} {prefix}
COPY --from=openbmc-sdeventplus {prefix} {prefix}
COPY --from=openbmc-boost {prefix} {prefix}
COPY --from=openbmc-phosphor-dbus-interfaces {prefix} {prefix}
COPY --from=openbmc-phosphor-logging {prefix} {prefix}
COPY --from=openbmc-json {prefix} {prefix}
COPY --from=openbmc-CLI11 {prefix} {prefix}
RUN curl -L https://github.com/openbmc/pldm/archive/{pkg_rev['openbmc/pldm']}.tar.gz | tar -xz && \
cd pldm-* && \
meson build {meson_flags} -Dlibpldm-only=enabled -Doem-ibm=enabled -Dtests=disabled && \
ninja -C build && \
ninja -C build install

# Build the final output image
FROM openbmc-base
{copy_cmds}

# Some of our infrastructure still relies on the presence of this file
# even though it is no longer needed to rebuild the docker environment
# NOTE: The file is sorted to ensure the ordering is stable.
RUN echo '{depcache}' > /tmp/depcache

# Final configuration for the workspace
RUN grep -q {gid} /etc/group || groupadd -g {gid} {username}
RUN mkdir -p "{os.path.dirname(homedir)}"
RUN grep -q {uid} /etc/passwd || useradd -d {homedir} -m -u {uid} -g {gid} {username}
RUN sed -i '1iDefaults umask=000' /etc/sudoers
RUN echo "{username} ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers

{proxy_cmd}

RUN /bin/bash
"""

# Do the docker build.
for line in docker.build(
    proxy_args,
    "--network=host",
    "-t",
    docker_image_name,
    "-",
    _in=dockerfile,
    _iter=True,
):
    print(line, end="", flush=True)