poky/scripts/crosstap

#!/usr/bin/env python3
#
# Build a systemtap script for a given image, kernel
#
# Effectively script extracts needed information from set of
# 'bitbake -e' commands and contructs proper invocation of stap on
# host to build systemtap script for a given target.
#
# By default script will compile scriptname.ko that could be copied
# to taget and activated with 'staprun scriptname.ko' command. Or if
# --remote user@hostname option is specified script will build, load
# execute script on target.
#
# This script is very similar and inspired by crosstap shell script.
# The major difference that this script supports user-land related
# systemtap script, whereas crosstap could deal only with scripts
# related to kernel.
#
# Copyright (c) 2018, Cisco Systems.
#
# SPDX-License-Identifier: GPL-2.0-only
#

import sys
import re
import subprocess
import os
import optparse

class Stap(object):
    def __init__(self, script, module, remote):
        self.script = script
        self.module = module
        self.remote = remote
        self.stap = None
        self.sysroot = None
        self.runtime = None
        self.tapset = None
        self.arch = None
        self.cross_compile = None
        self.kernel_release = None
        self.target_path = None
        self.target_ld_library_path = None

        if not self.remote:
            if not self.module:
                # derive module name from script
                self.module = os.path.basename(self.script)
                if self.module[-4:] == ".stp":
                    self.module = self.module[:-4]
                    # replace - if any with _
                    self.module = self.module.replace("-", "_")

    def command(self, args):
        ret = []
        ret.append(self.stap)

        if self.remote:
            ret.append("--remote")
            ret.append(self.remote)
        else:
            ret.append("-p4")
            ret.append("-m")
            ret.append(self.module)

        ret.append("-a")
        ret.append(self.arch)

        ret.append("-B")
        ret.append("CROSS_COMPILE=" + self.cross_compile)

        ret.append("-r")
        ret.append(self.kernel_release)

        ret.append("-I")
        ret.append(self.tapset)

        ret.append("-R")
        ret.append(self.runtime)

        if self.sysroot:
            ret.append("--sysroot")
            ret.append(self.sysroot)

            ret.append("--sysenv=PATH=" + self.target_path)
            ret.append("--sysenv=LD_LIBRARY_PATH=" + self.target_ld_library_path)

        ret = ret + args

        ret.append(self.script)
        return ret

    def additional_environment(self):
        ret = {}
        ret["SYSTEMTAP_DEBUGINFO_PATH"] = "+:.debug:build"
        return ret

    def environment(self):
        ret = os.environ.copy()
        additional = self.additional_environment()
        for e in additional:
            ret[e] = additional[e]
        return ret

    def display_command(self, args):
        additional_env = self.additional_environment()
        command = self.command(args)

        print("#!/bin/sh")
        for e in additional_env:
            print("export %s=\"%s\"" % (e, additional_env[e]))
        print(" ".join(command))

class BitbakeEnvInvocationException(Exception):
    def __init__(self, message):
        self.message = message

class BitbakeEnv(object):
    BITBAKE="bitbake"

    def __init__(self, package):
        self.package = package
        self.cmd = BitbakeEnv.BITBAKE + " -e " + self.package
        self.popen = subprocess.Popen(self.cmd, shell=True,
                                      stdout=subprocess.PIPE,
                                      stderr=subprocess.STDOUT)
        self.__lines = self.popen.stdout.readlines()
        self.popen.wait()

        self.lines = []
        for line in self.__lines:
                self.lines.append(line.decode('utf-8'))

    def get_vars(self, vars):
        if self.popen.returncode:
            raise BitbakeEnvInvocationException(
                "\nFailed to execute '" + self.cmd +
                "' with the following message:\n" +
                ''.join(self.lines))

        search_patterns = []
        retdict = {}
        for var in vars:
            # regular not exported variable
            rexpr = "^" + var + "=\"(.*)\""
            re_compiled = re.compile(rexpr)
            search_patterns.append((var, re_compiled))

            # exported variable
            rexpr = "^export " + var + "=\"(.*)\""
            re_compiled = re.compile(rexpr)
            search_patterns.append((var, re_compiled))

            for line in self.lines:
                for var, rexpr in search_patterns:
                    m = rexpr.match(line)
                    if m:
                        value = m.group(1)
                        retdict[var] = value

        # fill variables values in order how they were requested
        ret = []
        for var in vars:
            ret.append(retdict.get(var))

        # if it is single value list return it as scalar, not the list
        if len(ret) == 1:
            ret = ret[0]

        return ret

class ParamDiscovery(object):
    SYMBOLS_CHECK_MESSAGE = """
WARNING: image '%s' does not have dbg-pkgs IMAGE_FEATURES enabled and no
"image-combined-dbg" in inherited classes is specified. As result the image
does not have symbols for user-land processes DWARF based probes. Consider
adding 'dbg-pkgs' to EXTRA_IMAGE_FEATURES or adding "image-combined-dbg" to
USER_CLASSES. I.e add this line 'USER_CLASSES += "image-combined-dbg"' to
local.conf file.

Or you may use IMAGE_GEN_DEBUGFS="1" option, and then after build you need
recombine/unpack image and image-dbg tarballs and pass resulting dir location
with --sysroot option.
"""

    def __init__(self, image):
        self.image = image

        self.image_rootfs = None
        self.image_features = None
        self.image_gen_debugfs = None
        self.inherit = None
        self.base_bindir = None
        self.base_sbindir = None
        self.base_libdir = None
        self.bindir = None
        self.sbindir = None
        self.libdir = None

        self.staging_bindir_toolchain = None
        self.target_prefix = None
        self.target_arch = None
        self.target_kernel_builddir = None

        self.staging_dir_native = None

        self.image_combined_dbg = False

    def discover(self):
        if self.image:
            benv_image = BitbakeEnv(self.image)
            (self.image_rootfs,
             self.image_features,
             self.image_gen_debugfs,
             self.inherit,
             self.base_bindir,
             self.base_sbindir,
             self.base_libdir,
             self.bindir,
             self.sbindir,
             self.libdir
            ) = benv_image.get_vars(
                 ("IMAGE_ROOTFS",
                  "IMAGE_FEATURES",
                  "IMAGE_GEN_DEBUGFS",
                  "INHERIT",
                  "base_bindir",
                  "base_sbindir",
                  "base_libdir",
                  "bindir",
                  "sbindir",
                  "libdir"
                  ))

        benv_kernel = BitbakeEnv("virtual/kernel")
        (self.staging_bindir_toolchain,
         self.target_prefix,
         self.target_arch,
         self.target_kernel_builddir
        ) = benv_kernel.get_vars(
             ("STAGING_BINDIR_TOOLCHAIN",
              "TARGET_PREFIX",
              "TRANSLATED_TARGET_ARCH",
              "B"
            ))

        benv_systemtap = BitbakeEnv("systemtap-native")
        (self.staging_dir_native
        ) = benv_systemtap.get_vars(["STAGING_DIR_NATIVE"])

        if self.inherit:
            if "image-combined-dbg" in self.inherit.split():
                self.image_combined_dbg = True

    def check(self, sysroot_option):
        ret = True
        if self.image_rootfs:
            sysroot = self.image_rootfs
            if not os.path.isdir(self.image_rootfs):
                print("ERROR: Cannot find '" + sysroot +
                      "' directory. Was '" + self.image + "' image built?")
                ret = False

        stap = self.staging_dir_native + "/usr/bin/stap"
        if not os.path.isfile(stap):
            print("ERROR: Cannot find '" + stap +
                  "'. Was 'systemtap-native' built?")
            ret = False

        if not os.path.isdir(self.target_kernel_builddir):
            print("ERROR: Cannot find '" + self.target_kernel_builddir +
                  "' directory. Was 'kernel/virtual' built?")
            ret = False

        if not sysroot_option and self.image_rootfs:
            dbg_pkgs_found = False

            if self.image_features:
                image_features = self.image_features.split()
                if "dbg-pkgs" in image_features:
                    dbg_pkgs_found = True

            if not dbg_pkgs_found \
               and not self.image_combined_dbg:
                print(ParamDiscovery.SYMBOLS_CHECK_MESSAGE % (self.image))

        if not ret:
            print("")

        return ret

    def __map_systemtap_arch(self):
        a = self.target_arch
        ret = a
        if   re.match('(athlon|x86.64)$', a):
            ret = 'x86_64'
        elif re.match('i.86$', a):
            ret = 'i386'
        elif re.match('arm$', a):
            ret = 'arm'
        elif re.match('aarch64$', a):
            ret = 'arm64'
        elif re.match('mips(isa|)(32|64|)(r6|)(el|)$', a):
            ret = 'mips'
        elif re.match('p(pc|owerpc)(|64)', a):
            ret = 'powerpc'
        return ret

    def fill_stap(self, stap):
        stap.stap = self.staging_dir_native + "/usr/bin/stap"
        if not stap.sysroot:
            if self.image_rootfs:
                if self.image_combined_dbg:
                    stap.sysroot = self.image_rootfs + "-dbg"
                else:
                    stap.sysroot = self.image_rootfs
        stap.runtime = self.staging_dir_native + "/usr/share/systemtap/runtime"
        stap.tapset = self.staging_dir_native + "/usr/share/systemtap/tapset"
        stap.arch = self.__map_systemtap_arch()
        stap.cross_compile = self.staging_bindir_toolchain + "/" + \
                             self.target_prefix
        stap.kernel_release = self.target_kernel_builddir

        # do we have standard that tells in which order these need to appear
        target_path = []
        if self.sbindir:
            target_path.append(self.sbindir)
        if self.bindir:
            target_path.append(self.bindir)
        if self.base_sbindir:
            target_path.append(self.base_sbindir)
        if self.base_bindir:
            target_path.append(self.base_bindir)
        stap.target_path = ":".join(target_path)

        target_ld_library_path = []
        if self.libdir:
            target_ld_library_path.append(self.libdir)
        if self.base_libdir:
            target_ld_library_path.append(self.base_libdir)
        stap.target_ld_library_path = ":".join(target_ld_library_path)


def main():
    usage = """usage: %prog -s <systemtap-script> [options] [-- [systemtap options]]

%prog cross compile given SystemTap script against given image, kernel

It needs to run in environtment set for bitbake - it uses bitbake -e
invocations to retrieve information to construct proper stap cross build
invocation arguments. It assumes that systemtap-native is built in given
bitbake workspace.

Anything after -- option is passed directly to stap.

Legacy script invocation style supported but depreciated:
  %prog <user@hostname> <sytemtap-script> [systemtap options]

To enable most out of systemtap the following site.conf or local.conf
configuration is recommended:

# enables symbol + target binaries rootfs-dbg in workspace
IMAGE_GEN_DEBUGFS = "1"
IMAGE_FSTYPES_DEBUGFS = "tar.bz2"
USER_CLASSES += "image-combined-dbg"

# enables kernel debug symbols
KERNEL_EXTRA_FEATURES:append = " features/debug/debug-kernel.scc"

# minimal, just run-time systemtap configuration in target image
PACKAGECONFIG:pn-systemtap = "monitor"

# add systemtap run-time into target image if it is not there yet
IMAGE_INSTALL:append = " systemtap"
"""
    option_parser = optparse.OptionParser(usage=usage)

    option_parser.add_option("-s", "--script", dest="script",
                             help="specify input script FILE name",
                             metavar="FILE")

    option_parser.add_option("-i", "--image", dest="image",
                             help="specify image name for which script should be compiled")

    option_parser.add_option("-r", "--remote", dest="remote",
                             help="specify username@hostname of remote target to run script "
                             "optional, it assumes that remote target can be accessed through ssh")

    option_parser.add_option("-m", "--module", dest="module",
                             help="specify module name, optional, has effect only if --remote is not used, "
                             "if not specified module name will be derived from passed script name")

    option_parser.add_option("-y", "--sysroot", dest="sysroot",
                             help="explicitely specify image sysroot location. May need to use it in case "
                             "when IMAGE_GEN_DEBUGFS=\"1\" option is used and recombined with symbols "
                             "in different location",
                             metavar="DIR")

    option_parser.add_option("-o", "--out", dest="out",
                             action="store_true",
                             help="output shell script that equvivalent invocation of this script with "
                             "given set of arguments, in given bitbake environment. It could be stored in "
                             "separate shell script and could be repeated without incuring bitbake -e "
                             "invocation overhead",
                             default=False)

    option_parser.add_option("-d", "--debug", dest="debug",
                             action="store_true",
                             help="enable debug output. Use this option to see resulting stap invocation",
                             default=False)

    # is invocation follow syntax from orignal crosstap shell script
    legacy_args = False

    # check if we called the legacy way
    if len(sys.argv) >= 3:
        if sys.argv[1].find("@") != -1 and os.path.exists(sys.argv[2]):
            legacy_args = True

            # fill options values for legacy invocation case
            options = optparse.Values
            options.script = sys.argv[2]
            options.remote = sys.argv[1]
            options.image = None
            options.module = None
            options.sysroot = None
            options.out = None
            options.debug = None
            remaining_args = sys.argv[3:]

    if not legacy_args:
        (options, remaining_args) = option_parser.parse_args()

    if not options.script or not os.path.exists(options.script):
        print("'-s FILE' option is missing\n")
        option_parser.print_help()
    else:
        stap = Stap(options.script, options.module, options.remote)
        discovery = ParamDiscovery(options.image)
        discovery.discover()
        if not discovery.check(options.sysroot):
            option_parser.print_help()
        else:
            stap.sysroot = options.sysroot
            discovery.fill_stap(stap)

            if options.out:
                stap.display_command(remaining_args)
            else:
                cmd = stap.command(remaining_args)
                env = stap.environment()

                if options.debug:
                    print(" ".join(cmd))

                os.execve(cmd[0], cmd, env)

main()