Import 80d60e7 from yoctoproject.org meta-arm
To support ARMv8 SoCs.
meta-arm has several patch files. Since they are maintained by the
upstream meta-arm community, add meta-arm to the ignore list in
run-repotest.
Change-Id: Ia87a2e947bbabd347d256eccc47a343e1c885479
Signed-off-by: Brad Bishop <bradleyb@fuzziesquirrel.com>
diff --git a/meta-arm/scripts/layer-overview.py b/meta-arm/scripts/layer-overview.py
new file mode 100755
index 0000000..326470e
--- /dev/null
+++ b/meta-arm/scripts/layer-overview.py
@@ -0,0 +1,74 @@
+#! /usr/bin/env python3
+
+"""
+Print an overview of the layer to help writing release notes.
+
+Output includes sublayers, machines, recipes.
+"""
+
+import argparse
+import sys
+
+# TODO:
+# - More human-readable output
+# - Diff mode, give two revisions and list the changes
+
+def is_layer(path):
+ """
+ Determine if this path looks like a layer (is a directory and contains conf/layer.conf).
+ """
+ return path.is_dir() and (path / "conf" / "layer.conf").exists()
+
+
+def print_layer(layer):
+ """
+ Print a summary of the layer.
+ """
+ print(layer.name)
+
+ machines = sorted(p for p in layer.glob("conf/machine/*.conf"))
+ if machines:
+ print(" Machines")
+ for m in machines:
+ print(f" {m.stem}")
+ print()
+
+ recipes = sorted((p for p in layer.glob("recipes-*/*/*.bb")), key=lambda p:p.name)
+ if recipes:
+ print(" Recipes")
+ for r in recipes:
+ if "_" in r.stem:
+ pn, pv = r.stem.rsplit("_", 1)
+ print(f" {pn} {pv}")
+ else:
+ print(f" {r.stem}")
+ print()
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument("repository")
+parser.add_argument("revision", nargs="?")
+args = parser.parse_args()
+
+if args.revision:
+ import gitpathlib
+ base = gitpathlib.GitPath(args.repository, args.revision)
+else:
+ import pathlib
+ base = pathlib.Path(args.repository)
+
+if is_layer(base):
+ print_layer(base)
+else:
+ sublayers = sorted(p for p in base.glob("meta-*") if is_layer(p))
+ if sublayers:
+ print("Sub-Layers")
+ for l in sublayers:
+ print(f" {l.name}")
+ print()
+
+ for layer in sublayers:
+ print_layer(layer)
+ else:
+ print(f"No layers found in {base}", file=sys.stderr)
+ sys.exit(1)
diff --git a/meta-arm/scripts/machine-summary-overview.txt.jinja b/meta-arm/scripts/machine-summary-overview.txt.jinja
new file mode 100644
index 0000000..d585065
--- /dev/null
+++ b/meta-arm/scripts/machine-summary-overview.txt.jinja
@@ -0,0 +1,12 @@
+Machine Overview
+Generated at {{ timestamp }}.
+{% for machine, data in data|dictsort %}
+
+MACHINE: {{ machine }}
+{% for recipe in recipes|sort %}
+{% if recipe in data %}
+{% set details = data[recipe] %}
+{{ details.recipe }}: {{ details.version }}
+{% endif %}
+{% endfor %}
+{% endfor %}
diff --git a/meta-arm/scripts/machine-summary.py b/meta-arm/scripts/machine-summary.py
new file mode 100755
index 0000000..c2b43da
--- /dev/null
+++ b/meta-arm/scripts/machine-summary.py
@@ -0,0 +1,232 @@
+#! /usr/bin/env python3
+
+import argparse
+import datetime
+import os
+import pathlib
+import re
+import sys
+
+import jinja2
+
+def trim_pv(pv):
+ """
+ Strip anything after +git from the PV
+ """
+ return "".join(pv.partition("+git")[:2])
+
+def needs_update(version, upstream):
+ """
+ Do a dumb comparison to determine if the version needs to be updated.
+ """
+ if "+git" in version:
+ # strip +git and see if this is a post-release snapshot
+ version = version.replace("+git", "")
+ return version != upstream
+
+def safe_patches(patches):
+ for info in patches:
+ if info["status"] in ("Denied", "Pending", "Unknown"):
+ return False
+ return True
+
+def layer_path(layername: str, d) -> pathlib.Path:
+ """
+ Return the path to the specified layer, or None if the layer isn't present.
+ """
+ if not hasattr(layer_path, "cache"):
+ # Don't use functools.lru_cache as we don't want d changing to invalidate the cache
+ layer_path.cache = {}
+
+ if layername in layer_path.cache:
+ return layer_path.cache[layername]
+
+ bbpath = d.getVar("BBPATH").split(":")
+ pattern = d.getVar('BBFILE_PATTERN_' + layername)
+ for path in reversed(sorted(bbpath)):
+ if re.match(pattern, path + "/"):
+ layer_path.cache[layername] = pathlib.Path(path)
+ return layer_path.cache[layername]
+ return None
+
+def get_url_for_patch(layer: str, localpath: pathlib.Path, d) -> str:
+ relative = localpath.relative_to(layer_path(layer, d))
+
+ # TODO: use layerindexlib
+ # TODO: assumes default branch
+ if layer == "core":
+ return f"https://git.openembedded.org/openembedded-core/tree/meta/{relative}"
+ elif layer in ("meta-arm", "meta-arm-bsp", "arm-toolchain", "meta-atp", "meta-gem5"):
+ return f"https://git.yoctoproject.org/meta-arm/tree/{layer}/{relative}"
+ else:
+ print(f"WARNING: Don't know web URL for layer {layer}", file=sys.stderr)
+ return None
+
+def extract_patch_info(src_uri, d):
+ """
+ Parse the specified patch entry from a SRC_URI and return (base name, layer name, status) tuple
+ """
+ import bb.fetch, bb.utils
+
+ info = {}
+ localpath = pathlib.Path(bb.fetch.decodeurl(src_uri)[2])
+ info["name"] = localpath.name
+ info["layer"] = bb.utils.get_file_layer(str(localpath), d)
+ info["url"] = get_url_for_patch(info["layer"], localpath, d)
+
+ status = "Unknown"
+ with open(localpath, errors="ignore") as f:
+ m = re.search(r"^[\t ]*Upstream[-_ ]Status:?[\t ]*(\w*)", f.read(), re.IGNORECASE | re.MULTILINE)
+ if m:
+ # TODO: validate
+ status = m.group(1)
+ info["status"] = status
+ return info
+
+def harvest_data(machines, recipes):
+ import bb.tinfoil
+ with bb.tinfoil.Tinfoil() as tinfoil:
+ tinfoil.prepare(config_only=True)
+ corepath = layer_path("core", tinfoil.config_data)
+ sys.path.append(os.path.join(corepath, "lib"))
+ import oe.recipeutils
+ import oe.patch
+
+ # Queue of recipes that we're still looking for upstream releases for
+ to_check = list(recipes)
+
+ # Upstream releases
+ upstreams = {}
+ # Machines to recipes to versions
+ versions = {}
+
+ for machine in machines:
+ print(f"Gathering data for {machine}...")
+ os.environ["MACHINE"] = machine
+ with bb.tinfoil.Tinfoil() as tinfoil:
+ versions[machine] = {}
+
+ tinfoil.prepare(quiet=2)
+ for recipe in recipes:
+ try:
+ d = tinfoil.parse_recipe(recipe)
+ except bb.providers.NoProvider:
+ continue
+
+ if recipe in to_check:
+ try:
+ info = oe.recipeutils.get_recipe_upstream_version(d)
+ upstreams[recipe] = info["version"]
+ to_check.remove(recipe)
+ except (bb.providers.NoProvider, KeyError):
+ pass
+
+ details = versions[machine][recipe] = {}
+ details["recipe"] = d.getVar("PN")
+ details["version"] = trim_pv(d.getVar("PV"))
+ details["fullversion"] = d.getVar("PV")
+ details["patches"] = [extract_patch_info(p, d) for p in oe.patch.src_patches(d)]
+ details["patched"] = bool(details["patches"])
+ details["patches_safe"] = safe_patches(details["patches"])
+
+ # Now backfill the upstream versions
+ for machine in versions:
+ for recipe in versions[machine]:
+ data = versions[machine][recipe]
+ data["upstream"] = upstreams[recipe]
+ data["needs_update"] = needs_update(data["version"], data["upstream"])
+ return upstreams, versions
+
+# TODO can this be inferred from the list of recipes in the layer
+recipes = ("virtual/kernel",
+ "scp-firmware",
+ "trusted-firmware-a",
+ "trusted-firmware-m",
+ "edk2-firmware",
+ "u-boot",
+ "optee-os",
+ "gcc-aarch64-none-elf-native",
+ "gcc-arm-none-eabi-native")
+
+
+class Format:
+ """
+ The name of this format
+ """
+ name = None
+ """
+ Registry of names to classes
+ """
+ registry = {}
+
+ def __init_subclass__(cls, **kwargs):
+ super().__init_subclass__(**kwargs)
+ assert cls.name
+ cls.registry[cls.name] = cls
+
+ @classmethod
+ def get_format(cls, name):
+ return cls.registry[name]()
+
+ def render(self, context, output: pathlib.Path):
+ pass
+
+ def get_template(self, name):
+ template_dir = os.path.dirname(os.path.abspath(__file__))
+ env = jinja2.Environment(
+ loader=jinja2.FileSystemLoader(template_dir),
+ extensions=['jinja2.ext.i18n'],
+ autoescape=jinja2.select_autoescape(),
+ trim_blocks=True,
+ lstrip_blocks=True
+ )
+
+ # We only need i18n for plurals
+ env.install_null_translations()
+
+ return env.get_template(name)
+
+class TextOverview(Format):
+ name = "overview.txt"
+
+ def render(self, context, output: pathlib.Path):
+ with open(output, "wt") as f:
+ f.write(self.get_template(f"machine-summary-overview.txt.jinja").render(context))
+
+class HtmlUpdates(Format):
+ name = "report"
+
+ def render(self, context, output: pathlib.Path):
+ if output.exists() and not output.is_dir():
+ print(f"{output} is not a directory", file=sys.stderr)
+ sys.exit(1)
+
+ if not output.exists():
+ output.mkdir(parents=True)
+
+ with open(output / "index.html", "wt") as f:
+ f.write(self.get_template(f"report-index.html.jinja").render(context))
+
+ subcontext = context.copy()
+ del subcontext["data"]
+ for machine, subdata in context["data"].items():
+ subcontext["machine"] = machine
+ subcontext["data"] = subdata
+ with open(output / f"{machine}.html", "wt") as f:
+ f.write(self.get_template(f"report-details.html.jinja").render(subcontext))
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(description="machine-summary")
+ parser.add_argument("machines", nargs="+", help="machine names", metavar="MACHINE")
+ parser.add_argument("-t", "--type", required=True, choices=Format.registry.keys())
+ parser.add_argument("-o", "--output", type=pathlib.Path, required=True)
+ args = parser.parse_args()
+
+ context = {}
+ # TODO: include git describe for meta-arm
+ context["timestamp"] = str(datetime.datetime.now().strftime("%c"))
+ context["recipes"] = sorted(recipes)
+ context["releases"], context["data"] = harvest_data(args.machines, recipes)
+
+ formatter = Format.get_format(args.type)
+ formatter.render(context, args.output)
diff --git a/meta-arm/scripts/report-base.html.jinja b/meta-arm/scripts/report-base.html.jinja
new file mode 100644
index 0000000..be08125
--- /dev/null
+++ b/meta-arm/scripts/report-base.html.jinja
@@ -0,0 +1,35 @@
+<!DOCTYPE html>
+<html>
+ <head>
+ <title>{% block title %}{% endblock %}</title>
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+ <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bulma@0.9.1/css/bulma.min.css">
+ </head>
+ <body>
+ <section class="section">
+ {# TODO use position: sticky to glue this to the top #}
+ <nav class="breadcrumb is-large">
+ <ul>
+ <li class="{{ "is-active" if machine is undefined }}">
+ <a href="index.html">Recipe Report</a>
+ </li>
+ {% if machine is defined %}
+ <li class="is-active">
+ <a href="#">{{machine}}</a>
+ </li>
+ {% endif %}
+ </ul>
+ </nav>
+
+ <div class="content">
+ {% block content %}{% endblock %}
+ </div>
+ </section>
+
+ <footer class="footer">
+ <div class="content has-text-centered">
+ Generated by <code>machine-summary</code> at {{ timestamp }}.
+ </div>
+ </footer>
+ </body>
+</html>
diff --git a/meta-arm/scripts/report-details.html.jinja b/meta-arm/scripts/report-details.html.jinja
new file mode 100644
index 0000000..a656c26
--- /dev/null
+++ b/meta-arm/scripts/report-details.html.jinja
@@ -0,0 +1,64 @@
+{% extends "report-base.html.jinja" %}
+{% block title %}Recipe Report for {{ machine }}{% endblock %}
+
+{# Write a tag element using the Upstream-Status to determine the class. #}
+{% macro make_patch_tag(status) -%}
+ {% set status = status.split()[0] %}
+ {% if status in ("Unknown", "Pending") %}
+ {% set class = "is-danger" %}
+ {% elif status in ("Backport", "Accepted", "Inappropriate", "Denied") %}
+ {% set class = "is-success" %}
+ {% elif status in ("Submitted",) %}
+ {% set class = "is-info" %}
+ {% else %}
+ {% set class = "is-info" %}
+ {% endif %}
+ <span class="tag {{ class }}">{{ status }}</span>
+{%- endmacro %}
+
+{% block content %}
+ <!-- TODO table of contents -->
+
+ {% for name, data in data|dictsort if data.needs_update or data.patched %}
+ <h2 class="title is-4">
+ {{ data.recipe }} {{ data.fullversion }}
+ {% if name != data.recipe %}
+ (provides {{ name }})
+ {% endif %}
+ {% if data.needs_update %}<span class="tag is-danger">Upgrade Needed</span>{% endif %}
+ <a id="recipe-{{ data.recipe }}" class="has-text-grey-lighter">#</a>
+ </h2>
+
+ {% if data.needs_update %}
+ <p>
+ Recipe is version {{ data.fullversion }}, latest upstream release is <strong>{{ data.upstream }}</strong>.
+ </p>
+ {% endif%}
+
+ {% if data.patched %}
+ <table class="table is-striped is-bordered">
+ <thead>
+ <tr>
+ <th>Patch</th>
+ <th style="width: 20em">Layer</th>
+ <th style="width: 10em">Status</th>
+ </tr>
+ </thead>
+ <tbody>
+ {% for pinfo in data.patches %}
+ <tr>
+ <td>
+ {% if pinfo.url %}<a href="{{pinfo.url}}">{% endif %}
+ {{ pinfo.name }}
+ {% if pinfo.url %}</a>{% endif %}
+ </td>
+ <td>{{ pinfo.layer }}</td>
+ <!-- TODO: tooltip with full status? -->
+ <td class="has-text-centered">{{ make_patch_tag(pinfo.status)}}</td>
+ </tr>
+ {% endfor %}
+ </tbody>
+ </table>
+ {% endif %}
+ {% endfor %}
+{% endblock %}
diff --git a/meta-arm/scripts/report-index.html.jinja b/meta-arm/scripts/report-index.html.jinja
new file mode 100644
index 0000000..4453758
--- /dev/null
+++ b/meta-arm/scripts/report-index.html.jinja
@@ -0,0 +1,50 @@
+{% extends "report-base.html.jinja" %}
+{% block title %}Recipe Report{% endblock %}
+
+{% block content %}
+ <table class="table is-striped">
+ <thead>
+ <tr>
+ <th>Machine</th>
+ {% for recipe in recipes|sort %}
+ <th>{{ recipe }} ({{releases[recipe]|default("?")}})</th>
+ {% endfor %}
+ </tr>
+ </thead>
+ <tbody>
+ {% for machine, data in data|dictsort %}
+ <tr>
+ <th><a href="{{machine}}.html">{{ machine }}</a></th>
+ {% for recipe in recipes|sort %}
+ {% if recipe in data %}
+ {% set details = data[recipe] %}
+ <td style="text-align: center">
+ <a href="{{machine}}.html#recipe-{{details.recipe}}">
+ {{ details.recipe if details.recipe != recipe}}
+ {{ details.version }}
+ </a>
+ {% if details.patches or details.needs_update %}
+ <br>
+ {% if details.patches %}
+ <span class="tag {{ "is-info" if details.patches_safe else "is-danger" }}">
+ {% trans count=details.patches|length %}
+ {{ count }} Patch
+ {% pluralize %}
+ {{ count }} Patches
+ {% endtrans %}
+ </span>
+ {% endif %}
+ {% if details.needs_update %}
+ <span class="tag is-danger">Upgrade</span>
+ {% endif %}
+ {% endif %}
+ </td>
+ {% else %}
+ <td>-</td>
+ {% endif %}
+ {% endfor %}
+ </tr>
+ {% endfor %}
+ </tbody>
+ </table>
+{% endblock %}
diff --git a/meta-arm/scripts/runfvp b/meta-arm/scripts/runfvp
new file mode 100755
index 0000000..9fb77d3
--- /dev/null
+++ b/meta-arm/scripts/runfvp
@@ -0,0 +1,104 @@
+#! /usr/bin/env python3
+
+import asyncio
+import os
+import pathlib
+import signal
+import sys
+
+import logging
+logger = logging.getLogger("RunFVP")
+
+# Add meta-arm/lib/ to path
+libdir = pathlib.Path(__file__).parents[1] / "meta-arm" / "lib"
+sys.path.insert(0, str(libdir))
+
+from fvp import terminal, runner, conffile
+
+def parse_args(arguments):
+ import argparse
+ terminals = terminal.terminals
+
+ parser = argparse.ArgumentParser(description="Run images in a FVP")
+ parser.add_argument("config", nargs="?", help="Machine name or path to .fvpconf file")
+ group = parser.add_mutually_exclusive_group()
+ group.add_argument("-t", "--terminals", choices=terminals.all_terminals(), default=terminals.preferred_terminal(), help="Automatically start terminals (default: %(default)s)")
+ group.add_argument("-c", "--console", action="store_true", help="Attach the first uart to stdin/stdout")
+ parser.add_argument("--verbose", action="store_true", help="Output verbose logging")
+ parser.usage = f"{parser.format_usage().strip()} -- [ arguments passed to FVP ]"
+ # TODO option for telnet vs netcat
+
+ # If the arguments contains -- then everything after it should be passed to the FVP binary directly.
+ if "--" in arguments:
+ i = arguments.index("--")
+ fvp_args = arguments[i+1:]
+ arguments = arguments[:i]
+ else:
+ fvp_args = []
+
+ args = parser.parse_args(args=arguments)
+ logging.basicConfig(level=args.verbose and logging.DEBUG or logging.WARNING)
+
+ # If we're hooking up the console, don't start any terminals
+ if args.console:
+ args.terminals = "none"
+
+ logger.debug(f"Parsed arguments: {vars(args)}")
+ logger.debug(f"FVP arguments: {fvp_args}")
+ return args, fvp_args
+
+
+async def start_fvp(args, config, extra_args):
+ fvp = runner.FVPRunner(logger)
+ try:
+ await fvp.start(config, extra_args, args.terminals)
+
+ if args.console:
+ fvp.add_line_callback(lambda line: logger.debug(f"FVP output: {line}"))
+ expected_terminal = config["consoles"]["default"]
+ if not expected_terminal:
+ logger.error("--console used but FVP_CONSOLE not set in machine configuration")
+ return 1
+ telnet = await fvp.create_telnet(expected_terminal)
+ await telnet.wait()
+ logger.debug(f"Telnet quit, cancelling tasks")
+ else:
+ fvp.add_line_callback(lambda line: print(line))
+
+ await fvp.run()
+ finally:
+ await fvp.stop()
+
+def runfvp(cli_args):
+ args, extra_args = parse_args(cli_args)
+ if args.config and pathlib.Path(args.config).exists():
+ config_file = args.config
+ else:
+ config_file = conffile.find(args.config)
+ logger.debug(f"Loading {config_file}")
+ config = conffile.load(config_file)
+
+ try:
+ # When we can assume Py3.7+, this can simply be asyncio.run()
+ loop = asyncio.get_event_loop()
+ return loop.run_until_complete(start_fvp(args, config, extra_args))
+ except asyncio.CancelledError:
+ # This means telnet exited, which isn't an error
+ return 0
+
+if __name__ == "__main__":
+ try:
+ # Set the process group so that it's possible to kill runfvp and
+ # everything it spawns easily.
+ # Ignore permission errors happening when spawned from an other process
+ # for example run from except
+ try:
+ os.setpgid(0, 0)
+ except PermissionError:
+ pass
+ if sys.stdin.isatty():
+ signal.signal(signal.SIGTTOU, signal.SIG_IGN)
+ os.tcsetpgrp(sys.stdin.fileno(), os.getpgrp())
+ sys.exit(runfvp(sys.argv[1:]))
+ except KeyboardInterrupt:
+ pass