| # Recipe creation tool - go support plugin |
| # |
| # The code is based on golang internals. See the afftected |
| # methods for further reference and information. |
| # |
| # Copyright (C) 2023 Weidmueller GmbH & Co KG |
| # Author: Lukas Funke <lukas.funke@weidmueller.com> |
| # |
| # SPDX-License-Identifier: GPL-2.0-only |
| # |
| |
| |
| from collections import namedtuple |
| from enum import Enum |
| from html.parser import HTMLParser |
| from recipetool.create import RecipeHandler, handle_license_vars |
| from recipetool.create import guess_license, tidy_licenses, fixup_license |
| from recipetool.create import determine_from_url |
| from urllib.error import URLError |
| |
| import bb.utils |
| import json |
| import logging |
| import os |
| import re |
| import subprocess |
| import sys |
| import shutil |
| import tempfile |
| import urllib.parse |
| import urllib.request |
| |
| |
| GoImport = namedtuple('GoImport', 'root vcs url suffix') |
| logger = logging.getLogger('recipetool') |
| CodeRepo = namedtuple( |
| 'CodeRepo', 'path codeRoot codeDir pathMajor pathPrefix pseudoMajor') |
| |
| tinfoil = None |
| |
| # Regular expression to parse pseudo semantic version |
| # see https://go.dev/ref/mod#pseudo-versions |
| re_pseudo_semver = re.compile( |
| r"^v[0-9]+\.(0\.0-|\d+\.\d+-([^+]*\.)?0\.)(?P<utc>\d{14})-(?P<commithash>[A-Za-z0-9]+)(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$") |
| # Regular expression to parse semantic version |
| re_semver = re.compile( |
| r"^v(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$") |
| |
| |
| def tinfoil_init(instance): |
| global tinfoil |
| tinfoil = instance |
| |
| |
| class GoRecipeHandler(RecipeHandler): |
| """Class to handle the go recipe creation""" |
| |
| @staticmethod |
| def __ensure_go(): |
| """Check if the 'go' command is available in the recipes""" |
| recipe = "go-native" |
| if not tinfoil.recipes_parsed: |
| tinfoil.parse_recipes() |
| try: |
| rd = tinfoil.parse_recipe(recipe) |
| except bb.providers.NoProvider: |
| bb.error( |
| "Nothing provides '%s' which is required for the build" % (recipe)) |
| bb.note( |
| "You will likely need to add a layer that provides '%s'" % (recipe)) |
| return None |
| |
| bindir = rd.getVar('STAGING_BINDIR_NATIVE') |
| gopath = os.path.join(bindir, 'go') |
| |
| if not os.path.exists(gopath): |
| tinfoil.build_targets(recipe, 'addto_recipe_sysroot') |
| |
| if not os.path.exists(gopath): |
| logger.error( |
| '%s required to process specified source, but %s did not seem to populate it' % 'go', recipe) |
| return None |
| |
| return bindir |
| |
| def __resolve_repository_static(self, modulepath): |
| """Resolve the repository in a static manner |
| |
| The method is based on the go implementation of |
| `repoRootFromVCSPaths` in |
| https://github.com/golang/go/blob/master/src/cmd/go/internal/vcs/vcs.go |
| """ |
| |
| url = urllib.parse.urlparse("https://" + modulepath) |
| req = urllib.request.Request(url.geturl()) |
| |
| try: |
| resp = urllib.request.urlopen(req) |
| # Some modulepath are just redirects to github (or some other vcs |
| # hoster). Therefore, we check if this modulepath redirects to |
| # somewhere else |
| if resp.geturl() != url.geturl(): |
| bb.debug(1, "%s is redirectred to %s" % |
| (url.geturl(), resp.geturl())) |
| url = urllib.parse.urlparse(resp.geturl()) |
| modulepath = url.netloc + url.path |
| |
| except URLError as url_err: |
| # This is probably because the module path |
| # contains the subdir and major path. Thus, |
| # we ignore this error for now |
| logger.debug( |
| 1, "Failed to fetch page from [%s]: %s" % (url, str(url_err))) |
| |
| host, _, _ = modulepath.partition('/') |
| |
| class vcs(Enum): |
| pathprefix = "pathprefix" |
| regexp = "regexp" |
| type = "type" |
| repo = "repo" |
| check = "check" |
| schemelessRepo = "schemelessRepo" |
| |
| # GitHub |
| vcsGitHub = {} |
| vcsGitHub[vcs.pathprefix] = "github.com" |
| vcsGitHub[vcs.regexp] = re.compile( |
| r'^(?P<root>github\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/(?P<suffix>[A-Za-z0-9_.\-]+))*$') |
| vcsGitHub[vcs.type] = "git" |
| vcsGitHub[vcs.repo] = "https://\\g<root>" |
| |
| # Bitbucket |
| vcsBitbucket = {} |
| vcsBitbucket[vcs.pathprefix] = "bitbucket.org" |
| vcsBitbucket[vcs.regexp] = re.compile( |
| r'^(?P<root>bitbucket\.org/(?P<bitname>[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/(?P<suffix>[A-Za-z0-9_.\-]+))*$') |
| vcsBitbucket[vcs.type] = "git" |
| vcsBitbucket[vcs.repo] = "https://\\g<root>" |
| |
| # IBM DevOps Services (JazzHub) |
| vcsIBMDevOps = {} |
| vcsIBMDevOps[vcs.pathprefix] = "hub.jazz.net/git" |
| vcsIBMDevOps[vcs.regexp] = re.compile( |
| r'^(?P<root>hub\.jazz\.net/git/[a-z0-9]+/[A-Za-z0-9_.\-]+)(/(?P<suffix>[A-Za-z0-9_.\-]+))*$') |
| vcsIBMDevOps[vcs.type] = "git" |
| vcsIBMDevOps[vcs.repo] = "https://\\g<root>" |
| |
| # Git at Apache |
| vcsApacheGit = {} |
| vcsApacheGit[vcs.pathprefix] = "git.apache.org" |
| vcsApacheGit[vcs.regexp] = re.compile( |
| r'^(?P<root>git\.apache\.org/[a-z0-9_.\-]+\.git)(/(?P<suffix>[A-Za-z0-9_.\-]+))*$') |
| vcsApacheGit[vcs.type] = "git" |
| vcsApacheGit[vcs.repo] = "https://\\g<root>" |
| |
| # Git at OpenStack |
| vcsOpenStackGit = {} |
| vcsOpenStackGit[vcs.pathprefix] = "git.openstack.org" |
| vcsOpenStackGit[vcs.regexp] = re.compile( |
| r'^(?P<root>git\.openstack\.org/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(\.git)?(/(?P<suffix>[A-Za-z0-9_.\-]+))*$') |
| vcsOpenStackGit[vcs.type] = "git" |
| vcsOpenStackGit[vcs.repo] = "https://\\g<root>" |
| |
| # chiselapp.com for fossil |
| vcsChiselapp = {} |
| vcsChiselapp[vcs.pathprefix] = "chiselapp.com" |
| vcsChiselapp[vcs.regexp] = re.compile( |
| r'^(?P<root>chiselapp\.com/user/[A-Za-z0-9]+/repository/[A-Za-z0-9_.\-]+)$') |
| vcsChiselapp[vcs.type] = "fossil" |
| vcsChiselapp[vcs.repo] = "https://\\g<root>" |
| |
| # General syntax for any server. |
| # Must be last. |
| vcsGeneralServer = {} |
| vcsGeneralServer[vcs.regexp] = re.compile( |
| "(?P<root>(?P<repo>([a-z0-9.\\-]+\\.)+[a-z0-9.\\-]+(:[0-9]+)?(/~?[A-Za-z0-9_.\\-]+)+?)\\.(?P<vcs>bzr|fossil|git|hg|svn))(/~?(?P<suffix>[A-Za-z0-9_.\\-]+))*$") |
| vcsGeneralServer[vcs.schemelessRepo] = True |
| |
| vcsPaths = [vcsGitHub, vcsBitbucket, vcsIBMDevOps, |
| vcsApacheGit, vcsOpenStackGit, vcsChiselapp, |
| vcsGeneralServer] |
| |
| if modulepath.startswith("example.net") or modulepath == "rsc.io": |
| logger.warning("Suspicious module path %s" % modulepath) |
| return None |
| if modulepath.startswith("http:") or modulepath.startswith("https:"): |
| logger.warning("Import path should not start with %s %s" % |
| ("http", "https")) |
| return None |
| |
| rootpath = None |
| vcstype = None |
| repourl = None |
| suffix = None |
| |
| for srv in vcsPaths: |
| m = srv[vcs.regexp].match(modulepath) |
| if vcs.pathprefix in srv: |
| if host == srv[vcs.pathprefix]: |
| rootpath = m.group('root') |
| vcstype = srv[vcs.type] |
| repourl = m.expand(srv[vcs.repo]) |
| suffix = m.group('suffix') |
| break |
| elif m and srv[vcs.schemelessRepo]: |
| rootpath = m.group('root') |
| vcstype = m[vcs.type] |
| repourl = m[vcs.repo] |
| suffix = m.group('suffix') |
| break |
| |
| return GoImport(rootpath, vcstype, repourl, suffix) |
| |
| def __resolve_repository_dynamic(self, modulepath): |
| """Resolve the repository root in a dynamic manner. |
| |
| The method is based on the go implementation of |
| `repoRootForImportDynamic` in |
| https://github.com/golang/go/blob/master/src/cmd/go/internal/vcs/vcs.go |
| """ |
| url = urllib.parse.urlparse("https://" + modulepath) |
| |
| class GoImportHTMLParser(HTMLParser): |
| |
| def __init__(self): |
| super().__init__() |
| self.__srv = [] |
| |
| def handle_starttag(self, tag, attrs): |
| if tag == 'meta' and list( |
| filter(lambda a: (a[0] == 'name' and a[1] == 'go-import'), attrs)): |
| content = list( |
| filter(lambda a: (a[0] == 'content'), attrs)) |
| if content: |
| self.__srv = content[0][1].split() |
| |
| @property |
| def import_prefix(self): |
| return self.__srv[0] if len(self.__srv) else None |
| |
| @property |
| def vcs(self): |
| return self.__srv[1] if len(self.__srv) else None |
| |
| @property |
| def repourl(self): |
| return self.__srv[2] if len(self.__srv) else None |
| |
| url = url.geturl() + "?go-get=1" |
| req = urllib.request.Request(url) |
| |
| try: |
| resp = urllib.request.urlopen(req) |
| |
| except URLError as url_err: |
| logger.warning( |
| "Failed to fetch page from [%s]: %s", url, str(url_err)) |
| return None |
| |
| parser = GoImportHTMLParser() |
| parser.feed(resp.read().decode('utf-8')) |
| parser.close() |
| |
| return GoImport(parser.import_prefix, parser.vcs, parser.repourl, None) |
| |
| def __resolve_from_golang_proxy(self, modulepath, version): |
| """ |
| Resolves repository data from golang proxy |
| """ |
| url = urllib.parse.urlparse("https://proxy.golang.org/" |
| + modulepath |
| + "/@v/" |
| + version |
| + ".info") |
| |
| # Transform url to lower case, golang proxy doesn't like mixed case |
| req = urllib.request.Request(url.geturl().lower()) |
| |
| try: |
| resp = urllib.request.urlopen(req) |
| except URLError as url_err: |
| logger.warning( |
| "Failed to fetch page from [%s]: %s", url, str(url_err)) |
| return None |
| |
| golang_proxy_res = resp.read().decode('utf-8') |
| modinfo = json.loads(golang_proxy_res) |
| |
| if modinfo and 'Origin' in modinfo: |
| origin = modinfo['Origin'] |
| _root_url = urllib.parse.urlparse(origin['URL']) |
| |
| # We normalize the repo URL since we don't want the scheme in it |
| _subdir = origin['Subdir'] if 'Subdir' in origin else None |
| _root, _, _ = self.__split_path_version(modulepath) |
| if _subdir: |
| _root = _root[:-len(_subdir)].strip('/') |
| |
| _commit = origin['Hash'] |
| _vcs = origin['VCS'] |
| return (GoImport(_root, _vcs, _root_url.geturl(), None), _commit) |
| |
| return None |
| |
| def __resolve_repository(self, modulepath): |
| """ |
| Resolves src uri from go module-path |
| """ |
| repodata = self.__resolve_repository_static(modulepath) |
| if not repodata or not repodata.url: |
| repodata = self.__resolve_repository_dynamic(modulepath) |
| if not repodata or not repodata.url: |
| logger.error( |
| "Could not resolve repository for module path '%s'" % modulepath) |
| # There is no way to recover from this |
| sys.exit(14) |
| if repodata: |
| logger.debug(1, "Resolved download path for import '%s' => %s" % ( |
| modulepath, repodata.url)) |
| return repodata |
| |
| def __split_path_version(self, path): |
| i = len(path) |
| dot = False |
| for j in range(i, 0, -1): |
| if path[j - 1] < '0' or path[j - 1] > '9': |
| break |
| if path[j - 1] == '.': |
| dot = True |
| break |
| i = j - 1 |
| |
| if i <= 1 or i == len( |
| path) or path[i - 1] != 'v' or path[i - 2] != '/': |
| return path, "", True |
| |
| prefix, pathMajor = path[:i - 2], path[i - 2:] |
| if dot or len( |
| pathMajor) <= 2 or pathMajor[2] == '0' or pathMajor == "/v1": |
| return path, "", False |
| |
| return prefix, pathMajor, True |
| |
| def __get_path_major(self, pathMajor): |
| if not pathMajor: |
| return "" |
| |
| if pathMajor[0] != '/' and pathMajor[0] != '.': |
| logger.error( |
| "pathMajor suffix %s passed to PathMajorPrefix lacks separator", pathMajor) |
| |
| if pathMajor.startswith(".v") and pathMajor.endswith("-unstable"): |
| pathMajor = pathMajor[:len("-unstable") - 2] |
| |
| return pathMajor[1:] |
| |
| def __build_coderepo(self, repo, path): |
| codedir = "" |
| pathprefix, pathMajor, _ = self.__split_path_version(path) |
| if repo.root == path: |
| pathprefix = path |
| elif path.startswith(repo.root): |
| codedir = pathprefix[len(repo.root):].strip('/') |
| |
| pseudoMajor = self.__get_path_major(pathMajor) |
| |
| logger.debug("root='%s', codedir='%s', prefix='%s', pathMajor='%s', pseudoMajor='%s'", |
| repo.root, codedir, pathprefix, pathMajor, pseudoMajor) |
| |
| return CodeRepo(path, repo.root, codedir, |
| pathMajor, pathprefix, pseudoMajor) |
| |
| def __resolve_version(self, repo, path, version): |
| hash = None |
| coderoot = self.__build_coderepo(repo, path) |
| |
| def vcs_fetch_all(): |
| tmpdir = tempfile.mkdtemp() |
| clone_cmd = "%s clone --bare %s %s" % ('git', repo.url, tmpdir) |
| bb.process.run(clone_cmd) |
| log_cmd = "git log --all --pretty='%H %d' --decorate=short" |
| output, _ = bb.process.run( |
| log_cmd, shell=True, stderr=subprocess.PIPE, cwd=tmpdir) |
| bb.utils.prunedir(tmpdir) |
| return output.strip().split('\n') |
| |
| def vcs_fetch_remote(tag): |
| # add * to grab ^{} |
| refs = {} |
| ls_remote_cmd = "git ls-remote -q --tags {} {}*".format( |
| repo.url, tag) |
| output, _ = bb.process.run(ls_remote_cmd) |
| output = output.strip().split('\n') |
| for line in output: |
| f = line.split(maxsplit=1) |
| if len(f) != 2: |
| continue |
| |
| for prefix in ["HEAD", "refs/heads/", "refs/tags/"]: |
| if f[1].startswith(prefix): |
| refs[f[1][len(prefix):]] = f[0] |
| |
| for key, hash in refs.items(): |
| if key.endswith(r"^{}"): |
| refs[key.strip(r"^{}")] = hash |
| |
| return refs[tag] |
| |
| m_pseudo_semver = re_pseudo_semver.match(version) |
| |
| if m_pseudo_semver: |
| remote_refs = vcs_fetch_all() |
| short_commit = m_pseudo_semver.group('commithash') |
| for l in remote_refs: |
| r = l.split(maxsplit=1) |
| sha1 = r[0] if len(r) else None |
| if not sha1: |
| logger.error( |
| "Ups: could not resolve abbref commit for %s" % short_commit) |
| |
| elif sha1.startswith(short_commit): |
| hash = sha1 |
| break |
| else: |
| m_semver = re_semver.match(version) |
| if m_semver: |
| |
| def get_sha1_remote(re): |
| rsha1 = None |
| for line in remote_refs: |
| # Split lines of the following format: |
| # 22e90d9b964610628c10f673ca5f85b8c2a2ca9a (tag: sometag) |
| lineparts = line.split(maxsplit=1) |
| sha1 = lineparts[0] if len(lineparts) else None |
| refstring = lineparts[1] if len( |
| lineparts) == 2 else None |
| if refstring: |
| # Normalize tag string and split in case of multiple |
| # regs e.g. (tag: speech/v1.10.0, tag: orchestration/v1.5.0 ...) |
| refs = refstring.strip('(), ').split(',') |
| for ref in refs: |
| if re.match(ref.strip()): |
| rsha1 = sha1 |
| return rsha1 |
| |
| semver = "v" + m_semver.group('major') + "."\ |
| + m_semver.group('minor') + "."\ |
| + m_semver.group('patch') \ |
| + (("-" + m_semver.group('prerelease')) |
| if m_semver.group('prerelease') else "") |
| |
| tag = os.path.join( |
| coderoot.codeDir, semver) if coderoot.codeDir else semver |
| |
| # probe tag using 'ls-remote', which is faster than fetching |
| # complete history |
| hash = vcs_fetch_remote(tag) |
| if not hash: |
| # backup: fetch complete history |
| remote_refs = vcs_fetch_all() |
| hash = get_sha1_remote( |
| re.compile(fr"(tag:|HEAD ->) ({tag})")) |
| |
| logger.debug( |
| "Resolving commit for tag '%s' -> '%s'", tag, hash) |
| return hash |
| |
| def __generate_srcuri_inline_fcn(self, path, version, replaces=None): |
| """Generate SRC_URI functions for go imports""" |
| |
| logger.info("Resolving repository for module %s", path) |
| # First try to resolve repo and commit from golang proxy |
| # Most info is already there and we don't have to go through the |
| # repository or even perform the version resolve magic |
| golang_proxy_info = self.__resolve_from_golang_proxy(path, version) |
| if golang_proxy_info: |
| repo = golang_proxy_info[0] |
| commit = golang_proxy_info[1] |
| else: |
| # Fallback |
| # Resolve repository by 'hand' |
| repo = self.__resolve_repository(path) |
| commit = self.__resolve_version(repo, path, version) |
| |
| url = urllib.parse.urlparse(repo.url) |
| repo_url = url.netloc + url.path |
| |
| coderoot = self.__build_coderepo(repo, path) |
| |
| inline_fcn = "${@go_src_uri(" |
| inline_fcn += f"'{repo_url}','{version}'" |
| if repo_url != path: |
| inline_fcn += f",path='{path}'" |
| if coderoot.codeDir: |
| inline_fcn += f",subdir='{coderoot.codeDir}'" |
| if repo.vcs != 'git': |
| inline_fcn += f",vcs='{repo.vcs}'" |
| if replaces: |
| inline_fcn += f",replaces='{replaces}'" |
| if coderoot.pathMajor: |
| inline_fcn += f",pathmajor='{coderoot.pathMajor}'" |
| inline_fcn += ")}" |
| |
| return inline_fcn, commit |
| |
| def __go_handle_dependencies(self, go_mod, srctree, localfilesdir, extravalues, d): |
| |
| import re |
| src_uris = [] |
| src_revs = [] |
| |
| def generate_src_rev(path, version, commithash): |
| src_rev = f"# {path}@{version} => {commithash}\n" |
| # Ups...maybe someone manipulated the source repository and the |
| # version or commit could not be resolved. This is a sign of |
| # a) the supply chain was manipulated (bad) |
| # b) the implementation for the version resolving didn't work |
| # anymore (less bad) |
| if not commithash: |
| src_rev += f"#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n" |
| src_rev += f"#!!! Could not resolve version !!!\n" |
| src_rev += f"#!!! Possible supply chain attack !!!\n" |
| src_rev += f"#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n" |
| src_rev += f"SRCREV_{path.replace('/', '.')} = \"{commithash}\"" |
| |
| return src_rev |
| |
| # we first go over replacement list, because we are essentialy |
| # interested only in the replaced path |
| if go_mod['Replace']: |
| for replacement in go_mod['Replace']: |
| oldpath = replacement['Old']['Path'] |
| path = replacement['New']['Path'] |
| version = '' |
| if 'Version' in replacement['New']: |
| version = replacement['New']['Version'] |
| |
| if os.path.exists(os.path.join(srctree, path)): |
| # the module refers to the local path, remove it from requirement list |
| # because it's a local module |
| go_mod['Require'][:] = [v for v in go_mod['Require'] if v.get('Path') != oldpath] |
| else: |
| # Replace the path and the version, so we don't iterate replacement list anymore |
| for require in go_mod['Require']: |
| if require['Path'] == oldpath: |
| require.update({'Path': path, 'Version': version}) |
| break |
| |
| for require in go_mod['Require']: |
| path = require['Path'] |
| version = require['Version'] |
| |
| inline_fcn, commithash = self.__generate_srcuri_inline_fcn( |
| path, version) |
| src_uris.append(inline_fcn) |
| src_revs.append(generate_src_rev(path, version, commithash)) |
| |
| # strip version part from module URL /vXX |
| baseurl = re.sub(r'/v(\d+)$', '', go_mod['Module']['Path']) |
| pn, _ = determine_from_url(baseurl) |
| go_mods_basename = "%s-modules.inc" % pn |
| |
| go_mods_filename = os.path.join(localfilesdir, go_mods_basename) |
| with open(go_mods_filename, "w") as f: |
| # We introduce this indirection to make the tests a little easier |
| f.write("SRC_URI += \"${GO_DEPENDENCIES_SRC_URI}\"\n") |
| f.write("GO_DEPENDENCIES_SRC_URI = \"\\\n") |
| for uri in src_uris: |
| f.write(" " + uri + " \\\n") |
| f.write("\"\n\n") |
| for rev in src_revs: |
| f.write(rev + "\n") |
| |
| extravalues['extrafiles'][go_mods_basename] = go_mods_filename |
| |
| def __go_run_cmd(self, cmd, cwd, d): |
| return bb.process.run(cmd, env=dict(os.environ, PATH=d.getVar('PATH')), |
| shell=True, cwd=cwd) |
| |
| def __go_native_version(self, d): |
| stdout, _ = self.__go_run_cmd("go version", None, d) |
| m = re.match(r".*\sgo((\d+).(\d+).(\d+))\s([\w\/]*)", stdout) |
| major = int(m.group(2)) |
| minor = int(m.group(3)) |
| patch = int(m.group(4)) |
| |
| return major, minor, patch |
| |
| def __go_mod_patch(self, srctree, localfilesdir, extravalues, d): |
| |
| patchfilename = "go.mod.patch" |
| go_native_version_major, go_native_version_minor, _ = self.__go_native_version( |
| d) |
| self.__go_run_cmd("go mod tidy -go=%d.%d" % |
| (go_native_version_major, go_native_version_minor), srctree, d) |
| stdout, _ = self.__go_run_cmd("go mod edit -json", srctree, d) |
| |
| # Create patch in order to upgrade go version |
| self.__go_run_cmd("git diff go.mod > %s" % (patchfilename), srctree, d) |
| # Restore original state |
| self.__go_run_cmd("git checkout HEAD go.mod go.sum", srctree, d) |
| |
| go_mod = json.loads(stdout) |
| tmpfile = os.path.join(localfilesdir, patchfilename) |
| shutil.move(os.path.join(srctree, patchfilename), tmpfile) |
| |
| extravalues['extrafiles'][patchfilename] = tmpfile |
| |
| return go_mod, patchfilename |
| |
| def __go_mod_vendor(self, go_mod, srctree, localfilesdir, extravalues, d): |
| # Perform vendoring to retrieve the correct modules.txt |
| tmp_vendor_dir = tempfile.mkdtemp() |
| |
| # -v causes to go to print modules.txt to stderr |
| _, stderr = self.__go_run_cmd( |
| "go mod vendor -v -o %s" % (tmp_vendor_dir), srctree, d) |
| |
| modules_txt_basename = "modules.txt" |
| modules_txt_filename = os.path.join(localfilesdir, modules_txt_basename) |
| with open(modules_txt_filename, "w") as f: |
| f.write(stderr) |
| |
| extravalues['extrafiles'][modules_txt_basename] = modules_txt_filename |
| |
| licenses = [] |
| lic_files_chksum = [] |
| licvalues = guess_license(tmp_vendor_dir, d) |
| shutil.rmtree(tmp_vendor_dir) |
| |
| if licvalues: |
| for licvalue in licvalues: |
| license = licvalue[0] |
| lics = tidy_licenses(fixup_license(license)) |
| lics = [lic for lic in lics if lic not in licenses] |
| if len(lics): |
| licenses.extend(lics) |
| lic_files_chksum.append( |
| 'file://src/${GO_IMPORT}/vendor/%s;md5=%s' % (licvalue[1], licvalue[2])) |
| |
| # strip version part from module URL /vXX |
| baseurl = re.sub(r'/v(\d+)$', '', go_mod['Module']['Path']) |
| pn, _ = determine_from_url(baseurl) |
| licenses_basename = "%s-licenses.inc" % pn |
| |
| licenses_filename = os.path.join(localfilesdir, licenses_basename) |
| with open(licenses_filename, "w") as f: |
| f.write("GO_MOD_LICENSES = \"%s\"\n\n" % |
| ' & '.join(sorted(licenses, key=str.casefold))) |
| # We introduce this indirection to make the tests a little easier |
| f.write("LIC_FILES_CHKSUM += \"${VENDORED_LIC_FILES_CHKSUM}\"\n") |
| f.write("VENDORED_LIC_FILES_CHKSUM = \"\\\n") |
| for lic in lic_files_chksum: |
| f.write(" " + lic + " \\\n") |
| f.write("\"\n") |
| |
| extravalues['extrafiles'][licenses_basename] = licenses_filename |
| |
| def process(self, srctree, classes, lines_before, |
| lines_after, handled, extravalues): |
| |
| if 'buildsystem' in handled: |
| return False |
| |
| files = RecipeHandler.checkfiles(srctree, ['go.mod']) |
| if not files: |
| return False |
| |
| d = bb.data.createCopy(tinfoil.config_data) |
| go_bindir = self.__ensure_go() |
| if not go_bindir: |
| sys.exit(14) |
| |
| d.prependVar('PATH', '%s:' % go_bindir) |
| handled.append('buildsystem') |
| classes.append("go-vendor") |
| |
| stdout, _ = self.__go_run_cmd("go mod edit -json", srctree, d) |
| |
| go_mod = json.loads(stdout) |
| go_import = go_mod['Module']['Path'] |
| go_version_match = re.match("([0-9]+).([0-9]+)", go_mod['Go']) |
| go_version_major = int(go_version_match.group(1)) |
| go_version_minor = int(go_version_match.group(2)) |
| src_uris = [] |
| |
| localfilesdir = tempfile.mkdtemp(prefix='recipetool-go-') |
| extravalues.setdefault('extrafiles', {}) |
| |
| # Use an explicit name determined from the module name because it |
| # might differ from the actual URL for replaced modules |
| # strip version part from module URL /vXX |
| baseurl = re.sub(r'/v(\d+)$', '', go_mod['Module']['Path']) |
| pn, _ = determine_from_url(baseurl) |
| |
| # go.mod files with version < 1.17 may not include all indirect |
| # dependencies. Thus, we have to upgrade the go version. |
| if go_version_major == 1 and go_version_minor < 17: |
| logger.warning( |
| "go.mod files generated by Go < 1.17 might have incomplete indirect dependencies.") |
| go_mod, patchfilename = self.__go_mod_patch(srctree, localfilesdir, |
| extravalues, d) |
| src_uris.append( |
| "file://%s;patchdir=src/${GO_IMPORT}" % (patchfilename)) |
| |
| # Check whether the module is vendored. If so, we have nothing to do. |
| # Otherwise we gather all dependencies and add them to the recipe |
| if not os.path.exists(os.path.join(srctree, "vendor")): |
| |
| # Write additional $BPN-modules.inc file |
| self.__go_mod_vendor(go_mod, srctree, localfilesdir, extravalues, d) |
| lines_before.append("LICENSE += \" & ${GO_MOD_LICENSES}\"") |
| lines_before.append("require %s-licenses.inc" % (pn)) |
| |
| self.__rewrite_src_uri(lines_before, ["file://modules.txt"]) |
| |
| self.__go_handle_dependencies(go_mod, srctree, localfilesdir, extravalues, d) |
| lines_before.append("require %s-modules.inc" % (pn)) |
| |
| # Do generic license handling |
| handle_license_vars(srctree, lines_before, handled, extravalues, d) |
| self.__rewrite_lic_uri(lines_before) |
| |
| lines_before.append("GO_IMPORT = \"{}\"".format(baseurl)) |
| lines_before.append("SRCREV_FORMAT = \"${BPN}\"") |
| |
| def __update_lines_before(self, updated, newlines, lines_before): |
| if updated: |
| del lines_before[:] |
| for line in newlines: |
| # Hack to avoid newlines that edit_metadata inserts |
| if line.endswith('\n'): |
| line = line[:-1] |
| lines_before.append(line) |
| return updated |
| |
| def __rewrite_lic_uri(self, lines_before): |
| |
| def varfunc(varname, origvalue, op, newlines): |
| if varname == 'LIC_FILES_CHKSUM': |
| new_licenses = [] |
| licenses = origvalue.split('\\') |
| for license in licenses: |
| if not license: |
| logger.warning("No license file was detected for the main module!") |
| # the license list of the main recipe must be empty |
| # this can happen for example in case of CLOSED license |
| # Fall through to complete recipe generation |
| continue |
| license = license.strip() |
| uri, chksum = license.split(';', 1) |
| url = urllib.parse.urlparse(uri) |
| new_uri = os.path.join( |
| url.scheme + "://", "src", "${GO_IMPORT}", url.netloc + url.path) + ";" + chksum |
| new_licenses.append(new_uri) |
| |
| return new_licenses, None, -1, True |
| return origvalue, None, 0, True |
| |
| updated, newlines = bb.utils.edit_metadata( |
| lines_before, ['LIC_FILES_CHKSUM'], varfunc) |
| return self.__update_lines_before(updated, newlines, lines_before) |
| |
| def __rewrite_src_uri(self, lines_before, additional_uris = []): |
| |
| def varfunc(varname, origvalue, op, newlines): |
| if varname == 'SRC_URI': |
| src_uri = ["git://${GO_IMPORT};destsuffix=git/src/${GO_IMPORT};nobranch=1;name=${BPN};protocol=https"] |
| src_uri.extend(additional_uris) |
| return src_uri, None, -1, True |
| return origvalue, None, 0, True |
| |
| updated, newlines = bb.utils.edit_metadata(lines_before, ['SRC_URI'], varfunc) |
| return self.__update_lines_before(updated, newlines, lines_before) |
| |
| |
| def register_recipe_handlers(handlers): |
| handlers.append((GoRecipeHandler(), 60)) |