Yocto 2.3
Move OpenBMC to Yocto 2.3(pyro).
Tested: Built and verified Witherspoon and Palmetto images
Change-Id: I50744030e771f4850afc2a93a10d3507e76d36bc
Signed-off-by: Brad Bishop <bradleyb@fuzziesquirrel.com>
Resolves: openbmc/openbmc#2461
diff --git a/import-layers/yocto-poky/meta/classes/license.bbclass b/import-layers/yocto-poky/meta/classes/license.bbclass
index 721343d..b1fffe7 100644
--- a/import-layers/yocto-poky/meta/classes/license.bbclass
+++ b/import-layers/yocto-poky/meta/classes/license.bbclass
@@ -37,13 +37,13 @@
import oe.packagedata
from oe.rootfs import image_list_installed_packages
- build_images_from_feeds = d.getVar('BUILD_IMAGES_FROM_FEEDS', True)
+ build_images_from_feeds = d.getVar('BUILD_IMAGES_FROM_FEEDS')
if build_images_from_feeds == "1":
return 0
pkg_dic = {}
for pkg in sorted(image_list_installed_packages(d)):
- pkg_info = os.path.join(d.getVar('PKGDATA_DIR', True),
+ pkg_info = os.path.join(d.getVar('PKGDATA_DIR'),
'runtime-reverse', pkg)
pkg_name = os.path.basename(os.readlink(pkg_info))
@@ -52,15 +52,15 @@
pkg_lic_name = "LICENSE_" + pkg_name
pkg_dic[pkg_name]["LICENSE"] = pkg_dic[pkg_name][pkg_lic_name]
- rootfs_license_manifest = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
- d.getVar('IMAGE_NAME', True), 'license.manifest')
+ rootfs_license_manifest = os.path.join(d.getVar('LICENSE_DIRECTORY'),
+ d.getVar('IMAGE_NAME'), 'license.manifest')
write_license_files(d, rootfs_license_manifest, pkg_dic)
}
def write_license_files(d, license_manifest, pkg_dic):
import re
- bad_licenses = (d.getVar("INCOMPATIBLE_LICENSE", True) or "").split()
+ bad_licenses = (d.getVar("INCOMPATIBLE_LICENSE") or "").split()
bad_licenses = map(lambda l: canonical_license(d, l), bad_licenses)
bad_licenses = expand_wildcard_licenses(d, bad_licenses)
@@ -72,7 +72,7 @@
oe.license.manifest_licenses(pkg_dic[pkg]["LICENSE"],
bad_licenses, canonical_license, d)
except oe.license.LicenseError as exc:
- bb.fatal('%s: %s' % (d.getVar('P', True), exc))
+ bb.fatal('%s: %s' % (d.getVar('P'), exc))
else:
pkg_dic[pkg]["LICENSES"] = re.sub('[|&()*]', ' ', pkg_dic[pkg]["LICENSE"])
pkg_dic[pkg]["LICENSES"] = re.sub(' *', ' ', pkg_dic[pkg]["LICENSES"])
@@ -98,7 +98,7 @@
license_file.write("FILES: %s\n\n" % pkg_dic[pkg]["FILES"])
for lic in pkg_dic[pkg]["LICENSES"]:
- lic_file = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
+ lic_file = os.path.join(d.getVar('LICENSE_DIRECTORY'),
pkg_dic[pkg]["PN"], "generic_%s" %
re.sub('\+', '', lic))
# add explicity avoid of CLOSED license because isn't generic
@@ -114,10 +114,10 @@
# - Just copy the manifest
# - Copy the manifest and the license directories
# With both options set we see a .5 M increase in core-image-minimal
- copy_lic_manifest = d.getVar('COPY_LIC_MANIFEST', True)
- copy_lic_dirs = d.getVar('COPY_LIC_DIRS', True)
+ copy_lic_manifest = d.getVar('COPY_LIC_MANIFEST')
+ copy_lic_dirs = d.getVar('COPY_LIC_DIRS')
if copy_lic_manifest == "1":
- rootfs_license_dir = os.path.join(d.getVar('IMAGE_ROOTFS', 'True'),
+ rootfs_license_dir = os.path.join(d.getVar('IMAGE_ROOTFS'),
'usr', 'share', 'common-licenses')
bb.utils.mkdirhier(rootfs_license_dir)
rootfs_license_manifest = os.path.join(rootfs_license_dir,
@@ -129,8 +129,12 @@
for pkg in sorted(pkg_dic):
pkg_rootfs_license_dir = os.path.join(rootfs_license_dir, pkg)
bb.utils.mkdirhier(pkg_rootfs_license_dir)
- pkg_license_dir = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
+ pkg_license_dir = os.path.join(d.getVar('LICENSE_DIRECTORY'),
pkg_dic[pkg]["PN"])
+
+ pkg_manifest_licenses = [canonical_license(d, lic) \
+ for lic in pkg_dic[pkg]["LICENSES"]]
+
licenses = os.listdir(pkg_license_dir)
for lic in licenses:
rootfs_license = os.path.join(rootfs_license_dir, lic)
@@ -138,9 +142,18 @@
pkg_rootfs_license = os.path.join(pkg_rootfs_license_dir, lic)
if re.match("^generic_.*$", lic):
- generic_lic = re.search("^generic_(.*)$", lic).group(1)
- if oe.license.license_ok(canonical_license(d,
- generic_lic), bad_licenses) == False:
+ generic_lic = canonical_license(d,
+ re.search("^generic_(.*)$", lic).group(1))
+
+ # Do not copy generic license into package if isn't
+ # declared into LICENSES of the package.
+ if not re.sub('\+$', '', generic_lic) in \
+ [re.sub('\+', '', lic) for lic in \
+ pkg_manifest_licenses]:
+ continue
+
+ if oe.license.license_ok(generic_lic,
+ bad_licenses) == False:
continue
if not os.path.exists(rootfs_license):
@@ -166,7 +179,7 @@
dep_dic = {}
man_dic = {}
- lic_dir = d.getVar("LICENSE_DIRECTORY", True)
+ lic_dir = d.getVar("LICENSE_DIRECTORY")
dep_dic = get_deployed_dependencies(d)
for dep in dep_dic.keys():
@@ -181,8 +194,8 @@
key,val = line.split(": ", 1)
man_dic[dep][key] = val[:-1]
- lic_manifest_dir = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
- d.getVar('IMAGE_NAME', True))
+ lic_manifest_dir = os.path.join(d.getVar('LICENSE_DIRECTORY'),
+ d.getVar('IMAGE_NAME'))
bb.utils.mkdirhier(lic_manifest_dir)
image_license_manifest = os.path.join(lic_manifest_dir, 'image_license.manifest')
write_license_files(d, image_license_manifest, man_dic)
@@ -202,7 +215,7 @@
depends = list(set([dep[0] for dep
in list(taskdata.values())
if not dep[0].endswith("-native")]))
- extra_depends = d.getVar("EXTRA_IMAGEDEPENDS", True)
+ extra_depends = d.getVar("EXTRA_IMAGEDEPENDS")
boot_depends = get_boot_dependencies(d)
depends.extend(extra_depends.split())
depends.extend(boot_depends)
@@ -212,13 +225,13 @@
# the SSTATE_MANIFESTS for "deploy" task.
# The manifest file name contains the arch. Because we are not running
# in the recipe context it is necessary to check every arch used.
- sstate_manifest_dir = d.getVar("SSTATE_MANIFESTS", True)
- sstate_archs = d.getVar("SSTATE_ARCHS", True)
- extra_archs = d.getVar("PACKAGE_EXTRA_ARCHS", True)
+ sstate_manifest_dir = d.getVar("SSTATE_MANIFESTS")
+ sstate_archs = d.getVar("SSTATE_ARCHS")
+ extra_archs = d.getVar("PACKAGE_EXTRA_ARCHS")
archs = list(set(("%s %s" % (sstate_archs, extra_archs)).split()))
for dep in depends:
# Some recipes have an arch on their own, so we try that first.
- special_arch = d.getVar("PACKAGE_ARCH_pn-%s" % dep, True)
+ special_arch = d.getVar("PACKAGE_ARCH_pn-%s" % dep)
if special_arch:
sstate_manifest_file = os.path.join(sstate_manifest_dir,
"manifest-%s-%s.deploy" % (special_arch, dep))
@@ -249,12 +262,12 @@
for task in boot_tasks:
boot_depends_string = "%s %s" % (boot_depends_string,
- d.getVarFlag(task, "depends", True) or "")
+ d.getVarFlag(task, "depends") or "")
boot_depends = [dep.split(":")[0] for dep
in boot_depends_string.split()
if not dep.split(":")[0].endswith("-native")]
for dep in boot_depends:
- info_file = os.path.join(d.getVar("LICENSE_DIRECTORY", True),
+ info_file = os.path.join(d.getVar("LICENSE_DIRECTORY"),
dep, "recipeinfo")
# If the recipe and dependency name is the same
if os.path.exists(info_file):
@@ -265,7 +278,7 @@
# The fifth field contains what the task provides
if dep in taskdep[4]:
info_file = os.path.join(
- d.getVar("LICENSE_DIRECTORY", True),
+ d.getVar("LICENSE_DIRECTORY"),
taskdep[0], "recipeinfo")
if os.path.exists(info_file):
depends.append(taskdep[0])
@@ -295,7 +308,7 @@
lic_files_paths = find_license_files(d)
# The base directory we wrangle licenses to
- destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
+ destdir = os.path.join(d.getVar('LICSSTATEDIR'), d.getVar('PN'))
copy_license_files(lic_files_paths, destdir)
info = get_recipe_info(d)
with open(os.path.join(destdir, "recipeinfo"), "w") as f:
@@ -306,11 +319,11 @@
# it would be better to copy them in do_install_append, but find_license_filesa is python
python perform_packagecopy_prepend () {
enabled = oe.data.typed_value('LICENSE_CREATE_PACKAGE', d)
- if d.getVar('CLASSOVERRIDE', True) == 'class-target' and enabled:
+ if d.getVar('CLASSOVERRIDE') == 'class-target' and enabled:
lic_files_paths = find_license_files(d)
# LICENSE_FILES_DIRECTORY starts with '/' so os.path.join cannot be used to join D and LICENSE_FILES_DIRECTORY
- destdir = d.getVar('D', True) + os.path.join(d.getVar('LICENSE_FILES_DIRECTORY', True), d.getVar('PN', True))
+ destdir = d.getVar('D') + os.path.join(d.getVar('LICENSE_FILES_DIRECTORY'), d.getVar('PN'))
copy_license_files(lic_files_paths, destdir)
add_package_and_files(d)
}
@@ -318,15 +331,15 @@
def get_recipe_info(d):
info = {}
- info["PV"] = d.getVar("PV", True)
- info["PR"] = d.getVar("PR", True)
- info["LICENSE"] = d.getVar("LICENSE", True)
+ info["PV"] = d.getVar("PV")
+ info["PR"] = d.getVar("PR")
+ info["LICENSE"] = d.getVar("LICENSE")
return info
def add_package_and_files(d):
- packages = d.getVar('PACKAGES', True)
- files = d.getVar('LICENSE_FILES_DIRECTORY', True)
- pn = d.getVar('PN', True)
+ packages = d.getVar('PACKAGES')
+ files = d.getVar('LICENSE_FILES_DIRECTORY')
+ pn = d.getVar('PN')
pn_lic = "%s%s" % (pn, d.getVar('LICENSE_PACKAGE_SUFFIX', False))
if pn_lic in packages:
bb.warn("%s package already existed in %s." % (pn_lic, pn))
@@ -334,7 +347,7 @@
# first in PACKAGES to be sure that nothing else gets LICENSE_FILES_DIRECTORY
d.setVar('PACKAGES', "%s %s" % (pn_lic, packages))
d.setVar('FILES_' + pn_lic, files)
- rrecommends_pn = d.getVar('RRECOMMENDS_' + pn, True)
+ rrecommends_pn = d.getVar('RRECOMMENDS_' + pn)
if rrecommends_pn:
d.setVar('RRECOMMENDS_' + pn, "%s %s" % (pn_lic, rrecommends_pn))
else:
@@ -345,7 +358,7 @@
import errno
bb.utils.mkdirhier(destdir)
- for (basename, path) in lic_files_paths:
+ for (basename, path, beginline, endline) in lic_files_paths:
try:
src = path
dst = os.path.join(destdir, basename)
@@ -353,7 +366,7 @@
os.remove(dst)
if os.path.islink(src):
src = os.path.realpath(src)
- canlink = os.access(src, os.W_OK) and (os.stat(src).st_dev == os.stat(destdir).st_dev)
+ canlink = os.access(src, os.W_OK) and (os.stat(src).st_dev == os.stat(destdir).st_dev) and beginline is None and endline is None
if canlink:
try:
os.link(src, dst)
@@ -364,20 +377,19 @@
canlink = False
else:
raise
- try:
- if canlink:
- os.chown(dst,0,0)
- except OSError as err:
- if err.errno in (errno.EPERM, errno.EINVAL):
- # Suppress "Operation not permitted" error, as
- # sometimes this function is not executed under pseudo.
- # Also ignore "Invalid argument" errors that happen in
- # some (unprivileged) container environments (no root).
- pass
- else:
- raise
+ # Only chown if we did hardling, and, we're running under pseudo
+ if canlink and os.environ.get('PSEUDO_DISABLED') == '0':
+ os.chown(dst,0,0)
if not canlink:
- shutil.copyfile(src, dst)
+ begin_idx = int(beginline)-1 if beginline is not None else None
+ end_idx = int(endline) if endline is not None else None
+ if begin_idx is None and end_idx is None:
+ shutil.copyfile(src, dst)
+ else:
+ with open(src, 'rb') as src_f:
+ with open(dst, 'wb') as dst_f:
+ dst_f.write(b''.join(src_f.readlines()[begin_idx:end_idx]))
+
except Exception as e:
bb.warn("Could not copy license file %s to %s: %s" % (src, dst, e))
@@ -390,20 +402,22 @@
from collections import defaultdict, OrderedDict
# All the license files for the package
- lic_files = d.getVar('LIC_FILES_CHKSUM', True)
- pn = d.getVar('PN', True)
+ lic_files = d.getVar('LIC_FILES_CHKSUM') or ""
+ pn = d.getVar('PN')
# The license files are located in S/LIC_FILE_CHECKSUM.
- srcdir = d.getVar('S', True)
+ srcdir = d.getVar('S')
# Directory we store the generic licenses as set in the distro configuration
- generic_directory = d.getVar('COMMON_LICENSE_DIR', True)
+ generic_directory = d.getVar('COMMON_LICENSE_DIR')
# List of basename, path tuples
lic_files_paths = []
+ # hash for keep track generic lics mappings
+ non_generic_lics = {}
# Entries from LIC_FILES_CHKSUM
lic_chksums = {}
license_source_dirs = []
license_source_dirs.append(generic_directory)
try:
- additional_lic_dirs = d.getVar('LICENSE_PATH', True).split()
+ additional_lic_dirs = d.getVar('LICENSE_PATH').split()
for lic_dir in additional_lic_dirs:
license_source_dirs.append(lic_dir)
except:
@@ -431,10 +445,10 @@
# unless NO_GENERIC_LICENSE is set.
for lic_dir in license_source_dirs:
if not os.path.isfile(os.path.join(lic_dir, license_type)):
- if d.getVarFlag('SPDXLICENSEMAP', license_type, True) != None:
+ if d.getVarFlag('SPDXLICENSEMAP', license_type) != None:
# Great, there is an SPDXLICENSEMAP. We can copy!
bb.debug(1, "We need to use a SPDXLICENSEMAP for %s" % (license_type))
- spdx_generic = d.getVarFlag('SPDXLICENSEMAP', license_type, True)
+ spdx_generic = d.getVarFlag('SPDXLICENSEMAP', license_type)
license_source = lic_dir
break
elif os.path.isfile(os.path.join(lic_dir, license_type)):
@@ -442,23 +456,25 @@
license_source = lic_dir
break
- non_generic_lic = d.getVarFlag('NO_GENERIC_LICENSE', license_type, True)
+ non_generic_lic = d.getVarFlag('NO_GENERIC_LICENSE', license_type)
if spdx_generic and license_source:
# we really should copy to generic_ + spdx_generic, however, that ends up messing the manifest
# audit up. This should be fixed in emit_pkgdata (or, we actually got and fix all the recipes)
- lic_files_paths.append(("generic_" + license_type, os.path.join(license_source, spdx_generic)))
+ lic_files_paths.append(("generic_" + license_type, os.path.join(license_source, spdx_generic),
+ None, None))
# The user may attempt to use NO_GENERIC_LICENSE for a generic license which doesn't make sense
# and should not be allowed, warn the user in this case.
- if d.getVarFlag('NO_GENERIC_LICENSE', license_type, True):
+ if d.getVarFlag('NO_GENERIC_LICENSE', license_type):
bb.warn("%s: %s is a generic license, please don't use NO_GENERIC_LICENSE for it." % (pn, license_type))
elif non_generic_lic and non_generic_lic in lic_chksums:
# if NO_GENERIC_LICENSE is set, we copy the license files from the fetched source
# of the package rather than the license_source_dirs.
lic_files_paths.append(("generic_" + license_type,
- os.path.join(srcdir, non_generic_lic)))
+ os.path.join(srcdir, non_generic_lic), None, None))
+ non_generic_lics[non_generic_lic] = license_type
else:
# Add explicity avoid of CLOSED license because this isn't generic
if license_type != 'CLOSED':
@@ -469,41 +485,40 @@
if not generic_directory:
bb.fatal("COMMON_LICENSE_DIR is unset. Please set this in your distro config")
- if not lic_files:
- # No recipe should have an invalid license file. This is checked else
- # where, but let's be pedantic
- bb.note(pn + ": Recipe file does not have license file information.")
- return lic_files_paths
-
for url in lic_files.split():
try:
(type, host, path, user, pswd, parm) = bb.fetch.decodeurl(url)
except bb.fetch.MalformedUrl:
- bb.fatal("%s: LIC_FILES_CHKSUM contains an invalid URL: %s" % (d.getVar('PF', True), url))
+ bb.fatal("%s: LIC_FILES_CHKSUM contains an invalid URL: %s" % (d.getVar('PF'), url))
# We want the license filename and path
- chksum = parm['md5'] if 'md5' in parm else parm['sha256']
- lic_chksums[path] = chksum
+ chksum = parm.get('md5', None)
+ beginline = parm.get('beginline')
+ endline = parm.get('endline')
+ lic_chksums[path] = (chksum, beginline, endline)
v = FindVisitor()
try:
- v.visit_string(d.getVar('LICENSE', True))
+ v.visit_string(d.getVar('LICENSE'))
except oe.license.InvalidLicense as exc:
- bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
+ bb.fatal('%s: %s' % (d.getVar('PF'), exc))
except SyntaxError:
- bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
-
+ bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF')))
# Add files from LIC_FILES_CHKSUM to list of license files
lic_chksum_paths = defaultdict(OrderedDict)
- for path, chksum in lic_chksums.items():
- lic_chksum_paths[os.path.basename(path)][chksum] = os.path.join(srcdir, path)
+ for path, data in sorted(lic_chksums.items()):
+ lic_chksum_paths[os.path.basename(path)][data] = (os.path.join(srcdir, path), data[1], data[2])
for basename, files in lic_chksum_paths.items():
if len(files) == 1:
- lic_files_paths.append((basename, list(files.values())[0]))
+ # Don't copy again a LICENSE already handled as non-generic
+ if basename in non_generic_lics:
+ continue
+ data = list(files.values())[0]
+ lic_files_paths.append(tuple([basename] + list(data)))
else:
# If there are multiple different license files with identical
# basenames we rename them to <file>.0, <file>.1, ...
- for i, path in enumerate(files.values()):
- lic_files_paths.append(("%s.%d" % (basename, i), path))
+ for i, data in enumerate(files.values()):
+ lic_files_paths.append(tuple(["%s.%d" % (basename, i)] + list(data)))
return lic_files_paths
@@ -511,7 +526,7 @@
"""
This function returns the spdx mapping of a license if it exists.
"""
- return d.getVarFlag('SPDXLICENSEMAP', license, True)
+ return d.getVarFlag('SPDXLICENSEMAP', license)
def canonical_license(d, license):
"""
@@ -520,9 +535,9 @@
'X' if availabel and the tailing '+' (so GPLv3+ becomes GPL-3.0+),
or the passed license if there is no canonical form.
"""
- lic = d.getVarFlag('SPDXLICENSEMAP', license, True) or ""
+ lic = d.getVarFlag('SPDXLICENSEMAP', license) or ""
if not lic and license.endswith('+'):
- lic = d.getVarFlag('SPDXLICENSEMAP', license.rstrip('+'), True)
+ lic = d.getVarFlag('SPDXLICENSEMAP', license.rstrip('+'))
if lic:
lic += '+'
return lic or license
@@ -537,7 +552,7 @@
spdxmapkeys = d.getVarFlags('SPDXLICENSEMAP').keys()
for wld_lic in wildcard_licenses:
spdxflags = fnmatch.filter(spdxmapkeys, wld_lic)
- licenses += [d.getVarFlag('SPDXLICENSEMAP', flag, True) for flag in spdxflags]
+ licenses += [d.getVarFlag('SPDXLICENSEMAP', flag) for flag in spdxflags]
spdx_lics = (d.getVar('SRC_DISTRIBUTE_LICENSES', False) or '').split()
for wld_lic in wildcard_licenses:
@@ -548,7 +563,7 @@
def incompatible_license_contains(license, truevalue, falsevalue, d):
license = canonical_license(d, license)
- bad_licenses = (d.getVar('INCOMPATIBLE_LICENSE', True) or "").split()
+ bad_licenses = (d.getVar('INCOMPATIBLE_LICENSE') or "").split()
bad_licenses = expand_wildcard_licenses(d, bad_licenses)
return truevalue if license in bad_licenses else falsevalue
@@ -559,9 +574,9 @@
as canonical (SPDX) names.
"""
import oe.license
- license = d.getVar("LICENSE_%s" % package, True) if package else None
+ license = d.getVar("LICENSE_%s" % package) if package else None
if not license:
- license = d.getVar('LICENSE', True)
+ license = d.getVar('LICENSE')
# Handles an "or" or two license sets provided by
# flattened_licenses(), pick one that works if possible.
@@ -572,7 +587,7 @@
try:
licenses = oe.license.flattened_licenses(license, choose_lic_set)
except oe.license.LicenseError as exc:
- bb.fatal('%s: %s' % (d.getVar('P', True), exc))
+ bb.fatal('%s: %s' % (d.getVar('P'), exc))
return any(not oe.license.license_ok(canonical_license(d, l), \
dont_want_licenses) for l in licenses)
@@ -620,16 +635,16 @@
def all_license_flags_match(license_flags, whitelist):
""" Return first unmatched flag, None if all flags match """
- pn = d.getVar('PN', True)
+ pn = d.getVar('PN')
split_whitelist = whitelist.split()
for flag in license_flags.split():
if not license_flag_matches(flag, split_whitelist, pn):
return flag
return None
- license_flags = d.getVar('LICENSE_FLAGS', True)
+ license_flags = d.getVar('LICENSE_FLAGS')
if license_flags:
- whitelist = d.getVar('LICENSE_FLAGS_WHITELIST', True)
+ whitelist = d.getVar('LICENSE_FLAGS_WHITELIST')
if not whitelist:
return license_flags
unmatched_flag = all_license_flags_match(license_flags, whitelist)
@@ -643,8 +658,8 @@
Validate operators in LICENSES.
No spaces are allowed between LICENSES.
"""
- pn = d.getVar('PN', True)
- licenses = d.getVar('LICENSE', True)
+ pn = d.getVar('PN')
+ licenses = d.getVar('LICENSE')
from oe.license import license_operator, license_operator_chars, license_pattern
elements = list(filter(lambda x: x.strip(), license_operator.split(licenses)))