Yocto 2.4
Move OpenBMC to Yocto 2.4(rocko)
Tested: Built and verified Witherspoon and Palmetto images
Change-Id: I12057b18610d6fb0e6903c60213690301e9b0c67
Signed-off-by: Brad Bishop <bradleyb@fuzziesquirrel.com>
diff --git a/import-layers/yocto-poky/meta/lib/oe/buildhistory_analysis.py b/import-layers/yocto-poky/meta/lib/oe/buildhistory_analysis.py
index 3a5b7b6..3e86a46 100644
--- a/import-layers/yocto-poky/meta/lib/oe/buildhistory_analysis.py
+++ b/import-layers/yocto-poky/meta/lib/oe/buildhistory_analysis.py
@@ -143,22 +143,25 @@
out += '\n '.join(list(diff)[2:])
out += '\n --'
elif self.fieldname in img_monitor_files or '/image-files/' in self.path:
- fieldname = self.fieldname
- if '/image-files/' in self.path:
- fieldname = os.path.join('/' + self.path.split('/image-files/')[1], self.fieldname)
- out = 'Changes to %s:\n ' % fieldname
+ if self.filechanges or (self.oldvalue and self.newvalue):
+ fieldname = self.fieldname
+ if '/image-files/' in self.path:
+ fieldname = os.path.join('/' + self.path.split('/image-files/')[1], self.fieldname)
+ out = 'Changes to %s:\n ' % fieldname
+ else:
+ if outer:
+ prefix = 'Changes to %s ' % self.path
+ out = '(%s):\n ' % self.fieldname
+ if self.filechanges:
+ out += '\n '.join(['%s' % i for i in self.filechanges])
+ else:
+ alines = self.oldvalue.splitlines()
+ blines = self.newvalue.splitlines()
+ diff = difflib.unified_diff(alines, blines, fieldname, fieldname, lineterm='')
+ out += '\n '.join(list(diff))
+ out += '\n --'
else:
- if outer:
- prefix = 'Changes to %s ' % self.path
- out = '(%s):\n ' % self.fieldname
- if self.filechanges:
- out += '\n '.join(['%s' % i for i in self.filechanges])
- else:
- alines = self.oldvalue.splitlines()
- blines = self.newvalue.splitlines()
- diff = difflib.unified_diff(alines, blines, fieldname, fieldname, lineterm='')
- out += '\n '.join(list(diff))
- out += '\n --'
+ out = ''
else:
out = '%s changed from "%s" to "%s"' % (self.fieldname, self.oldvalue, self.newvalue)
@@ -169,7 +172,7 @@
for line in chg._str_internal(False).splitlines():
out += '\n * %s' % line
- return '%s%s' % (prefix, out)
+ return '%s%s' % (prefix, out) if out else ''
class FileChange:
changetype_add = 'A'
@@ -508,7 +511,8 @@
return '\n'.join(out)
-def process_changes(repopath, revision1, revision2='HEAD', report_all=False, report_ver=False, sigs=False, sigsdiff=False):
+def process_changes(repopath, revision1, revision2='HEAD', report_all=False, report_ver=False,
+ sigs=False, sigsdiff=False, exclude_path=None):
repo = git.Repo(repopath)
assert repo.bare == False
commit = repo.commit(revision1)
@@ -601,6 +605,19 @@
elif chg.path == chg2.path and chg.path.startswith('packages/') and chg2.fieldname in ['PE', 'PV', 'PR']:
chg.related.append(chg2)
+ # filter out unwanted paths
+ if exclude_path:
+ for chg in changes:
+ if chg.filechanges:
+ fchgs = []
+ for fchg in chg.filechanges:
+ for epath in exclude_path:
+ if fchg.path.startswith(epath):
+ break
+ else:
+ fchgs.append(fchg)
+ chg.filechanges = fchgs
+
if report_all:
return changes
else:
diff --git a/import-layers/yocto-poky/meta/lib/oe/copy_buildsystem.py b/import-layers/yocto-poky/meta/lib/oe/copy_buildsystem.py
index a372904..ac2fae1 100644
--- a/import-layers/yocto-poky/meta/lib/oe/copy_buildsystem.py
+++ b/import-layers/yocto-poky/meta/lib/oe/copy_buildsystem.py
@@ -32,6 +32,10 @@
corebase = os.path.abspath(self.d.getVar('COREBASE'))
layers.append(corebase)
+ # The bitbake build system uses the meta-skeleton layer as a layout
+ # for common recipies, e.g: the recipetool script to create kernel recipies
+ # Add the meta-skeleton layer to be included as part of the eSDK installation
+ layers.append(os.path.join(corebase, 'meta-skeleton'))
# Exclude layers
for layer_exclude in self.layers_exclude:
@@ -71,6 +75,11 @@
layerdestpath = destdir
if corebase == os.path.dirname(layer):
layerdestpath += '/' + os.path.basename(corebase)
+ else:
+ layer_relative = os.path.basename(corebase) + '/' + os.path.relpath(layer, corebase)
+ if os.path.dirname(layer_relative) != layernewname:
+ layerdestpath += '/' + os.path.dirname(layer_relative)
+
layerdestpath += '/' + layernewname
layer_relative = os.path.relpath(layerdestpath,
@@ -123,6 +132,14 @@
line = line.replace('workspacelayer', workspace_newname)
f.write(line)
+ # meta-skeleton layer is added as part of the build system
+ # but not as a layer included in the build, therefore it is
+ # not reported to the function caller.
+ for layer in layers_copied:
+ if layer.endswith('/meta-skeleton'):
+ layers_copied.remove(layer)
+ break
+
return layers_copied
def generate_locked_sigs(sigfile, d):
@@ -239,6 +256,7 @@
cmd = "%sBB_SETSCENE_ENFORCE=1 PSEUDO_DISABLED=1 oe-check-sstate %s -s -o %s %s" % (cmdprefix, targets, filteroutfile, logparam)
env = dict(d.getVar('BB_ORIGENV', False))
env.pop('BUILDDIR', '')
+ env.pop('BBPATH', '')
pathitems = env['PATH'].split(':')
env['PATH'] = ':'.join([item for item in pathitems if not item.endswith('/bitbake/bin')])
bb.process.run(cmd, stderr=subprocess.STDOUT, env=env, cwd=cwd, executable='/bin/bash')
diff --git a/import-layers/yocto-poky/meta/lib/oe/distro_check.py b/import-layers/yocto-poky/meta/lib/oe/distro_check.py
index 37f04ed..e775c3a 100644
--- a/import-layers/yocto-poky/meta/lib/oe/distro_check.py
+++ b/import-layers/yocto-poky/meta/lib/oe/distro_check.py
@@ -77,17 +77,10 @@
def get_latest_released_opensuse_source_package_list(d):
"Returns list of all the name os packages in the latest opensuse distro"
- latest = find_latest_numeric_release("http://download.opensuse.org/source/distribution/",d)
+ latest = find_latest_numeric_release("http://download.opensuse.org/source/distribution/leap", d)
- package_names = get_source_package_list_from_url("http://download.opensuse.org/source/distribution/%s/repo/oss/suse/src/" % latest, "main", d)
- package_names |= get_source_package_list_from_url("http://download.opensuse.org/update/%s/src/" % latest, "updates", d)
- return latest, package_names
-
-def get_latest_released_mandriva_source_package_list(d):
- "Returns list of all the name os packages in the latest mandriva distro"
- latest = find_latest_numeric_release("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/", d)
- package_names = get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/release/" % latest, "main", d)
- package_names |= get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/updates/" % latest, "updates", d)
+ package_names = get_source_package_list_from_url("http://download.opensuse.org/source/distribution/leap/%s/repo/oss/suse/src/" % latest, "main", d)
+ package_names |= get_source_package_list_from_url("http://download.opensuse.org/update/leap/%s/oss/src/" % latest, "updates", d)
return latest, package_names
def get_latest_released_clear_source_package_list(d):
@@ -161,8 +154,7 @@
("Debian", get_latest_released_debian_source_package_list),
("Ubuntu", get_latest_released_ubuntu_source_package_list),
("Fedora", get_latest_released_fedora_source_package_list),
- ("OpenSuSE", get_latest_released_opensuse_source_package_list),
- ("Mandriva", get_latest_released_mandriva_source_package_list),
+ ("openSUSE", get_latest_released_opensuse_source_package_list),
("Clear", get_latest_released_clear_source_package_list),
)
diff --git a/import-layers/yocto-poky/meta/lib/oe/gpg_sign.py b/import-layers/yocto-poky/meta/lib/oe/gpg_sign.py
index 7ce767e..9cc88f0 100644
--- a/import-layers/yocto-poky/meta/lib/oe/gpg_sign.py
+++ b/import-layers/yocto-poky/meta/lib/oe/gpg_sign.py
@@ -15,7 +15,7 @@
def export_pubkey(self, output_file, keyid, armor=True):
"""Export GPG public key to a file"""
- cmd = '%s --batch --yes --export -o %s ' % \
+ cmd = '%s --no-permission-warning --batch --yes --export -o %s ' % \
(self.gpg_bin, output_file)
if self.gpg_path:
cmd += "--homedir %s " % self.gpg_path
@@ -27,22 +27,27 @@
raise bb.build.FuncFailed('Failed to export gpg public key (%s): %s' %
(keyid, output))
- def sign_rpms(self, files, keyid, passphrase):
+ def sign_rpms(self, files, keyid, passphrase, digest, sign_chunk, fsk=None, fsk_password=None):
"""Sign RPM files"""
cmd = self.rpm_bin + " --addsign --define '_gpg_name %s' " % keyid
- gpg_args = '--batch --passphrase=%s' % passphrase
+ gpg_args = '--no-permission-warning --batch --passphrase=%s' % passphrase
if self.gpg_version > (2,1,):
gpg_args += ' --pinentry-mode=loopback'
cmd += "--define '_gpg_sign_cmd_extra_args %s' " % gpg_args
+ cmd += "--define '_binary_filedigest_algorithm %s' " % digest
if self.gpg_bin:
- cmd += "--define '%%__gpg %s' " % self.gpg_bin
+ cmd += "--define '__gpg %s' " % self.gpg_bin
if self.gpg_path:
cmd += "--define '_gpg_path %s' " % self.gpg_path
+ if fsk:
+ cmd += "--signfiles --fskpath %s " % fsk
+ if fsk_password:
+ cmd += "--define '_file_signing_key_password %s' " % fsk_password
- # Sign in chunks of 100 packages
- for i in range(0, len(files), 100):
- status, output = oe.utils.getstatusoutput(cmd + ' '.join(files[i:i+100]))
+ # Sign in chunks
+ for i in range(0, len(files), sign_chunk):
+ status, output = oe.utils.getstatusoutput(cmd + ' '.join(files[i:i+sign_chunk]))
if status:
raise bb.build.FuncFailed("Failed to sign RPM packages: %s" % output)
@@ -53,8 +58,8 @@
if passphrase_file and passphrase:
raise Exception("You should use either passphrase_file of passphrase, not both")
- cmd = [self.gpg_bin, '--detach-sign', '--batch', '--no-tty', '--yes',
- '--passphrase-fd', '0', '-u', keyid]
+ cmd = [self.gpg_bin, '--detach-sign', '--no-permission-warning', '--batch',
+ '--no-tty', '--yes', '--passphrase-fd', '0', '-u', keyid]
if self.gpg_path:
cmd += ['--homedir', self.gpg_path]
@@ -93,7 +98,7 @@
"""Return the gpg version as a tuple of ints"""
import subprocess
try:
- ver_str = subprocess.check_output((self.gpg_bin, "--version")).split()[2].decode("utf-8")
+ ver_str = subprocess.check_output((self.gpg_bin, "--version", "--no-permission-warning")).split()[2].decode("utf-8")
return tuple([int(i) for i in ver_str.split('.')])
except subprocess.CalledProcessError as e:
raise bb.build.FuncFailed("Could not get gpg version: %s" % e)
@@ -101,7 +106,7 @@
def verify(self, sig_file):
"""Verify signature"""
- cmd = self.gpg_bin + " --verify "
+ cmd = self.gpg_bin + " --verify --no-permission-warning "
if self.gpg_path:
cmd += "--homedir %s " % self.gpg_path
cmd += sig_file
diff --git a/import-layers/yocto-poky/meta/lib/oe/license.py b/import-layers/yocto-poky/meta/lib/oe/license.py
index 8d2fd17..ca385d5 100644
--- a/import-layers/yocto-poky/meta/lib/oe/license.py
+++ b/import-layers/yocto-poky/meta/lib/oe/license.py
@@ -106,7 +106,8 @@
license string matches the whitelist and does not match the blacklist.
Returns a tuple holding the boolean state and a list of the applicable
- licenses which were excluded (or None, if the state is True)
+ licenses that were excluded if state is False, or the licenses that were
+ included if the state is True.
"""
def include_license(license):
@@ -117,10 +118,17 @@
def choose_licenses(alpha, beta):
"""Select the option in an OR which is the 'best' (has the most
- included licenses)."""
- alpha_weight = len(list(filter(include_license, alpha)))
- beta_weight = len(list(filter(include_license, beta)))
- if alpha_weight > beta_weight:
+ included licenses and no excluded licenses)."""
+ # The factor 1000 below is arbitrary, just expected to be much larger
+ # that the number of licenses actually specified. That way the weight
+ # will be negative if the list of licenses contains an excluded license,
+ # but still gives a higher weight to the list with the most included
+ # licenses.
+ alpha_weight = (len(list(filter(include_license, alpha))) -
+ 1000 * (len(list(filter(exclude_license, alpha))) > 0))
+ beta_weight = (len(list(filter(include_license, beta))) -
+ 1000 * (len(list(filter(exclude_license, beta))) > 0))
+ if alpha_weight >= beta_weight:
return alpha
else:
return beta
diff --git a/import-layers/yocto-poky/meta/lib/oe/lsb.py b/import-layers/yocto-poky/meta/lib/oe/lsb.py
index 3a945e0..71c0992 100644
--- a/import-layers/yocto-poky/meta/lib/oe/lsb.py
+++ b/import-layers/yocto-poky/meta/lib/oe/lsb.py
@@ -1,19 +1,26 @@
+def get_os_release():
+ """Get all key-value pairs from /etc/os-release as a dict"""
+ from collections import OrderedDict
+
+ data = OrderedDict()
+ if os.path.exists('/etc/os-release'):
+ with open('/etc/os-release') as f:
+ for line in f:
+ try:
+ key, val = line.rstrip().split('=', 1)
+ except ValueError:
+ continue
+ data[key.strip()] = val.strip('"')
+ return data
+
def release_dict_osr():
""" Populate a dict with pertinent values from /etc/os-release """
- if not os.path.exists('/etc/os-release'):
- return None
-
data = {}
- with open('/etc/os-release') as f:
- for line in f:
- try:
- key, val = line.rstrip().split('=', 1)
- except ValueError:
- continue
- if key == 'ID':
- data['DISTRIB_ID'] = val.strip('"')
- if key == 'VERSION_ID':
- data['DISTRIB_RELEASE'] = val.strip('"')
+ os_release = get_os_release()
+ if 'ID' in os_release:
+ data['DISTRIB_ID'] = os_release['ID']
+ if 'VERSION_ID' in os_release:
+ data['DISTRIB_RELEASE'] = os_release['VERSION_ID']
return data
diff --git a/import-layers/yocto-poky/meta/lib/oe/package.py b/import-layers/yocto-poky/meta/lib/oe/package.py
index 4797e7d..1e5c3aa 100644
--- a/import-layers/yocto-poky/meta/lib/oe/package.py
+++ b/import-layers/yocto-poky/meta/lib/oe/package.py
@@ -45,6 +45,115 @@
return
+def strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, qa_already_stripped=False):
+ """
+ Strip executable code (like executables, shared libraries) _in_place_
+ - Based on sysroot_strip in staging.bbclass
+ :param dstdir: directory in which to strip files
+ :param strip_cmd: Strip command (usually ${STRIP})
+ :param libdir: ${libdir} - strip .so files in this directory
+ :param base_libdir: ${base_libdir} - strip .so files in this directory
+ :param qa_already_stripped: Set to True if already-stripped' in ${INSANE_SKIP}
+ This is for proper logging and messages only.
+ """
+ import stat, errno, oe.path, oe.utils, mmap
+
+ # Detect .ko module by searching for "vermagic=" string
+ def is_kernel_module(path):
+ with open(path) as f:
+ return mmap.mmap(f.fileno(), 0, prot=mmap.PROT_READ).find(b"vermagic=") >= 0
+
+ # Return type (bits):
+ # 0 - not elf
+ # 1 - ELF
+ # 2 - stripped
+ # 4 - executable
+ # 8 - shared library
+ # 16 - kernel module
+ def is_elf(path):
+ exec_type = 0
+ ret, result = oe.utils.getstatusoutput(
+ "file \"%s\"" % path.replace("\"", "\\\""))
+
+ if ret:
+ bb.error("split_and_strip_files: 'file %s' failed" % path)
+ return exec_type
+
+ if "ELF" in result:
+ exec_type |= 1
+ if "not stripped" not in result:
+ exec_type |= 2
+ if "executable" in result:
+ exec_type |= 4
+ if "shared" in result:
+ exec_type |= 8
+ if "relocatable" in result and is_kernel_module(path):
+ exec_type |= 16
+ return exec_type
+
+ elffiles = {}
+ inodes = {}
+ libdir = os.path.abspath(dstdir + os.sep + libdir)
+ base_libdir = os.path.abspath(dstdir + os.sep + base_libdir)
+ exec_mask = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
+ #
+ # First lets figure out all of the files we may have to process
+ #
+ for root, dirs, files in os.walk(dstdir):
+ for f in files:
+ file = os.path.join(root, f)
+
+ try:
+ ltarget = oe.path.realpath(file, dstdir, False)
+ s = os.lstat(ltarget)
+ except OSError as e:
+ (err, strerror) = e.args
+ if err != errno.ENOENT:
+ raise
+ # Skip broken symlinks
+ continue
+ if not s:
+ continue
+ # Check its an excutable
+ if s[stat.ST_MODE] & exec_mask \
+ or ((file.startswith(libdir) or file.startswith(base_libdir)) and ".so" in f) \
+ or file.endswith('.ko'):
+ # If it's a symlink, and points to an ELF file, we capture the readlink target
+ if os.path.islink(file):
+ continue
+
+ # It's a file (or hardlink), not a link
+ # ...but is it ELF, and is it already stripped?
+ elf_file = is_elf(file)
+ if elf_file & 1:
+ if elf_file & 2:
+ if qa_already_stripped:
+ bb.note("Skipping file %s from %s for already-stripped QA test" % (file[len(dstdir):], pn))
+ else:
+ bb.warn("File '%s' from %s was already stripped, this will prevent future debugging!" % (file[len(dstdir):], pn))
+ continue
+
+ if s.st_ino in inodes:
+ os.unlink(file)
+ os.link(inodes[s.st_ino], file)
+ else:
+ # break hardlinks so that we do not strip the original.
+ inodes[s.st_ino] = file
+ bb.utils.copyfile(file, file)
+ elffiles[file] = elf_file
+
+ #
+ # Now strip them (in parallel)
+ #
+ sfiles = []
+ for file in elffiles:
+ elf_file = int(elffiles[file])
+ sfiles.append((file, elf_file, strip_cmd))
+
+ oe.utils.multiprocess_exec(sfiles, runstrip)
+
+
+
def file_translate(file):
ft = file.replace("@", "@at@")
ft = ft.replace(" ", "@space@")
@@ -67,8 +176,7 @@
def process_deps(pipe, pkg, pkgdest, provides, requires):
file = None
- for line in pipe:
- line = line.decode("utf-8")
+ for line in pipe.split("\n"):
m = file_re.match(line)
if m:
@@ -117,12 +225,8 @@
return provides, requires
- try:
- dep_popen = subprocess.Popen(shlex.split(rpmdeps) + pkgfiles, stdout=subprocess.PIPE)
- provides, requires = process_deps(dep_popen.stdout, pkg, pkgdest, provides, requires)
- except OSError as e:
- bb.error("rpmdeps: '%s' command failed, '%s'" % (shlex.split(rpmdeps) + pkgfiles, e))
- raise e
+ output = subprocess.check_output(shlex.split(rpmdeps) + pkgfiles, stderr=subprocess.STDOUT).decode("utf-8")
+ provides, requires = process_deps(output, pkg, pkgdest, provides, requires)
return (pkg, provides, requires)
diff --git a/import-layers/yocto-poky/meta/lib/oe/package_manager.py b/import-layers/yocto-poky/meta/lib/oe/package_manager.py
index 3a2daad..0c5d907 100644
--- a/import-layers/yocto-poky/meta/lib/oe/package_manager.py
+++ b/import-layers/yocto-poky/meta/lib/oe/package_manager.py
@@ -17,18 +17,11 @@
def create_index(arg):
index_cmd = arg
- try:
- bb.note("Executing '%s' ..." % index_cmd)
- result = subprocess.check_output(index_cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8")
- except subprocess.CalledProcessError as e:
- return("Index creation command '%s' failed with return code %d:\n%s" %
- (e.cmd, e.returncode, e.output.decode("utf-8")))
-
+ bb.note("Executing '%s' ..." % index_cmd)
+ result = subprocess.check_output(index_cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8")
if result:
bb.note(result)
- return None
-
"""
This method parse the output from the package managerand return
a dictionary with the information of the packages. This is used
@@ -104,13 +97,25 @@
class RpmIndexer(Indexer):
def write_index(self):
if self.d.getVar('PACKAGE_FEED_SIGN') == '1':
- raise NotImplementedError('Package feed signing not yet implementd for rpm')
+ signer = get_signer(self.d, self.d.getVar('PACKAGE_FEED_GPG_BACKEND'))
+ else:
+ signer = None
createrepo_c = bb.utils.which(os.environ['PATH'], "createrepo_c")
result = create_index("%s --update -q %s" % (createrepo_c, self.deploy_dir))
if result:
bb.fatal(result)
+ # Sign repomd
+ if signer:
+ sig_type = self.d.getVar('PACKAGE_FEED_GPG_SIGNATURE_TYPE')
+ is_ascii_sig = (sig_type.upper() != "BIN")
+ signer.detach_sign(os.path.join(self.deploy_dir, 'repodata', 'repomd.xml'),
+ self.d.getVar('PACKAGE_FEED_GPG_NAME'),
+ self.d.getVar('PACKAGE_FEED_GPG_PASSPHRASE_FILE'),
+ armor=is_ascii_sig)
+
+
class OpkgIndexer(Indexer):
def write_index(self):
arch_vars = ["ALL_MULTILIB_PACKAGE_ARCHS",
@@ -152,9 +157,7 @@
bb.note("There are no packages in %s!" % self.deploy_dir)
return
- result = oe.utils.multiprocess_exec(index_cmds, create_index)
- if result:
- bb.fatal('%s' % ('\n'.join(result)))
+ oe.utils.multiprocess_exec(index_cmds, create_index)
if signer:
feed_sig_type = self.d.getVar('PACKAGE_FEED_GPG_SIGNATURE_TYPE')
@@ -220,7 +223,7 @@
cmd = "cd %s; PSEUDO_UNLOAD=1 %s packages . > Packages;" % (arch_dir, apt_ftparchive)
- cmd += "%s -fc Packages > Packages.gz;" % gzip
+ cmd += "%s -fcn Packages > Packages.gz;" % gzip
with open(os.path.join(arch_dir, "Release"), "w+") as release:
release.write("Label: %s\n" % arch)
@@ -235,9 +238,7 @@
bb.note("There are no packages in %s" % self.deploy_dir)
return
- result = oe.utils.multiprocess_exec(index_cmds, create_index)
- if result:
- bb.fatal('%s' % ('\n'.join(result)))
+ oe.utils.multiprocess_exec(index_cmds, create_index)
if self.d.getVar('PACKAGE_FEED_SIGN') == '1':
raise NotImplementedError('Package feed signing not implementd for dpkg')
@@ -548,6 +549,14 @@
if feed_uris == "":
return
+ gpg_opts = ''
+ if self.d.getVar('PACKAGE_FEED_SIGN') == '1':
+ gpg_opts += 'repo_gpgcheck=1\n'
+ gpg_opts += 'gpgkey=file://%s/pki/packagefeed-gpg/PACKAGEFEED-GPG-KEY-%s-%s\n' % (self.d.getVar('sysconfdir'), self.d.getVar('DISTRO'), self.d.getVar('DISTRO_CODENAME'))
+
+ if self.d.getVar('RPM_SIGN_PACKAGES') == '0':
+ gpg_opts += 'gpgcheck=0\n'
+
bb.utils.mkdirhier(oe.path.join(self.target_rootfs, "etc", "yum.repos.d"))
remote_uris = self.construct_uris(feed_uris.split(), feed_base_paths.split())
for uri in remote_uris:
@@ -558,12 +567,12 @@
repo_id = "oe-remote-repo" + "-".join(urlparse(repo_uri).path.split("/"))
repo_name = "OE Remote Repo:" + " ".join(urlparse(repo_uri).path.split("/"))
open(oe.path.join(self.target_rootfs, "etc", "yum.repos.d", repo_base + ".repo"), 'a').write(
- "[%s]\nname=%s\nbaseurl=%s\n\n" % (repo_id, repo_name, repo_uri))
+ "[%s]\nname=%s\nbaseurl=%s\n%s\n" % (repo_id, repo_name, repo_uri, gpg_opts))
else:
repo_name = "OE Remote Repo:" + " ".join(urlparse(uri).path.split("/"))
repo_uri = uri
open(oe.path.join(self.target_rootfs, "etc", "yum.repos.d", repo_base + ".repo"), 'w').write(
- "[%s]\nname=%s\nbaseurl=%s\n" % (repo_base, repo_name, repo_uri))
+ "[%s]\nname=%s\nbaseurl=%s\n%s" % (repo_base, repo_name, repo_uri, gpg_opts))
def _prepare_pkg_transaction(self):
os.environ['D'] = self.target_rootfs
@@ -608,10 +617,12 @@
self._invoke_dnf(["remove"] + pkgs)
else:
cmd = bb.utils.which(os.getenv('PATH'), "rpm")
- args = ["-e", "--nodeps", "--root=%s" %self.target_rootfs]
+ args = ["-e", "-v", "--nodeps", "--root=%s" %self.target_rootfs]
try:
+ bb.note("Running %s" % ' '.join([cmd] + args + pkgs))
output = subprocess.check_output([cmd] + args + pkgs, stderr=subprocess.STDOUT).decode("utf-8")
+ bb.note(output)
except subprocess.CalledProcessError as e:
bb.fatal("Could not invoke rpm. Command "
"'%s' returned %d:\n%s" % (' '.join([cmd] + args + pkgs), e.returncode, e.output.decode("utf-8")))
@@ -682,7 +693,7 @@
return packages
def update(self):
- self._invoke_dnf(["makecache"])
+ self._invoke_dnf(["makecache", "--refresh"])
def _invoke_dnf(self, dnf_args, fatal = True, print_output = True ):
os.environ['RPM_ETCCONFIGDIR'] = self.target_rootfs
@@ -1145,7 +1156,7 @@
# Create an temp dir as opkg root for dummy installation
temp_rootfs = self.d.expand('${T}/opkg')
- opkg_lib_dir = self.d.getVar('OPKGLIBDIR', True)
+ opkg_lib_dir = self.d.getVar('OPKGLIBDIR')
if opkg_lib_dir[0] == "/":
opkg_lib_dir = opkg_lib_dir[1:]
temp_opkg_dir = os.path.join(temp_rootfs, opkg_lib_dir, 'opkg')
diff --git a/import-layers/yocto-poky/meta/lib/oe/patch.py b/import-layers/yocto-poky/meta/lib/oe/patch.py
index f1ab3dd..584bf6c 100644
--- a/import-layers/yocto-poky/meta/lib/oe/patch.py
+++ b/import-layers/yocto-poky/meta/lib/oe/patch.py
@@ -1,4 +1,5 @@
import oe.path
+import oe.types
class NotFoundError(bb.BBHandledException):
def __init__(self, path):
diff --git a/import-layers/yocto-poky/meta/lib/oe/path.py b/import-layers/yocto-poky/meta/lib/oe/path.py
index 448a2b9..1ea03d5 100644
--- a/import-layers/yocto-poky/meta/lib/oe/path.py
+++ b/import-layers/yocto-poky/meta/lib/oe/path.py
@@ -98,7 +98,7 @@
if (os.stat(src).st_dev == os.stat(dst).st_dev):
# Need to copy directories only with tar first since cp will error if two
# writers try and create a directory at the same time
- cmd = "cd %s; find . -type d -print | tar --xattrs --xattrs-include='*' -cf - -C %s -p --no-recursion --files-from - | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, src, dst)
+ cmd = "cd %s; find . -type d -print | tar --xattrs --xattrs-include='*' -cf - -C %s -p --no-recursion --files-from - | tar --xattrs --xattrs-include='*' -xhf - -C %s" % (src, src, dst)
subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
source = ''
if os.path.isdir(src):
diff --git a/import-layers/yocto-poky/meta/lib/oe/recipeutils.py b/import-layers/yocto-poky/meta/lib/oe/recipeutils.py
index a7fdd36..cab8e40 100644
--- a/import-layers/yocto-poky/meta/lib/oe/recipeutils.py
+++ b/import-layers/yocto-poky/meta/lib/oe/recipeutils.py
@@ -2,7 +2,7 @@
#
# Some code borrowed from the OE layer index
#
-# Copyright (C) 2013-2016 Intel Corporation
+# Copyright (C) 2013-2017 Intel Corporation
#
import sys
@@ -188,6 +188,11 @@
for wrapline in wrapped[:-1]:
addlines.append('%s \\%s' % (wrapline, newline))
addlines.append('%s%s' % (wrapped[-1], newline))
+
+ # Split on newlines - this isn't strictly necessary if you are only
+ # going to write the output to disk, but if you want to compare it
+ # (as patch_recipe_file() will do if patch=True) then it's important.
+ addlines = [line for l in addlines for line in l.splitlines(True)]
if rewindcomments:
# Ensure we insert the lines before any leading comments
# (that we'd want to ensure remain leading the next value)
@@ -320,7 +325,7 @@
-def copy_recipe_files(d, tgt_dir, whole_dir=False, download=True):
+def copy_recipe_files(d, tgt_dir, whole_dir=False, download=True, all_variants=False):
"""Copy (local) recipe files, including both files included via include/require,
and files referred to in the SRC_URI variable."""
import bb.fetch2
@@ -328,18 +333,41 @@
# FIXME need a warning if the unexpanded SRC_URI value contains variable references
- uris = (d.getVar('SRC_URI') or "").split()
- fetch = bb.fetch2.Fetch(uris, d)
- if download:
- fetch.download()
+ uri_values = []
+ localpaths = []
+ def fetch_urls(rdata):
+ # Collect the local paths from SRC_URI
+ srcuri = rdata.getVar('SRC_URI') or ""
+ if srcuri not in uri_values:
+ fetch = bb.fetch2.Fetch(srcuri.split(), rdata)
+ if download:
+ fetch.download()
+ for pth in fetch.localpaths():
+ if pth not in localpaths:
+ localpaths.append(pth)
+ uri_values.append(srcuri)
+
+ fetch_urls(d)
+ if all_variants:
+ # Get files for other variants e.g. in the case of a SRC_URI_append
+ localdata = bb.data.createCopy(d)
+ variants = (localdata.getVar('BBCLASSEXTEND') or '').split()
+ if variants:
+ # Ensure we handle class-target if we're dealing with one of the variants
+ variants.append('target')
+ for variant in variants:
+ localdata.setVar('CLASSOVERRIDE', 'class-%s' % variant)
+ fetch_urls(localdata)
# Copy local files to target directory and gather any remote files
- bb_dir = os.path.dirname(d.getVar('FILE')) + os.sep
+ bb_dir = os.path.abspath(os.path.dirname(d.getVar('FILE'))) + os.sep
remotes = []
copied = []
- includes = [path for path in d.getVar('BBINCLUDED').split() if
- path.startswith(bb_dir) and os.path.exists(path)]
- for path in fetch.localpaths() + includes:
+ # Need to do this in two steps since we want to check against the absolute path
+ includes = [os.path.abspath(path) for path in d.getVar('BBINCLUDED').split() if os.path.exists(path)]
+ # We also check this below, but we don't want any items in this list being considered remotes
+ includes = [path for path in includes if path.startswith(bb_dir)]
+ for path in localpaths + includes:
# Only import files that are under the meta directory
if path.startswith(bb_dir):
if not whole_dir:
@@ -778,7 +806,7 @@
def find_layerdir(fn):
""" Figure out the path to the base of the layer containing a file (e.g. a recipe)"""
- pth = fn
+ pth = os.path.abspath(fn)
layerdir = ''
while pth:
if os.path.exists(os.path.join(pth, 'conf', 'layer.conf')):
diff --git a/import-layers/yocto-poky/meta/lib/oe/rootfs.py b/import-layers/yocto-poky/meta/lib/oe/rootfs.py
index 96591f3..754ef56 100644
--- a/import-layers/yocto-poky/meta/lib/oe/rootfs.py
+++ b/import-layers/yocto-poky/meta/lib/oe/rootfs.py
@@ -261,15 +261,22 @@
# Remove components that we don't need if it's a read-only rootfs
unneeded_pkgs = self.d.getVar("ROOTFS_RO_UNNEEDED").split()
pkgs_installed = image_list_installed_packages(self.d)
- # Make sure update-alternatives is last on the command line, so
- # that it is removed last. This makes sure that its database is
- # available while uninstalling packages, allowing alternative
- # symlinks of packages to be uninstalled to be managed correctly.
+ # Make sure update-alternatives is removed last. This is
+ # because its database has to available while uninstalling
+ # other packages, allowing alternative symlinks of packages
+ # to be uninstalled or to be managed correctly otherwise.
provider = self.d.getVar("VIRTUAL-RUNTIME_update-alternatives")
pkgs_to_remove = sorted([pkg for pkg in pkgs_installed if pkg in unneeded_pkgs], key=lambda x: x == provider)
+ # update-alternatives provider is removed in its own remove()
+ # call because all package managers do not guarantee the packages
+ # are removed in the order they given in the list (which is
+ # passed to the command line). The sorting done earlier is
+ # utilized to implement the 2-stage removal.
+ if len(pkgs_to_remove) > 1:
+ self.pm.remove(pkgs_to_remove[:-1], False)
if len(pkgs_to_remove) > 0:
- self.pm.remove(pkgs_to_remove, False)
+ self.pm.remove([pkgs_to_remove[-1]], False)
if delayed_postinsts:
self._save_postinsts()
@@ -302,10 +309,11 @@
bb.note("> Executing %s intercept ..." % script)
try:
- subprocess.check_output(script_full)
+ output = subprocess.check_output(script_full, stderr=subprocess.STDOUT)
+ if output: bb.note(output.decode("utf-8"))
except subprocess.CalledProcessError as e:
- bb.warn("The postinstall intercept hook '%s' failed (exit code: %d)! See log for details! (Output: %s)" %
- (script, e.returncode, e.output))
+ bb.warn("The postinstall intercept hook '%s' failed, details in log.do_rootfs" % script)
+ bb.note("Exit code %d. Output:\n%s" % (e.returncode, e.output.decode("utf-8")))
with open(script_full) as intercept:
registered_pkgs = None
@@ -524,7 +532,8 @@
self.pm.save_rpmpostinst(pkg)
def _cleanup(self):
- pass
+ self.pm._invoke_dnf(["clean", "all"])
+
class DpkgOpkgRootfs(Rootfs):
def __init__(self, d, progress_reporter=None, logcatcher=None):
diff --git a/import-layers/yocto-poky/meta/lib/oe/sdk.py b/import-layers/yocto-poky/meta/lib/oe/sdk.py
index 9fe1687..a3a6c39 100644
--- a/import-layers/yocto-poky/meta/lib/oe/sdk.py
+++ b/import-layers/yocto-poky/meta/lib/oe/sdk.py
@@ -152,6 +152,8 @@
pm.install(pkgs_attempt, True)
def _populate(self):
+ execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_PRE_TARGET_COMMAND"))
+
bb.note("Installing TARGET packages")
self._populate_sysroot(self.target_pm, self.target_manifest)
@@ -233,6 +235,8 @@
[False, True][pkg_type == Manifest.PKG_TYPE_ATTEMPT_ONLY])
def _populate(self):
+ execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_PRE_TARGET_COMMAND"))
+
bb.note("Installing TARGET packages")
self._populate_sysroot(self.target_pm, self.target_manifest)
@@ -315,6 +319,8 @@
[False, True][pkg_type == Manifest.PKG_TYPE_ATTEMPT_ONLY])
def _populate(self):
+ execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_PRE_TARGET_COMMAND"))
+
bb.note("Installing TARGET packages")
self._populate_sysroot(self.target_pm, self.target_manifest)
@@ -379,5 +385,24 @@
os.environ.clear()
os.environ.update(env_bkp)
+def get_extra_sdkinfo(sstate_dir):
+ """
+ This function is going to be used for generating the target and host manifest files packages of eSDK.
+ """
+ import math
+
+ extra_info = {}
+ extra_info['tasksizes'] = {}
+ extra_info['filesizes'] = {}
+ for root, _, files in os.walk(sstate_dir):
+ for fn in files:
+ if fn.endswith('.tgz'):
+ fsize = int(math.ceil(float(os.path.getsize(os.path.join(root, fn))) / 1024))
+ task = fn.rsplit(':',1)[1].split('_',1)[1].split(',')[0]
+ origtotal = extra_info['tasksizes'].get(task, 0)
+ extra_info['tasksizes'][task] = origtotal + fsize
+ extra_info['filesizes'][fn] = fsize
+ return extra_info
+
if __name__ == "__main__":
pass
diff --git a/import-layers/yocto-poky/meta/lib/oe/sstatesig.py b/import-layers/yocto-poky/meta/lib/oe/sstatesig.py
index b8dd4c8..3a8778e 100644
--- a/import-layers/yocto-poky/meta/lib/oe/sstatesig.py
+++ b/import-layers/yocto-poky/meta/lib/oe/sstatesig.py
@@ -29,7 +29,7 @@
return True
# Quilt (patch application) changing isn't likely to affect anything
- excludelist = ['quilt-native', 'subversion-native', 'git-native']
+ excludelist = ['quilt-native', 'subversion-native', 'git-native', 'ccache-native']
if depname in excludelist and recipename != depname:
return False
@@ -320,7 +320,7 @@
if not taskhashlist or (len(filedates) < 2 and not foundall):
# That didn't work, look in sstate-cache
- hashes = taskhashlist or ['*']
+ hashes = taskhashlist or ['?' * 32]
localdata = bb.data.createCopy(d)
for hashval in hashes:
localdata.setVar('PACKAGE_ARCH', '*')
diff --git a/import-layers/yocto-poky/meta/lib/oe/terminal.py b/import-layers/yocto-poky/meta/lib/oe/terminal.py
index 2f18ec0..94afe39 100644
--- a/import-layers/yocto-poky/meta/lib/oe/terminal.py
+++ b/import-layers/yocto-poky/meta/lib/oe/terminal.py
@@ -292,6 +292,8 @@
vernum = ver.split(' ')[-1]
if ver.startswith('GNOME Terminal'):
vernum = ver.split(' ')[-1]
+ if ver.startswith('MATE Terminal'):
+ vernum = ver.split(' ')[-1]
if ver.startswith('tmux'):
vernum = ver.split()[-1]
return vernum
diff --git a/import-layers/yocto-poky/meta/lib/oe/useradd.py b/import-layers/yocto-poky/meta/lib/oe/useradd.py
new file mode 100644
index 0000000..179ac76
--- /dev/null
+++ b/import-layers/yocto-poky/meta/lib/oe/useradd.py
@@ -0,0 +1,68 @@
+import argparse
+import re
+
+class myArgumentParser(argparse.ArgumentParser):
+ def _print_message(self, message, file=None):
+ bb.warn("%s - %s: %s" % (d.getVar('PN'), pkg, message))
+
+ # This should never be called...
+ def exit(self, status=0, message=None):
+ message = message or ("%s - %s: useradd.bbclass: Argument parsing exited" % (d.getVar('PN'), pkg))
+ error(message)
+
+ def error(self, message):
+ raise bb.build.FuncFailed(message)
+
+def split_commands(params):
+ params = re.split('''[ \t]*;[ \t]*(?=(?:[^'"]|'[^']*'|"[^"]*")*$)''', params.strip())
+ # Remove any empty items
+ return [x for x in params if x]
+
+def split_args(params):
+ params = re.split('''[ \t]+(?=(?:[^'"]|'[^']*'|"[^"]*")*$)''', params.strip())
+ # Remove any empty items
+ return [x for x in params if x]
+
+def build_useradd_parser():
+ # The following comes from --help on useradd from shadow
+ parser = myArgumentParser(prog='useradd')
+ parser.add_argument("-b", "--base-dir", metavar="BASE_DIR", help="base directory for the home directory of the new account")
+ parser.add_argument("-c", "--comment", metavar="COMMENT", help="GECOS field of the new account")
+ parser.add_argument("-d", "--home-dir", metavar="HOME_DIR", help="home directory of the new account")
+ parser.add_argument("-D", "--defaults", help="print or change default useradd configuration", action="store_true")
+ parser.add_argument("-e", "--expiredate", metavar="EXPIRE_DATE", help="expiration date of the new account")
+ parser.add_argument("-f", "--inactive", metavar="INACTIVE", help="password inactivity period of the new account")
+ parser.add_argument("-g", "--gid", metavar="GROUP", help="name or ID of the primary group of the new account")
+ parser.add_argument("-G", "--groups", metavar="GROUPS", help="list of supplementary groups of the new account")
+ parser.add_argument("-k", "--skel", metavar="SKEL_DIR", help="use this alternative skeleton directory")
+ parser.add_argument("-K", "--key", metavar="KEY=VALUE", help="override /etc/login.defs defaults")
+ parser.add_argument("-l", "--no-log-init", help="do not add the user to the lastlog and faillog databases", action="store_true")
+ parser.add_argument("-m", "--create-home", help="create the user's home directory", action="store_const", const=True)
+ parser.add_argument("-M", "--no-create-home", dest="create_home", help="do not create the user's home directory", action="store_const", const=False)
+ parser.add_argument("-N", "--no-user-group", dest="user_group", help="do not create a group with the same name as the user", action="store_const", const=False)
+ parser.add_argument("-o", "--non-unique", help="allow to create users with duplicate (non-unique UID)", action="store_true")
+ parser.add_argument("-p", "--password", metavar="PASSWORD", help="encrypted password of the new account")
+ parser.add_argument("-P", "--clear-password", metavar="CLEAR_PASSWORD", help="use this clear password for the new account")
+ parser.add_argument("-R", "--root", metavar="CHROOT_DIR", help="directory to chroot into")
+ parser.add_argument("-r", "--system", help="create a system account", action="store_true")
+ parser.add_argument("-s", "--shell", metavar="SHELL", help="login shell of the new account")
+ parser.add_argument("-u", "--uid", metavar="UID", help="user ID of the new account")
+ parser.add_argument("-U", "--user-group", help="create a group with the same name as the user", action="store_const", const=True)
+ parser.add_argument("LOGIN", help="Login name of the new user")
+
+ return parser
+
+def build_groupadd_parser():
+ # The following comes from --help on groupadd from shadow
+ parser = myArgumentParser(prog='groupadd')
+ parser.add_argument("-f", "--force", help="exit successfully if the group already exists, and cancel -g if the GID is already used", action="store_true")
+ parser.add_argument("-g", "--gid", metavar="GID", help="use GID for the new group")
+ parser.add_argument("-K", "--key", metavar="KEY=VALUE", help="override /etc/login.defs defaults")
+ parser.add_argument("-o", "--non-unique", help="allow to create groups with duplicate (non-unique) GID", action="store_true")
+ parser.add_argument("-p", "--password", metavar="PASSWORD", help="use this encrypted password for the new group")
+ parser.add_argument("-P", "--clear-password", metavar="CLEAR_PASSWORD", help="use this clear password for the new group")
+ parser.add_argument("-R", "--root", metavar="CHROOT_DIR", help="directory to chroot into")
+ parser.add_argument("-r", "--system", help="create a system account", action="store_true")
+ parser.add_argument("GROUP", help="Group name of the new group")
+
+ return parser
diff --git a/import-layers/yocto-poky/meta/lib/oe/utils.py b/import-layers/yocto-poky/meta/lib/oe/utils.py
index 330a5ff..643ab78 100644
--- a/import-layers/yocto-poky/meta/lib/oe/utils.py
+++ b/import-layers/yocto-poky/meta/lib/oe/utils.py
@@ -126,6 +126,46 @@
if addfeatures:
d.appendVar(var, " " + " ".join(addfeatures))
+def all_distro_features(d, features, truevalue="1", falsevalue=""):
+ """
+ Returns truevalue if *all* given features are set in DISTRO_FEATURES,
+ else falsevalue. The features can be given as single string or anything
+ that can be turned into a set.
+
+ This is a shorter, more flexible version of
+ bb.utils.contains("DISTRO_FEATURES", features, truevalue, falsevalue, d).
+
+ Without explicit true/false values it can be used directly where
+ Python expects a boolean:
+ if oe.utils.all_distro_features(d, "foo bar"):
+ bb.fatal("foo and bar are mutually exclusive DISTRO_FEATURES")
+
+ With just a truevalue, it can be used to include files that are meant to be
+ used only when requested via DISTRO_FEATURES:
+ require ${@ oe.utils.all_distro_features(d, "foo bar", "foo-and-bar.inc")
+ """
+ return bb.utils.contains("DISTRO_FEATURES", features, truevalue, falsevalue, d)
+
+def any_distro_features(d, features, truevalue="1", falsevalue=""):
+ """
+ Returns truevalue if at least *one* of the given features is set in DISTRO_FEATURES,
+ else falsevalue. The features can be given as single string or anything
+ that can be turned into a set.
+
+ This is a shorter, more flexible version of
+ bb.utils.contains_any("DISTRO_FEATURES", features, truevalue, falsevalue, d).
+
+ Without explicit true/false values it can be used directly where
+ Python expects a boolean:
+ if not oe.utils.any_distro_features(d, "foo bar"):
+ bb.fatal("foo, bar or both must be set in DISTRO_FEATURES")
+
+ With just a truevalue, it can be used to include files that are meant to be
+ used only when requested via DISTRO_FEATURES:
+ require ${@ oe.utils.any_distro_features(d, "foo bar", "foo-or-bar.inc")
+
+ """
+ return bb.utils.contains_any("DISTRO_FEATURES", features, truevalue, falsevalue, d)
def packages_filter_out_system(d):
"""
@@ -184,25 +224,30 @@
def init_worker():
signal.signal(signal.SIGINT, signal.SIG_IGN)
+ fails = []
+
+ def failures(res):
+ fails.append(res)
+
nproc = min(multiprocessing.cpu_count(), len(commands))
pool = bb.utils.multiprocessingpool(nproc, init_worker)
- imap = pool.imap(function, commands)
try:
- res = list(imap)
+ mapresult = pool.map_async(function, commands, error_callback=failures)
+
pool.close()
pool.join()
- results = []
- for result in res:
- if result is not None:
- results.append(result)
- return results
-
+ results = mapresult.get()
except KeyboardInterrupt:
pool.terminate()
pool.join()
raise
+ if fails:
+ raise fails[0]
+
+ return results
+
def squashspaces(string):
import re
return re.sub("\s+", " ", string).strip()