poky: subtree update:b66b9f7548..26ae42ded7
Adrian Bunk (1):
dpkg: Remove workaound patch for host tar < 1.27
Alexander Kanavin (39):
linux-yocto: exclude from version checks/automated version updates
pciutils: upgrade 3.6.4 -> 3.7.0
createrepo-c: upgrade 0.15.10 -> 0.15.11
librepo: upgrade 1.11.3 -> 1.12.0
pkgconf: upgrade 1.6.3 -> 1.7.3
python3-numpy: upgrade 1.18.4 -> 1.18.5
python3-git: upgrade 3.1.2 -> 3.1.3
strace: upgrade 5.6 -> 5.7
acpica: upgrade 20200430 -> 20200528
man-db: upgrade 2.9.1 -> 2.9.2
msmtp: upgrade 1.8.10 -> 1.8.11
epiphany: upgrade 3.36.1 -> 3.36.2
cogl-1.0: upgrade 1.22.6 -> 1.22.8
libdrm: upgrade 2.4.101 -> 2.4.102
vulkan-demos: upgrade to latest revision
xkeyboard-config: upgrade 2.29 -> 2.30
linux-firmware: upgrade 20200421 -> 20200519
babeltrace2: upgrade 2.0.2 -> 2.0.3
lttng-tools: upgrade 2.12.0 -> 2.12.1
ffmpeg: upgrade 4.2.2 -> 4.2.3
wpebackend-fdo: upgrade 1.6.0 -> 1.6.1
gnutls: upgrade 3.6.13 -> 3.6.14
libcap: upgrade 2.34 -> 2.36
bison: upgrade 3.6.2 -> 3.6.3
asciidoc: 8.6.10 -> 9.0.0
debianutils: 4.9.1 -> 4.11
git: upgrade 2.26.2 -> 2.27.0
go: 1.14.3 -> 1.14.4
iproute2: upgrade 5.6.0 -> 5.7.0
libksba: 1.3.5 -> 1.4.0
lttng-modules: update to 2.12.1
mpg123: update to 1.26.1
ovmf: update to 202005
shared-mime-info: upgrade 1.15 -> 2.0
subversion: upgrade 1.13.0 -> 1.14.0
xinetd: 2.3.15 -> 2.3.15.4
init-system-helpers: use https for fetching
ca-certificates: correct upstream version check
build-sysroots: add sysroot paths with native binaries to PATH
Andreas Müller (4):
vte: tiny cleanup / renumber patch
vte: upgrade 0.60.2 -> 0.60.3
harfbuzz: upgrade 2.6.4 -> 2.6.7
sqlite3: upgrade 3.32.1 -> 3.32.2
Changqing Li (1):
cups.inc: remove template service from SYSTEMD_SERVICE
Chen Qi (2):
db: do not install db_verify if 'verify' is not enabled
vim: restore the 'chmod -x' workaround in do_install
Hongxu Jia (1):
glib-networking/btrfs-tools/dosfstools/parted/bmap-tools/libsoup-2.4: add nativesdk support
Jacob Kroon (4):
features_check: Factorize code for checking features
meta: Don't inherit 'features_check' in recipes that don't utilize it
features_check: Warn if not used
insane: Check for feature check variables not being used
Joe Slater (2):
qemu: force build type to production
vim: _FORTIFY_SOURCE=2 be gone
Joshua Watt (12):
bitbake: bitbake: cooker: Split file collections per multiconfig
bitbake: bitbake: cache: Use multiconfig aware caches
bitbake: bitbake: lib: Add support for Logging Adapters
bitbake: bitbake: lib: Add PrefixLoggerAdapter helper
bitbake: bitbake: cache: Improve logging
bitbake: bitbake: cache: Cache size optimization
bitbake: bitbake: tests: Add tests for BBMASK in multiconfig
bitbake: bitbake: command: Move split_mc_pn to runqueue
bitbake: bitbake: cache: Fix error when cache is rebuilt
wic: Fix --extra-space argument handling
bitbake: bitbake: siggen: Pass all data caches to hash functions
bitbake: bitbake: tests: Add mcdepends test
Kai Kang (4):
mdadm: remove service template from SYSTEMD_SERVICE
wpa-supplicant: remove service templates from SYSTEMD_SERVICE
encodings: clear postinst script
avahi-dnsconfd: rdepends on avahi-daemon
Khem Raj (2):
libunwind: Fix build on aarch64/musl
stress-ng: Fix build on musl
Lee Chee Yang (1):
qemu: fix CVE-2020-13361
Ming Liu (1):
u-boot: support merging .cfg files for UBOOT_CONFIG
Mingli Yu (2):
python3-magic: add the missing rdepends
python3-setuptools: add missing rdepends for python3-pkg-resources
Paul Barker (5):
selftest: git-submodule-test: New recipe for testing a gitsm SRC_URI
archiver: Capture git submodules in mirror archiver
selftest-ed: Support native builds
selftest-nopackages: New recipe in meta-selftest
archiver: Speed up tests
Pierre-Jean Texier (2):
libarchive: upgrade 3.4.2 -> 3.4.3
iptables: upgrade 1.8.4 -> 1.8.5
Rasmus Villemoes (1):
glibc: move ld.so.conf back to main package
Richard Purdie (1):
Revert "bitbake.conf: Remove unused DEPLOY_DIR_TOOLS variable"
Stefan Agner (1):
initramfs-framework: check successful mount using mountpoint
Signed-off-by: Andrew Geissler <geissonator@yahoo.com>
Change-Id: I047d0fa664dcc2864fd7c1a09d124e3d8c197e9f
diff --git a/poky/bitbake/lib/bb/cooker.py b/poky/bitbake/lib/bb/cooker.py
index e527e23..effd024 100644
--- a/poky/bitbake/lib/bb/cooker.py
+++ b/poky/bitbake/lib/bb/cooker.py
@@ -525,7 +525,7 @@
self.parseConfiguration()
fn, cls, mc = bb.cache.virtualfn2realfn(buildfile)
- fn = self.matchFile(fn)
+ fn = self.matchFile(fn, mc)
fn = bb.cache.realfn2virtual(fn, cls, mc)
elif len(pkgs_to_build) == 1:
mc = mc_base(pkgs_to_build[0])
@@ -541,8 +541,8 @@
if fn:
try:
- bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array)
- envdata = bb_cache.loadDataFull(fn, self.collection.get_file_appends(fn))
+ bb_caches = bb.cache.MulticonfigCache(self.databuilder, self.data_hash, self.caches_array)
+ envdata = bb_caches[mc].loadDataFull(fn, self.collections[mc].get_file_appends(fn))
except Exception as e:
parselog.exception("Unable to read %s", fn)
raise
@@ -929,26 +929,33 @@
logger.info("Task dependencies saved to 'task-depends.dot'")
def show_appends_with_no_recipes(self):
+ appends_without_recipes = {}
# Determine which bbappends haven't been applied
+ for mc in self.multiconfigs:
+ # First get list of recipes, including skipped
+ recipefns = list(self.recipecaches[mc].pkg_fn.keys())
+ recipefns.extend(self.skiplist.keys())
- # First get list of recipes, including skipped
- recipefns = list(self.recipecaches[''].pkg_fn.keys())
- recipefns.extend(self.skiplist.keys())
+ # Work out list of bbappends that have been applied
+ applied_appends = []
+ for fn in recipefns:
+ applied_appends.extend(self.collections[mc].get_file_appends(fn))
- # Work out list of bbappends that have been applied
- applied_appends = []
- for fn in recipefns:
- applied_appends.extend(self.collection.get_file_appends(fn))
+ appends_without_recipes[mc] = []
+ for _, appendfn in self.collections[mc].bbappends:
+ if not appendfn in applied_appends:
+ appends_without_recipes[mc].append(appendfn)
- appends_without_recipes = []
- for _, appendfn in self.collection.bbappends:
- if not appendfn in applied_appends:
- appends_without_recipes.append(appendfn)
+ msgs = []
+ for mc in sorted(appends_without_recipes.keys()):
+ if appends_without_recipes[mc]:
+ msgs.append('No recipes in %s available for:\n %s' % (mc if mc else 'default',
+ '\n '.join(appends_without_recipes[mc])))
- if appends_without_recipes:
- msg = 'No recipes available for:\n %s' % '\n '.join(appends_without_recipes)
- warn_only = self.data.getVar("BB_DANGLINGAPPENDS_WARNONLY", \
- False) or "no"
+ if msgs:
+ msg = "\n".join(msgs)
+ warn_only = self.databuilder.mcdata[mc].getVar("BB_DANGLINGAPPENDS_WARNONLY", \
+ False) or "no"
if warn_only.lower() in ("1", "yes", "true"):
bb.warn(msg)
else:
@@ -1249,15 +1256,15 @@
if siggen_cache:
bb.parse.siggen.checksum_cache.mtime_cache.clear()
- def matchFiles(self, bf):
+ def matchFiles(self, bf, mc=''):
"""
Find the .bb files which match the expression in 'buildfile'.
"""
if bf.startswith("/") or bf.startswith("../"):
bf = os.path.abspath(bf)
- self.collection = CookerCollectFiles(self.bbfile_config_priorities)
- filelist, masked, searchdirs = self.collection.collect_bbfiles(self.data, self.data)
+ self.collections = {mc: CookerCollectFiles(self.bbfile_config_priorities, mc)}
+ filelist, masked, searchdirs = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc])
try:
os.stat(bf)
bf = os.path.abspath(bf)
@@ -1270,12 +1277,12 @@
matches.append(f)
return matches
- def matchFile(self, buildfile):
+ def matchFile(self, buildfile, mc=''):
"""
Find the .bb file which matches the expression in 'buildfile'.
Raise an error if multiple files
"""
- matches = self.matchFiles(buildfile)
+ matches = self.matchFiles(buildfile, mc)
if len(matches) != 1:
if matches:
msg = "Unable to match '%s' to a specific recipe file - %s matches found:" % (buildfile, len(matches))
@@ -1316,14 +1323,14 @@
task = "do_%s" % task
fn, cls, mc = bb.cache.virtualfn2realfn(buildfile)
- fn = self.matchFile(fn)
+ fn = self.matchFile(fn, mc)
self.buildSetVars()
self.reset_mtime_caches()
- bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array)
+ bb_caches = bb.cache.MulticonfigCache(self.databuilder, self.data_hash, self.caches_array)
- infos = bb_cache.parse(fn, self.collection.get_file_appends(fn))
+ infos = bb_caches[mc].parse(fn, self.collections[mc].get_file_appends(fn))
infos = dict(infos)
fn = bb.cache.realfn2virtual(fn, cls, mc)
@@ -1552,14 +1559,24 @@
for dep in self.configuration.extra_assume_provided:
self.recipecaches[mc].ignored_dependencies.add(dep)
- self.collection = CookerCollectFiles(self.bbfile_config_priorities)
- (filelist, masked, searchdirs) = self.collection.collect_bbfiles(self.data, self.data)
+ self.collections = {}
+
+ mcfilelist = {}
+ total_masked = 0
+ searchdirs = set()
+ for mc in self.multiconfigs:
+ self.collections[mc] = CookerCollectFiles(self.bbfile_config_priorities, mc)
+ (filelist, masked, search) = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc])
+
+ mcfilelist[mc] = filelist
+ total_masked += masked
+ searchdirs |= set(search)
# Add inotify watches for directories searched for bb/bbappend files
for dirent in searchdirs:
self.add_filewatch([[dirent]], dirs=True)
- self.parser = CookerParser(self, filelist, masked)
+ self.parser = CookerParser(self, mcfilelist, total_masked)
self.parsecache_valid = True
self.state = state.parsing
@@ -1571,7 +1588,7 @@
self.show_appends_with_no_recipes()
self.handlePrefProviders()
for mc in self.multiconfigs:
- self.recipecaches[mc].bbfile_priority = self.collection.collection_priorities(self.recipecaches[mc].pkg_fn, self.data)
+ self.recipecaches[mc].bbfile_priority = self.collections[mc].collection_priorities(self.recipecaches[mc].pkg_fn, self.data)
self.state = state.running
# Send an event listing all stamps reachable after parsing
@@ -1679,7 +1696,8 @@
class CookerCollectFiles(object):
- def __init__(self, priorities):
+ def __init__(self, priorities, mc=''):
+ self.mc = mc
self.bbappends = []
# Priorities is a list of tupples, with the second element as the pattern.
# We need to sort the list with the longest pattern first, and so on to
@@ -1846,7 +1864,7 @@
(bbappend, filename) = b
if (bbappend == f) or ('%' in bbappend and bbappend.startswith(f[:bbappend.index('%')])):
filelist.append(filename)
- return filelist
+ return tuple(filelist)
def collection_priorities(self, pkgfns, d):
@@ -1882,7 +1900,8 @@
for collection, pattern, regex, _ in self.bbfile_config_priorities:
if regex in unmatched:
if d.getVar('BBFILE_PATTERN_IGNORE_EMPTY_%s' % collection) != '1':
- collectlog.warning("No bb files matched BBFILE_PATTERN_%s '%s'" % (collection, pattern))
+ collectlog.warning("No bb files in %s matched BBFILE_PATTERN_%s '%s'" % (self.mc if self.mc else 'default',
+ collection, pattern))
return priorities
@@ -1949,7 +1968,7 @@
except queue.Full:
pending.append(result)
- def parse(self, filename, appends):
+ def parse(self, mc, cache, filename, appends):
try:
origfilter = bb.event.LogHandler.filter
# Record the filename we're parsing into any events generated
@@ -1963,7 +1982,7 @@
bb.event.set_class_handlers(self.handlers.copy())
bb.event.LogHandler.filter = parse_filter
- return True, self.bb_cache.parse(filename, appends)
+ return True, mc, cache.parse(filename, appends)
except Exception as exc:
tb = sys.exc_info()[2]
exc.recipe = filename
@@ -1978,8 +1997,8 @@
bb.event.LogHandler.filter = origfilter
class CookerParser(object):
- def __init__(self, cooker, filelist, masked):
- self.filelist = filelist
+ def __init__(self, cooker, mcfilelist, masked):
+ self.mcfilelist = mcfilelist
self.cooker = cooker
self.cfgdata = cooker.data
self.cfghash = cooker.data_hash
@@ -1993,25 +2012,27 @@
self.skipped = 0
self.virtuals = 0
- self.total = len(filelist)
self.current = 0
self.process_names = []
- self.bb_cache = bb.cache.Cache(self.cfgbuilder, self.cfghash, cooker.caches_array)
- self.fromcache = []
- self.willparse = []
- for filename in self.filelist:
- appends = self.cooker.collection.get_file_appends(filename)
- if not self.bb_cache.cacheValid(filename, appends):
- self.willparse.append((filename, appends))
- else:
- self.fromcache.append((filename, appends))
- self.toparse = self.total - len(self.fromcache)
+ self.bb_caches = bb.cache.MulticonfigCache(self.cfgbuilder, self.cfghash, cooker.caches_array)
+ self.fromcache = set()
+ self.willparse = set()
+ for mc in self.cooker.multiconfigs:
+ for filename in self.mcfilelist[mc]:
+ appends = self.cooker.collections[mc].get_file_appends(filename)
+ if not self.bb_caches[mc].cacheValid(filename, appends):
+ self.willparse.add((mc, self.bb_caches[mc], filename, appends))
+ else:
+ self.fromcache.add((mc, self.bb_caches[mc], filename, appends))
+
+ self.total = len(self.fromcache) + len(self.willparse)
+ self.toparse = len(self.willparse)
self.progress_chunk = int(max(self.toparse / 100, 1))
self.num_processes = min(int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS") or
- multiprocessing.cpu_count()), len(self.willparse))
+ multiprocessing.cpu_count()), self.toparse)
self.start()
self.haveshutdown = False
@@ -2022,7 +2043,6 @@
if self.toparse:
bb.event.fire(bb.event.ParseStarted(self.toparse), self.cfgdata)
def init():
- Parser.bb_cache = self.bb_cache
bb.utils.set_process_name(multiprocessing.current_process().name)
multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, exitpriority=1)
multiprocessing.util.Finalize(None, bb.fetch.fetcher_parse_save, exitpriority=1)
@@ -2032,7 +2052,7 @@
def chunkify(lst,n):
return [lst[i::n] for i in range(n)]
- self.jobs = chunkify(self.willparse, self.num_processes)
+ self.jobs = chunkify(list(self.willparse), self.num_processes)
for i in range(0, self.num_processes):
parser = Parser(self.jobs[i], self.result_queue, self.parser_quit, init, self.cooker.configuration.profile)
@@ -2078,7 +2098,11 @@
else:
process.join()
- sync = threading.Thread(target=self.bb_cache.sync)
+ def sync_caches():
+ for c in self.bb_caches.values():
+ c.sync()
+
+ sync = threading.Thread(target=sync_caches)
sync.start()
multiprocessing.util.Finalize(None, sync.join, exitpriority=-100)
bb.codeparser.parser_cache_savemerge()
@@ -2095,9 +2119,9 @@
print("Processed parsing statistics saved to %s" % (pout))
def load_cached(self):
- for filename, appends in self.fromcache:
- cached, infos = self.bb_cache.load(filename, appends)
- yield not cached, infos
+ for mc, cache, filename, appends in self.fromcache:
+ cached, infos = cache.load(filename, appends)
+ yield not cached, mc, infos
def parse_generator(self):
while True:
@@ -2119,7 +2143,7 @@
result = []
parsed = None
try:
- parsed, result = next(self.results)
+ parsed, mc, result = next(self.results)
except StopIteration:
self.shutdown()
return False
@@ -2175,13 +2199,16 @@
if info_array[0].skipped:
self.skipped += 1
self.cooker.skiplist[virtualfn] = SkippedPackage(info_array[0])
- (fn, cls, mc) = bb.cache.virtualfn2realfn(virtualfn)
- self.bb_cache.add_info(virtualfn, info_array, self.cooker.recipecaches[mc],
+ self.bb_caches[mc].add_info(virtualfn, info_array, self.cooker.recipecaches[mc],
parsed=parsed, watcher = self.cooker.add_filewatch)
return True
def reparse(self, filename):
- infos = self.bb_cache.parse(filename, self.cooker.collection.get_file_appends(filename))
- for vfn, info_array in infos:
- (fn, cls, mc) = bb.cache.virtualfn2realfn(vfn)
- self.cooker.recipecaches[mc].add_from_recipeinfo(vfn, info_array)
+ to_reparse = set()
+ for mc in self.cooker.multiconfigs:
+ to_reparse.add((mc, filename, self.cooker.collections[mc].get_file_appends(filename)))
+
+ for mc, filename, appends in to_reparse:
+ infos = self.bb_caches[mc].parse(filename, appends)
+ for vfn, info_array in infos:
+ self.cooker.recipecaches[mc].add_from_recipeinfo(vfn, info_array)