poky: refresh thud: 506ec088e5..e4c0a8a7cb
Update poky to thud HEAD.
Alexander Kanavin (1):
ca-certificates: upgrade 20180409 -> 20190110
André Draszik (1):
systemd: RDEPENDS on util-linux-umount
Changqing Li (1):
libsndfile1: Security fix CVE-2018-19432
Chen Qi (1):
target-sdk-provides-dummy: add more perl modules to avoid populate_sdk failure
Douglas Royds (1):
libpam: libpamc is licensed under its own BSD-style licence
George McCollister (1):
systemd: fix CVE-2019-6454
Jonathan Rajotte-Julien (3):
lttng-ust: update to 2.10.3
lttng-modules: update to 2.10.9
lttng-tools: update to 2.9.11
Mark Hatle (10):
bitbake: gitsm.py: Fix when a submodule is defined, but not initialized
bitbake: gitsm.py: Add support for alternative URL formats from submodule files
bitbake: tests/fetch.py: Add alternative gitsm test case
bitbake: gitsm.py: Optimize code and attempt to resolve locking issue
bitbake: gitsm.py: revise unpack
bitbake: gitsm.py: Rework the shallow fetcher and test case
bitbake: gitsm.py: Refactor the functions and simplify the class
bitbake: gitsm.py: Fix relative URLs
bitbake: gitsmy.py: Fix unpack of submodules of submodules
bitbake: gitsm: The fetcher did not process some recursive submodules properly.
Ming Liu (1):
rm_work: sort the value of do_build dependencies
Oleksandr Kravchuk (1):
target-sdk-provides-dummy: add perl-module-overload
Richard Purdie (3):
target-sdk-provides-dummy: Extend to -dev and -src packages
systemd: Update recent CVE patches
kernel: Ensure an initramfs is added if configured
Robert Yang (1):
send-error-report: Add --no-ssl to use http protocol
Ross Burton (1):
libpng: fix CVE-2019-7317
Change-Id: I3e03c837688d49703b4989a561f3728d616abbec
Signed-off-by: Brad Bishop <bradleyb@fuzziesquirrel.com>
diff --git a/poky/bitbake/lib/bb/fetch2/gitsm.py b/poky/bitbake/lib/bb/fetch2/gitsm.py
index 35729db..b21fed2 100644
--- a/poky/bitbake/lib/bb/fetch2/gitsm.py
+++ b/poky/bitbake/lib/bb/fetch2/gitsm.py
@@ -45,60 +45,97 @@
"""
return ud.type in ['gitsm']
- @staticmethod
- def parse_gitmodules(gitmodules):
- modules = {}
- module = ""
- for line in gitmodules.splitlines():
- if line.startswith('[submodule'):
- module = line.split('"')[1]
- modules[module] = {}
- elif module and line.strip().startswith('path'):
- path = line.split('=')[1].strip()
- modules[module]['path'] = path
- elif module and line.strip().startswith('url'):
- url = line.split('=')[1].strip()
- modules[module]['url'] = url
- return modules
+ def process_submodules(self, ud, workdir, function, d):
+ """
+ Iterate over all of the submodules in this repository and execute
+ the 'function' for each of them.
+ """
- def update_submodules(self, ud, d):
submodules = []
paths = {}
+ revision = {}
uris = {}
- local_paths = {}
+ subrevision = {}
+ def parse_gitmodules(gitmodules):
+ modules = {}
+ module = ""
+ for line in gitmodules.splitlines():
+ if line.startswith('[submodule'):
+ module = line.split('"')[1]
+ modules[module] = {}
+ elif module and line.strip().startswith('path'):
+ path = line.split('=')[1].strip()
+ modules[module]['path'] = path
+ elif module and line.strip().startswith('url'):
+ url = line.split('=')[1].strip()
+ modules[module]['url'] = url
+ return modules
+
+ # Collect the defined submodules, and their attributes
for name in ud.names:
try:
- gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=ud.clonedir)
+ gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=workdir)
except:
# No submodules to update
continue
- for m, md in self.parse_gitmodules(gitmodules).items():
+ for m, md in parse_gitmodules(gitmodules).items():
+ try:
+ module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=workdir)
+ except:
+ # If the command fails, we don't have a valid file to check. If it doesn't
+ # fail -- it still might be a failure, see next check...
+ module_hash = ""
+
+ if not module_hash:
+ logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", m)
+ continue
+
submodules.append(m)
paths[m] = md['path']
+ revision[m] = ud.revisions[name]
uris[m] = md['url']
+ subrevision[m] = module_hash.split()[2]
+
+ # Convert relative to absolute uri based on parent uri
if uris[m].startswith('..'):
newud = copy.copy(ud)
- newud.path = os.path.realpath(os.path.join(newud.path, md['url']))
+ newud.path = os.path.realpath(os.path.join(newud.path, uris[m]))
uris[m] = Git._get_repo_url(self, newud)
for module in submodules:
- module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], paths[module]), d, quiet=True, workdir=ud.clonedir)
- module_hash = module_hash.split()[2]
+ # Translate the module url into a SRC_URI
- # Build new SRC_URI
- proto = uris[module].split(':', 1)[0]
- url = uris[module].replace('%s:' % proto, 'gitsm:', 1)
+ if "://" in uris[module]:
+ # Properly formated URL already
+ proto = uris[module].split(':', 1)[0]
+ url = uris[module].replace('%s:' % proto, 'gitsm:', 1)
+ else:
+ if ":" in uris[module]:
+ # Most likely an SSH style reference
+ proto = "ssh"
+ if ":/" in uris[module]:
+ # Absolute reference, easy to convert..
+ url = "gitsm://" + uris[module].replace(':/', '/', 1)
+ else:
+ # Relative reference, no way to know if this is right!
+ logger.warning("Submodule included by %s refers to relative ssh reference %s. References may fail if not absolute." % (ud.url, uris[module]))
+ url = "gitsm://" + uris[module].replace(':', '/', 1)
+ else:
+ # This has to be a file reference
+ proto = "file"
+ url = "gitsm://" + uris[module]
+
url += ';protocol=%s' % proto
url += ";name=%s" % module
- url += ";bareclone=1;nocheckout=1;nobranch=1"
+ url += ";subpath=%s" % paths[module]
ld = d.createCopy()
# Not necessary to set SRC_URI, since we're passing the URI to
# Fetch.
#ld.setVar('SRC_URI', url)
- ld.setVar('SRCREV_%s' % module, module_hash)
+ ld.setVar('SRCREV_%s' % module, subrevision[module])
# Workaround for issues with SRCPV/SRCREV_FORMAT errors
# error refer to 'multiple' repositories. Only the repository
@@ -106,145 +143,63 @@
ld.setVar('SRCPV', d.getVar('SRCPV'))
ld.setVar('SRCREV_FORMAT', module)
- newfetch = Fetch([url], ld, cache=False)
- newfetch.download()
- local_paths[module] = newfetch.localpath(url)
+ function(ud, url, module, paths[module], ld)
- # Correct the submodule references to the local download version...
- runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_paths[module]}, d, workdir=ud.clonedir)
-
- symlink_path = os.path.join(ud.clonedir, 'modules', paths[module])
- if not os.path.exists(symlink_path):
- try:
- os.makedirs(os.path.dirname(symlink_path), exist_ok=True)
- except OSError:
- pass
- os.symlink(local_paths[module], symlink_path)
-
- return True
-
- def need_update(self, ud, d):
- main_repo_needs_update = Git.need_update(self, ud, d)
-
- # First check that the main repository has enough history fetched. If it doesn't, then we don't
- # even have the .gitmodules and gitlinks for the submodules to attempt asking whether the
- # submodules' histories are recent enough.
- if main_repo_needs_update:
- return True
-
- # Now check that the submodule histories are new enough. The git-submodule command doesn't have
- # any clean interface for doing this aside from just attempting the checkout (with network
- # fetched disabled).
- return not self.update_submodules(ud, d)
+ return submodules != []
def download(self, ud, d):
- Git.download(self, ud, d)
+ def download_submodule(ud, url, module, modpath, d):
+ url += ";bareclone=1;nobranch=1"
- if not ud.shallow or ud.localpath != ud.fullshallow:
- self.update_submodules(ud, d)
+ # Is the following still needed?
+ #url += ";nocheckout=1"
- def copy_submodules(self, submodules, ud, destdir, d):
- if ud.bareclone:
- repo_conf = destdir
- else:
- repo_conf = os.path.join(destdir, '.git')
-
- if submodules and not os.path.exists(os.path.join(repo_conf, 'modules')):
- os.mkdir(os.path.join(repo_conf, 'modules'))
-
- for module, md in submodules.items():
- srcpath = os.path.join(ud.clonedir, 'modules', md['path'])
- modpath = os.path.join(repo_conf, 'modules', md['path'])
-
- if os.path.exists(srcpath):
- if os.path.exists(os.path.join(srcpath, '.git')):
- srcpath = os.path.join(srcpath, '.git')
-
- target = modpath
- if os.path.exists(modpath):
- target = os.path.dirname(modpath)
-
- os.makedirs(os.path.dirname(target), exist_ok=True)
- runfetchcmd("cp -fpLR %s %s" % (srcpath, target), d)
- elif os.path.exists(modpath):
- # Module already exists, likely unpacked from a shallow mirror clone
- pass
- else:
- # This is fatal, as we do NOT want git-submodule to hit the network
- raise bb.fetch2.FetchError('Submodule %s does not exist in %s or %s.' % (module, srcpath, modpath))
-
- def clone_shallow_local(self, ud, dest, d):
- super(GitSM, self).clone_shallow_local(ud, dest, d)
-
- # Copy over the submodules' fetched histories too.
- repo_conf = os.path.join(dest, '.git')
-
- submodules = []
- for name in ud.names:
try:
- gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revision), d, quiet=True, workdir=dest)
- except:
- # No submodules to update
- continue
+ newfetch = Fetch([url], d, cache=False)
+ newfetch.download()
+ except Exception as e:
+ logger.error('gitsm: submodule download failed: %s %s' % (type(e).__name__, str(e)))
+ raise
- submodules = self.parse_gitmodules(gitmodules)
- self.copy_submodules(submodules, ud, dest, d)
+ Git.download(self, ud, d)
+ self.process_submodules(ud, ud.clonedir, download_submodule, d)
def unpack(self, ud, destdir, d):
+ def unpack_submodules(ud, url, module, modpath, d):
+ url += ";bareclone=1;nobranch=1"
+
+ # Figure out where we clone over the bare submodules...
+ if ud.bareclone:
+ repo_conf = ud.destdir
+ else:
+ repo_conf = os.path.join(ud.destdir, '.git')
+
+ try:
+ newfetch = Fetch([url], d, cache=False)
+ newfetch.unpack(root=os.path.dirname(os.path.join(repo_conf, 'modules', modpath)))
+ except Exception as e:
+ logger.error('gitsm: submodule unpack failed: %s %s' % (type(e).__name__, str(e)))
+ raise
+
+ local_path = newfetch.localpath(url)
+
+ # Correct the submodule references to the local download version...
+ runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_path}, d, workdir=ud.destdir)
+
+ if ud.shallow:
+ runfetchcmd("%(basecmd)s config submodule.%(module)s.shallow true" % {'basecmd': ud.basecmd, 'module': module}, d, workdir=ud.destdir)
+
+ # Ensure the submodule repository is NOT set to bare, since we're checking it out...
+ try:
+ runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=os.path.join(repo_conf, 'modules', modpath))
+ except:
+ logger.error("Unable to set git config core.bare to false for %s" % os.path.join(repo_conf, 'modules', modpath))
+ raise
+
Git.unpack(self, ud, destdir, d)
- # Copy over the submodules' fetched histories too.
- if ud.bareclone:
- repo_conf = ud.destdir
- else:
- repo_conf = os.path.join(ud.destdir, '.git')
+ ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d)
- update_submodules = False
- paths = {}
- uris = {}
- local_paths = {}
- for name in ud.names:
- try:
- gitmodules = runfetchcmd("%s show HEAD:.gitmodules" % (ud.basecmd), d, quiet=True, workdir=ud.destdir)
- except:
- # No submodules to update
- continue
-
- submodules = self.parse_gitmodules(gitmodules)
- self.copy_submodules(submodules, ud, ud.destdir, d)
-
- submodules_queue = [(module, os.path.join(repo_conf, 'modules', md['path'])) for module, md in submodules.items()]
- while len(submodules_queue) != 0:
- module, modpath = submodules_queue.pop()
-
- # add submodule children recursively
- try:
- gitmodules = runfetchcmd("%s show HEAD:.gitmodules" % (ud.basecmd), d, quiet=True, workdir=modpath)
- for m, md in self.parse_gitmodules(gitmodules).items():
- submodules_queue.append([m, os.path.join(modpath, 'modules', md['path'])])
- except:
- # no children
- pass
-
-
- # There are submodules to update
- update_submodules = True
-
- # Determine (from the submodule) the correct url to reference
- try:
- output = runfetchcmd("%(basecmd)s config remote.origin.url" % {'basecmd': ud.basecmd}, d, workdir=modpath)
- except bb.fetch2.FetchError as e:
- # No remote url defined in this submodule
- continue
-
- local_paths[module] = output
-
- # Setup the local URL properly (like git submodule init or sync would do...)
- runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_paths[module]}, d, workdir=ud.destdir)
-
- # Ensure the submodule repository is NOT set to bare, since we're checking it out...
- runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=modpath)
-
- if update_submodules:
+ if not ud.bareclone and ret:
# Run submodule update, this sets up the directories -- without touching the config
runfetchcmd("%s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir)
diff --git a/poky/bitbake/lib/bb/tests/fetch.py b/poky/bitbake/lib/bb/tests/fetch.py
index 6848095..522d202 100644
--- a/poky/bitbake/lib/bb/tests/fetch.py
+++ b/poky/bitbake/lib/bb/tests/fetch.py
@@ -893,12 +893,70 @@
@skipIfNoNetwork()
def test_git_submodule(self):
- fetcher = bb.fetch.Fetch(["gitsm://git.yoctoproject.org/git-submodule-test;rev=f12e57f2edf0aa534cf1616fa983d165a92b0842"], self.d)
+ # URL with ssh submodules
+ url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=ssh-gitsm-tests;rev=049da4a6cb198d7c0302e9e8b243a1443cb809a7"
+ # Original URL (comment this if you have ssh access to git.yoctoproject.org)
+ url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=master;rev=a2885dd7d25380d23627e7544b7bbb55014b16ee"
+ fetcher = bb.fetch.Fetch([url], self.d)
fetcher.download()
# Previous cwd has been deleted
os.chdir(os.path.dirname(self.unpackdir))
fetcher.unpack(self.unpackdir)
+ repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
+ self.assertTrue(os.path.exists(repo_path), msg='Unpacked repository missing')
+ self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake')), msg='bitbake submodule missing')
+ self.assertFalse(os.path.exists(os.path.join(repo_path, 'na')), msg='uninitialized submodule present')
+
+ # Only when we're running the extended test with a submodule's submodule, can we check this.
+ if os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1')):
+ self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1', 'bitbake')), msg='submodule of submodule missing')
+
+ def test_git_submodule_dbus_broker(self):
+ # The following external repositories have show failures in fetch and unpack operations
+ # We want to avoid regressions!
+ url = "gitsm://github.com/bus1/dbus-broker;protocol=git;rev=fc874afa0992d0c75ec25acb43d344679f0ee7d2"
+ fetcher = bb.fetch.Fetch([url], self.d)
+ fetcher.download()
+ # Previous cwd has been deleted
+ os.chdir(os.path.dirname(self.unpackdir))
+ fetcher.unpack(self.unpackdir)
+
+ repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
+ self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-dvar/config')), msg='Missing submodule config "subprojects/c-dvar"')
+ self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-list/config')), msg='Missing submodule config "subprojects/c-list"')
+ self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-rbtree/config')), msg='Missing submodule config "subprojects/c-rbtree"')
+ self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-sundry/config')), msg='Missing submodule config "subprojects/c-sundry"')
+ self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-utf8/config')), msg='Missing submodule config "subprojects/c-utf8"')
+
+ def test_git_submodule_CLI11(self):
+ url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf"
+ fetcher = bb.fetch.Fetch([url], self.d)
+ fetcher.download()
+ # Previous cwd has been deleted
+ os.chdir(os.path.dirname(self.unpackdir))
+ fetcher.unpack(self.unpackdir)
+
+ repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
+ self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/googletest/config')), msg='Missing submodule config "extern/googletest"')
+ self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"')
+ self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"')
+
+ def test_git_submodule_aktualizr(self):
+ url = "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=git;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44"
+ fetcher = bb.fetch.Fetch([url], self.d)
+ fetcher.download()
+ # Previous cwd has been deleted
+ os.chdir(os.path.dirname(self.unpackdir))
+ fetcher.unpack(self.unpackdir)
+
+ repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
+ self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/partial/extern/isotp-c/config')), msg='Missing submodule config "partial/extern/isotp-c/config"')
+ self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/partial/extern/isotp-c/modules/deps/bitfield-c/config')), msg='Missing submodule config "partial/extern/isotp-c/modules/deps/bitfield-c/config"')
+ self.assertTrue(os.path.exists(os.path.join(repo_path, 'partial/extern/isotp-c/deps/bitfield-c/.git')), msg="Submodule of submodule isotp-c did not unpack properly")
+ self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/tests/tuf-test-vectors/config')), msg='Missing submodule config "tests/tuf-test-vectors/config"')
+ self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/googletest/config')), msg='Missing submodule config "third_party/googletest/config"')
+ self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/HdrHistogram_c/config')), msg='Missing submodule config "third_party/HdrHistogram_c/config"')
class TrustedNetworksTest(FetcherTest):
def test_trusted_network(self):
@@ -1312,6 +1370,7 @@
# fetch and unpack, from the shallow tarball
bb.utils.remove(self.gitdir, recurse=True)
bb.utils.remove(ud.clonedir, recurse=True)
+ bb.utils.remove(ud.clonedir.replace('gitsource', 'gitsubmodule'), recurse=True)
# confirm that the unpacked repo is used when no git clone or git
# mirror tarball is available
@@ -1466,6 +1525,7 @@
self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir)
self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir)
self.add_empty_file('asub', cwd=smdir)
+ self.add_empty_file('bsub', cwd=smdir)
self.git('submodule init', cwd=self.srcdir)
self.git('submodule add file://%s' % smdir, cwd=self.srcdir)
@@ -1475,10 +1535,16 @@
uri = 'gitsm://%s;protocol=file;subdir=${S}' % self.srcdir
fetcher, ud = self.fetch_shallow(uri)
+ # Verify the main repository is shallow
self.assertRevCount(1)
- assert './.git/modules/' in bb.process.run('tar -tzf %s' % os.path.join(self.dldir, ud.mirrortarballs[0]))[0]
+
+ # Verify the gitsubmodule directory is present
assert os.listdir(os.path.join(self.gitdir, 'gitsubmodule'))
+ # Verify the submodule is also shallow
+ self.assertRevCount(1, cwd=os.path.join(self.gitdir, 'gitsubmodule'))
+
+
if any(os.path.exists(os.path.join(p, 'git-annex')) for p in os.environ.get('PATH').split(':')):
def test_shallow_annex(self):
self.add_empty_file('a')