Yocto 2.4
Move OpenBMC to Yocto 2.4(rocko)
Tested: Built and verified Witherspoon and Palmetto images
Change-Id: I12057b18610d6fb0e6903c60213690301e9b0c67
Signed-off-by: Brad Bishop <bradleyb@fuzziesquirrel.com>
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/git.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/git.py
index 7442f84..5ef8cd6 100644
--- a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/git.py
+++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/git.py
@@ -70,11 +70,14 @@
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+import collections
import errno
+import fnmatch
import os
import re
+import subprocess
+import tempfile
import bb
-import errno
import bb.progress
from bb.fetch2 import FetchMethod
from bb.fetch2 import runfetchcmd
@@ -172,18 +175,66 @@
branches = ud.parm.get("branch", "master").split(',')
if len(branches) != len(ud.names):
raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url)
+
+ ud.cloneflags = "-s -n"
+ if ud.bareclone:
+ ud.cloneflags += " --mirror"
+
+ ud.shallow = d.getVar("BB_GIT_SHALLOW") == "1"
+ ud.shallow_extra_refs = (d.getVar("BB_GIT_SHALLOW_EXTRA_REFS") or "").split()
+
+ depth_default = d.getVar("BB_GIT_SHALLOW_DEPTH")
+ if depth_default is not None:
+ try:
+ depth_default = int(depth_default or 0)
+ except ValueError:
+ raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH: %s" % depth_default)
+ else:
+ if depth_default < 0:
+ raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH: %s" % depth_default)
+ else:
+ depth_default = 1
+ ud.shallow_depths = collections.defaultdict(lambda: depth_default)
+
+ revs_default = d.getVar("BB_GIT_SHALLOW_REVS", True)
+ ud.shallow_revs = []
ud.branches = {}
for pos, name in enumerate(ud.names):
branch = branches[pos]
ud.branches[name] = branch
ud.unresolvedrev[name] = branch
+ shallow_depth = d.getVar("BB_GIT_SHALLOW_DEPTH_%s" % name)
+ if shallow_depth is not None:
+ try:
+ shallow_depth = int(shallow_depth or 0)
+ except ValueError:
+ raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (name, shallow_depth))
+ else:
+ if shallow_depth < 0:
+ raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (name, shallow_depth))
+ ud.shallow_depths[name] = shallow_depth
+
+ revs = d.getVar("BB_GIT_SHALLOW_REVS_%s" % name)
+ if revs is not None:
+ ud.shallow_revs.extend(revs.split())
+ elif revs_default is not None:
+ ud.shallow_revs.extend(revs_default.split())
+
+ if (ud.shallow and
+ not ud.shallow_revs and
+ all(ud.shallow_depths[n] == 0 for n in ud.names)):
+ # Shallow disabled for this URL
+ ud.shallow = False
+
if ud.usehead:
ud.unresolvedrev['default'] = 'HEAD'
ud.basecmd = d.getVar("FETCHCMD_git") or "git -c core.fsyncobjectfiles=0"
- ud.write_tarballs = ((d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0") != "0") or ud.rebaseable
+ write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0"
+ ud.write_tarballs = write_tarballs != "0" or ud.rebaseable
+ ud.write_shallow_tarballs = (d.getVar("BB_GENERATE_SHALLOW_TARBALLS") or write_tarballs) != "0"
ud.setup_revisions(d)
@@ -205,13 +256,42 @@
if ud.rebaseable:
for name in ud.names:
gitsrcname = gitsrcname + '_' + ud.revisions[name]
- ud.mirrortarball = 'git2_%s.tar.gz' % gitsrcname
- ud.fullmirror = os.path.join(d.getVar("DL_DIR"), ud.mirrortarball)
- gitdir = d.getVar("GITDIR") or (d.getVar("DL_DIR") + "/git2/")
- ud.clonedir = os.path.join(gitdir, gitsrcname)
+ dl_dir = d.getVar("DL_DIR")
+ gitdir = d.getVar("GITDIR") or (dl_dir + "/git2/")
+ ud.clonedir = os.path.join(gitdir, gitsrcname)
ud.localfile = ud.clonedir
+ mirrortarball = 'git2_%s.tar.gz' % gitsrcname
+ ud.fullmirror = os.path.join(dl_dir, mirrortarball)
+ ud.mirrortarballs = [mirrortarball]
+ if ud.shallow:
+ tarballname = gitsrcname
+ if ud.bareclone:
+ tarballname = "%s_bare" % tarballname
+
+ if ud.shallow_revs:
+ tarballname = "%s_%s" % (tarballname, "_".join(sorted(ud.shallow_revs)))
+
+ for name, revision in sorted(ud.revisions.items()):
+ tarballname = "%s_%s" % (tarballname, ud.revisions[name][:7])
+ depth = ud.shallow_depths[name]
+ if depth:
+ tarballname = "%s-%s" % (tarballname, depth)
+
+ shallow_refs = []
+ if not ud.nobranch:
+ shallow_refs.extend(ud.branches.values())
+ if ud.shallow_extra_refs:
+ shallow_refs.extend(r.replace('refs/heads/', '').replace('*', 'ALL') for r in ud.shallow_extra_refs)
+ if shallow_refs:
+ tarballname = "%s_%s" % (tarballname, "_".join(sorted(shallow_refs)).replace('/', '.'))
+
+ fetcher = self.__class__.__name__.lower()
+ ud.shallowtarball = '%sshallow_%s.tar.gz' % (fetcher, tarballname)
+ ud.fullshallow = os.path.join(dl_dir, ud.shallowtarball)
+ ud.mirrortarballs.insert(0, ud.shallowtarball)
+
def localpath(self, ud, d):
return ud.clonedir
@@ -221,6 +301,8 @@
for name in ud.names:
if not self._contains_ref(ud, d, name, ud.clonedir):
return True
+ if ud.shallow and ud.write_shallow_tarballs and not os.path.exists(ud.fullshallow):
+ return True
if ud.write_tarballs and not os.path.exists(ud.fullmirror):
return True
return False
@@ -237,8 +319,16 @@
def download(self, ud, d):
"""Fetch url"""
- # If the checkout doesn't exist and the mirror tarball does, extract it
- if not os.path.exists(ud.clonedir) and os.path.exists(ud.fullmirror):
+ no_clone = not os.path.exists(ud.clonedir)
+ need_update = no_clone or self.need_update(ud, d)
+
+ # A current clone is preferred to either tarball, a shallow tarball is
+ # preferred to an out of date clone, and a missing clone will use
+ # either tarball.
+ if ud.shallow and os.path.exists(ud.fullshallow) and need_update:
+ ud.localpath = ud.fullshallow
+ return
+ elif os.path.exists(ud.fullmirror) and no_clone:
bb.utils.mkdirhier(ud.clonedir)
runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir)
@@ -284,9 +374,21 @@
raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revisions[name], ud.branches[name]))
def build_mirror_data(self, ud, d):
- # Generate a mirror tarball if needed
- if ud.write_tarballs and not os.path.exists(ud.fullmirror):
- # it's possible that this symlink points to read-only filesystem with PREMIRROR
+ if ud.shallow and ud.write_shallow_tarballs:
+ if not os.path.exists(ud.fullshallow):
+ if os.path.islink(ud.fullshallow):
+ os.unlink(ud.fullshallow)
+ tempdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
+ shallowclone = os.path.join(tempdir, 'git')
+ try:
+ self.clone_shallow_local(ud, shallowclone, d)
+
+ logger.info("Creating tarball of git repository")
+ runfetchcmd("tar -czf %s ." % ud.fullshallow, d, workdir=shallowclone)
+ runfetchcmd("touch %s.done" % ud.fullshallow, d)
+ finally:
+ bb.utils.remove(tempdir, recurse=True)
+ elif ud.write_tarballs and not os.path.exists(ud.fullmirror):
if os.path.islink(ud.fullmirror):
os.unlink(ud.fullmirror)
@@ -294,6 +396,62 @@
runfetchcmd("tar -czf %s ." % ud.fullmirror, d, workdir=ud.clonedir)
runfetchcmd("touch %s.done" % ud.fullmirror, d)
+ def clone_shallow_local(self, ud, dest, d):
+ """Clone the repo and make it shallow.
+
+ The upstream url of the new clone isn't set at this time, as it'll be
+ set correctly when unpacked."""
+ runfetchcmd("%s clone %s %s %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, dest), d)
+
+ to_parse, shallow_branches = [], []
+ for name in ud.names:
+ revision = ud.revisions[name]
+ depth = ud.shallow_depths[name]
+ if depth:
+ to_parse.append('%s~%d^{}' % (revision, depth - 1))
+
+ # For nobranch, we need a ref, otherwise the commits will be
+ # removed, and for non-nobranch, we truncate the branch to our
+ # srcrev, to avoid keeping unnecessary history beyond that.
+ branch = ud.branches[name]
+ if ud.nobranch:
+ ref = "refs/shallow/%s" % name
+ elif ud.bareclone:
+ ref = "refs/heads/%s" % branch
+ else:
+ ref = "refs/remotes/origin/%s" % branch
+
+ shallow_branches.append(ref)
+ runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest)
+
+ # Map srcrev+depths to revisions
+ parsed_depths = runfetchcmd("%s rev-parse %s" % (ud.basecmd, " ".join(to_parse)), d, workdir=dest)
+
+ # Resolve specified revisions
+ parsed_revs = runfetchcmd("%s rev-parse %s" % (ud.basecmd, " ".join('"%s^{}"' % r for r in ud.shallow_revs)), d, workdir=dest)
+ shallow_revisions = parsed_depths.splitlines() + parsed_revs.splitlines()
+
+ # Apply extra ref wildcards
+ all_refs = runfetchcmd('%s for-each-ref "--format=%%(refname)"' % ud.basecmd,
+ d, workdir=dest).splitlines()
+ for r in ud.shallow_extra_refs:
+ if not ud.bareclone:
+ r = r.replace('refs/heads/', 'refs/remotes/origin/')
+
+ if '*' in r:
+ matches = filter(lambda a: fnmatch.fnmatchcase(a, r), all_refs)
+ shallow_branches.extend(matches)
+ else:
+ shallow_branches.append(r)
+
+ # Make the repository shallow
+ shallow_cmd = ['git', 'make-shallow', '-s']
+ for b in shallow_branches:
+ shallow_cmd.append('-r')
+ shallow_cmd.append(b)
+ shallow_cmd.extend(shallow_revisions)
+ runfetchcmd(subprocess.list2cmdline(shallow_cmd), d, workdir=dest)
+
def unpack(self, ud, destdir, d):
""" unpack the downloaded src to destdir"""
@@ -310,11 +468,12 @@
if os.path.exists(destdir):
bb.utils.prunedir(destdir)
- cloneflags = "-s -n"
- if ud.bareclone:
- cloneflags += " --mirror"
+ if ud.shallow and (not os.path.exists(ud.clonedir) or self.need_update(ud, d)):
+ bb.utils.mkdirhier(destdir)
+ runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=destdir)
+ else:
+ runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d)
- runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, cloneflags, ud.clonedir, destdir), d)
repourl = self._get_repo_url(ud)
runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, repourl), d, workdir=destdir)
if not ud.nocheckout: