Squashed 'yocto-poky/' content from commit ea562de
git-subtree-dir: yocto-poky
git-subtree-split: ea562de57590c966cd5a75fda8defecd397e6436
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
new file mode 100644
index 0000000..40658ff
--- /dev/null
+++ b/bitbake/lib/bb/fetch2/git.py
@@ -0,0 +1,447 @@
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+"""
+BitBake 'Fetch' git implementation
+
+git fetcher support the SRC_URI with format of:
+SRC_URI = "git://some.host/somepath;OptionA=xxx;OptionB=xxx;..."
+
+Supported SRC_URI options are:
+
+- branch
+ The git branch to retrieve from. The default is "master"
+
+ This option also supports multiple branch fetching, with branches
+ separated by commas. In multiple branches case, the name option
+ must have the same number of names to match the branches, which is
+ used to specify the SRC_REV for the branch
+ e.g:
+ SRC_URI="git://some.host/somepath;branch=branchX,branchY;name=nameX,nameY"
+ SRCREV_nameX = "xxxxxxxxxxxxxxxxxxxx"
+ SRCREV_nameY = "YYYYYYYYYYYYYYYYYYYY"
+
+- tag
+ The git tag to retrieve. The default is "master"
+
+- protocol
+ The method to use to access the repository. Common options are "git",
+ "http", "https", "file", "ssh" and "rsync". The default is "git".
+
+- rebaseable
+ rebaseable indicates that the upstream git repo may rebase in the future,
+ and current revision may disappear from upstream repo. This option will
+ remind fetcher to preserve local cache carefully for future use.
+ The default value is "0", set rebaseable=1 for rebaseable git repo.
+
+- nocheckout
+ Don't checkout source code when unpacking. set this option for the recipe
+ who has its own routine to checkout code.
+ The default is "0", set nocheckout=1 if needed.
+
+- bareclone
+ Create a bare clone of the source code and don't checkout the source code
+ when unpacking. Set this option for the recipe who has its own routine to
+ checkout code and tracking branch requirements.
+ The default is "0", set bareclone=1 if needed.
+
+- nobranch
+ Don't check the SHA validation for branch. set this option for the recipe
+ referring to commit which is valid in tag instead of branch.
+ The default is "0", set nobranch=1 if needed.
+
+"""
+
+#Copyright (C) 2005 Richard Purdie
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import os
+import re
+import bb
+from bb import data
+from bb.fetch2 import FetchMethod
+from bb.fetch2 import runfetchcmd
+from bb.fetch2 import logger
+
+class Git(FetchMethod):
+ """Class to fetch a module or modules from git repositories"""
+ def init(self, d):
+ pass
+
+ def supports(self, ud, d):
+ """
+ Check to see if a given url can be fetched with git.
+ """
+ return ud.type in ['git']
+
+ def supports_checksum(self, urldata):
+ return False
+
+ def urldata_init(self, ud, d):
+ """
+ init git specific variable within url data
+ so that the git method like latest_revision() can work
+ """
+ if 'protocol' in ud.parm:
+ ud.proto = ud.parm['protocol']
+ elif not ud.host:
+ ud.proto = 'file'
+ else:
+ ud.proto = "git"
+
+ if not ud.proto in ('git', 'file', 'ssh', 'http', 'https', 'rsync'):
+ raise bb.fetch2.ParameterError("Invalid protocol type", ud.url)
+
+ ud.nocheckout = ud.parm.get("nocheckout","0") == "1"
+
+ ud.rebaseable = ud.parm.get("rebaseable","0") == "1"
+
+ ud.nobranch = ud.parm.get("nobranch","0") == "1"
+
+ # bareclone implies nocheckout
+ ud.bareclone = ud.parm.get("bareclone","0") == "1"
+ if ud.bareclone:
+ ud.nocheckout = 1
+
+ ud.unresolvedrev = {}
+ branches = ud.parm.get("branch", "master").split(',')
+ if len(branches) != len(ud.names):
+ raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url)
+ ud.branches = {}
+ for name in ud.names:
+ branch = branches[ud.names.index(name)]
+ ud.branches[name] = branch
+ ud.unresolvedrev[name] = branch
+
+ ud.basecmd = data.getVar("FETCHCMD_git", d, True) or "git -c core.fsyncobjectfiles=0"
+
+ ud.write_tarballs = ((data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True) or "0") != "0") or ud.rebaseable
+
+ ud.setup_revisons(d)
+
+ for name in ud.names:
+ # Ensure anything that doesn't look like a sha256 checksum/revision is translated into one
+ if not ud.revisions[name] or len(ud.revisions[name]) != 40 or (False in [c in "abcdef0123456789" for c in ud.revisions[name]]):
+ if ud.revisions[name]:
+ ud.unresolvedrev[name] = ud.revisions[name]
+ ud.revisions[name] = self.latest_revision(ud, d, name)
+
+ gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.'))
+ if gitsrcname.startswith('.'):
+ gitsrcname = gitsrcname[1:]
+
+ # for rebaseable git repo, it is necessary to keep mirror tar ball
+ # per revision, so that even the revision disappears from the
+ # upstream repo in the future, the mirror will remain intact and still
+ # contains the revision
+ if ud.rebaseable:
+ for name in ud.names:
+ gitsrcname = gitsrcname + '_' + ud.revisions[name]
+ ud.mirrortarball = 'git2_%s.tar.gz' % (gitsrcname)
+ ud.fullmirror = os.path.join(d.getVar("DL_DIR", True), ud.mirrortarball)
+ gitdir = d.getVar("GITDIR", True) or (d.getVar("DL_DIR", True) + "/git2/")
+ ud.clonedir = os.path.join(gitdir, gitsrcname)
+
+ ud.localfile = ud.clonedir
+
+ def localpath(self, ud, d):
+ return ud.clonedir
+
+ def need_update(self, ud, d):
+ if not os.path.exists(ud.clonedir):
+ return True
+ os.chdir(ud.clonedir)
+ for name in ud.names:
+ if not self._contains_ref(ud, d, name):
+ return True
+ if ud.write_tarballs and not os.path.exists(ud.fullmirror):
+ return True
+ return False
+
+ def try_premirror(self, ud, d):
+ # If we don't do this, updating an existing checkout with only premirrors
+ # is not possible
+ if d.getVar("BB_FETCH_PREMIRRORONLY", True) is not None:
+ return True
+ if os.path.exists(ud.clonedir):
+ return False
+ return True
+
+ def download(self, ud, d):
+ """Fetch url"""
+
+ ud.repochanged = not os.path.exists(ud.fullmirror)
+
+ # If the checkout doesn't exist and the mirror tarball does, extract it
+ if not os.path.exists(ud.clonedir) and os.path.exists(ud.fullmirror):
+ bb.utils.mkdirhier(ud.clonedir)
+ os.chdir(ud.clonedir)
+ runfetchcmd("tar -xzf %s" % (ud.fullmirror), d)
+
+ repourl = self._get_repo_url(ud)
+
+ # If the repo still doesn't exist, fallback to cloning it
+ if not os.path.exists(ud.clonedir):
+ # We do this since git will use a "-l" option automatically for local urls where possible
+ if repourl.startswith("file://"):
+ repourl = repourl[7:]
+ clone_cmd = "%s clone --bare --mirror %s %s" % (ud.basecmd, repourl, ud.clonedir)
+ if ud.proto.lower() != 'file':
+ bb.fetch2.check_network_access(d, clone_cmd)
+ runfetchcmd(clone_cmd, d)
+
+ os.chdir(ud.clonedir)
+ # Update the checkout if needed
+ needupdate = False
+ for name in ud.names:
+ if not self._contains_ref(ud, d, name):
+ needupdate = True
+ if needupdate:
+ try:
+ runfetchcmd("%s remote rm origin" % ud.basecmd, d)
+ except bb.fetch2.FetchError:
+ logger.debug(1, "No Origin")
+
+ runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, repourl), d)
+ fetch_cmd = "%s fetch -f --prune %s refs/*:refs/*" % (ud.basecmd, repourl)
+ if ud.proto.lower() != 'file':
+ bb.fetch2.check_network_access(d, fetch_cmd, ud.url)
+ runfetchcmd(fetch_cmd, d)
+ runfetchcmd("%s prune-packed" % ud.basecmd, d)
+ runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d)
+ ud.repochanged = True
+ os.chdir(ud.clonedir)
+ for name in ud.names:
+ if not self._contains_ref(ud, d, name):
+ raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revisions[name], ud.branches[name]))
+
+ def build_mirror_data(self, ud, d):
+ # Generate a mirror tarball if needed
+ if ud.write_tarballs and (ud.repochanged or not os.path.exists(ud.fullmirror)):
+ # it's possible that this symlink points to read-only filesystem with PREMIRROR
+ if os.path.islink(ud.fullmirror):
+ os.unlink(ud.fullmirror)
+
+ os.chdir(ud.clonedir)
+ logger.info("Creating tarball of git repository")
+ runfetchcmd("tar -czf %s %s" % (ud.fullmirror, os.path.join(".") ), d)
+ runfetchcmd("touch %s.done" % (ud.fullmirror), d)
+
+ def unpack(self, ud, destdir, d):
+ """ unpack the downloaded src to destdir"""
+
+ subdir = ud.parm.get("subpath", "")
+ if subdir != "":
+ readpathspec = ":%s" % (subdir)
+ def_destsuffix = "%s/" % os.path.basename(subdir.rstrip('/'))
+ else:
+ readpathspec = ""
+ def_destsuffix = "git/"
+
+ destsuffix = ud.parm.get("destsuffix", def_destsuffix)
+ destdir = ud.destdir = os.path.join(destdir, destsuffix)
+ if os.path.exists(destdir):
+ bb.utils.prunedir(destdir)
+
+ cloneflags = "-s -n"
+ if ud.bareclone:
+ cloneflags += " --mirror"
+
+ # Versions of git prior to 1.7.9.2 have issues where foo.git and foo get confused
+ # and you end up with some horrible union of the two when you attempt to clone it
+ # The least invasive workaround seems to be a symlink to the real directory to
+ # fool git into ignoring any .git version that may also be present.
+ #
+ # The issue is fixed in more recent versions of git so we can drop this hack in future
+ # when that version becomes common enough.
+ clonedir = ud.clonedir
+ if not ud.path.endswith(".git"):
+ indirectiondir = destdir[:-1] + ".indirectionsymlink"
+ if os.path.exists(indirectiondir):
+ os.remove(indirectiondir)
+ bb.utils.mkdirhier(os.path.dirname(indirectiondir))
+ os.symlink(ud.clonedir, indirectiondir)
+ clonedir = indirectiondir
+
+ runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, cloneflags, clonedir, destdir), d)
+ os.chdir(destdir)
+ repourl = self._get_repo_url(ud)
+ runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, repourl), d)
+ if not ud.nocheckout:
+ if subdir != "":
+ runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revisions[ud.names[0]], readpathspec), d)
+ runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d)
+ elif not ud.nobranch:
+ branchname = ud.branches[ud.names[0]]
+ runfetchcmd("%s checkout -B %s %s" % (ud.basecmd, branchname, \
+ ud.revisions[ud.names[0]]), d)
+ runfetchcmd("%s branch --set-upstream %s origin/%s" % (ud.basecmd, branchname, \
+ branchname), d)
+ else:
+ runfetchcmd("%s checkout %s" % (ud.basecmd, ud.revisions[ud.names[0]]), d)
+
+ return True
+
+ def clean(self, ud, d):
+ """ clean the git directory """
+
+ bb.utils.remove(ud.localpath, True)
+ bb.utils.remove(ud.fullmirror)
+ bb.utils.remove(ud.fullmirror + ".done")
+
+ def supports_srcrev(self):
+ return True
+
+ def _contains_ref(self, ud, d, name):
+ cmd = ""
+ if ud.nobranch:
+ cmd = "%s log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % (
+ ud.basecmd, ud.revisions[name])
+ else:
+ cmd = "%s branch --contains %s --list %s 2> /dev/null | wc -l" % (
+ ud.basecmd, ud.revisions[name], ud.branches[name])
+ try:
+ output = runfetchcmd(cmd, d, quiet=True)
+ except bb.fetch2.FetchError:
+ return False
+ if len(output.split()) > 1:
+ raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output))
+ return output.split()[0] != "0"
+
+ def _get_repo_url(self, ud):
+ """
+ Return the repository URL
+ """
+ if ud.user:
+ username = ud.user + '@'
+ else:
+ username = ""
+ return "%s://%s%s%s" % (ud.proto, username, ud.host, ud.path)
+
+ def _revision_key(self, ud, d, name):
+ """
+ Return a unique key for the url
+ """
+ return "git:" + ud.host + ud.path.replace('/', '.') + ud.unresolvedrev[name]
+
+ def _lsremote(self, ud, d, search):
+ """
+ Run git ls-remote with the specified search string
+ """
+ repourl = self._get_repo_url(ud)
+ cmd = "%s ls-remote %s %s" % \
+ (ud.basecmd, repourl, search)
+ if ud.proto.lower() != 'file':
+ bb.fetch2.check_network_access(d, cmd)
+ output = runfetchcmd(cmd, d, True)
+ if not output:
+ raise bb.fetch2.FetchError("The command %s gave empty output unexpectedly" % cmd, ud.url)
+ return output
+
+ def _latest_revision(self, ud, d, name):
+ """
+ Compute the HEAD revision for the url
+ """
+ output = self._lsremote(ud, d, "")
+ # Tags of the form ^{} may not work, need to fallback to other form
+ if ud.unresolvedrev[name][:5] == "refs/":
+ head = ud.unresolvedrev[name]
+ tag = ud.unresolvedrev[name]
+ else:
+ head = "refs/heads/%s" % ud.unresolvedrev[name]
+ tag = "refs/tags/%s" % ud.unresolvedrev[name]
+ for s in [head, tag + "^{}", tag]:
+ for l in output.split('\n'):
+ if s in l:
+ return l.split()[0]
+ raise bb.fetch2.FetchError("Unable to resolve '%s' in upstream git repository in git ls-remote output for %s" % \
+ (ud.unresolvedrev[name], ud.host+ud.path))
+
+ def latest_versionstring(self, ud, d):
+ """
+ Compute the latest release name like "x.y.x" in "x.y.x+gitHASH"
+ by searching through the tags output of ls-remote, comparing
+ versions and returning the highest match.
+ """
+ pupver = ('', '')
+
+ tagregex = re.compile(d.getVar('GITTAGREGEX', True) or "(?P<pver>([0-9][\.|_]?)+)")
+ try:
+ output = self._lsremote(ud, d, "refs/tags/*")
+ except bb.fetch2.FetchError or bb.fetch2.NetworkAccess:
+ return pupver
+
+ verstring = ""
+ revision = ""
+ for line in output.split("\n"):
+ if not line:
+ break
+
+ tag_head = line.split("/")[-1]
+ # Ignore non-released branches
+ m = re.search("(alpha|beta|rc|final)+", tag_head)
+ if m:
+ continue
+
+ # search for version in the line
+ tag = tagregex.search(tag_head)
+ if tag == None:
+ continue
+
+ tag = tag.group('pver')
+ tag = tag.replace("_", ".")
+
+ if verstring and bb.utils.vercmp(("0", tag, ""), ("0", verstring, "")) < 0:
+ continue
+
+ verstring = tag
+ revision = line.split()[0]
+ pupver = (verstring, revision)
+
+ return pupver
+
+ def _build_revision(self, ud, d, name):
+ return ud.revisions[name]
+
+ def gitpkgv_revision(self, ud, d, name):
+ """
+ Return a sortable revision number by counting commits in the history
+ Based on gitpkgv.bblass in meta-openembedded
+ """
+ rev = self._build_revision(ud, d, name)
+ localpath = ud.localpath
+ rev_file = os.path.join(localpath, "oe-gitpkgv_" + rev)
+ if not os.path.exists(localpath):
+ commits = None
+ else:
+ if not os.path.exists(rev_file) or not os.path.getsize(rev_file):
+ from pipes import quote
+ commits = bb.fetch2.runfetchcmd(
+ "git rev-list %s -- | wc -l" % (quote(rev)),
+ d, quiet=True).strip().lstrip('0')
+ if commits:
+ open(rev_file, "w").write("%d\n" % int(commits))
+ else:
+ commits = open(rev_file, "r").readline(128).strip()
+ if commits:
+ return False, "%s+%s" % (commits, rev[:7])
+ else:
+ return True, str(rev)
+
+ def checkstatus(self, fetch, ud, d):
+ try:
+ self._lsremote(ud, d, "")
+ return True
+ except FetchError:
+ return False