Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1 | # ex:ts=4:sw=4:sts=4:et |
| 2 | # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- |
| 3 | """ |
| 4 | BitBake 'Fetch' git implementation |
| 5 | |
| 6 | git fetcher support the SRC_URI with format of: |
| 7 | SRC_URI = "git://some.host/somepath;OptionA=xxx;OptionB=xxx;..." |
| 8 | |
| 9 | Supported SRC_URI options are: |
| 10 | |
| 11 | - branch |
| 12 | The git branch to retrieve from. The default is "master" |
| 13 | |
| 14 | This option also supports multiple branch fetching, with branches |
| 15 | separated by commas. In multiple branches case, the name option |
| 16 | must have the same number of names to match the branches, which is |
| 17 | used to specify the SRC_REV for the branch |
| 18 | e.g: |
| 19 | SRC_URI="git://some.host/somepath;branch=branchX,branchY;name=nameX,nameY" |
| 20 | SRCREV_nameX = "xxxxxxxxxxxxxxxxxxxx" |
| 21 | SRCREV_nameY = "YYYYYYYYYYYYYYYYYYYY" |
| 22 | |
| 23 | - tag |
| 24 | The git tag to retrieve. The default is "master" |
| 25 | |
| 26 | - protocol |
| 27 | The method to use to access the repository. Common options are "git", |
| 28 | "http", "https", "file", "ssh" and "rsync". The default is "git". |
| 29 | |
| 30 | - rebaseable |
| 31 | rebaseable indicates that the upstream git repo may rebase in the future, |
| 32 | and current revision may disappear from upstream repo. This option will |
| 33 | remind fetcher to preserve local cache carefully for future use. |
| 34 | The default value is "0", set rebaseable=1 for rebaseable git repo. |
| 35 | |
| 36 | - nocheckout |
| 37 | Don't checkout source code when unpacking. set this option for the recipe |
| 38 | who has its own routine to checkout code. |
| 39 | The default is "0", set nocheckout=1 if needed. |
| 40 | |
| 41 | - bareclone |
| 42 | Create a bare clone of the source code and don't checkout the source code |
| 43 | when unpacking. Set this option for the recipe who has its own routine to |
| 44 | checkout code and tracking branch requirements. |
| 45 | The default is "0", set bareclone=1 if needed. |
| 46 | |
| 47 | - nobranch |
| 48 | Don't check the SHA validation for branch. set this option for the recipe |
| 49 | referring to commit which is valid in tag instead of branch. |
| 50 | The default is "0", set nobranch=1 if needed. |
| 51 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame^] | 52 | - usehead |
| 53 | For local git:// urls to use the current branch HEAD as the revsion for use with |
| 54 | AUTOREV. Implies nobranch. |
| 55 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 56 | """ |
| 57 | |
| 58 | #Copyright (C) 2005 Richard Purdie |
| 59 | # |
| 60 | # This program is free software; you can redistribute it and/or modify |
| 61 | # it under the terms of the GNU General Public License version 2 as |
| 62 | # published by the Free Software Foundation. |
| 63 | # |
| 64 | # This program is distributed in the hope that it will be useful, |
| 65 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 66 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 67 | # GNU General Public License for more details. |
| 68 | # |
| 69 | # You should have received a copy of the GNU General Public License along |
| 70 | # with this program; if not, write to the Free Software Foundation, Inc., |
| 71 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. |
| 72 | |
Patrick Williams | d7e9631 | 2015-09-22 08:09:05 -0500 | [diff] [blame] | 73 | import errno |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 74 | import os |
| 75 | import re |
| 76 | import bb |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 77 | import errno |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame^] | 78 | import bb.progress |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 79 | from bb import data |
| 80 | from bb.fetch2 import FetchMethod |
| 81 | from bb.fetch2 import runfetchcmd |
| 82 | from bb.fetch2 import logger |
| 83 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame^] | 84 | |
| 85 | class GitProgressHandler(bb.progress.LineFilterProgressHandler): |
| 86 | """Extract progress information from git output""" |
| 87 | def __init__(self, d): |
| 88 | self._buffer = '' |
| 89 | self._count = 0 |
| 90 | super(GitProgressHandler, self).__init__(d) |
| 91 | # Send an initial progress event so the bar gets shown |
| 92 | self._fire_progress(-1) |
| 93 | |
| 94 | def write(self, string): |
| 95 | self._buffer += string |
| 96 | stages = ['Counting objects', 'Compressing objects', 'Receiving objects', 'Resolving deltas'] |
| 97 | stage_weights = [0.2, 0.05, 0.5, 0.25] |
| 98 | stagenum = 0 |
| 99 | for i, stage in reversed(list(enumerate(stages))): |
| 100 | if stage in self._buffer: |
| 101 | stagenum = i |
| 102 | self._buffer = '' |
| 103 | break |
| 104 | self._status = stages[stagenum] |
| 105 | percs = re.findall(r'(\d+)%', string) |
| 106 | if percs: |
| 107 | progress = int(round((int(percs[-1]) * stage_weights[stagenum]) + (sum(stage_weights[:stagenum]) * 100))) |
| 108 | rates = re.findall(r'([\d.]+ [a-zA-Z]*/s+)', string) |
| 109 | if rates: |
| 110 | rate = rates[-1] |
| 111 | else: |
| 112 | rate = None |
| 113 | self.update(progress, rate) |
| 114 | else: |
| 115 | if stagenum == 0: |
| 116 | percs = re.findall(r': (\d+)', string) |
| 117 | if percs: |
| 118 | count = int(percs[-1]) |
| 119 | if count > self._count: |
| 120 | self._count = count |
| 121 | self._fire_progress(-count) |
| 122 | super(GitProgressHandler, self).write(string) |
| 123 | |
| 124 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 125 | class Git(FetchMethod): |
| 126 | """Class to fetch a module or modules from git repositories""" |
| 127 | def init(self, d): |
| 128 | pass |
| 129 | |
| 130 | def supports(self, ud, d): |
| 131 | """ |
| 132 | Check to see if a given url can be fetched with git. |
| 133 | """ |
| 134 | return ud.type in ['git'] |
| 135 | |
| 136 | def supports_checksum(self, urldata): |
| 137 | return False |
| 138 | |
| 139 | def urldata_init(self, ud, d): |
| 140 | """ |
| 141 | init git specific variable within url data |
| 142 | so that the git method like latest_revision() can work |
| 143 | """ |
| 144 | if 'protocol' in ud.parm: |
| 145 | ud.proto = ud.parm['protocol'] |
| 146 | elif not ud.host: |
| 147 | ud.proto = 'file' |
| 148 | else: |
| 149 | ud.proto = "git" |
| 150 | |
| 151 | if not ud.proto in ('git', 'file', 'ssh', 'http', 'https', 'rsync'): |
| 152 | raise bb.fetch2.ParameterError("Invalid protocol type", ud.url) |
| 153 | |
| 154 | ud.nocheckout = ud.parm.get("nocheckout","0") == "1" |
| 155 | |
| 156 | ud.rebaseable = ud.parm.get("rebaseable","0") == "1" |
| 157 | |
| 158 | ud.nobranch = ud.parm.get("nobranch","0") == "1" |
| 159 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame^] | 160 | # usehead implies nobranch |
| 161 | ud.usehead = ud.parm.get("usehead","0") == "1" |
| 162 | if ud.usehead: |
| 163 | if ud.proto != "file": |
| 164 | raise bb.fetch2.ParameterError("The usehead option is only for use with local ('protocol=file') git repositories", ud.url) |
| 165 | ud.nobranch = 1 |
| 166 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 167 | # bareclone implies nocheckout |
| 168 | ud.bareclone = ud.parm.get("bareclone","0") == "1" |
| 169 | if ud.bareclone: |
| 170 | ud.nocheckout = 1 |
| 171 | |
| 172 | ud.unresolvedrev = {} |
| 173 | branches = ud.parm.get("branch", "master").split(',') |
| 174 | if len(branches) != len(ud.names): |
| 175 | raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url) |
| 176 | ud.branches = {} |
| 177 | for name in ud.names: |
| 178 | branch = branches[ud.names.index(name)] |
| 179 | ud.branches[name] = branch |
| 180 | ud.unresolvedrev[name] = branch |
| 181 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame^] | 182 | if ud.usehead: |
| 183 | ud.unresolvedrev['default'] = 'HEAD' |
| 184 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 185 | ud.basecmd = data.getVar("FETCHCMD_git", d, True) or "git -c core.fsyncobjectfiles=0" |
| 186 | |
| 187 | ud.write_tarballs = ((data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True) or "0") != "0") or ud.rebaseable |
| 188 | |
| 189 | ud.setup_revisons(d) |
| 190 | |
| 191 | for name in ud.names: |
| 192 | # Ensure anything that doesn't look like a sha256 checksum/revision is translated into one |
| 193 | if not ud.revisions[name] or len(ud.revisions[name]) != 40 or (False in [c in "abcdef0123456789" for c in ud.revisions[name]]): |
| 194 | if ud.revisions[name]: |
| 195 | ud.unresolvedrev[name] = ud.revisions[name] |
| 196 | ud.revisions[name] = self.latest_revision(ud, d, name) |
| 197 | |
| 198 | gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.')) |
| 199 | if gitsrcname.startswith('.'): |
| 200 | gitsrcname = gitsrcname[1:] |
| 201 | |
| 202 | # for rebaseable git repo, it is necessary to keep mirror tar ball |
| 203 | # per revision, so that even the revision disappears from the |
| 204 | # upstream repo in the future, the mirror will remain intact and still |
| 205 | # contains the revision |
| 206 | if ud.rebaseable: |
| 207 | for name in ud.names: |
| 208 | gitsrcname = gitsrcname + '_' + ud.revisions[name] |
| 209 | ud.mirrortarball = 'git2_%s.tar.gz' % (gitsrcname) |
| 210 | ud.fullmirror = os.path.join(d.getVar("DL_DIR", True), ud.mirrortarball) |
| 211 | gitdir = d.getVar("GITDIR", True) or (d.getVar("DL_DIR", True) + "/git2/") |
| 212 | ud.clonedir = os.path.join(gitdir, gitsrcname) |
| 213 | |
| 214 | ud.localfile = ud.clonedir |
| 215 | |
| 216 | def localpath(self, ud, d): |
| 217 | return ud.clonedir |
| 218 | |
| 219 | def need_update(self, ud, d): |
| 220 | if not os.path.exists(ud.clonedir): |
| 221 | return True |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 222 | for name in ud.names: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame^] | 223 | if not self._contains_ref(ud, d, name, ud.clonedir): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 224 | return True |
| 225 | if ud.write_tarballs and not os.path.exists(ud.fullmirror): |
| 226 | return True |
| 227 | return False |
| 228 | |
| 229 | def try_premirror(self, ud, d): |
| 230 | # If we don't do this, updating an existing checkout with only premirrors |
| 231 | # is not possible |
| 232 | if d.getVar("BB_FETCH_PREMIRRORONLY", True) is not None: |
| 233 | return True |
| 234 | if os.path.exists(ud.clonedir): |
| 235 | return False |
| 236 | return True |
| 237 | |
| 238 | def download(self, ud, d): |
| 239 | """Fetch url""" |
| 240 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 241 | # If the checkout doesn't exist and the mirror tarball does, extract it |
| 242 | if not os.path.exists(ud.clonedir) and os.path.exists(ud.fullmirror): |
| 243 | bb.utils.mkdirhier(ud.clonedir) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame^] | 244 | runfetchcmd("tar -xzf %s" % (ud.fullmirror), d, workdir=ud.clonedir) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 245 | |
| 246 | repourl = self._get_repo_url(ud) |
| 247 | |
| 248 | # If the repo still doesn't exist, fallback to cloning it |
| 249 | if not os.path.exists(ud.clonedir): |
| 250 | # We do this since git will use a "-l" option automatically for local urls where possible |
| 251 | if repourl.startswith("file://"): |
| 252 | repourl = repourl[7:] |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame^] | 253 | clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, repourl, ud.clonedir) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 254 | if ud.proto.lower() != 'file': |
| 255 | bb.fetch2.check_network_access(d, clone_cmd) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame^] | 256 | progresshandler = GitProgressHandler(d) |
| 257 | runfetchcmd(clone_cmd, d, log=progresshandler) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 258 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 259 | # Update the checkout if needed |
| 260 | needupdate = False |
| 261 | for name in ud.names: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame^] | 262 | if not self._contains_ref(ud, d, name, ud.clonedir): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 263 | needupdate = True |
| 264 | if needupdate: |
| 265 | try: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame^] | 266 | runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 267 | except bb.fetch2.FetchError: |
| 268 | logger.debug(1, "No Origin") |
| 269 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame^] | 270 | runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, repourl), d, workdir=ud.clonedir) |
| 271 | fetch_cmd = "LANG=C %s fetch -f --prune --progress %s refs/*:refs/*" % (ud.basecmd, repourl) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 272 | if ud.proto.lower() != 'file': |
| 273 | bb.fetch2.check_network_access(d, fetch_cmd, ud.url) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame^] | 274 | progresshandler = GitProgressHandler(d) |
| 275 | runfetchcmd(fetch_cmd, d, log=progresshandler, workdir=ud.clonedir) |
| 276 | runfetchcmd("%s prune-packed" % ud.basecmd, d, workdir=ud.clonedir) |
| 277 | runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d, workdir=ud.clonedir) |
Patrick Williams | d7e9631 | 2015-09-22 08:09:05 -0500 | [diff] [blame] | 278 | try: |
| 279 | os.unlink(ud.fullmirror) |
| 280 | except OSError as exc: |
| 281 | if exc.errno != errno.ENOENT: |
| 282 | raise |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 283 | for name in ud.names: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame^] | 284 | if not self._contains_ref(ud, d, name, ud.clonedir): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 285 | raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revisions[name], ud.branches[name])) |
| 286 | |
| 287 | def build_mirror_data(self, ud, d): |
| 288 | # Generate a mirror tarball if needed |
Patrick Williams | d7e9631 | 2015-09-22 08:09:05 -0500 | [diff] [blame] | 289 | if ud.write_tarballs and not os.path.exists(ud.fullmirror): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 290 | # it's possible that this symlink points to read-only filesystem with PREMIRROR |
| 291 | if os.path.islink(ud.fullmirror): |
| 292 | os.unlink(ud.fullmirror) |
| 293 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 294 | logger.info("Creating tarball of git repository") |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame^] | 295 | runfetchcmd("tar -czf %s %s" % (ud.fullmirror, os.path.join(".") ), d, workdir=ud.clonedir) |
| 296 | runfetchcmd("touch %s.done" % (ud.fullmirror), d, workdir=ud.clonedir) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 297 | |
| 298 | def unpack(self, ud, destdir, d): |
| 299 | """ unpack the downloaded src to destdir""" |
| 300 | |
| 301 | subdir = ud.parm.get("subpath", "") |
| 302 | if subdir != "": |
| 303 | readpathspec = ":%s" % (subdir) |
| 304 | def_destsuffix = "%s/" % os.path.basename(subdir.rstrip('/')) |
| 305 | else: |
| 306 | readpathspec = "" |
| 307 | def_destsuffix = "git/" |
| 308 | |
| 309 | destsuffix = ud.parm.get("destsuffix", def_destsuffix) |
| 310 | destdir = ud.destdir = os.path.join(destdir, destsuffix) |
| 311 | if os.path.exists(destdir): |
| 312 | bb.utils.prunedir(destdir) |
| 313 | |
| 314 | cloneflags = "-s -n" |
| 315 | if ud.bareclone: |
| 316 | cloneflags += " --mirror" |
| 317 | |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 318 | runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, cloneflags, ud.clonedir, destdir), d) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 319 | repourl = self._get_repo_url(ud) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame^] | 320 | runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, repourl), d, workdir=destdir) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 321 | if not ud.nocheckout: |
| 322 | if subdir != "": |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame^] | 323 | runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revisions[ud.names[0]], readpathspec), d, |
| 324 | workdir=destdir) |
| 325 | runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d, workdir=destdir) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 326 | elif not ud.nobranch: |
| 327 | branchname = ud.branches[ud.names[0]] |
| 328 | runfetchcmd("%s checkout -B %s %s" % (ud.basecmd, branchname, \ |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame^] | 329 | ud.revisions[ud.names[0]]), d, workdir=destdir) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 330 | runfetchcmd("%s branch --set-upstream %s origin/%s" % (ud.basecmd, branchname, \ |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame^] | 331 | branchname), d, workdir=destdir) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 332 | else: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame^] | 333 | runfetchcmd("%s checkout %s" % (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=destdir) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 334 | |
| 335 | return True |
| 336 | |
| 337 | def clean(self, ud, d): |
| 338 | """ clean the git directory """ |
| 339 | |
| 340 | bb.utils.remove(ud.localpath, True) |
| 341 | bb.utils.remove(ud.fullmirror) |
| 342 | bb.utils.remove(ud.fullmirror + ".done") |
| 343 | |
| 344 | def supports_srcrev(self): |
| 345 | return True |
| 346 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame^] | 347 | def _contains_ref(self, ud, d, name, wd): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 348 | cmd = "" |
| 349 | if ud.nobranch: |
| 350 | cmd = "%s log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % ( |
| 351 | ud.basecmd, ud.revisions[name]) |
| 352 | else: |
| 353 | cmd = "%s branch --contains %s --list %s 2> /dev/null | wc -l" % ( |
| 354 | ud.basecmd, ud.revisions[name], ud.branches[name]) |
| 355 | try: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame^] | 356 | output = runfetchcmd(cmd, d, quiet=True, workdir=wd) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 357 | except bb.fetch2.FetchError: |
| 358 | return False |
| 359 | if len(output.split()) > 1: |
| 360 | raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output)) |
| 361 | return output.split()[0] != "0" |
| 362 | |
| 363 | def _get_repo_url(self, ud): |
| 364 | """ |
| 365 | Return the repository URL |
| 366 | """ |
| 367 | if ud.user: |
| 368 | username = ud.user + '@' |
| 369 | else: |
| 370 | username = "" |
| 371 | return "%s://%s%s%s" % (ud.proto, username, ud.host, ud.path) |
| 372 | |
| 373 | def _revision_key(self, ud, d, name): |
| 374 | """ |
| 375 | Return a unique key for the url |
| 376 | """ |
| 377 | return "git:" + ud.host + ud.path.replace('/', '.') + ud.unresolvedrev[name] |
| 378 | |
| 379 | def _lsremote(self, ud, d, search): |
| 380 | """ |
| 381 | Run git ls-remote with the specified search string |
| 382 | """ |
| 383 | repourl = self._get_repo_url(ud) |
| 384 | cmd = "%s ls-remote %s %s" % \ |
| 385 | (ud.basecmd, repourl, search) |
| 386 | if ud.proto.lower() != 'file': |
| 387 | bb.fetch2.check_network_access(d, cmd) |
| 388 | output = runfetchcmd(cmd, d, True) |
| 389 | if not output: |
| 390 | raise bb.fetch2.FetchError("The command %s gave empty output unexpectedly" % cmd, ud.url) |
| 391 | return output |
| 392 | |
| 393 | def _latest_revision(self, ud, d, name): |
| 394 | """ |
| 395 | Compute the HEAD revision for the url |
| 396 | """ |
| 397 | output = self._lsremote(ud, d, "") |
| 398 | # Tags of the form ^{} may not work, need to fallback to other form |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame^] | 399 | if ud.unresolvedrev[name][:5] == "refs/" or ud.usehead: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 400 | head = ud.unresolvedrev[name] |
| 401 | tag = ud.unresolvedrev[name] |
| 402 | else: |
| 403 | head = "refs/heads/%s" % ud.unresolvedrev[name] |
| 404 | tag = "refs/tags/%s" % ud.unresolvedrev[name] |
| 405 | for s in [head, tag + "^{}", tag]: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame^] | 406 | for l in output.strip().split('\n'): |
| 407 | sha1, ref = l.split() |
| 408 | if s == ref: |
| 409 | return sha1 |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 410 | raise bb.fetch2.FetchError("Unable to resolve '%s' in upstream git repository in git ls-remote output for %s" % \ |
| 411 | (ud.unresolvedrev[name], ud.host+ud.path)) |
| 412 | |
| 413 | def latest_versionstring(self, ud, d): |
| 414 | """ |
| 415 | Compute the latest release name like "x.y.x" in "x.y.x+gitHASH" |
| 416 | by searching through the tags output of ls-remote, comparing |
| 417 | versions and returning the highest match. |
| 418 | """ |
| 419 | pupver = ('', '') |
| 420 | |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 421 | tagregex = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX', True) or "(?P<pver>([0-9][\.|_]?)+)") |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 422 | try: |
| 423 | output = self._lsremote(ud, d, "refs/tags/*") |
| 424 | except bb.fetch2.FetchError or bb.fetch2.NetworkAccess: |
| 425 | return pupver |
| 426 | |
| 427 | verstring = "" |
| 428 | revision = "" |
| 429 | for line in output.split("\n"): |
| 430 | if not line: |
| 431 | break |
| 432 | |
| 433 | tag_head = line.split("/")[-1] |
| 434 | # Ignore non-released branches |
| 435 | m = re.search("(alpha|beta|rc|final)+", tag_head) |
| 436 | if m: |
| 437 | continue |
| 438 | |
| 439 | # search for version in the line |
| 440 | tag = tagregex.search(tag_head) |
| 441 | if tag == None: |
| 442 | continue |
| 443 | |
| 444 | tag = tag.group('pver') |
| 445 | tag = tag.replace("_", ".") |
| 446 | |
| 447 | if verstring and bb.utils.vercmp(("0", tag, ""), ("0", verstring, "")) < 0: |
| 448 | continue |
| 449 | |
| 450 | verstring = tag |
| 451 | revision = line.split()[0] |
| 452 | pupver = (verstring, revision) |
| 453 | |
| 454 | return pupver |
| 455 | |
| 456 | def _build_revision(self, ud, d, name): |
| 457 | return ud.revisions[name] |
| 458 | |
| 459 | def gitpkgv_revision(self, ud, d, name): |
| 460 | """ |
| 461 | Return a sortable revision number by counting commits in the history |
| 462 | Based on gitpkgv.bblass in meta-openembedded |
| 463 | """ |
| 464 | rev = self._build_revision(ud, d, name) |
| 465 | localpath = ud.localpath |
| 466 | rev_file = os.path.join(localpath, "oe-gitpkgv_" + rev) |
| 467 | if not os.path.exists(localpath): |
| 468 | commits = None |
| 469 | else: |
| 470 | if not os.path.exists(rev_file) or not os.path.getsize(rev_file): |
| 471 | from pipes import quote |
| 472 | commits = bb.fetch2.runfetchcmd( |
| 473 | "git rev-list %s -- | wc -l" % (quote(rev)), |
| 474 | d, quiet=True).strip().lstrip('0') |
| 475 | if commits: |
| 476 | open(rev_file, "w").write("%d\n" % int(commits)) |
| 477 | else: |
| 478 | commits = open(rev_file, "r").readline(128).strip() |
| 479 | if commits: |
| 480 | return False, "%s+%s" % (commits, rev[:7]) |
| 481 | else: |
| 482 | return True, str(rev) |
| 483 | |
| 484 | def checkstatus(self, fetch, ud, d): |
| 485 | try: |
| 486 | self._lsremote(ud, d, "") |
| 487 | return True |
| 488 | except FetchError: |
| 489 | return False |