blob: 5ef8cd69e275c20c7a12f285708c03089ff464c2 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# ex:ts=4:sw=4:sts=4:et
2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
3"""
4BitBake 'Fetch' git implementation
5
6git fetcher support the SRC_URI with format of:
7SRC_URI = "git://some.host/somepath;OptionA=xxx;OptionB=xxx;..."
8
9Supported SRC_URI options are:
10
11- branch
12 The git branch to retrieve from. The default is "master"
13
14 This option also supports multiple branch fetching, with branches
15 separated by commas. In multiple branches case, the name option
16 must have the same number of names to match the branches, which is
17 used to specify the SRC_REV for the branch
18 e.g:
19 SRC_URI="git://some.host/somepath;branch=branchX,branchY;name=nameX,nameY"
20 SRCREV_nameX = "xxxxxxxxxxxxxxxxxxxx"
21 SRCREV_nameY = "YYYYYYYYYYYYYYYYYYYY"
22
23- tag
24 The git tag to retrieve. The default is "master"
25
26- protocol
27 The method to use to access the repository. Common options are "git",
28 "http", "https", "file", "ssh" and "rsync". The default is "git".
29
30- rebaseable
31 rebaseable indicates that the upstream git repo may rebase in the future,
32 and current revision may disappear from upstream repo. This option will
33 remind fetcher to preserve local cache carefully for future use.
34 The default value is "0", set rebaseable=1 for rebaseable git repo.
35
36- nocheckout
37 Don't checkout source code when unpacking. set this option for the recipe
38 who has its own routine to checkout code.
39 The default is "0", set nocheckout=1 if needed.
40
41- bareclone
42 Create a bare clone of the source code and don't checkout the source code
43 when unpacking. Set this option for the recipe who has its own routine to
44 checkout code and tracking branch requirements.
45 The default is "0", set bareclone=1 if needed.
46
47- nobranch
48 Don't check the SHA validation for branch. set this option for the recipe
49 referring to commit which is valid in tag instead of branch.
50 The default is "0", set nobranch=1 if needed.
51
Patrick Williamsc0f7c042017-02-23 20:41:17 -060052- usehead
Brad Bishop6e60e8b2018-02-01 10:27:11 -050053 For local git:// urls to use the current branch HEAD as the revision for use with
Patrick Williamsc0f7c042017-02-23 20:41:17 -060054 AUTOREV. Implies nobranch.
55
Patrick Williamsc124f4f2015-09-15 14:41:29 -050056"""
57
58#Copyright (C) 2005 Richard Purdie
59#
60# This program is free software; you can redistribute it and/or modify
61# it under the terms of the GNU General Public License version 2 as
62# published by the Free Software Foundation.
63#
64# This program is distributed in the hope that it will be useful,
65# but WITHOUT ANY WARRANTY; without even the implied warranty of
66# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
67# GNU General Public License for more details.
68#
69# You should have received a copy of the GNU General Public License along
70# with this program; if not, write to the Free Software Foundation, Inc.,
71# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
72
Brad Bishopd7bf8c12018-02-25 22:55:05 -050073import collections
Patrick Williamsd7e96312015-09-22 08:09:05 -050074import errno
Brad Bishopd7bf8c12018-02-25 22:55:05 -050075import fnmatch
Patrick Williamsc124f4f2015-09-15 14:41:29 -050076import os
77import re
Brad Bishopd7bf8c12018-02-25 22:55:05 -050078import subprocess
79import tempfile
Patrick Williamsc124f4f2015-09-15 14:41:29 -050080import bb
Patrick Williamsc0f7c042017-02-23 20:41:17 -060081import bb.progress
Patrick Williamsc124f4f2015-09-15 14:41:29 -050082from bb.fetch2 import FetchMethod
83from bb.fetch2 import runfetchcmd
84from bb.fetch2 import logger
85
Patrick Williamsc0f7c042017-02-23 20:41:17 -060086
87class GitProgressHandler(bb.progress.LineFilterProgressHandler):
88 """Extract progress information from git output"""
89 def __init__(self, d):
90 self._buffer = ''
91 self._count = 0
92 super(GitProgressHandler, self).__init__(d)
93 # Send an initial progress event so the bar gets shown
94 self._fire_progress(-1)
95
96 def write(self, string):
97 self._buffer += string
98 stages = ['Counting objects', 'Compressing objects', 'Receiving objects', 'Resolving deltas']
99 stage_weights = [0.2, 0.05, 0.5, 0.25]
100 stagenum = 0
101 for i, stage in reversed(list(enumerate(stages))):
102 if stage in self._buffer:
103 stagenum = i
104 self._buffer = ''
105 break
106 self._status = stages[stagenum]
107 percs = re.findall(r'(\d+)%', string)
108 if percs:
109 progress = int(round((int(percs[-1]) * stage_weights[stagenum]) + (sum(stage_weights[:stagenum]) * 100)))
110 rates = re.findall(r'([\d.]+ [a-zA-Z]*/s+)', string)
111 if rates:
112 rate = rates[-1]
113 else:
114 rate = None
115 self.update(progress, rate)
116 else:
117 if stagenum == 0:
118 percs = re.findall(r': (\d+)', string)
119 if percs:
120 count = int(percs[-1])
121 if count > self._count:
122 self._count = count
123 self._fire_progress(-count)
124 super(GitProgressHandler, self).write(string)
125
126
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500127class Git(FetchMethod):
128 """Class to fetch a module or modules from git repositories"""
129 def init(self, d):
130 pass
131
132 def supports(self, ud, d):
133 """
134 Check to see if a given url can be fetched with git.
135 """
136 return ud.type in ['git']
137
138 def supports_checksum(self, urldata):
139 return False
140
141 def urldata_init(self, ud, d):
142 """
143 init git specific variable within url data
144 so that the git method like latest_revision() can work
145 """
146 if 'protocol' in ud.parm:
147 ud.proto = ud.parm['protocol']
148 elif not ud.host:
149 ud.proto = 'file'
150 else:
151 ud.proto = "git"
152
153 if not ud.proto in ('git', 'file', 'ssh', 'http', 'https', 'rsync'):
154 raise bb.fetch2.ParameterError("Invalid protocol type", ud.url)
155
156 ud.nocheckout = ud.parm.get("nocheckout","0") == "1"
157
158 ud.rebaseable = ud.parm.get("rebaseable","0") == "1"
159
160 ud.nobranch = ud.parm.get("nobranch","0") == "1"
161
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600162 # usehead implies nobranch
163 ud.usehead = ud.parm.get("usehead","0") == "1"
164 if ud.usehead:
165 if ud.proto != "file":
166 raise bb.fetch2.ParameterError("The usehead option is only for use with local ('protocol=file') git repositories", ud.url)
167 ud.nobranch = 1
168
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500169 # bareclone implies nocheckout
170 ud.bareclone = ud.parm.get("bareclone","0") == "1"
171 if ud.bareclone:
172 ud.nocheckout = 1
173
174 ud.unresolvedrev = {}
175 branches = ud.parm.get("branch", "master").split(',')
176 if len(branches) != len(ud.names):
177 raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500178
179 ud.cloneflags = "-s -n"
180 if ud.bareclone:
181 ud.cloneflags += " --mirror"
182
183 ud.shallow = d.getVar("BB_GIT_SHALLOW") == "1"
184 ud.shallow_extra_refs = (d.getVar("BB_GIT_SHALLOW_EXTRA_REFS") or "").split()
185
186 depth_default = d.getVar("BB_GIT_SHALLOW_DEPTH")
187 if depth_default is not None:
188 try:
189 depth_default = int(depth_default or 0)
190 except ValueError:
191 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH: %s" % depth_default)
192 else:
193 if depth_default < 0:
194 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH: %s" % depth_default)
195 else:
196 depth_default = 1
197 ud.shallow_depths = collections.defaultdict(lambda: depth_default)
198
199 revs_default = d.getVar("BB_GIT_SHALLOW_REVS", True)
200 ud.shallow_revs = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500201 ud.branches = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500202 for pos, name in enumerate(ud.names):
203 branch = branches[pos]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500204 ud.branches[name] = branch
205 ud.unresolvedrev[name] = branch
206
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500207 shallow_depth = d.getVar("BB_GIT_SHALLOW_DEPTH_%s" % name)
208 if shallow_depth is not None:
209 try:
210 shallow_depth = int(shallow_depth or 0)
211 except ValueError:
212 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (name, shallow_depth))
213 else:
214 if shallow_depth < 0:
215 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (name, shallow_depth))
216 ud.shallow_depths[name] = shallow_depth
217
218 revs = d.getVar("BB_GIT_SHALLOW_REVS_%s" % name)
219 if revs is not None:
220 ud.shallow_revs.extend(revs.split())
221 elif revs_default is not None:
222 ud.shallow_revs.extend(revs_default.split())
223
224 if (ud.shallow and
225 not ud.shallow_revs and
226 all(ud.shallow_depths[n] == 0 for n in ud.names)):
227 # Shallow disabled for this URL
228 ud.shallow = False
229
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600230 if ud.usehead:
231 ud.unresolvedrev['default'] = 'HEAD'
232
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500233 ud.basecmd = d.getVar("FETCHCMD_git") or "git -c core.fsyncobjectfiles=0"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500234
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500235 write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0"
236 ud.write_tarballs = write_tarballs != "0" or ud.rebaseable
237 ud.write_shallow_tarballs = (d.getVar("BB_GENERATE_SHALLOW_TARBALLS") or write_tarballs) != "0"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500238
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500239 ud.setup_revisions(d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500240
241 for name in ud.names:
242 # Ensure anything that doesn't look like a sha256 checksum/revision is translated into one
243 if not ud.revisions[name] or len(ud.revisions[name]) != 40 or (False in [c in "abcdef0123456789" for c in ud.revisions[name]]):
244 if ud.revisions[name]:
245 ud.unresolvedrev[name] = ud.revisions[name]
246 ud.revisions[name] = self.latest_revision(ud, d, name)
247
248 gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.'))
249 if gitsrcname.startswith('.'):
250 gitsrcname = gitsrcname[1:]
251
252 # for rebaseable git repo, it is necessary to keep mirror tar ball
253 # per revision, so that even the revision disappears from the
254 # upstream repo in the future, the mirror will remain intact and still
255 # contains the revision
256 if ud.rebaseable:
257 for name in ud.names:
258 gitsrcname = gitsrcname + '_' + ud.revisions[name]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500259
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500260 dl_dir = d.getVar("DL_DIR")
261 gitdir = d.getVar("GITDIR") or (dl_dir + "/git2/")
262 ud.clonedir = os.path.join(gitdir, gitsrcname)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500263 ud.localfile = ud.clonedir
264
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500265 mirrortarball = 'git2_%s.tar.gz' % gitsrcname
266 ud.fullmirror = os.path.join(dl_dir, mirrortarball)
267 ud.mirrortarballs = [mirrortarball]
268 if ud.shallow:
269 tarballname = gitsrcname
270 if ud.bareclone:
271 tarballname = "%s_bare" % tarballname
272
273 if ud.shallow_revs:
274 tarballname = "%s_%s" % (tarballname, "_".join(sorted(ud.shallow_revs)))
275
276 for name, revision in sorted(ud.revisions.items()):
277 tarballname = "%s_%s" % (tarballname, ud.revisions[name][:7])
278 depth = ud.shallow_depths[name]
279 if depth:
280 tarballname = "%s-%s" % (tarballname, depth)
281
282 shallow_refs = []
283 if not ud.nobranch:
284 shallow_refs.extend(ud.branches.values())
285 if ud.shallow_extra_refs:
286 shallow_refs.extend(r.replace('refs/heads/', '').replace('*', 'ALL') for r in ud.shallow_extra_refs)
287 if shallow_refs:
288 tarballname = "%s_%s" % (tarballname, "_".join(sorted(shallow_refs)).replace('/', '.'))
289
290 fetcher = self.__class__.__name__.lower()
291 ud.shallowtarball = '%sshallow_%s.tar.gz' % (fetcher, tarballname)
292 ud.fullshallow = os.path.join(dl_dir, ud.shallowtarball)
293 ud.mirrortarballs.insert(0, ud.shallowtarball)
294
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500295 def localpath(self, ud, d):
296 return ud.clonedir
297
298 def need_update(self, ud, d):
299 if not os.path.exists(ud.clonedir):
300 return True
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500301 for name in ud.names:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600302 if not self._contains_ref(ud, d, name, ud.clonedir):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500303 return True
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500304 if ud.shallow and ud.write_shallow_tarballs and not os.path.exists(ud.fullshallow):
305 return True
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500306 if ud.write_tarballs and not os.path.exists(ud.fullmirror):
307 return True
308 return False
309
310 def try_premirror(self, ud, d):
311 # If we don't do this, updating an existing checkout with only premirrors
312 # is not possible
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500313 if d.getVar("BB_FETCH_PREMIRRORONLY") is not None:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500314 return True
315 if os.path.exists(ud.clonedir):
316 return False
317 return True
318
319 def download(self, ud, d):
320 """Fetch url"""
321
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500322 no_clone = not os.path.exists(ud.clonedir)
323 need_update = no_clone or self.need_update(ud, d)
324
325 # A current clone is preferred to either tarball, a shallow tarball is
326 # preferred to an out of date clone, and a missing clone will use
327 # either tarball.
328 if ud.shallow and os.path.exists(ud.fullshallow) and need_update:
329 ud.localpath = ud.fullshallow
330 return
331 elif os.path.exists(ud.fullmirror) and no_clone:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500332 bb.utils.mkdirhier(ud.clonedir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500333 runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500334
335 repourl = self._get_repo_url(ud)
336
337 # If the repo still doesn't exist, fallback to cloning it
338 if not os.path.exists(ud.clonedir):
339 # We do this since git will use a "-l" option automatically for local urls where possible
340 if repourl.startswith("file://"):
341 repourl = repourl[7:]
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600342 clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, repourl, ud.clonedir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500343 if ud.proto.lower() != 'file':
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500344 bb.fetch2.check_network_access(d, clone_cmd, ud.url)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600345 progresshandler = GitProgressHandler(d)
346 runfetchcmd(clone_cmd, d, log=progresshandler)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500347
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500348 # Update the checkout if needed
349 needupdate = False
350 for name in ud.names:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600351 if not self._contains_ref(ud, d, name, ud.clonedir):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500352 needupdate = True
353 if needupdate:
354 try:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600355 runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500356 except bb.fetch2.FetchError:
357 logger.debug(1, "No Origin")
358
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600359 runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, repourl), d, workdir=ud.clonedir)
360 fetch_cmd = "LANG=C %s fetch -f --prune --progress %s refs/*:refs/*" % (ud.basecmd, repourl)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500361 if ud.proto.lower() != 'file':
362 bb.fetch2.check_network_access(d, fetch_cmd, ud.url)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600363 progresshandler = GitProgressHandler(d)
364 runfetchcmd(fetch_cmd, d, log=progresshandler, workdir=ud.clonedir)
365 runfetchcmd("%s prune-packed" % ud.basecmd, d, workdir=ud.clonedir)
366 runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d, workdir=ud.clonedir)
Patrick Williamsd7e96312015-09-22 08:09:05 -0500367 try:
368 os.unlink(ud.fullmirror)
369 except OSError as exc:
370 if exc.errno != errno.ENOENT:
371 raise
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500372 for name in ud.names:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600373 if not self._contains_ref(ud, d, name, ud.clonedir):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500374 raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revisions[name], ud.branches[name]))
375
376 def build_mirror_data(self, ud, d):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500377 if ud.shallow and ud.write_shallow_tarballs:
378 if not os.path.exists(ud.fullshallow):
379 if os.path.islink(ud.fullshallow):
380 os.unlink(ud.fullshallow)
381 tempdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
382 shallowclone = os.path.join(tempdir, 'git')
383 try:
384 self.clone_shallow_local(ud, shallowclone, d)
385
386 logger.info("Creating tarball of git repository")
387 runfetchcmd("tar -czf %s ." % ud.fullshallow, d, workdir=shallowclone)
388 runfetchcmd("touch %s.done" % ud.fullshallow, d)
389 finally:
390 bb.utils.remove(tempdir, recurse=True)
391 elif ud.write_tarballs and not os.path.exists(ud.fullmirror):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500392 if os.path.islink(ud.fullmirror):
393 os.unlink(ud.fullmirror)
394
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500395 logger.info("Creating tarball of git repository")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500396 runfetchcmd("tar -czf %s ." % ud.fullmirror, d, workdir=ud.clonedir)
397 runfetchcmd("touch %s.done" % ud.fullmirror, d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500398
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500399 def clone_shallow_local(self, ud, dest, d):
400 """Clone the repo and make it shallow.
401
402 The upstream url of the new clone isn't set at this time, as it'll be
403 set correctly when unpacked."""
404 runfetchcmd("%s clone %s %s %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, dest), d)
405
406 to_parse, shallow_branches = [], []
407 for name in ud.names:
408 revision = ud.revisions[name]
409 depth = ud.shallow_depths[name]
410 if depth:
411 to_parse.append('%s~%d^{}' % (revision, depth - 1))
412
413 # For nobranch, we need a ref, otherwise the commits will be
414 # removed, and for non-nobranch, we truncate the branch to our
415 # srcrev, to avoid keeping unnecessary history beyond that.
416 branch = ud.branches[name]
417 if ud.nobranch:
418 ref = "refs/shallow/%s" % name
419 elif ud.bareclone:
420 ref = "refs/heads/%s" % branch
421 else:
422 ref = "refs/remotes/origin/%s" % branch
423
424 shallow_branches.append(ref)
425 runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest)
426
427 # Map srcrev+depths to revisions
428 parsed_depths = runfetchcmd("%s rev-parse %s" % (ud.basecmd, " ".join(to_parse)), d, workdir=dest)
429
430 # Resolve specified revisions
431 parsed_revs = runfetchcmd("%s rev-parse %s" % (ud.basecmd, " ".join('"%s^{}"' % r for r in ud.shallow_revs)), d, workdir=dest)
432 shallow_revisions = parsed_depths.splitlines() + parsed_revs.splitlines()
433
434 # Apply extra ref wildcards
435 all_refs = runfetchcmd('%s for-each-ref "--format=%%(refname)"' % ud.basecmd,
436 d, workdir=dest).splitlines()
437 for r in ud.shallow_extra_refs:
438 if not ud.bareclone:
439 r = r.replace('refs/heads/', 'refs/remotes/origin/')
440
441 if '*' in r:
442 matches = filter(lambda a: fnmatch.fnmatchcase(a, r), all_refs)
443 shallow_branches.extend(matches)
444 else:
445 shallow_branches.append(r)
446
447 # Make the repository shallow
448 shallow_cmd = ['git', 'make-shallow', '-s']
449 for b in shallow_branches:
450 shallow_cmd.append('-r')
451 shallow_cmd.append(b)
452 shallow_cmd.extend(shallow_revisions)
453 runfetchcmd(subprocess.list2cmdline(shallow_cmd), d, workdir=dest)
454
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500455 def unpack(self, ud, destdir, d):
456 """ unpack the downloaded src to destdir"""
457
458 subdir = ud.parm.get("subpath", "")
459 if subdir != "":
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500460 readpathspec = ":%s" % subdir
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500461 def_destsuffix = "%s/" % os.path.basename(subdir.rstrip('/'))
462 else:
463 readpathspec = ""
464 def_destsuffix = "git/"
465
466 destsuffix = ud.parm.get("destsuffix", def_destsuffix)
467 destdir = ud.destdir = os.path.join(destdir, destsuffix)
468 if os.path.exists(destdir):
469 bb.utils.prunedir(destdir)
470
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500471 if ud.shallow and (not os.path.exists(ud.clonedir) or self.need_update(ud, d)):
472 bb.utils.mkdirhier(destdir)
473 runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=destdir)
474 else:
475 runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500476
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500477 repourl = self._get_repo_url(ud)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600478 runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, repourl), d, workdir=destdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500479 if not ud.nocheckout:
480 if subdir != "":
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600481 runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revisions[ud.names[0]], readpathspec), d,
482 workdir=destdir)
483 runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d, workdir=destdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500484 elif not ud.nobranch:
485 branchname = ud.branches[ud.names[0]]
486 runfetchcmd("%s checkout -B %s %s" % (ud.basecmd, branchname, \
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600487 ud.revisions[ud.names[0]]), d, workdir=destdir)
Andre Rosa49271d42017-09-07 11:15:55 +0200488 runfetchcmd("%s branch %s --set-upstream-to origin/%s" % (ud.basecmd, branchname, \
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600489 branchname), d, workdir=destdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500490 else:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600491 runfetchcmd("%s checkout %s" % (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=destdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500492
493 return True
494
495 def clean(self, ud, d):
496 """ clean the git directory """
497
498 bb.utils.remove(ud.localpath, True)
499 bb.utils.remove(ud.fullmirror)
500 bb.utils.remove(ud.fullmirror + ".done")
501
502 def supports_srcrev(self):
503 return True
504
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600505 def _contains_ref(self, ud, d, name, wd):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500506 cmd = ""
507 if ud.nobranch:
508 cmd = "%s log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % (
509 ud.basecmd, ud.revisions[name])
510 else:
511 cmd = "%s branch --contains %s --list %s 2> /dev/null | wc -l" % (
512 ud.basecmd, ud.revisions[name], ud.branches[name])
513 try:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600514 output = runfetchcmd(cmd, d, quiet=True, workdir=wd)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500515 except bb.fetch2.FetchError:
516 return False
517 if len(output.split()) > 1:
518 raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output))
519 return output.split()[0] != "0"
520
521 def _get_repo_url(self, ud):
522 """
523 Return the repository URL
524 """
525 if ud.user:
526 username = ud.user + '@'
527 else:
528 username = ""
529 return "%s://%s%s%s" % (ud.proto, username, ud.host, ud.path)
530
531 def _revision_key(self, ud, d, name):
532 """
533 Return a unique key for the url
534 """
535 return "git:" + ud.host + ud.path.replace('/', '.') + ud.unresolvedrev[name]
536
537 def _lsremote(self, ud, d, search):
538 """
539 Run git ls-remote with the specified search string
540 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500541 # Prevent recursion e.g. in OE if SRCPV is in PV, PV is in WORKDIR,
542 # and WORKDIR is in PATH (as a result of RSS), our call to
543 # runfetchcmd() exports PATH so this function will get called again (!)
544 # In this scenario the return call of the function isn't actually
545 # important - WORKDIR isn't needed in PATH to call git ls-remote
546 # anyway.
547 if d.getVar('_BB_GIT_IN_LSREMOTE', False):
548 return ''
549 d.setVar('_BB_GIT_IN_LSREMOTE', '1')
550 try:
551 repourl = self._get_repo_url(ud)
552 cmd = "%s ls-remote %s %s" % \
553 (ud.basecmd, repourl, search)
554 if ud.proto.lower() != 'file':
555 bb.fetch2.check_network_access(d, cmd, repourl)
556 output = runfetchcmd(cmd, d, True)
557 if not output:
558 raise bb.fetch2.FetchError("The command %s gave empty output unexpectedly" % cmd, ud.url)
559 finally:
560 d.delVar('_BB_GIT_IN_LSREMOTE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500561 return output
562
563 def _latest_revision(self, ud, d, name):
564 """
565 Compute the HEAD revision for the url
566 """
567 output = self._lsremote(ud, d, "")
568 # Tags of the form ^{} may not work, need to fallback to other form
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600569 if ud.unresolvedrev[name][:5] == "refs/" or ud.usehead:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500570 head = ud.unresolvedrev[name]
571 tag = ud.unresolvedrev[name]
572 else:
573 head = "refs/heads/%s" % ud.unresolvedrev[name]
574 tag = "refs/tags/%s" % ud.unresolvedrev[name]
575 for s in [head, tag + "^{}", tag]:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600576 for l in output.strip().split('\n'):
577 sha1, ref = l.split()
578 if s == ref:
579 return sha1
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500580 raise bb.fetch2.FetchError("Unable to resolve '%s' in upstream git repository in git ls-remote output for %s" % \
581 (ud.unresolvedrev[name], ud.host+ud.path))
582
583 def latest_versionstring(self, ud, d):
584 """
585 Compute the latest release name like "x.y.x" in "x.y.x+gitHASH"
586 by searching through the tags output of ls-remote, comparing
587 versions and returning the highest match.
588 """
589 pupver = ('', '')
590
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500591 tagregex = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX') or "(?P<pver>([0-9][\.|_]?)+)")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500592 try:
593 output = self._lsremote(ud, d, "refs/tags/*")
594 except bb.fetch2.FetchError or bb.fetch2.NetworkAccess:
595 return pupver
596
597 verstring = ""
598 revision = ""
599 for line in output.split("\n"):
600 if not line:
601 break
602
603 tag_head = line.split("/")[-1]
604 # Ignore non-released branches
605 m = re.search("(alpha|beta|rc|final)+", tag_head)
606 if m:
607 continue
608
609 # search for version in the line
610 tag = tagregex.search(tag_head)
611 if tag == None:
612 continue
613
614 tag = tag.group('pver')
615 tag = tag.replace("_", ".")
616
617 if verstring and bb.utils.vercmp(("0", tag, ""), ("0", verstring, "")) < 0:
618 continue
619
620 verstring = tag
621 revision = line.split()[0]
622 pupver = (verstring, revision)
623
624 return pupver
625
626 def _build_revision(self, ud, d, name):
627 return ud.revisions[name]
628
629 def gitpkgv_revision(self, ud, d, name):
630 """
631 Return a sortable revision number by counting commits in the history
632 Based on gitpkgv.bblass in meta-openembedded
633 """
634 rev = self._build_revision(ud, d, name)
635 localpath = ud.localpath
636 rev_file = os.path.join(localpath, "oe-gitpkgv_" + rev)
637 if not os.path.exists(localpath):
638 commits = None
639 else:
640 if not os.path.exists(rev_file) or not os.path.getsize(rev_file):
641 from pipes import quote
642 commits = bb.fetch2.runfetchcmd(
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500643 "git rev-list %s -- | wc -l" % quote(rev),
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500644 d, quiet=True).strip().lstrip('0')
645 if commits:
646 open(rev_file, "w").write("%d\n" % int(commits))
647 else:
648 commits = open(rev_file, "r").readline(128).strip()
649 if commits:
650 return False, "%s+%s" % (commits, rev[:7])
651 else:
652 return True, str(rev)
653
654 def checkstatus(self, fetch, ud, d):
655 try:
656 self._lsremote(ud, d, "")
657 return True
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500658 except bb.fetch2.FetchError:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500659 return False