blob: dbf8715674e63f8682595706d44b08814a4761a7 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001"""
2BitBake 'Fetch' git implementation
3
4git fetcher support the SRC_URI with format of:
5SRC_URI = "git://some.host/somepath;OptionA=xxx;OptionB=xxx;..."
6
7Supported SRC_URI options are:
8
9- branch
10 The git branch to retrieve from. The default is "master"
11
12 This option also supports multiple branch fetching, with branches
13 separated by commas. In multiple branches case, the name option
14 must have the same number of names to match the branches, which is
15 used to specify the SRC_REV for the branch
16 e.g:
17 SRC_URI="git://some.host/somepath;branch=branchX,branchY;name=nameX,nameY"
18 SRCREV_nameX = "xxxxxxxxxxxxxxxxxxxx"
19 SRCREV_nameY = "YYYYYYYYYYYYYYYYYYYY"
20
21- tag
22 The git tag to retrieve. The default is "master"
23
24- protocol
25 The method to use to access the repository. Common options are "git",
26 "http", "https", "file", "ssh" and "rsync". The default is "git".
27
28- rebaseable
29 rebaseable indicates that the upstream git repo may rebase in the future,
30 and current revision may disappear from upstream repo. This option will
31 remind fetcher to preserve local cache carefully for future use.
32 The default value is "0", set rebaseable=1 for rebaseable git repo.
33
34- nocheckout
35 Don't checkout source code when unpacking. set this option for the recipe
36 who has its own routine to checkout code.
37 The default is "0", set nocheckout=1 if needed.
38
39- bareclone
40 Create a bare clone of the source code and don't checkout the source code
41 when unpacking. Set this option for the recipe who has its own routine to
42 checkout code and tracking branch requirements.
43 The default is "0", set bareclone=1 if needed.
44
45- nobranch
46 Don't check the SHA validation for branch. set this option for the recipe
47 referring to commit which is valid in tag instead of branch.
48 The default is "0", set nobranch=1 if needed.
49
Patrick Williamsc0f7c042017-02-23 20:41:17 -060050- usehead
Brad Bishop6e60e8b2018-02-01 10:27:11 -050051 For local git:// urls to use the current branch HEAD as the revision for use with
Patrick Williamsc0f7c042017-02-23 20:41:17 -060052 AUTOREV. Implies nobranch.
53
Patrick Williamsc124f4f2015-09-15 14:41:29 -050054"""
55
Brad Bishopc342db32019-05-15 21:57:59 -040056# Copyright (C) 2005 Richard Purdie
Patrick Williamsc124f4f2015-09-15 14:41:29 -050057#
Brad Bishopc342db32019-05-15 21:57:59 -040058# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -050059#
Patrick Williamsc124f4f2015-09-15 14:41:29 -050060
Brad Bishopd7bf8c12018-02-25 22:55:05 -050061import collections
Patrick Williamsd7e96312015-09-22 08:09:05 -050062import errno
Brad Bishopd7bf8c12018-02-25 22:55:05 -050063import fnmatch
Patrick Williamsc124f4f2015-09-15 14:41:29 -050064import os
65import re
Brad Bishopd7bf8c12018-02-25 22:55:05 -050066import subprocess
67import tempfile
Patrick Williamsc124f4f2015-09-15 14:41:29 -050068import bb
Patrick Williamsc0f7c042017-02-23 20:41:17 -060069import bb.progress
Patrick Williamsc124f4f2015-09-15 14:41:29 -050070from bb.fetch2 import FetchMethod
71from bb.fetch2 import runfetchcmd
72from bb.fetch2 import logger
73
Patrick Williamsc0f7c042017-02-23 20:41:17 -060074
75class GitProgressHandler(bb.progress.LineFilterProgressHandler):
76 """Extract progress information from git output"""
77 def __init__(self, d):
78 self._buffer = ''
79 self._count = 0
80 super(GitProgressHandler, self).__init__(d)
81 # Send an initial progress event so the bar gets shown
82 self._fire_progress(-1)
83
84 def write(self, string):
85 self._buffer += string
86 stages = ['Counting objects', 'Compressing objects', 'Receiving objects', 'Resolving deltas']
87 stage_weights = [0.2, 0.05, 0.5, 0.25]
88 stagenum = 0
89 for i, stage in reversed(list(enumerate(stages))):
90 if stage in self._buffer:
91 stagenum = i
92 self._buffer = ''
93 break
94 self._status = stages[stagenum]
95 percs = re.findall(r'(\d+)%', string)
96 if percs:
97 progress = int(round((int(percs[-1]) * stage_weights[stagenum]) + (sum(stage_weights[:stagenum]) * 100)))
98 rates = re.findall(r'([\d.]+ [a-zA-Z]*/s+)', string)
99 if rates:
100 rate = rates[-1]
101 else:
102 rate = None
103 self.update(progress, rate)
104 else:
105 if stagenum == 0:
106 percs = re.findall(r': (\d+)', string)
107 if percs:
108 count = int(percs[-1])
109 if count > self._count:
110 self._count = count
111 self._fire_progress(-count)
112 super(GitProgressHandler, self).write(string)
113
114
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500115class Git(FetchMethod):
Brad Bishop316dfdd2018-06-25 12:45:53 -0400116 bitbake_dir = os.path.abspath(os.path.join(os.path.dirname(os.path.join(os.path.abspath(__file__))), '..', '..', '..'))
117 make_shallow_path = os.path.join(bitbake_dir, 'bin', 'git-make-shallow')
118
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500119 """Class to fetch a module or modules from git repositories"""
120 def init(self, d):
121 pass
122
123 def supports(self, ud, d):
124 """
125 Check to see if a given url can be fetched with git.
126 """
127 return ud.type in ['git']
128
129 def supports_checksum(self, urldata):
130 return False
131
132 def urldata_init(self, ud, d):
133 """
134 init git specific variable within url data
135 so that the git method like latest_revision() can work
136 """
137 if 'protocol' in ud.parm:
138 ud.proto = ud.parm['protocol']
139 elif not ud.host:
140 ud.proto = 'file'
141 else:
142 ud.proto = "git"
143
144 if not ud.proto in ('git', 'file', 'ssh', 'http', 'https', 'rsync'):
145 raise bb.fetch2.ParameterError("Invalid protocol type", ud.url)
146
147 ud.nocheckout = ud.parm.get("nocheckout","0") == "1"
148
149 ud.rebaseable = ud.parm.get("rebaseable","0") == "1"
150
151 ud.nobranch = ud.parm.get("nobranch","0") == "1"
152
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600153 # usehead implies nobranch
154 ud.usehead = ud.parm.get("usehead","0") == "1"
155 if ud.usehead:
156 if ud.proto != "file":
157 raise bb.fetch2.ParameterError("The usehead option is only for use with local ('protocol=file') git repositories", ud.url)
158 ud.nobranch = 1
159
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500160 # bareclone implies nocheckout
161 ud.bareclone = ud.parm.get("bareclone","0") == "1"
162 if ud.bareclone:
163 ud.nocheckout = 1
164
165 ud.unresolvedrev = {}
166 branches = ud.parm.get("branch", "master").split(',')
167 if len(branches) != len(ud.names):
168 raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500169
170 ud.cloneflags = "-s -n"
171 if ud.bareclone:
172 ud.cloneflags += " --mirror"
173
174 ud.shallow = d.getVar("BB_GIT_SHALLOW") == "1"
175 ud.shallow_extra_refs = (d.getVar("BB_GIT_SHALLOW_EXTRA_REFS") or "").split()
176
177 depth_default = d.getVar("BB_GIT_SHALLOW_DEPTH")
178 if depth_default is not None:
179 try:
180 depth_default = int(depth_default or 0)
181 except ValueError:
182 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH: %s" % depth_default)
183 else:
184 if depth_default < 0:
185 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH: %s" % depth_default)
186 else:
187 depth_default = 1
188 ud.shallow_depths = collections.defaultdict(lambda: depth_default)
189
Brad Bishop19323692019-04-05 15:28:33 -0400190 revs_default = d.getVar("BB_GIT_SHALLOW_REVS")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500191 ud.shallow_revs = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500192 ud.branches = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500193 for pos, name in enumerate(ud.names):
194 branch = branches[pos]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500195 ud.branches[name] = branch
196 ud.unresolvedrev[name] = branch
197
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500198 shallow_depth = d.getVar("BB_GIT_SHALLOW_DEPTH_%s" % name)
199 if shallow_depth is not None:
200 try:
201 shallow_depth = int(shallow_depth or 0)
202 except ValueError:
203 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (name, shallow_depth))
204 else:
205 if shallow_depth < 0:
206 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (name, shallow_depth))
207 ud.shallow_depths[name] = shallow_depth
208
209 revs = d.getVar("BB_GIT_SHALLOW_REVS_%s" % name)
210 if revs is not None:
211 ud.shallow_revs.extend(revs.split())
212 elif revs_default is not None:
213 ud.shallow_revs.extend(revs_default.split())
214
215 if (ud.shallow and
216 not ud.shallow_revs and
217 all(ud.shallow_depths[n] == 0 for n in ud.names)):
218 # Shallow disabled for this URL
219 ud.shallow = False
220
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600221 if ud.usehead:
222 ud.unresolvedrev['default'] = 'HEAD'
223
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500224 ud.basecmd = d.getVar("FETCHCMD_git") or "git -c core.fsyncobjectfiles=0"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500225
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500226 write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0"
227 ud.write_tarballs = write_tarballs != "0" or ud.rebaseable
228 ud.write_shallow_tarballs = (d.getVar("BB_GENERATE_SHALLOW_TARBALLS") or write_tarballs) != "0"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500229
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500230 ud.setup_revisions(d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500231
232 for name in ud.names:
233 # Ensure anything that doesn't look like a sha256 checksum/revision is translated into one
234 if not ud.revisions[name] or len(ud.revisions[name]) != 40 or (False in [c in "abcdef0123456789" for c in ud.revisions[name]]):
235 if ud.revisions[name]:
236 ud.unresolvedrev[name] = ud.revisions[name]
237 ud.revisions[name] = self.latest_revision(ud, d, name)
238
239 gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.'))
240 if gitsrcname.startswith('.'):
241 gitsrcname = gitsrcname[1:]
242
243 # for rebaseable git repo, it is necessary to keep mirror tar ball
244 # per revision, so that even the revision disappears from the
245 # upstream repo in the future, the mirror will remain intact and still
246 # contains the revision
247 if ud.rebaseable:
248 for name in ud.names:
249 gitsrcname = gitsrcname + '_' + ud.revisions[name]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500250
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500251 dl_dir = d.getVar("DL_DIR")
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800252 gitdir = d.getVar("GITDIR") or (dl_dir + "/git2")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500253 ud.clonedir = os.path.join(gitdir, gitsrcname)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500254 ud.localfile = ud.clonedir
255
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500256 mirrortarball = 'git2_%s.tar.gz' % gitsrcname
257 ud.fullmirror = os.path.join(dl_dir, mirrortarball)
258 ud.mirrortarballs = [mirrortarball]
259 if ud.shallow:
260 tarballname = gitsrcname
261 if ud.bareclone:
262 tarballname = "%s_bare" % tarballname
263
264 if ud.shallow_revs:
265 tarballname = "%s_%s" % (tarballname, "_".join(sorted(ud.shallow_revs)))
266
267 for name, revision in sorted(ud.revisions.items()):
268 tarballname = "%s_%s" % (tarballname, ud.revisions[name][:7])
269 depth = ud.shallow_depths[name]
270 if depth:
271 tarballname = "%s-%s" % (tarballname, depth)
272
273 shallow_refs = []
274 if not ud.nobranch:
275 shallow_refs.extend(ud.branches.values())
276 if ud.shallow_extra_refs:
277 shallow_refs.extend(r.replace('refs/heads/', '').replace('*', 'ALL') for r in ud.shallow_extra_refs)
278 if shallow_refs:
279 tarballname = "%s_%s" % (tarballname, "_".join(sorted(shallow_refs)).replace('/', '.'))
280
281 fetcher = self.__class__.__name__.lower()
282 ud.shallowtarball = '%sshallow_%s.tar.gz' % (fetcher, tarballname)
283 ud.fullshallow = os.path.join(dl_dir, ud.shallowtarball)
284 ud.mirrortarballs.insert(0, ud.shallowtarball)
285
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500286 def localpath(self, ud, d):
287 return ud.clonedir
288
289 def need_update(self, ud, d):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800290 return self.clonedir_need_update(ud, d) or self.shallow_tarball_need_update(ud) or self.tarball_need_update(ud)
291
292 def clonedir_need_update(self, ud, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500293 if not os.path.exists(ud.clonedir):
294 return True
Brad Bishop64c979e2019-11-04 13:55:29 -0500295 if ud.shallow and ud.write_shallow_tarballs and self.clonedir_need_shallow_revs(ud, d):
296 return True
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500297 for name in ud.names:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600298 if not self._contains_ref(ud, d, name, ud.clonedir):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500299 return True
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500300 return False
301
Brad Bishop64c979e2019-11-04 13:55:29 -0500302 def clonedir_need_shallow_revs(self, ud, d):
303 for rev in ud.shallow_revs:
304 try:
305 runfetchcmd('%s rev-parse -q --verify %s' % (ud.basecmd, rev), d, quiet=True, workdir=ud.clonedir)
306 except bb.fetch2.FetchError:
307 return rev
308 return None
309
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800310 def shallow_tarball_need_update(self, ud):
311 return ud.shallow and ud.write_shallow_tarballs and not os.path.exists(ud.fullshallow)
312
313 def tarball_need_update(self, ud):
314 return ud.write_tarballs and not os.path.exists(ud.fullmirror)
315
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500316 def try_premirror(self, ud, d):
317 # If we don't do this, updating an existing checkout with only premirrors
318 # is not possible
Brad Bishop19323692019-04-05 15:28:33 -0400319 if bb.utils.to_boolean(d.getVar("BB_FETCH_PREMIRRORONLY")):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500320 return True
321 if os.path.exists(ud.clonedir):
322 return False
323 return True
324
325 def download(self, ud, d):
326 """Fetch url"""
327
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500328 # A current clone is preferred to either tarball, a shallow tarball is
329 # preferred to an out of date clone, and a missing clone will use
330 # either tarball.
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800331 if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500332 ud.localpath = ud.fullshallow
333 return
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800334 elif os.path.exists(ud.fullmirror) and not os.path.exists(ud.clonedir):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500335 bb.utils.mkdirhier(ud.clonedir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500336 runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500337
338 repourl = self._get_repo_url(ud)
339
340 # If the repo still doesn't exist, fallback to cloning it
341 if not os.path.exists(ud.clonedir):
342 # We do this since git will use a "-l" option automatically for local urls where possible
343 if repourl.startswith("file://"):
344 repourl = repourl[7:]
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600345 clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, repourl, ud.clonedir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500346 if ud.proto.lower() != 'file':
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500347 bb.fetch2.check_network_access(d, clone_cmd, ud.url)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600348 progresshandler = GitProgressHandler(d)
349 runfetchcmd(clone_cmd, d, log=progresshandler)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500350
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500351 # Update the checkout if needed
Brad Bishop64c979e2019-11-04 13:55:29 -0500352 if self.clonedir_need_update(ud, d):
Brad Bishop6ef32652018-10-09 18:59:25 +0100353 output = runfetchcmd("%s remote" % ud.basecmd, d, quiet=True, workdir=ud.clonedir)
354 if "origin" in output:
355 runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500356
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600357 runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, repourl), d, workdir=ud.clonedir)
358 fetch_cmd = "LANG=C %s fetch -f --prune --progress %s refs/*:refs/*" % (ud.basecmd, repourl)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500359 if ud.proto.lower() != 'file':
360 bb.fetch2.check_network_access(d, fetch_cmd, ud.url)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600361 progresshandler = GitProgressHandler(d)
362 runfetchcmd(fetch_cmd, d, log=progresshandler, workdir=ud.clonedir)
363 runfetchcmd("%s prune-packed" % ud.basecmd, d, workdir=ud.clonedir)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400364 runfetchcmd("%s pack-refs --all" % ud.basecmd, d, workdir=ud.clonedir)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600365 runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d, workdir=ud.clonedir)
Patrick Williamsd7e96312015-09-22 08:09:05 -0500366 try:
367 os.unlink(ud.fullmirror)
368 except OSError as exc:
369 if exc.errno != errno.ENOENT:
370 raise
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800371
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500372 for name in ud.names:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600373 if not self._contains_ref(ud, d, name, ud.clonedir):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500374 raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revisions[name], ud.branches[name]))
375
Brad Bishop64c979e2019-11-04 13:55:29 -0500376 if ud.shallow and ud.write_shallow_tarballs:
377 missing_rev = self.clonedir_need_shallow_revs(ud, d)
378 if missing_rev:
379 raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev)
380
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500381 def build_mirror_data(self, ud, d):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500382 if ud.shallow and ud.write_shallow_tarballs:
383 if not os.path.exists(ud.fullshallow):
384 if os.path.islink(ud.fullshallow):
385 os.unlink(ud.fullshallow)
386 tempdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
387 shallowclone = os.path.join(tempdir, 'git')
388 try:
389 self.clone_shallow_local(ud, shallowclone, d)
390
391 logger.info("Creating tarball of git repository")
392 runfetchcmd("tar -czf %s ." % ud.fullshallow, d, workdir=shallowclone)
393 runfetchcmd("touch %s.done" % ud.fullshallow, d)
394 finally:
395 bb.utils.remove(tempdir, recurse=True)
396 elif ud.write_tarballs and not os.path.exists(ud.fullmirror):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500397 if os.path.islink(ud.fullmirror):
398 os.unlink(ud.fullmirror)
399
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500400 logger.info("Creating tarball of git repository")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500401 runfetchcmd("tar -czf %s ." % ud.fullmirror, d, workdir=ud.clonedir)
402 runfetchcmd("touch %s.done" % ud.fullmirror, d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500403
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500404 def clone_shallow_local(self, ud, dest, d):
405 """Clone the repo and make it shallow.
406
407 The upstream url of the new clone isn't set at this time, as it'll be
408 set correctly when unpacked."""
409 runfetchcmd("%s clone %s %s %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, dest), d)
410
411 to_parse, shallow_branches = [], []
412 for name in ud.names:
413 revision = ud.revisions[name]
414 depth = ud.shallow_depths[name]
415 if depth:
416 to_parse.append('%s~%d^{}' % (revision, depth - 1))
417
418 # For nobranch, we need a ref, otherwise the commits will be
419 # removed, and for non-nobranch, we truncate the branch to our
420 # srcrev, to avoid keeping unnecessary history beyond that.
421 branch = ud.branches[name]
422 if ud.nobranch:
423 ref = "refs/shallow/%s" % name
424 elif ud.bareclone:
425 ref = "refs/heads/%s" % branch
426 else:
427 ref = "refs/remotes/origin/%s" % branch
428
429 shallow_branches.append(ref)
430 runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest)
431
432 # Map srcrev+depths to revisions
433 parsed_depths = runfetchcmd("%s rev-parse %s" % (ud.basecmd, " ".join(to_parse)), d, workdir=dest)
434
435 # Resolve specified revisions
436 parsed_revs = runfetchcmd("%s rev-parse %s" % (ud.basecmd, " ".join('"%s^{}"' % r for r in ud.shallow_revs)), d, workdir=dest)
437 shallow_revisions = parsed_depths.splitlines() + parsed_revs.splitlines()
438
439 # Apply extra ref wildcards
440 all_refs = runfetchcmd('%s for-each-ref "--format=%%(refname)"' % ud.basecmd,
441 d, workdir=dest).splitlines()
442 for r in ud.shallow_extra_refs:
443 if not ud.bareclone:
444 r = r.replace('refs/heads/', 'refs/remotes/origin/')
445
446 if '*' in r:
447 matches = filter(lambda a: fnmatch.fnmatchcase(a, r), all_refs)
448 shallow_branches.extend(matches)
449 else:
450 shallow_branches.append(r)
451
452 # Make the repository shallow
Brad Bishop316dfdd2018-06-25 12:45:53 -0400453 shallow_cmd = [self.make_shallow_path, '-s']
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500454 for b in shallow_branches:
455 shallow_cmd.append('-r')
456 shallow_cmd.append(b)
457 shallow_cmd.extend(shallow_revisions)
458 runfetchcmd(subprocess.list2cmdline(shallow_cmd), d, workdir=dest)
459
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500460 def unpack(self, ud, destdir, d):
461 """ unpack the downloaded src to destdir"""
462
463 subdir = ud.parm.get("subpath", "")
464 if subdir != "":
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500465 readpathspec = ":%s" % subdir
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500466 def_destsuffix = "%s/" % os.path.basename(subdir.rstrip('/'))
467 else:
468 readpathspec = ""
469 def_destsuffix = "git/"
470
471 destsuffix = ud.parm.get("destsuffix", def_destsuffix)
472 destdir = ud.destdir = os.path.join(destdir, destsuffix)
473 if os.path.exists(destdir):
474 bb.utils.prunedir(destdir)
475
Brad Bishopa34c0302019-09-23 22:34:48 -0400476 need_lfs = ud.parm.get("lfs", "1") == "1"
477
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500478 if not need_lfs:
479 ud.basecmd = "GIT_LFS_SKIP_SMUDGE=1 " + ud.basecmd
480
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800481 source_found = False
482 source_error = []
483
484 if not source_found:
485 clonedir_is_up_to_date = not self.clonedir_need_update(ud, d)
486 if clonedir_is_up_to_date:
487 runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d)
488 source_found = True
489 else:
490 source_error.append("clone directory not available or not up to date: " + ud.clonedir)
491
492 if not source_found:
493 if ud.shallow:
494 if os.path.exists(ud.fullshallow):
495 bb.utils.mkdirhier(destdir)
496 runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=destdir)
497 source_found = True
498 else:
499 source_error.append("shallow clone not available: " + ud.fullshallow)
500 else:
501 source_error.append("shallow clone not enabled")
502
503 if not source_found:
504 raise bb.fetch2.UnpackError("No up to date source found: " + "; ".join(source_error), ud.url)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500505
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500506 repourl = self._get_repo_url(ud)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600507 runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, repourl), d, workdir=destdir)
Brad Bishopc342db32019-05-15 21:57:59 -0400508
509 if self._contains_lfs(ud, d, destdir):
Brad Bishop00e122a2019-10-05 11:10:57 -0400510 if need_lfs and not self._find_git_lfs(d):
511 raise bb.fetch2.FetchError("Repository %s has LFS content, install git-lfs on host to download (or set lfs=0 to ignore it)" % (repourl))
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500512 elif not need_lfs:
Brad Bishopa34c0302019-09-23 22:34:48 -0400513 bb.note("Repository %s has LFS content but it is not being fetched" % (repourl))
Brad Bishopc342db32019-05-15 21:57:59 -0400514
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500515 if not ud.nocheckout:
516 if subdir != "":
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600517 runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revisions[ud.names[0]], readpathspec), d,
518 workdir=destdir)
519 runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d, workdir=destdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500520 elif not ud.nobranch:
521 branchname = ud.branches[ud.names[0]]
522 runfetchcmd("%s checkout -B %s %s" % (ud.basecmd, branchname, \
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600523 ud.revisions[ud.names[0]]), d, workdir=destdir)
Andre Rosa49271d42017-09-07 11:15:55 +0200524 runfetchcmd("%s branch %s --set-upstream-to origin/%s" % (ud.basecmd, branchname, \
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600525 branchname), d, workdir=destdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500526 else:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600527 runfetchcmd("%s checkout %s" % (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=destdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500528
529 return True
530
531 def clean(self, ud, d):
532 """ clean the git directory """
533
Brad Bishop19323692019-04-05 15:28:33 -0400534 to_remove = [ud.localpath, ud.fullmirror, ud.fullmirror + ".done"]
535 # The localpath is a symlink to clonedir when it is cloned from a
536 # mirror, so remove both of them.
537 if os.path.islink(ud.localpath):
538 clonedir = os.path.realpath(ud.localpath)
539 to_remove.append(clonedir)
540
541 for r in to_remove:
542 if os.path.exists(r):
543 bb.note('Removing %s' % r)
544 bb.utils.remove(r, True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500545
546 def supports_srcrev(self):
547 return True
548
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600549 def _contains_ref(self, ud, d, name, wd):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500550 cmd = ""
551 if ud.nobranch:
552 cmd = "%s log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % (
553 ud.basecmd, ud.revisions[name])
554 else:
555 cmd = "%s branch --contains %s --list %s 2> /dev/null | wc -l" % (
556 ud.basecmd, ud.revisions[name], ud.branches[name])
557 try:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600558 output = runfetchcmd(cmd, d, quiet=True, workdir=wd)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500559 except bb.fetch2.FetchError:
560 return False
561 if len(output.split()) > 1:
562 raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output))
563 return output.split()[0] != "0"
564
Brad Bishopc342db32019-05-15 21:57:59 -0400565 def _contains_lfs(self, ud, d, wd):
566 """
567 Check if the repository has 'lfs' (large file) content
568 """
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500569
570 if not ud.nobranch:
571 branchname = ud.branches[ud.names[0]]
572 else:
573 branchname = "master"
574
575 cmd = "%s grep lfs origin/%s:.gitattributes | wc -l" % (
576 ud.basecmd, ud.branches[ud.names[0]])
577
Brad Bishopc342db32019-05-15 21:57:59 -0400578 try:
579 output = runfetchcmd(cmd, d, quiet=True, workdir=wd)
580 if int(output) > 0:
581 return True
582 except (bb.fetch2.FetchError,ValueError):
583 pass
584 return False
585
Brad Bishop00e122a2019-10-05 11:10:57 -0400586 def _find_git_lfs(self, d):
587 """
588 Return True if git-lfs can be found, False otherwise.
589 """
590 import shutil
591 return shutil.which("git-lfs", path=d.getVar('PATH')) is not None
592
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500593 def _get_repo_url(self, ud):
594 """
595 Return the repository URL
596 """
597 if ud.user:
598 username = ud.user + '@'
599 else:
600 username = ""
601 return "%s://%s%s%s" % (ud.proto, username, ud.host, ud.path)
602
603 def _revision_key(self, ud, d, name):
604 """
605 Return a unique key for the url
606 """
Andrew Geissler82c905d2020-04-13 13:39:40 -0500607 # Collapse adjacent slashes
608 slash_re = re.compile(r"/+")
609 return "git:" + ud.host + slash_re.sub(".", ud.path) + ud.unresolvedrev[name]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500610
611 def _lsremote(self, ud, d, search):
612 """
613 Run git ls-remote with the specified search string
614 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500615 # Prevent recursion e.g. in OE if SRCPV is in PV, PV is in WORKDIR,
616 # and WORKDIR is in PATH (as a result of RSS), our call to
617 # runfetchcmd() exports PATH so this function will get called again (!)
618 # In this scenario the return call of the function isn't actually
619 # important - WORKDIR isn't needed in PATH to call git ls-remote
620 # anyway.
621 if d.getVar('_BB_GIT_IN_LSREMOTE', False):
622 return ''
623 d.setVar('_BB_GIT_IN_LSREMOTE', '1')
624 try:
625 repourl = self._get_repo_url(ud)
626 cmd = "%s ls-remote %s %s" % \
627 (ud.basecmd, repourl, search)
628 if ud.proto.lower() != 'file':
629 bb.fetch2.check_network_access(d, cmd, repourl)
630 output = runfetchcmd(cmd, d, True)
631 if not output:
632 raise bb.fetch2.FetchError("The command %s gave empty output unexpectedly" % cmd, ud.url)
633 finally:
634 d.delVar('_BB_GIT_IN_LSREMOTE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500635 return output
636
637 def _latest_revision(self, ud, d, name):
638 """
639 Compute the HEAD revision for the url
640 """
641 output = self._lsremote(ud, d, "")
642 # Tags of the form ^{} may not work, need to fallback to other form
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600643 if ud.unresolvedrev[name][:5] == "refs/" or ud.usehead:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500644 head = ud.unresolvedrev[name]
645 tag = ud.unresolvedrev[name]
646 else:
647 head = "refs/heads/%s" % ud.unresolvedrev[name]
648 tag = "refs/tags/%s" % ud.unresolvedrev[name]
649 for s in [head, tag + "^{}", tag]:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600650 for l in output.strip().split('\n'):
651 sha1, ref = l.split()
652 if s == ref:
653 return sha1
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500654 raise bb.fetch2.FetchError("Unable to resolve '%s' in upstream git repository in git ls-remote output for %s" % \
655 (ud.unresolvedrev[name], ud.host+ud.path))
656
657 def latest_versionstring(self, ud, d):
658 """
659 Compute the latest release name like "x.y.x" in "x.y.x+gitHASH"
660 by searching through the tags output of ls-remote, comparing
661 versions and returning the highest match.
662 """
663 pupver = ('', '')
664
Brad Bishop19323692019-04-05 15:28:33 -0400665 tagregex = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX') or r"(?P<pver>([0-9][\.|_]?)+)")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500666 try:
667 output = self._lsremote(ud, d, "refs/tags/*")
Brad Bishop316dfdd2018-06-25 12:45:53 -0400668 except (bb.fetch2.FetchError, bb.fetch2.NetworkAccess) as e:
669 bb.note("Could not list remote: %s" % str(e))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500670 return pupver
671
672 verstring = ""
673 revision = ""
674 for line in output.split("\n"):
675 if not line:
676 break
677
678 tag_head = line.split("/")[-1]
679 # Ignore non-released branches
Brad Bishop19323692019-04-05 15:28:33 -0400680 m = re.search(r"(alpha|beta|rc|final)+", tag_head)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500681 if m:
682 continue
683
684 # search for version in the line
685 tag = tagregex.search(tag_head)
Andrew Geissler82c905d2020-04-13 13:39:40 -0500686 if tag is None:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500687 continue
688
689 tag = tag.group('pver')
690 tag = tag.replace("_", ".")
691
692 if verstring and bb.utils.vercmp(("0", tag, ""), ("0", verstring, "")) < 0:
693 continue
694
695 verstring = tag
696 revision = line.split()[0]
697 pupver = (verstring, revision)
698
699 return pupver
700
701 def _build_revision(self, ud, d, name):
702 return ud.revisions[name]
703
704 def gitpkgv_revision(self, ud, d, name):
705 """
706 Return a sortable revision number by counting commits in the history
707 Based on gitpkgv.bblass in meta-openembedded
708 """
709 rev = self._build_revision(ud, d, name)
710 localpath = ud.localpath
711 rev_file = os.path.join(localpath, "oe-gitpkgv_" + rev)
712 if not os.path.exists(localpath):
713 commits = None
714 else:
715 if not os.path.exists(rev_file) or not os.path.getsize(rev_file):
716 from pipes import quote
717 commits = bb.fetch2.runfetchcmd(
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500718 "git rev-list %s -- | wc -l" % quote(rev),
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500719 d, quiet=True).strip().lstrip('0')
720 if commits:
721 open(rev_file, "w").write("%d\n" % int(commits))
722 else:
723 commits = open(rev_file, "r").readline(128).strip()
724 if commits:
725 return False, "%s+%s" % (commits, rev[:7])
726 else:
727 return True, str(rev)
728
729 def checkstatus(self, fetch, ud, d):
730 try:
731 self._lsremote(ud, d, "")
732 return True
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500733 except bb.fetch2.FetchError:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500734 return False