blob: 490d57fbbf847188a193ba02b2961aa7ff213104 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001"""
2BitBake 'Fetch' git implementation
3
4git fetcher support the SRC_URI with format of:
5SRC_URI = "git://some.host/somepath;OptionA=xxx;OptionB=xxx;..."
6
7Supported SRC_URI options are:
8
9- branch
10 The git branch to retrieve from. The default is "master"
11
12 This option also supports multiple branch fetching, with branches
13 separated by commas. In multiple branches case, the name option
14 must have the same number of names to match the branches, which is
15 used to specify the SRC_REV for the branch
16 e.g:
17 SRC_URI="git://some.host/somepath;branch=branchX,branchY;name=nameX,nameY"
18 SRCREV_nameX = "xxxxxxxxxxxxxxxxxxxx"
19 SRCREV_nameY = "YYYYYYYYYYYYYYYYYYYY"
20
21- tag
22 The git tag to retrieve. The default is "master"
23
24- protocol
25 The method to use to access the repository. Common options are "git",
26 "http", "https", "file", "ssh" and "rsync". The default is "git".
27
28- rebaseable
29 rebaseable indicates that the upstream git repo may rebase in the future,
30 and current revision may disappear from upstream repo. This option will
31 remind fetcher to preserve local cache carefully for future use.
32 The default value is "0", set rebaseable=1 for rebaseable git repo.
33
34- nocheckout
35 Don't checkout source code when unpacking. set this option for the recipe
36 who has its own routine to checkout code.
37 The default is "0", set nocheckout=1 if needed.
38
39- bareclone
40 Create a bare clone of the source code and don't checkout the source code
41 when unpacking. Set this option for the recipe who has its own routine to
42 checkout code and tracking branch requirements.
43 The default is "0", set bareclone=1 if needed.
44
45- nobranch
46 Don't check the SHA validation for branch. set this option for the recipe
47 referring to commit which is valid in tag instead of branch.
48 The default is "0", set nobranch=1 if needed.
49
Patrick Williamsc0f7c042017-02-23 20:41:17 -060050- usehead
Brad Bishop6e60e8b2018-02-01 10:27:11 -050051 For local git:// urls to use the current branch HEAD as the revision for use with
Patrick Williamsc0f7c042017-02-23 20:41:17 -060052 AUTOREV. Implies nobranch.
53
Patrick Williamsc124f4f2015-09-15 14:41:29 -050054"""
55
Brad Bishopc342db32019-05-15 21:57:59 -040056# Copyright (C) 2005 Richard Purdie
Patrick Williamsc124f4f2015-09-15 14:41:29 -050057#
Brad Bishopc342db32019-05-15 21:57:59 -040058# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -050059#
Patrick Williamsc124f4f2015-09-15 14:41:29 -050060
Brad Bishopd7bf8c12018-02-25 22:55:05 -050061import collections
Patrick Williamsd7e96312015-09-22 08:09:05 -050062import errno
Brad Bishopd7bf8c12018-02-25 22:55:05 -050063import fnmatch
Patrick Williamsc124f4f2015-09-15 14:41:29 -050064import os
65import re
Andrew Geissler4c19ea12020-10-27 13:52:24 -050066import shlex
Brad Bishopd7bf8c12018-02-25 22:55:05 -050067import subprocess
68import tempfile
Patrick Williamsc124f4f2015-09-15 14:41:29 -050069import bb
Patrick Williamsc0f7c042017-02-23 20:41:17 -060070import bb.progress
Patrick Williamsc124f4f2015-09-15 14:41:29 -050071from bb.fetch2 import FetchMethod
72from bb.fetch2 import runfetchcmd
73from bb.fetch2 import logger
74
Patrick Williamsc0f7c042017-02-23 20:41:17 -060075
76class GitProgressHandler(bb.progress.LineFilterProgressHandler):
77 """Extract progress information from git output"""
78 def __init__(self, d):
79 self._buffer = ''
80 self._count = 0
81 super(GitProgressHandler, self).__init__(d)
82 # Send an initial progress event so the bar gets shown
83 self._fire_progress(-1)
84
85 def write(self, string):
86 self._buffer += string
87 stages = ['Counting objects', 'Compressing objects', 'Receiving objects', 'Resolving deltas']
88 stage_weights = [0.2, 0.05, 0.5, 0.25]
89 stagenum = 0
90 for i, stage in reversed(list(enumerate(stages))):
91 if stage in self._buffer:
92 stagenum = i
93 self._buffer = ''
94 break
95 self._status = stages[stagenum]
96 percs = re.findall(r'(\d+)%', string)
97 if percs:
98 progress = int(round((int(percs[-1]) * stage_weights[stagenum]) + (sum(stage_weights[:stagenum]) * 100)))
99 rates = re.findall(r'([\d.]+ [a-zA-Z]*/s+)', string)
100 if rates:
101 rate = rates[-1]
102 else:
103 rate = None
104 self.update(progress, rate)
105 else:
106 if stagenum == 0:
107 percs = re.findall(r': (\d+)', string)
108 if percs:
109 count = int(percs[-1])
110 if count > self._count:
111 self._count = count
112 self._fire_progress(-count)
113 super(GitProgressHandler, self).write(string)
114
115
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500116class Git(FetchMethod):
Brad Bishop316dfdd2018-06-25 12:45:53 -0400117 bitbake_dir = os.path.abspath(os.path.join(os.path.dirname(os.path.join(os.path.abspath(__file__))), '..', '..', '..'))
118 make_shallow_path = os.path.join(bitbake_dir, 'bin', 'git-make-shallow')
119
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500120 """Class to fetch a module or modules from git repositories"""
121 def init(self, d):
122 pass
123
124 def supports(self, ud, d):
125 """
126 Check to see if a given url can be fetched with git.
127 """
128 return ud.type in ['git']
129
130 def supports_checksum(self, urldata):
131 return False
132
133 def urldata_init(self, ud, d):
134 """
135 init git specific variable within url data
136 so that the git method like latest_revision() can work
137 """
138 if 'protocol' in ud.parm:
139 ud.proto = ud.parm['protocol']
140 elif not ud.host:
141 ud.proto = 'file'
142 else:
143 ud.proto = "git"
144
145 if not ud.proto in ('git', 'file', 'ssh', 'http', 'https', 'rsync'):
146 raise bb.fetch2.ParameterError("Invalid protocol type", ud.url)
147
148 ud.nocheckout = ud.parm.get("nocheckout","0") == "1"
149
150 ud.rebaseable = ud.parm.get("rebaseable","0") == "1"
151
152 ud.nobranch = ud.parm.get("nobranch","0") == "1"
153
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600154 # usehead implies nobranch
155 ud.usehead = ud.parm.get("usehead","0") == "1"
156 if ud.usehead:
157 if ud.proto != "file":
158 raise bb.fetch2.ParameterError("The usehead option is only for use with local ('protocol=file') git repositories", ud.url)
159 ud.nobranch = 1
160
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500161 # bareclone implies nocheckout
162 ud.bareclone = ud.parm.get("bareclone","0") == "1"
163 if ud.bareclone:
164 ud.nocheckout = 1
165
166 ud.unresolvedrev = {}
167 branches = ud.parm.get("branch", "master").split(',')
168 if len(branches) != len(ud.names):
169 raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500170
171 ud.cloneflags = "-s -n"
172 if ud.bareclone:
173 ud.cloneflags += " --mirror"
174
175 ud.shallow = d.getVar("BB_GIT_SHALLOW") == "1"
176 ud.shallow_extra_refs = (d.getVar("BB_GIT_SHALLOW_EXTRA_REFS") or "").split()
177
178 depth_default = d.getVar("BB_GIT_SHALLOW_DEPTH")
179 if depth_default is not None:
180 try:
181 depth_default = int(depth_default or 0)
182 except ValueError:
183 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH: %s" % depth_default)
184 else:
185 if depth_default < 0:
186 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH: %s" % depth_default)
187 else:
188 depth_default = 1
189 ud.shallow_depths = collections.defaultdict(lambda: depth_default)
190
Brad Bishop19323692019-04-05 15:28:33 -0400191 revs_default = d.getVar("BB_GIT_SHALLOW_REVS")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500192 ud.shallow_revs = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500193 ud.branches = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500194 for pos, name in enumerate(ud.names):
195 branch = branches[pos]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500196 ud.branches[name] = branch
197 ud.unresolvedrev[name] = branch
198
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500199 shallow_depth = d.getVar("BB_GIT_SHALLOW_DEPTH_%s" % name)
200 if shallow_depth is not None:
201 try:
202 shallow_depth = int(shallow_depth or 0)
203 except ValueError:
204 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (name, shallow_depth))
205 else:
206 if shallow_depth < 0:
207 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (name, shallow_depth))
208 ud.shallow_depths[name] = shallow_depth
209
210 revs = d.getVar("BB_GIT_SHALLOW_REVS_%s" % name)
211 if revs is not None:
212 ud.shallow_revs.extend(revs.split())
213 elif revs_default is not None:
214 ud.shallow_revs.extend(revs_default.split())
215
216 if (ud.shallow and
217 not ud.shallow_revs and
218 all(ud.shallow_depths[n] == 0 for n in ud.names)):
219 # Shallow disabled for this URL
220 ud.shallow = False
221
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600222 if ud.usehead:
223 ud.unresolvedrev['default'] = 'HEAD'
224
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500225 ud.basecmd = d.getVar("FETCHCMD_git") or "git -c core.fsyncobjectfiles=0"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500226
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500227 write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0"
228 ud.write_tarballs = write_tarballs != "0" or ud.rebaseable
229 ud.write_shallow_tarballs = (d.getVar("BB_GENERATE_SHALLOW_TARBALLS") or write_tarballs) != "0"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500230
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500231 ud.setup_revisions(d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500232
233 for name in ud.names:
234 # Ensure anything that doesn't look like a sha256 checksum/revision is translated into one
235 if not ud.revisions[name] or len(ud.revisions[name]) != 40 or (False in [c in "abcdef0123456789" for c in ud.revisions[name]]):
236 if ud.revisions[name]:
237 ud.unresolvedrev[name] = ud.revisions[name]
238 ud.revisions[name] = self.latest_revision(ud, d, name)
239
Andrew Geisslerc3d88e42020-10-02 09:45:00 -0500240 gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500241 if gitsrcname.startswith('.'):
242 gitsrcname = gitsrcname[1:]
243
244 # for rebaseable git repo, it is necessary to keep mirror tar ball
245 # per revision, so that even the revision disappears from the
246 # upstream repo in the future, the mirror will remain intact and still
247 # contains the revision
248 if ud.rebaseable:
249 for name in ud.names:
250 gitsrcname = gitsrcname + '_' + ud.revisions[name]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500251
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500252 dl_dir = d.getVar("DL_DIR")
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800253 gitdir = d.getVar("GITDIR") or (dl_dir + "/git2")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500254 ud.clonedir = os.path.join(gitdir, gitsrcname)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500255 ud.localfile = ud.clonedir
256
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500257 mirrortarball = 'git2_%s.tar.gz' % gitsrcname
258 ud.fullmirror = os.path.join(dl_dir, mirrortarball)
259 ud.mirrortarballs = [mirrortarball]
260 if ud.shallow:
261 tarballname = gitsrcname
262 if ud.bareclone:
263 tarballname = "%s_bare" % tarballname
264
265 if ud.shallow_revs:
266 tarballname = "%s_%s" % (tarballname, "_".join(sorted(ud.shallow_revs)))
267
268 for name, revision in sorted(ud.revisions.items()):
269 tarballname = "%s_%s" % (tarballname, ud.revisions[name][:7])
270 depth = ud.shallow_depths[name]
271 if depth:
272 tarballname = "%s-%s" % (tarballname, depth)
273
274 shallow_refs = []
275 if not ud.nobranch:
276 shallow_refs.extend(ud.branches.values())
277 if ud.shallow_extra_refs:
278 shallow_refs.extend(r.replace('refs/heads/', '').replace('*', 'ALL') for r in ud.shallow_extra_refs)
279 if shallow_refs:
280 tarballname = "%s_%s" % (tarballname, "_".join(sorted(shallow_refs)).replace('/', '.'))
281
282 fetcher = self.__class__.__name__.lower()
283 ud.shallowtarball = '%sshallow_%s.tar.gz' % (fetcher, tarballname)
284 ud.fullshallow = os.path.join(dl_dir, ud.shallowtarball)
285 ud.mirrortarballs.insert(0, ud.shallowtarball)
286
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500287 def localpath(self, ud, d):
288 return ud.clonedir
289
290 def need_update(self, ud, d):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800291 return self.clonedir_need_update(ud, d) or self.shallow_tarball_need_update(ud) or self.tarball_need_update(ud)
292
293 def clonedir_need_update(self, ud, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500294 if not os.path.exists(ud.clonedir):
295 return True
Brad Bishop64c979e2019-11-04 13:55:29 -0500296 if ud.shallow and ud.write_shallow_tarballs and self.clonedir_need_shallow_revs(ud, d):
297 return True
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500298 for name in ud.names:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600299 if not self._contains_ref(ud, d, name, ud.clonedir):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500300 return True
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500301 return False
302
Brad Bishop64c979e2019-11-04 13:55:29 -0500303 def clonedir_need_shallow_revs(self, ud, d):
304 for rev in ud.shallow_revs:
305 try:
306 runfetchcmd('%s rev-parse -q --verify %s' % (ud.basecmd, rev), d, quiet=True, workdir=ud.clonedir)
307 except bb.fetch2.FetchError:
308 return rev
309 return None
310
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800311 def shallow_tarball_need_update(self, ud):
312 return ud.shallow and ud.write_shallow_tarballs and not os.path.exists(ud.fullshallow)
313
314 def tarball_need_update(self, ud):
315 return ud.write_tarballs and not os.path.exists(ud.fullmirror)
316
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500317 def try_premirror(self, ud, d):
318 # If we don't do this, updating an existing checkout with only premirrors
319 # is not possible
Brad Bishop19323692019-04-05 15:28:33 -0400320 if bb.utils.to_boolean(d.getVar("BB_FETCH_PREMIRRORONLY")):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500321 return True
322 if os.path.exists(ud.clonedir):
323 return False
324 return True
325
326 def download(self, ud, d):
327 """Fetch url"""
328
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500329 # A current clone is preferred to either tarball, a shallow tarball is
330 # preferred to an out of date clone, and a missing clone will use
331 # either tarball.
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800332 if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500333 ud.localpath = ud.fullshallow
334 return
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800335 elif os.path.exists(ud.fullmirror) and not os.path.exists(ud.clonedir):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500336 bb.utils.mkdirhier(ud.clonedir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500337 runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500338
339 repourl = self._get_repo_url(ud)
340
341 # If the repo still doesn't exist, fallback to cloning it
342 if not os.path.exists(ud.clonedir):
343 # We do this since git will use a "-l" option automatically for local urls where possible
344 if repourl.startswith("file://"):
345 repourl = repourl[7:]
Andrew Geissler4c19ea12020-10-27 13:52:24 -0500346 clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, shlex.quote(repourl), ud.clonedir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500347 if ud.proto.lower() != 'file':
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500348 bb.fetch2.check_network_access(d, clone_cmd, ud.url)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600349 progresshandler = GitProgressHandler(d)
350 runfetchcmd(clone_cmd, d, log=progresshandler)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500351
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500352 # Update the checkout if needed
Brad Bishop64c979e2019-11-04 13:55:29 -0500353 if self.clonedir_need_update(ud, d):
Brad Bishop6ef32652018-10-09 18:59:25 +0100354 output = runfetchcmd("%s remote" % ud.basecmd, d, quiet=True, workdir=ud.clonedir)
355 if "origin" in output:
356 runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500357
Andrew Geissler4c19ea12020-10-27 13:52:24 -0500358 runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=ud.clonedir)
359 fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, shlex.quote(repourl))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500360 if ud.proto.lower() != 'file':
361 bb.fetch2.check_network_access(d, fetch_cmd, ud.url)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600362 progresshandler = GitProgressHandler(d)
363 runfetchcmd(fetch_cmd, d, log=progresshandler, workdir=ud.clonedir)
364 runfetchcmd("%s prune-packed" % ud.basecmd, d, workdir=ud.clonedir)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400365 runfetchcmd("%s pack-refs --all" % ud.basecmd, d, workdir=ud.clonedir)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600366 runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d, workdir=ud.clonedir)
Patrick Williamsd7e96312015-09-22 08:09:05 -0500367 try:
368 os.unlink(ud.fullmirror)
369 except OSError as exc:
370 if exc.errno != errno.ENOENT:
371 raise
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800372
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500373 for name in ud.names:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600374 if not self._contains_ref(ud, d, name, ud.clonedir):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500375 raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revisions[name], ud.branches[name]))
376
Brad Bishop64c979e2019-11-04 13:55:29 -0500377 if ud.shallow and ud.write_shallow_tarballs:
378 missing_rev = self.clonedir_need_shallow_revs(ud, d)
379 if missing_rev:
380 raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev)
381
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500382 def build_mirror_data(self, ud, d):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500383 if ud.shallow and ud.write_shallow_tarballs:
384 if not os.path.exists(ud.fullshallow):
385 if os.path.islink(ud.fullshallow):
386 os.unlink(ud.fullshallow)
387 tempdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
388 shallowclone = os.path.join(tempdir, 'git')
389 try:
390 self.clone_shallow_local(ud, shallowclone, d)
391
392 logger.info("Creating tarball of git repository")
393 runfetchcmd("tar -czf %s ." % ud.fullshallow, d, workdir=shallowclone)
394 runfetchcmd("touch %s.done" % ud.fullshallow, d)
395 finally:
396 bb.utils.remove(tempdir, recurse=True)
397 elif ud.write_tarballs and not os.path.exists(ud.fullmirror):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500398 if os.path.islink(ud.fullmirror):
399 os.unlink(ud.fullmirror)
400
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500401 logger.info("Creating tarball of git repository")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500402 runfetchcmd("tar -czf %s ." % ud.fullmirror, d, workdir=ud.clonedir)
403 runfetchcmd("touch %s.done" % ud.fullmirror, d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500404
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500405 def clone_shallow_local(self, ud, dest, d):
406 """Clone the repo and make it shallow.
407
408 The upstream url of the new clone isn't set at this time, as it'll be
409 set correctly when unpacked."""
410 runfetchcmd("%s clone %s %s %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, dest), d)
411
412 to_parse, shallow_branches = [], []
413 for name in ud.names:
414 revision = ud.revisions[name]
415 depth = ud.shallow_depths[name]
416 if depth:
417 to_parse.append('%s~%d^{}' % (revision, depth - 1))
418
419 # For nobranch, we need a ref, otherwise the commits will be
420 # removed, and for non-nobranch, we truncate the branch to our
421 # srcrev, to avoid keeping unnecessary history beyond that.
422 branch = ud.branches[name]
423 if ud.nobranch:
424 ref = "refs/shallow/%s" % name
425 elif ud.bareclone:
426 ref = "refs/heads/%s" % branch
427 else:
428 ref = "refs/remotes/origin/%s" % branch
429
430 shallow_branches.append(ref)
431 runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest)
432
433 # Map srcrev+depths to revisions
434 parsed_depths = runfetchcmd("%s rev-parse %s" % (ud.basecmd, " ".join(to_parse)), d, workdir=dest)
435
436 # Resolve specified revisions
437 parsed_revs = runfetchcmd("%s rev-parse %s" % (ud.basecmd, " ".join('"%s^{}"' % r for r in ud.shallow_revs)), d, workdir=dest)
438 shallow_revisions = parsed_depths.splitlines() + parsed_revs.splitlines()
439
440 # Apply extra ref wildcards
441 all_refs = runfetchcmd('%s for-each-ref "--format=%%(refname)"' % ud.basecmd,
442 d, workdir=dest).splitlines()
443 for r in ud.shallow_extra_refs:
444 if not ud.bareclone:
445 r = r.replace('refs/heads/', 'refs/remotes/origin/')
446
447 if '*' in r:
448 matches = filter(lambda a: fnmatch.fnmatchcase(a, r), all_refs)
449 shallow_branches.extend(matches)
450 else:
451 shallow_branches.append(r)
452
453 # Make the repository shallow
Brad Bishop316dfdd2018-06-25 12:45:53 -0400454 shallow_cmd = [self.make_shallow_path, '-s']
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500455 for b in shallow_branches:
456 shallow_cmd.append('-r')
457 shallow_cmd.append(b)
458 shallow_cmd.extend(shallow_revisions)
459 runfetchcmd(subprocess.list2cmdline(shallow_cmd), d, workdir=dest)
460
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500461 def unpack(self, ud, destdir, d):
462 """ unpack the downloaded src to destdir"""
463
464 subdir = ud.parm.get("subpath", "")
465 if subdir != "":
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500466 readpathspec = ":%s" % subdir
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500467 def_destsuffix = "%s/" % os.path.basename(subdir.rstrip('/'))
468 else:
469 readpathspec = ""
470 def_destsuffix = "git/"
471
472 destsuffix = ud.parm.get("destsuffix", def_destsuffix)
473 destdir = ud.destdir = os.path.join(destdir, destsuffix)
474 if os.path.exists(destdir):
475 bb.utils.prunedir(destdir)
476
Brad Bishopa34c0302019-09-23 22:34:48 -0400477 need_lfs = ud.parm.get("lfs", "1") == "1"
478
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500479 if not need_lfs:
480 ud.basecmd = "GIT_LFS_SKIP_SMUDGE=1 " + ud.basecmd
481
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800482 source_found = False
483 source_error = []
484
485 if not source_found:
486 clonedir_is_up_to_date = not self.clonedir_need_update(ud, d)
487 if clonedir_is_up_to_date:
488 runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d)
489 source_found = True
490 else:
491 source_error.append("clone directory not available or not up to date: " + ud.clonedir)
492
493 if not source_found:
494 if ud.shallow:
495 if os.path.exists(ud.fullshallow):
496 bb.utils.mkdirhier(destdir)
497 runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=destdir)
498 source_found = True
499 else:
500 source_error.append("shallow clone not available: " + ud.fullshallow)
501 else:
502 source_error.append("shallow clone not enabled")
503
504 if not source_found:
505 raise bb.fetch2.UnpackError("No up to date source found: " + "; ".join(source_error), ud.url)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500506
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500507 repourl = self._get_repo_url(ud)
Andrew Geissler4c19ea12020-10-27 13:52:24 -0500508 runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=destdir)
Brad Bishopc342db32019-05-15 21:57:59 -0400509
510 if self._contains_lfs(ud, d, destdir):
Brad Bishop00e122a2019-10-05 11:10:57 -0400511 if need_lfs and not self._find_git_lfs(d):
512 raise bb.fetch2.FetchError("Repository %s has LFS content, install git-lfs on host to download (or set lfs=0 to ignore it)" % (repourl))
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500513 elif not need_lfs:
Brad Bishopa34c0302019-09-23 22:34:48 -0400514 bb.note("Repository %s has LFS content but it is not being fetched" % (repourl))
Brad Bishopc342db32019-05-15 21:57:59 -0400515
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500516 if not ud.nocheckout:
517 if subdir != "":
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600518 runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revisions[ud.names[0]], readpathspec), d,
519 workdir=destdir)
520 runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d, workdir=destdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500521 elif not ud.nobranch:
522 branchname = ud.branches[ud.names[0]]
523 runfetchcmd("%s checkout -B %s %s" % (ud.basecmd, branchname, \
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600524 ud.revisions[ud.names[0]]), d, workdir=destdir)
Andre Rosa49271d42017-09-07 11:15:55 +0200525 runfetchcmd("%s branch %s --set-upstream-to origin/%s" % (ud.basecmd, branchname, \
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600526 branchname), d, workdir=destdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500527 else:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600528 runfetchcmd("%s checkout %s" % (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=destdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500529
530 return True
531
532 def clean(self, ud, d):
533 """ clean the git directory """
534
Brad Bishop19323692019-04-05 15:28:33 -0400535 to_remove = [ud.localpath, ud.fullmirror, ud.fullmirror + ".done"]
536 # The localpath is a symlink to clonedir when it is cloned from a
537 # mirror, so remove both of them.
538 if os.path.islink(ud.localpath):
539 clonedir = os.path.realpath(ud.localpath)
540 to_remove.append(clonedir)
541
542 for r in to_remove:
543 if os.path.exists(r):
544 bb.note('Removing %s' % r)
545 bb.utils.remove(r, True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500546
547 def supports_srcrev(self):
548 return True
549
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600550 def _contains_ref(self, ud, d, name, wd):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500551 cmd = ""
552 if ud.nobranch:
553 cmd = "%s log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % (
554 ud.basecmd, ud.revisions[name])
555 else:
556 cmd = "%s branch --contains %s --list %s 2> /dev/null | wc -l" % (
557 ud.basecmd, ud.revisions[name], ud.branches[name])
558 try:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600559 output = runfetchcmd(cmd, d, quiet=True, workdir=wd)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500560 except bb.fetch2.FetchError:
561 return False
562 if len(output.split()) > 1:
563 raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output))
564 return output.split()[0] != "0"
565
Brad Bishopc342db32019-05-15 21:57:59 -0400566 def _contains_lfs(self, ud, d, wd):
567 """
568 Check if the repository has 'lfs' (large file) content
569 """
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500570
571 if not ud.nobranch:
572 branchname = ud.branches[ud.names[0]]
573 else:
574 branchname = "master"
575
576 cmd = "%s grep lfs origin/%s:.gitattributes | wc -l" % (
577 ud.basecmd, ud.branches[ud.names[0]])
578
Brad Bishopc342db32019-05-15 21:57:59 -0400579 try:
580 output = runfetchcmd(cmd, d, quiet=True, workdir=wd)
581 if int(output) > 0:
582 return True
583 except (bb.fetch2.FetchError,ValueError):
584 pass
585 return False
586
Brad Bishop00e122a2019-10-05 11:10:57 -0400587 def _find_git_lfs(self, d):
588 """
589 Return True if git-lfs can be found, False otherwise.
590 """
591 import shutil
592 return shutil.which("git-lfs", path=d.getVar('PATH')) is not None
593
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500594 def _get_repo_url(self, ud):
595 """
596 Return the repository URL
597 """
Andrew Geissler6ce62a22020-11-30 19:58:47 -0600598 # Note that we do not support passwords directly in the git urls. There are several
599 # reasons. SRC_URI can be written out to things like buildhistory and people don't
600 # want to leak passwords like that. Its also all too easy to share metadata without
601 # removing the password. ssh keys, ~/.netrc and ~/.ssh/config files can be used as
602 # alternatives so we will not take patches adding password support here.
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500603 if ud.user:
604 username = ud.user + '@'
605 else:
606 username = ""
607 return "%s://%s%s%s" % (ud.proto, username, ud.host, ud.path)
608
609 def _revision_key(self, ud, d, name):
610 """
611 Return a unique key for the url
612 """
Andrew Geissler82c905d2020-04-13 13:39:40 -0500613 # Collapse adjacent slashes
614 slash_re = re.compile(r"/+")
615 return "git:" + ud.host + slash_re.sub(".", ud.path) + ud.unresolvedrev[name]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500616
617 def _lsremote(self, ud, d, search):
618 """
619 Run git ls-remote with the specified search string
620 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500621 # Prevent recursion e.g. in OE if SRCPV is in PV, PV is in WORKDIR,
622 # and WORKDIR is in PATH (as a result of RSS), our call to
623 # runfetchcmd() exports PATH so this function will get called again (!)
624 # In this scenario the return call of the function isn't actually
625 # important - WORKDIR isn't needed in PATH to call git ls-remote
626 # anyway.
627 if d.getVar('_BB_GIT_IN_LSREMOTE', False):
628 return ''
629 d.setVar('_BB_GIT_IN_LSREMOTE', '1')
630 try:
631 repourl = self._get_repo_url(ud)
Andrew Geissler4c19ea12020-10-27 13:52:24 -0500632 cmd = "%s ls-remote %s %s" % \
633 (ud.basecmd, shlex.quote(repourl), search)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500634 if ud.proto.lower() != 'file':
635 bb.fetch2.check_network_access(d, cmd, repourl)
636 output = runfetchcmd(cmd, d, True)
637 if not output:
638 raise bb.fetch2.FetchError("The command %s gave empty output unexpectedly" % cmd, ud.url)
639 finally:
640 d.delVar('_BB_GIT_IN_LSREMOTE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500641 return output
642
643 def _latest_revision(self, ud, d, name):
644 """
645 Compute the HEAD revision for the url
646 """
647 output = self._lsremote(ud, d, "")
648 # Tags of the form ^{} may not work, need to fallback to other form
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600649 if ud.unresolvedrev[name][:5] == "refs/" or ud.usehead:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500650 head = ud.unresolvedrev[name]
651 tag = ud.unresolvedrev[name]
652 else:
653 head = "refs/heads/%s" % ud.unresolvedrev[name]
654 tag = "refs/tags/%s" % ud.unresolvedrev[name]
655 for s in [head, tag + "^{}", tag]:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600656 for l in output.strip().split('\n'):
657 sha1, ref = l.split()
658 if s == ref:
659 return sha1
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500660 raise bb.fetch2.FetchError("Unable to resolve '%s' in upstream git repository in git ls-remote output for %s" % \
661 (ud.unresolvedrev[name], ud.host+ud.path))
662
663 def latest_versionstring(self, ud, d):
664 """
665 Compute the latest release name like "x.y.x" in "x.y.x+gitHASH"
666 by searching through the tags output of ls-remote, comparing
667 versions and returning the highest match.
668 """
669 pupver = ('', '')
670
Brad Bishop19323692019-04-05 15:28:33 -0400671 tagregex = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX') or r"(?P<pver>([0-9][\.|_]?)+)")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500672 try:
673 output = self._lsremote(ud, d, "refs/tags/*")
Brad Bishop316dfdd2018-06-25 12:45:53 -0400674 except (bb.fetch2.FetchError, bb.fetch2.NetworkAccess) as e:
675 bb.note("Could not list remote: %s" % str(e))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500676 return pupver
677
678 verstring = ""
679 revision = ""
680 for line in output.split("\n"):
681 if not line:
682 break
683
684 tag_head = line.split("/")[-1]
685 # Ignore non-released branches
Brad Bishop19323692019-04-05 15:28:33 -0400686 m = re.search(r"(alpha|beta|rc|final)+", tag_head)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500687 if m:
688 continue
689
690 # search for version in the line
691 tag = tagregex.search(tag_head)
Andrew Geissler82c905d2020-04-13 13:39:40 -0500692 if tag is None:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500693 continue
694
695 tag = tag.group('pver')
696 tag = tag.replace("_", ".")
697
698 if verstring and bb.utils.vercmp(("0", tag, ""), ("0", verstring, "")) < 0:
699 continue
700
701 verstring = tag
702 revision = line.split()[0]
703 pupver = (verstring, revision)
704
705 return pupver
706
707 def _build_revision(self, ud, d, name):
708 return ud.revisions[name]
709
710 def gitpkgv_revision(self, ud, d, name):
711 """
712 Return a sortable revision number by counting commits in the history
713 Based on gitpkgv.bblass in meta-openembedded
714 """
715 rev = self._build_revision(ud, d, name)
716 localpath = ud.localpath
717 rev_file = os.path.join(localpath, "oe-gitpkgv_" + rev)
718 if not os.path.exists(localpath):
719 commits = None
720 else:
721 if not os.path.exists(rev_file) or not os.path.getsize(rev_file):
722 from pipes import quote
723 commits = bb.fetch2.runfetchcmd(
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500724 "git rev-list %s -- | wc -l" % quote(rev),
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500725 d, quiet=True).strip().lstrip('0')
726 if commits:
727 open(rev_file, "w").write("%d\n" % int(commits))
728 else:
729 commits = open(rev_file, "r").readline(128).strip()
730 if commits:
731 return False, "%s+%s" % (commits, rev[:7])
732 else:
733 return True, str(rev)
734
735 def checkstatus(self, fetch, ud, d):
736 try:
737 self._lsremote(ud, d, "")
738 return True
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500739 except bb.fetch2.FetchError:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500740 return False