blob: 488f4c7414d3432f94e5a360247b4c572e401a3d [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001"""
2BitBake 'Fetch' git implementation
3
4git fetcher support the SRC_URI with format of:
5SRC_URI = "git://some.host/somepath;OptionA=xxx;OptionB=xxx;..."
6
7Supported SRC_URI options are:
8
9- branch
10 The git branch to retrieve from. The default is "master"
11
12 This option also supports multiple branch fetching, with branches
13 separated by commas. In multiple branches case, the name option
14 must have the same number of names to match the branches, which is
15 used to specify the SRC_REV for the branch
16 e.g:
17 SRC_URI="git://some.host/somepath;branch=branchX,branchY;name=nameX,nameY"
18 SRCREV_nameX = "xxxxxxxxxxxxxxxxxxxx"
19 SRCREV_nameY = "YYYYYYYYYYYYYYYYYYYY"
20
21- tag
22 The git tag to retrieve. The default is "master"
23
24- protocol
25 The method to use to access the repository. Common options are "git",
26 "http", "https", "file", "ssh" and "rsync". The default is "git".
27
28- rebaseable
29 rebaseable indicates that the upstream git repo may rebase in the future,
30 and current revision may disappear from upstream repo. This option will
31 remind fetcher to preserve local cache carefully for future use.
32 The default value is "0", set rebaseable=1 for rebaseable git repo.
33
34- nocheckout
35 Don't checkout source code when unpacking. set this option for the recipe
36 who has its own routine to checkout code.
37 The default is "0", set nocheckout=1 if needed.
38
39- bareclone
40 Create a bare clone of the source code and don't checkout the source code
41 when unpacking. Set this option for the recipe who has its own routine to
42 checkout code and tracking branch requirements.
43 The default is "0", set bareclone=1 if needed.
44
45- nobranch
46 Don't check the SHA validation for branch. set this option for the recipe
47 referring to commit which is valid in tag instead of branch.
48 The default is "0", set nobranch=1 if needed.
49
Patrick Williamsc0f7c042017-02-23 20:41:17 -060050- usehead
Brad Bishop6e60e8b2018-02-01 10:27:11 -050051 For local git:// urls to use the current branch HEAD as the revision for use with
Patrick Williamsc0f7c042017-02-23 20:41:17 -060052 AUTOREV. Implies nobranch.
53
Patrick Williamsc124f4f2015-09-15 14:41:29 -050054"""
55
Brad Bishopc342db32019-05-15 21:57:59 -040056# Copyright (C) 2005 Richard Purdie
Patrick Williamsc124f4f2015-09-15 14:41:29 -050057#
Brad Bishopc342db32019-05-15 21:57:59 -040058# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -050059#
Patrick Williamsc124f4f2015-09-15 14:41:29 -050060
Brad Bishopd7bf8c12018-02-25 22:55:05 -050061import collections
Patrick Williamsd7e96312015-09-22 08:09:05 -050062import errno
Brad Bishopd7bf8c12018-02-25 22:55:05 -050063import fnmatch
Patrick Williamsc124f4f2015-09-15 14:41:29 -050064import os
65import re
Andrew Geissler4c19ea12020-10-27 13:52:24 -050066import shlex
Brad Bishopd7bf8c12018-02-25 22:55:05 -050067import subprocess
68import tempfile
Patrick Williamsc124f4f2015-09-15 14:41:29 -050069import bb
Patrick Williamsc0f7c042017-02-23 20:41:17 -060070import bb.progress
Patrick Williamsc124f4f2015-09-15 14:41:29 -050071from bb.fetch2 import FetchMethod
72from bb.fetch2 import runfetchcmd
73from bb.fetch2 import logger
74
Patrick Williamsc0f7c042017-02-23 20:41:17 -060075
76class GitProgressHandler(bb.progress.LineFilterProgressHandler):
77 """Extract progress information from git output"""
78 def __init__(self, d):
79 self._buffer = ''
80 self._count = 0
81 super(GitProgressHandler, self).__init__(d)
82 # Send an initial progress event so the bar gets shown
83 self._fire_progress(-1)
84
85 def write(self, string):
86 self._buffer += string
87 stages = ['Counting objects', 'Compressing objects', 'Receiving objects', 'Resolving deltas']
88 stage_weights = [0.2, 0.05, 0.5, 0.25]
89 stagenum = 0
90 for i, stage in reversed(list(enumerate(stages))):
91 if stage in self._buffer:
92 stagenum = i
93 self._buffer = ''
94 break
95 self._status = stages[stagenum]
96 percs = re.findall(r'(\d+)%', string)
97 if percs:
98 progress = int(round((int(percs[-1]) * stage_weights[stagenum]) + (sum(stage_weights[:stagenum]) * 100)))
99 rates = re.findall(r'([\d.]+ [a-zA-Z]*/s+)', string)
100 if rates:
101 rate = rates[-1]
102 else:
103 rate = None
104 self.update(progress, rate)
105 else:
106 if stagenum == 0:
107 percs = re.findall(r': (\d+)', string)
108 if percs:
109 count = int(percs[-1])
110 if count > self._count:
111 self._count = count
112 self._fire_progress(-count)
113 super(GitProgressHandler, self).write(string)
114
115
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500116class Git(FetchMethod):
Brad Bishop316dfdd2018-06-25 12:45:53 -0400117 bitbake_dir = os.path.abspath(os.path.join(os.path.dirname(os.path.join(os.path.abspath(__file__))), '..', '..', '..'))
118 make_shallow_path = os.path.join(bitbake_dir, 'bin', 'git-make-shallow')
119
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500120 """Class to fetch a module or modules from git repositories"""
121 def init(self, d):
122 pass
123
124 def supports(self, ud, d):
125 """
126 Check to see if a given url can be fetched with git.
127 """
128 return ud.type in ['git']
129
130 def supports_checksum(self, urldata):
131 return False
132
133 def urldata_init(self, ud, d):
134 """
135 init git specific variable within url data
136 so that the git method like latest_revision() can work
137 """
138 if 'protocol' in ud.parm:
139 ud.proto = ud.parm['protocol']
140 elif not ud.host:
141 ud.proto = 'file'
142 else:
143 ud.proto = "git"
144
145 if not ud.proto in ('git', 'file', 'ssh', 'http', 'https', 'rsync'):
146 raise bb.fetch2.ParameterError("Invalid protocol type", ud.url)
147
148 ud.nocheckout = ud.parm.get("nocheckout","0") == "1"
149
150 ud.rebaseable = ud.parm.get("rebaseable","0") == "1"
151
152 ud.nobranch = ud.parm.get("nobranch","0") == "1"
153
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600154 # usehead implies nobranch
155 ud.usehead = ud.parm.get("usehead","0") == "1"
156 if ud.usehead:
157 if ud.proto != "file":
158 raise bb.fetch2.ParameterError("The usehead option is only for use with local ('protocol=file') git repositories", ud.url)
159 ud.nobranch = 1
160
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500161 # bareclone implies nocheckout
162 ud.bareclone = ud.parm.get("bareclone","0") == "1"
163 if ud.bareclone:
164 ud.nocheckout = 1
165
166 ud.unresolvedrev = {}
167 branches = ud.parm.get("branch", "master").split(',')
168 if len(branches) != len(ud.names):
169 raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500170
Andrew Geisslerc926e172021-05-07 16:11:35 -0500171 ud.noshared = d.getVar("BB_GIT_NOSHARED") == "1"
172
173 ud.cloneflags = "-n"
174 if not ud.noshared:
175 ud.cloneflags += " -s"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500176 if ud.bareclone:
177 ud.cloneflags += " --mirror"
178
179 ud.shallow = d.getVar("BB_GIT_SHALLOW") == "1"
180 ud.shallow_extra_refs = (d.getVar("BB_GIT_SHALLOW_EXTRA_REFS") or "").split()
181
182 depth_default = d.getVar("BB_GIT_SHALLOW_DEPTH")
183 if depth_default is not None:
184 try:
185 depth_default = int(depth_default or 0)
186 except ValueError:
187 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH: %s" % depth_default)
188 else:
189 if depth_default < 0:
190 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH: %s" % depth_default)
191 else:
192 depth_default = 1
193 ud.shallow_depths = collections.defaultdict(lambda: depth_default)
194
Brad Bishop19323692019-04-05 15:28:33 -0400195 revs_default = d.getVar("BB_GIT_SHALLOW_REVS")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500196 ud.shallow_revs = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500197 ud.branches = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500198 for pos, name in enumerate(ud.names):
199 branch = branches[pos]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500200 ud.branches[name] = branch
201 ud.unresolvedrev[name] = branch
202
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500203 shallow_depth = d.getVar("BB_GIT_SHALLOW_DEPTH_%s" % name)
204 if shallow_depth is not None:
205 try:
206 shallow_depth = int(shallow_depth or 0)
207 except ValueError:
208 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (name, shallow_depth))
209 else:
210 if shallow_depth < 0:
211 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (name, shallow_depth))
212 ud.shallow_depths[name] = shallow_depth
213
214 revs = d.getVar("BB_GIT_SHALLOW_REVS_%s" % name)
215 if revs is not None:
216 ud.shallow_revs.extend(revs.split())
217 elif revs_default is not None:
218 ud.shallow_revs.extend(revs_default.split())
219
220 if (ud.shallow and
221 not ud.shallow_revs and
222 all(ud.shallow_depths[n] == 0 for n in ud.names)):
223 # Shallow disabled for this URL
224 ud.shallow = False
225
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600226 if ud.usehead:
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600227 # When usehead is set let's associate 'HEAD' with the unresolved
228 # rev of this repository. This will get resolved into a revision
229 # later. If an actual revision happens to have also been provided
230 # then this setting will be overridden.
231 for name in ud.names:
232 ud.unresolvedrev[name] = 'HEAD'
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600233
Andrew Geissler5f350902021-07-23 13:09:54 -0400234 ud.basecmd = d.getVar("FETCHCMD_git") or "git -c core.fsyncobjectfiles=0 -c gc.autoDetach=false"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500235
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500236 write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0"
237 ud.write_tarballs = write_tarballs != "0" or ud.rebaseable
238 ud.write_shallow_tarballs = (d.getVar("BB_GENERATE_SHALLOW_TARBALLS") or write_tarballs) != "0"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500239
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500240 ud.setup_revisions(d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500241
242 for name in ud.names:
243 # Ensure anything that doesn't look like a sha256 checksum/revision is translated into one
244 if not ud.revisions[name] or len(ud.revisions[name]) != 40 or (False in [c in "abcdef0123456789" for c in ud.revisions[name]]):
245 if ud.revisions[name]:
246 ud.unresolvedrev[name] = ud.revisions[name]
247 ud.revisions[name] = self.latest_revision(ud, d, name)
248
Andrew Geisslerc3d88e42020-10-02 09:45:00 -0500249 gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500250 if gitsrcname.startswith('.'):
251 gitsrcname = gitsrcname[1:]
252
253 # for rebaseable git repo, it is necessary to keep mirror tar ball
254 # per revision, so that even the revision disappears from the
255 # upstream repo in the future, the mirror will remain intact and still
256 # contains the revision
257 if ud.rebaseable:
258 for name in ud.names:
259 gitsrcname = gitsrcname + '_' + ud.revisions[name]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500260
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500261 dl_dir = d.getVar("DL_DIR")
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800262 gitdir = d.getVar("GITDIR") or (dl_dir + "/git2")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500263 ud.clonedir = os.path.join(gitdir, gitsrcname)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500264 ud.localfile = ud.clonedir
265
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500266 mirrortarball = 'git2_%s.tar.gz' % gitsrcname
267 ud.fullmirror = os.path.join(dl_dir, mirrortarball)
268 ud.mirrortarballs = [mirrortarball]
269 if ud.shallow:
270 tarballname = gitsrcname
271 if ud.bareclone:
272 tarballname = "%s_bare" % tarballname
273
274 if ud.shallow_revs:
275 tarballname = "%s_%s" % (tarballname, "_".join(sorted(ud.shallow_revs)))
276
277 for name, revision in sorted(ud.revisions.items()):
278 tarballname = "%s_%s" % (tarballname, ud.revisions[name][:7])
279 depth = ud.shallow_depths[name]
280 if depth:
281 tarballname = "%s-%s" % (tarballname, depth)
282
283 shallow_refs = []
284 if not ud.nobranch:
285 shallow_refs.extend(ud.branches.values())
286 if ud.shallow_extra_refs:
287 shallow_refs.extend(r.replace('refs/heads/', '').replace('*', 'ALL') for r in ud.shallow_extra_refs)
288 if shallow_refs:
289 tarballname = "%s_%s" % (tarballname, "_".join(sorted(shallow_refs)).replace('/', '.'))
290
291 fetcher = self.__class__.__name__.lower()
292 ud.shallowtarball = '%sshallow_%s.tar.gz' % (fetcher, tarballname)
293 ud.fullshallow = os.path.join(dl_dir, ud.shallowtarball)
294 ud.mirrortarballs.insert(0, ud.shallowtarball)
295
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500296 def localpath(self, ud, d):
297 return ud.clonedir
298
299 def need_update(self, ud, d):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800300 return self.clonedir_need_update(ud, d) or self.shallow_tarball_need_update(ud) or self.tarball_need_update(ud)
301
302 def clonedir_need_update(self, ud, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500303 if not os.path.exists(ud.clonedir):
304 return True
Brad Bishop64c979e2019-11-04 13:55:29 -0500305 if ud.shallow and ud.write_shallow_tarballs and self.clonedir_need_shallow_revs(ud, d):
306 return True
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500307 for name in ud.names:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600308 if not self._contains_ref(ud, d, name, ud.clonedir):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500309 return True
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500310 return False
311
Brad Bishop64c979e2019-11-04 13:55:29 -0500312 def clonedir_need_shallow_revs(self, ud, d):
313 for rev in ud.shallow_revs:
314 try:
315 runfetchcmd('%s rev-parse -q --verify %s' % (ud.basecmd, rev), d, quiet=True, workdir=ud.clonedir)
316 except bb.fetch2.FetchError:
317 return rev
318 return None
319
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800320 def shallow_tarball_need_update(self, ud):
321 return ud.shallow and ud.write_shallow_tarballs and not os.path.exists(ud.fullshallow)
322
323 def tarball_need_update(self, ud):
324 return ud.write_tarballs and not os.path.exists(ud.fullmirror)
325
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500326 def try_premirror(self, ud, d):
327 # If we don't do this, updating an existing checkout with only premirrors
328 # is not possible
Brad Bishop19323692019-04-05 15:28:33 -0400329 if bb.utils.to_boolean(d.getVar("BB_FETCH_PREMIRRORONLY")):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500330 return True
331 if os.path.exists(ud.clonedir):
332 return False
333 return True
334
335 def download(self, ud, d):
336 """Fetch url"""
337
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500338 # A current clone is preferred to either tarball, a shallow tarball is
339 # preferred to an out of date clone, and a missing clone will use
340 # either tarball.
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800341 if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500342 ud.localpath = ud.fullshallow
343 return
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800344 elif os.path.exists(ud.fullmirror) and not os.path.exists(ud.clonedir):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500345 bb.utils.mkdirhier(ud.clonedir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500346 runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500347
348 repourl = self._get_repo_url(ud)
349
350 # If the repo still doesn't exist, fallback to cloning it
351 if not os.path.exists(ud.clonedir):
352 # We do this since git will use a "-l" option automatically for local urls where possible
353 if repourl.startswith("file://"):
354 repourl = repourl[7:]
Andrew Geissler4c19ea12020-10-27 13:52:24 -0500355 clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, shlex.quote(repourl), ud.clonedir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500356 if ud.proto.lower() != 'file':
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500357 bb.fetch2.check_network_access(d, clone_cmd, ud.url)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600358 progresshandler = GitProgressHandler(d)
359 runfetchcmd(clone_cmd, d, log=progresshandler)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500360
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500361 # Update the checkout if needed
Brad Bishop64c979e2019-11-04 13:55:29 -0500362 if self.clonedir_need_update(ud, d):
Brad Bishop6ef32652018-10-09 18:59:25 +0100363 output = runfetchcmd("%s remote" % ud.basecmd, d, quiet=True, workdir=ud.clonedir)
364 if "origin" in output:
365 runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500366
Andrew Geissler4c19ea12020-10-27 13:52:24 -0500367 runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=ud.clonedir)
368 fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, shlex.quote(repourl))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500369 if ud.proto.lower() != 'file':
370 bb.fetch2.check_network_access(d, fetch_cmd, ud.url)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600371 progresshandler = GitProgressHandler(d)
372 runfetchcmd(fetch_cmd, d, log=progresshandler, workdir=ud.clonedir)
373 runfetchcmd("%s prune-packed" % ud.basecmd, d, workdir=ud.clonedir)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400374 runfetchcmd("%s pack-refs --all" % ud.basecmd, d, workdir=ud.clonedir)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600375 runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d, workdir=ud.clonedir)
Patrick Williamsd7e96312015-09-22 08:09:05 -0500376 try:
377 os.unlink(ud.fullmirror)
378 except OSError as exc:
379 if exc.errno != errno.ENOENT:
380 raise
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800381
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500382 for name in ud.names:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600383 if not self._contains_ref(ud, d, name, ud.clonedir):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500384 raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revisions[name], ud.branches[name]))
385
Brad Bishop64c979e2019-11-04 13:55:29 -0500386 if ud.shallow and ud.write_shallow_tarballs:
387 missing_rev = self.clonedir_need_shallow_revs(ud, d)
388 if missing_rev:
389 raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev)
390
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600391 if self._contains_lfs(ud, d, ud.clonedir) and self._need_lfs(ud):
392 # Unpack temporary working copy, use it to run 'git checkout' to force pre-fetching
393 # of all LFS blobs needed at the the srcrev.
394 #
395 # It would be nice to just do this inline here by running 'git-lfs fetch'
396 # on the bare clonedir, but that operation requires a working copy on some
397 # releases of Git LFS.
398 tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
399 try:
400 # Do the checkout. This implicitly involves a Git LFS fetch.
Andrew Geisslerc926e172021-05-07 16:11:35 -0500401 Git.unpack(self, ud, tmpdir, d)
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600402
403 # Scoop up a copy of any stuff that Git LFS downloaded. Merge them into
404 # the bare clonedir.
405 #
406 # As this procedure is invoked repeatedly on incremental fetches as
407 # a recipe's SRCREV is bumped throughout its lifetime, this will
408 # result in a gradual accumulation of LFS blobs in <ud.clonedir>/lfs
409 # corresponding to all the blobs reachable from the different revs
410 # fetched across time.
411 #
412 # Only do this if the unpack resulted in a .git/lfs directory being
413 # created; this only happens if at least one blob needed to be
414 # downloaded.
415 if os.path.exists(os.path.join(tmpdir, "git", ".git", "lfs")):
416 runfetchcmd("tar -cf - lfs | tar -xf - -C %s" % ud.clonedir, d, workdir="%s/git/.git" % tmpdir)
417 finally:
418 bb.utils.remove(tmpdir, recurse=True)
419
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500420 def build_mirror_data(self, ud, d):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500421 if ud.shallow and ud.write_shallow_tarballs:
422 if not os.path.exists(ud.fullshallow):
423 if os.path.islink(ud.fullshallow):
424 os.unlink(ud.fullshallow)
425 tempdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
426 shallowclone = os.path.join(tempdir, 'git')
427 try:
428 self.clone_shallow_local(ud, shallowclone, d)
429
430 logger.info("Creating tarball of git repository")
431 runfetchcmd("tar -czf %s ." % ud.fullshallow, d, workdir=shallowclone)
432 runfetchcmd("touch %s.done" % ud.fullshallow, d)
433 finally:
434 bb.utils.remove(tempdir, recurse=True)
435 elif ud.write_tarballs and not os.path.exists(ud.fullmirror):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500436 if os.path.islink(ud.fullmirror):
437 os.unlink(ud.fullmirror)
438
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500439 logger.info("Creating tarball of git repository")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500440 runfetchcmd("tar -czf %s ." % ud.fullmirror, d, workdir=ud.clonedir)
441 runfetchcmd("touch %s.done" % ud.fullmirror, d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500442
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500443 def clone_shallow_local(self, ud, dest, d):
444 """Clone the repo and make it shallow.
445
446 The upstream url of the new clone isn't set at this time, as it'll be
447 set correctly when unpacked."""
448 runfetchcmd("%s clone %s %s %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, dest), d)
449
450 to_parse, shallow_branches = [], []
451 for name in ud.names:
452 revision = ud.revisions[name]
453 depth = ud.shallow_depths[name]
454 if depth:
455 to_parse.append('%s~%d^{}' % (revision, depth - 1))
456
457 # For nobranch, we need a ref, otherwise the commits will be
458 # removed, and for non-nobranch, we truncate the branch to our
459 # srcrev, to avoid keeping unnecessary history beyond that.
460 branch = ud.branches[name]
461 if ud.nobranch:
462 ref = "refs/shallow/%s" % name
463 elif ud.bareclone:
464 ref = "refs/heads/%s" % branch
465 else:
466 ref = "refs/remotes/origin/%s" % branch
467
468 shallow_branches.append(ref)
469 runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest)
470
471 # Map srcrev+depths to revisions
472 parsed_depths = runfetchcmd("%s rev-parse %s" % (ud.basecmd, " ".join(to_parse)), d, workdir=dest)
473
474 # Resolve specified revisions
475 parsed_revs = runfetchcmd("%s rev-parse %s" % (ud.basecmd, " ".join('"%s^{}"' % r for r in ud.shallow_revs)), d, workdir=dest)
476 shallow_revisions = parsed_depths.splitlines() + parsed_revs.splitlines()
477
478 # Apply extra ref wildcards
479 all_refs = runfetchcmd('%s for-each-ref "--format=%%(refname)"' % ud.basecmd,
480 d, workdir=dest).splitlines()
481 for r in ud.shallow_extra_refs:
482 if not ud.bareclone:
483 r = r.replace('refs/heads/', 'refs/remotes/origin/')
484
485 if '*' in r:
486 matches = filter(lambda a: fnmatch.fnmatchcase(a, r), all_refs)
487 shallow_branches.extend(matches)
488 else:
489 shallow_branches.append(r)
490
491 # Make the repository shallow
Brad Bishop316dfdd2018-06-25 12:45:53 -0400492 shallow_cmd = [self.make_shallow_path, '-s']
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500493 for b in shallow_branches:
494 shallow_cmd.append('-r')
495 shallow_cmd.append(b)
496 shallow_cmd.extend(shallow_revisions)
497 runfetchcmd(subprocess.list2cmdline(shallow_cmd), d, workdir=dest)
498
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500499 def unpack(self, ud, destdir, d):
500 """ unpack the downloaded src to destdir"""
501
502 subdir = ud.parm.get("subpath", "")
503 if subdir != "":
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500504 readpathspec = ":%s" % subdir
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500505 def_destsuffix = "%s/" % os.path.basename(subdir.rstrip('/'))
506 else:
507 readpathspec = ""
508 def_destsuffix = "git/"
509
510 destsuffix = ud.parm.get("destsuffix", def_destsuffix)
511 destdir = ud.destdir = os.path.join(destdir, destsuffix)
512 if os.path.exists(destdir):
513 bb.utils.prunedir(destdir)
514
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600515 need_lfs = self._need_lfs(ud)
Brad Bishopa34c0302019-09-23 22:34:48 -0400516
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500517 if not need_lfs:
518 ud.basecmd = "GIT_LFS_SKIP_SMUDGE=1 " + ud.basecmd
519
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800520 source_found = False
521 source_error = []
522
523 if not source_found:
524 clonedir_is_up_to_date = not self.clonedir_need_update(ud, d)
525 if clonedir_is_up_to_date:
526 runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d)
527 source_found = True
528 else:
529 source_error.append("clone directory not available or not up to date: " + ud.clonedir)
530
531 if not source_found:
532 if ud.shallow:
533 if os.path.exists(ud.fullshallow):
534 bb.utils.mkdirhier(destdir)
535 runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=destdir)
536 source_found = True
537 else:
538 source_error.append("shallow clone not available: " + ud.fullshallow)
539 else:
540 source_error.append("shallow clone not enabled")
541
542 if not source_found:
543 raise bb.fetch2.UnpackError("No up to date source found: " + "; ".join(source_error), ud.url)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500544
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500545 repourl = self._get_repo_url(ud)
Andrew Geissler4c19ea12020-10-27 13:52:24 -0500546 runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=destdir)
Brad Bishopc342db32019-05-15 21:57:59 -0400547
548 if self._contains_lfs(ud, d, destdir):
Brad Bishop00e122a2019-10-05 11:10:57 -0400549 if need_lfs and not self._find_git_lfs(d):
550 raise bb.fetch2.FetchError("Repository %s has LFS content, install git-lfs on host to download (or set lfs=0 to ignore it)" % (repourl))
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500551 elif not need_lfs:
Brad Bishopa34c0302019-09-23 22:34:48 -0400552 bb.note("Repository %s has LFS content but it is not being fetched" % (repourl))
Brad Bishopc342db32019-05-15 21:57:59 -0400553
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500554 if not ud.nocheckout:
555 if subdir != "":
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600556 runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revisions[ud.names[0]], readpathspec), d,
557 workdir=destdir)
558 runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d, workdir=destdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500559 elif not ud.nobranch:
560 branchname = ud.branches[ud.names[0]]
561 runfetchcmd("%s checkout -B %s %s" % (ud.basecmd, branchname, \
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600562 ud.revisions[ud.names[0]]), d, workdir=destdir)
Andre Rosa49271d42017-09-07 11:15:55 +0200563 runfetchcmd("%s branch %s --set-upstream-to origin/%s" % (ud.basecmd, branchname, \
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600564 branchname), d, workdir=destdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500565 else:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600566 runfetchcmd("%s checkout %s" % (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=destdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500567
568 return True
569
570 def clean(self, ud, d):
571 """ clean the git directory """
572
Brad Bishop19323692019-04-05 15:28:33 -0400573 to_remove = [ud.localpath, ud.fullmirror, ud.fullmirror + ".done"]
574 # The localpath is a symlink to clonedir when it is cloned from a
575 # mirror, so remove both of them.
576 if os.path.islink(ud.localpath):
577 clonedir = os.path.realpath(ud.localpath)
578 to_remove.append(clonedir)
579
580 for r in to_remove:
581 if os.path.exists(r):
582 bb.note('Removing %s' % r)
583 bb.utils.remove(r, True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500584
585 def supports_srcrev(self):
586 return True
587
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600588 def _contains_ref(self, ud, d, name, wd):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500589 cmd = ""
590 if ud.nobranch:
591 cmd = "%s log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % (
592 ud.basecmd, ud.revisions[name])
593 else:
594 cmd = "%s branch --contains %s --list %s 2> /dev/null | wc -l" % (
595 ud.basecmd, ud.revisions[name], ud.branches[name])
596 try:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600597 output = runfetchcmd(cmd, d, quiet=True, workdir=wd)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500598 except bb.fetch2.FetchError:
599 return False
600 if len(output.split()) > 1:
601 raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output))
602 return output.split()[0] != "0"
603
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600604 def _need_lfs(self, ud):
605 return ud.parm.get("lfs", "1") == "1"
606
Brad Bishopc342db32019-05-15 21:57:59 -0400607 def _contains_lfs(self, ud, d, wd):
608 """
609 Check if the repository has 'lfs' (large file) content
610 """
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500611
612 if not ud.nobranch:
613 branchname = ud.branches[ud.names[0]]
614 else:
615 branchname = "master"
616
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600617 # The bare clonedir doesn't use the remote names; it has the branch immediately.
618 if wd == ud.clonedir:
619 refname = ud.branches[ud.names[0]]
620 else:
621 refname = "origin/%s" % ud.branches[ud.names[0]]
622
623 cmd = "%s grep lfs %s:.gitattributes | wc -l" % (
624 ud.basecmd, refname)
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500625
Brad Bishopc342db32019-05-15 21:57:59 -0400626 try:
627 output = runfetchcmd(cmd, d, quiet=True, workdir=wd)
628 if int(output) > 0:
629 return True
630 except (bb.fetch2.FetchError,ValueError):
631 pass
632 return False
633
Brad Bishop00e122a2019-10-05 11:10:57 -0400634 def _find_git_lfs(self, d):
635 """
636 Return True if git-lfs can be found, False otherwise.
637 """
638 import shutil
639 return shutil.which("git-lfs", path=d.getVar('PATH')) is not None
640
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500641 def _get_repo_url(self, ud):
642 """
643 Return the repository URL
644 """
Andrew Geissler6ce62a22020-11-30 19:58:47 -0600645 # Note that we do not support passwords directly in the git urls. There are several
646 # reasons. SRC_URI can be written out to things like buildhistory and people don't
647 # want to leak passwords like that. Its also all too easy to share metadata without
648 # removing the password. ssh keys, ~/.netrc and ~/.ssh/config files can be used as
649 # alternatives so we will not take patches adding password support here.
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500650 if ud.user:
651 username = ud.user + '@'
652 else:
653 username = ""
654 return "%s://%s%s%s" % (ud.proto, username, ud.host, ud.path)
655
656 def _revision_key(self, ud, d, name):
657 """
658 Return a unique key for the url
659 """
Andrew Geissler82c905d2020-04-13 13:39:40 -0500660 # Collapse adjacent slashes
661 slash_re = re.compile(r"/+")
662 return "git:" + ud.host + slash_re.sub(".", ud.path) + ud.unresolvedrev[name]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500663
664 def _lsremote(self, ud, d, search):
665 """
666 Run git ls-remote with the specified search string
667 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500668 # Prevent recursion e.g. in OE if SRCPV is in PV, PV is in WORKDIR,
669 # and WORKDIR is in PATH (as a result of RSS), our call to
670 # runfetchcmd() exports PATH so this function will get called again (!)
671 # In this scenario the return call of the function isn't actually
672 # important - WORKDIR isn't needed in PATH to call git ls-remote
673 # anyway.
674 if d.getVar('_BB_GIT_IN_LSREMOTE', False):
675 return ''
676 d.setVar('_BB_GIT_IN_LSREMOTE', '1')
677 try:
678 repourl = self._get_repo_url(ud)
Andrew Geissler4c19ea12020-10-27 13:52:24 -0500679 cmd = "%s ls-remote %s %s" % \
680 (ud.basecmd, shlex.quote(repourl), search)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500681 if ud.proto.lower() != 'file':
682 bb.fetch2.check_network_access(d, cmd, repourl)
683 output = runfetchcmd(cmd, d, True)
684 if not output:
685 raise bb.fetch2.FetchError("The command %s gave empty output unexpectedly" % cmd, ud.url)
686 finally:
687 d.delVar('_BB_GIT_IN_LSREMOTE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500688 return output
689
690 def _latest_revision(self, ud, d, name):
691 """
692 Compute the HEAD revision for the url
693 """
694 output = self._lsremote(ud, d, "")
695 # Tags of the form ^{} may not work, need to fallback to other form
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600696 if ud.unresolvedrev[name][:5] == "refs/" or ud.usehead:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500697 head = ud.unresolvedrev[name]
698 tag = ud.unresolvedrev[name]
699 else:
700 head = "refs/heads/%s" % ud.unresolvedrev[name]
701 tag = "refs/tags/%s" % ud.unresolvedrev[name]
702 for s in [head, tag + "^{}", tag]:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600703 for l in output.strip().split('\n'):
704 sha1, ref = l.split()
705 if s == ref:
706 return sha1
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500707 raise bb.fetch2.FetchError("Unable to resolve '%s' in upstream git repository in git ls-remote output for %s" % \
708 (ud.unresolvedrev[name], ud.host+ud.path))
709
710 def latest_versionstring(self, ud, d):
711 """
712 Compute the latest release name like "x.y.x" in "x.y.x+gitHASH"
713 by searching through the tags output of ls-remote, comparing
714 versions and returning the highest match.
715 """
716 pupver = ('', '')
717
Brad Bishop19323692019-04-05 15:28:33 -0400718 tagregex = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX') or r"(?P<pver>([0-9][\.|_]?)+)")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500719 try:
720 output = self._lsremote(ud, d, "refs/tags/*")
Brad Bishop316dfdd2018-06-25 12:45:53 -0400721 except (bb.fetch2.FetchError, bb.fetch2.NetworkAccess) as e:
722 bb.note("Could not list remote: %s" % str(e))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500723 return pupver
724
725 verstring = ""
726 revision = ""
727 for line in output.split("\n"):
728 if not line:
729 break
730
731 tag_head = line.split("/")[-1]
732 # Ignore non-released branches
Brad Bishop19323692019-04-05 15:28:33 -0400733 m = re.search(r"(alpha|beta|rc|final)+", tag_head)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500734 if m:
735 continue
736
737 # search for version in the line
738 tag = tagregex.search(tag_head)
Andrew Geissler82c905d2020-04-13 13:39:40 -0500739 if tag is None:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500740 continue
741
742 tag = tag.group('pver')
743 tag = tag.replace("_", ".")
744
745 if verstring and bb.utils.vercmp(("0", tag, ""), ("0", verstring, "")) < 0:
746 continue
747
748 verstring = tag
749 revision = line.split()[0]
750 pupver = (verstring, revision)
751
752 return pupver
753
754 def _build_revision(self, ud, d, name):
755 return ud.revisions[name]
756
757 def gitpkgv_revision(self, ud, d, name):
758 """
759 Return a sortable revision number by counting commits in the history
760 Based on gitpkgv.bblass in meta-openembedded
761 """
762 rev = self._build_revision(ud, d, name)
763 localpath = ud.localpath
764 rev_file = os.path.join(localpath, "oe-gitpkgv_" + rev)
765 if not os.path.exists(localpath):
766 commits = None
767 else:
768 if not os.path.exists(rev_file) or not os.path.getsize(rev_file):
769 from pipes import quote
770 commits = bb.fetch2.runfetchcmd(
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500771 "git rev-list %s -- | wc -l" % quote(rev),
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500772 d, quiet=True).strip().lstrip('0')
773 if commits:
774 open(rev_file, "w").write("%d\n" % int(commits))
775 else:
776 commits = open(rev_file, "r").readline(128).strip()
777 if commits:
778 return False, "%s+%s" % (commits, rev[:7])
779 else:
780 return True, str(rev)
781
782 def checkstatus(self, fetch, ud, d):
783 try:
784 self._lsremote(ud, d, "")
785 return True
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500786 except bb.fetch2.FetchError:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500787 return False