blob: b3c5e6dacc9a053062eee41114c3108ebf7ec58c [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001"""
2BitBake 'Fetch' git implementation
3
4git fetcher support the SRC_URI with format of:
5SRC_URI = "git://some.host/somepath;OptionA=xxx;OptionB=xxx;..."
6
7Supported SRC_URI options are:
8
9- branch
10 The git branch to retrieve from. The default is "master"
11
12 This option also supports multiple branch fetching, with branches
13 separated by commas. In multiple branches case, the name option
14 must have the same number of names to match the branches, which is
15 used to specify the SRC_REV for the branch
16 e.g:
17 SRC_URI="git://some.host/somepath;branch=branchX,branchY;name=nameX,nameY"
18 SRCREV_nameX = "xxxxxxxxxxxxxxxxxxxx"
19 SRCREV_nameY = "YYYYYYYYYYYYYYYYYYYY"
20
21- tag
22 The git tag to retrieve. The default is "master"
23
24- protocol
25 The method to use to access the repository. Common options are "git",
26 "http", "https", "file", "ssh" and "rsync". The default is "git".
27
28- rebaseable
29 rebaseable indicates that the upstream git repo may rebase in the future,
30 and current revision may disappear from upstream repo. This option will
31 remind fetcher to preserve local cache carefully for future use.
32 The default value is "0", set rebaseable=1 for rebaseable git repo.
33
34- nocheckout
35 Don't checkout source code when unpacking. set this option for the recipe
36 who has its own routine to checkout code.
37 The default is "0", set nocheckout=1 if needed.
38
39- bareclone
40 Create a bare clone of the source code and don't checkout the source code
41 when unpacking. Set this option for the recipe who has its own routine to
42 checkout code and tracking branch requirements.
43 The default is "0", set bareclone=1 if needed.
44
45- nobranch
46 Don't check the SHA validation for branch. set this option for the recipe
47 referring to commit which is valid in tag instead of branch.
48 The default is "0", set nobranch=1 if needed.
49
Patrick Williamsc0f7c042017-02-23 20:41:17 -060050- usehead
Brad Bishop6e60e8b2018-02-01 10:27:11 -050051 For local git:// urls to use the current branch HEAD as the revision for use with
Patrick Williamsc0f7c042017-02-23 20:41:17 -060052 AUTOREV. Implies nobranch.
53
Patrick Williamsc124f4f2015-09-15 14:41:29 -050054"""
55
Brad Bishopc342db32019-05-15 21:57:59 -040056# Copyright (C) 2005 Richard Purdie
Patrick Williamsc124f4f2015-09-15 14:41:29 -050057#
Brad Bishopc342db32019-05-15 21:57:59 -040058# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -050059#
Patrick Williamsc124f4f2015-09-15 14:41:29 -050060
Brad Bishopd7bf8c12018-02-25 22:55:05 -050061import collections
Patrick Williamsd7e96312015-09-22 08:09:05 -050062import errno
Brad Bishopd7bf8c12018-02-25 22:55:05 -050063import fnmatch
Patrick Williamsc124f4f2015-09-15 14:41:29 -050064import os
65import re
Andrew Geissler4c19ea12020-10-27 13:52:24 -050066import shlex
Brad Bishopd7bf8c12018-02-25 22:55:05 -050067import subprocess
68import tempfile
Patrick Williamsc124f4f2015-09-15 14:41:29 -050069import bb
Patrick Williamsc0f7c042017-02-23 20:41:17 -060070import bb.progress
Andrew Geissler5199d832021-09-24 16:47:35 -050071from contextlib import contextmanager
Patrick Williamsc124f4f2015-09-15 14:41:29 -050072from bb.fetch2 import FetchMethod
73from bb.fetch2 import runfetchcmd
74from bb.fetch2 import logger
75
Patrick Williamsc0f7c042017-02-23 20:41:17 -060076
77class GitProgressHandler(bb.progress.LineFilterProgressHandler):
78 """Extract progress information from git output"""
79 def __init__(self, d):
80 self._buffer = ''
81 self._count = 0
82 super(GitProgressHandler, self).__init__(d)
83 # Send an initial progress event so the bar gets shown
84 self._fire_progress(-1)
85
86 def write(self, string):
87 self._buffer += string
88 stages = ['Counting objects', 'Compressing objects', 'Receiving objects', 'Resolving deltas']
89 stage_weights = [0.2, 0.05, 0.5, 0.25]
90 stagenum = 0
91 for i, stage in reversed(list(enumerate(stages))):
92 if stage in self._buffer:
93 stagenum = i
94 self._buffer = ''
95 break
96 self._status = stages[stagenum]
97 percs = re.findall(r'(\d+)%', string)
98 if percs:
99 progress = int(round((int(percs[-1]) * stage_weights[stagenum]) + (sum(stage_weights[:stagenum]) * 100)))
100 rates = re.findall(r'([\d.]+ [a-zA-Z]*/s+)', string)
101 if rates:
102 rate = rates[-1]
103 else:
104 rate = None
105 self.update(progress, rate)
106 else:
107 if stagenum == 0:
108 percs = re.findall(r': (\d+)', string)
109 if percs:
110 count = int(percs[-1])
111 if count > self._count:
112 self._count = count
113 self._fire_progress(-count)
114 super(GitProgressHandler, self).write(string)
115
116
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500117class Git(FetchMethod):
Brad Bishop316dfdd2018-06-25 12:45:53 -0400118 bitbake_dir = os.path.abspath(os.path.join(os.path.dirname(os.path.join(os.path.abspath(__file__))), '..', '..', '..'))
119 make_shallow_path = os.path.join(bitbake_dir, 'bin', 'git-make-shallow')
120
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500121 """Class to fetch a module or modules from git repositories"""
122 def init(self, d):
123 pass
124
125 def supports(self, ud, d):
126 """
127 Check to see if a given url can be fetched with git.
128 """
129 return ud.type in ['git']
130
131 def supports_checksum(self, urldata):
132 return False
133
134 def urldata_init(self, ud, d):
135 """
136 init git specific variable within url data
137 so that the git method like latest_revision() can work
138 """
139 if 'protocol' in ud.parm:
140 ud.proto = ud.parm['protocol']
141 elif not ud.host:
142 ud.proto = 'file'
143 else:
144 ud.proto = "git"
Andrew Geissler595f6302022-01-24 19:11:47 +0000145 if ud.host == "github.com" and ud.proto == "git":
146 # github stopped supporting git protocol
147 # https://github.blog/2021-09-01-improving-git-protocol-security-github/#no-more-unauthenticated-git
148 ud.proto = "https"
149 bb.warn("URL: %s uses git protocol which is no longer supported by github. Please change to ;protocol=https in the url." % ud.url)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500150
151 if not ud.proto in ('git', 'file', 'ssh', 'http', 'https', 'rsync'):
152 raise bb.fetch2.ParameterError("Invalid protocol type", ud.url)
153
154 ud.nocheckout = ud.parm.get("nocheckout","0") == "1"
155
156 ud.rebaseable = ud.parm.get("rebaseable","0") == "1"
157
158 ud.nobranch = ud.parm.get("nobranch","0") == "1"
159
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600160 # usehead implies nobranch
161 ud.usehead = ud.parm.get("usehead","0") == "1"
162 if ud.usehead:
163 if ud.proto != "file":
164 raise bb.fetch2.ParameterError("The usehead option is only for use with local ('protocol=file') git repositories", ud.url)
165 ud.nobranch = 1
166
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500167 # bareclone implies nocheckout
168 ud.bareclone = ud.parm.get("bareclone","0") == "1"
169 if ud.bareclone:
170 ud.nocheckout = 1
171
172 ud.unresolvedrev = {}
Andrew Geissler595f6302022-01-24 19:11:47 +0000173 branches = ud.parm.get("branch", "").split(',')
174 if branches == [""] and not ud.nobranch:
175 bb.warn("URL: %s does not set any branch parameter. The future default branch used by tools and repositories is uncertain and we will therefore soon require this is set in all git urls." % ud.url)
176 branches = ["master"]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500177 if len(branches) != len(ud.names):
178 raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500179
Andrew Geisslerc926e172021-05-07 16:11:35 -0500180 ud.noshared = d.getVar("BB_GIT_NOSHARED") == "1"
181
182 ud.cloneflags = "-n"
183 if not ud.noshared:
184 ud.cloneflags += " -s"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500185 if ud.bareclone:
186 ud.cloneflags += " --mirror"
187
188 ud.shallow = d.getVar("BB_GIT_SHALLOW") == "1"
189 ud.shallow_extra_refs = (d.getVar("BB_GIT_SHALLOW_EXTRA_REFS") or "").split()
190
191 depth_default = d.getVar("BB_GIT_SHALLOW_DEPTH")
192 if depth_default is not None:
193 try:
194 depth_default = int(depth_default or 0)
195 except ValueError:
196 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH: %s" % depth_default)
197 else:
198 if depth_default < 0:
199 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH: %s" % depth_default)
200 else:
201 depth_default = 1
202 ud.shallow_depths = collections.defaultdict(lambda: depth_default)
203
Brad Bishop19323692019-04-05 15:28:33 -0400204 revs_default = d.getVar("BB_GIT_SHALLOW_REVS")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500205 ud.shallow_revs = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500206 ud.branches = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500207 for pos, name in enumerate(ud.names):
208 branch = branches[pos]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500209 ud.branches[name] = branch
210 ud.unresolvedrev[name] = branch
211
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500212 shallow_depth = d.getVar("BB_GIT_SHALLOW_DEPTH_%s" % name)
213 if shallow_depth is not None:
214 try:
215 shallow_depth = int(shallow_depth or 0)
216 except ValueError:
217 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (name, shallow_depth))
218 else:
219 if shallow_depth < 0:
220 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (name, shallow_depth))
221 ud.shallow_depths[name] = shallow_depth
222
223 revs = d.getVar("BB_GIT_SHALLOW_REVS_%s" % name)
224 if revs is not None:
225 ud.shallow_revs.extend(revs.split())
226 elif revs_default is not None:
227 ud.shallow_revs.extend(revs_default.split())
228
229 if (ud.shallow and
230 not ud.shallow_revs and
231 all(ud.shallow_depths[n] == 0 for n in ud.names)):
232 # Shallow disabled for this URL
233 ud.shallow = False
234
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600235 if ud.usehead:
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600236 # When usehead is set let's associate 'HEAD' with the unresolved
237 # rev of this repository. This will get resolved into a revision
238 # later. If an actual revision happens to have also been provided
239 # then this setting will be overridden.
240 for name in ud.names:
241 ud.unresolvedrev[name] = 'HEAD'
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600242
Andrew Geissler5f350902021-07-23 13:09:54 -0400243 ud.basecmd = d.getVar("FETCHCMD_git") or "git -c core.fsyncobjectfiles=0 -c gc.autoDetach=false"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500244
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500245 write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0"
246 ud.write_tarballs = write_tarballs != "0" or ud.rebaseable
247 ud.write_shallow_tarballs = (d.getVar("BB_GENERATE_SHALLOW_TARBALLS") or write_tarballs) != "0"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500248
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500249 ud.setup_revisions(d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500250
251 for name in ud.names:
252 # Ensure anything that doesn't look like a sha256 checksum/revision is translated into one
253 if not ud.revisions[name] or len(ud.revisions[name]) != 40 or (False in [c in "abcdef0123456789" for c in ud.revisions[name]]):
254 if ud.revisions[name]:
255 ud.unresolvedrev[name] = ud.revisions[name]
256 ud.revisions[name] = self.latest_revision(ud, d, name)
257
Andrew Geisslerc3d88e42020-10-02 09:45:00 -0500258 gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500259 if gitsrcname.startswith('.'):
260 gitsrcname = gitsrcname[1:]
261
262 # for rebaseable git repo, it is necessary to keep mirror tar ball
263 # per revision, so that even the revision disappears from the
264 # upstream repo in the future, the mirror will remain intact and still
265 # contains the revision
266 if ud.rebaseable:
267 for name in ud.names:
268 gitsrcname = gitsrcname + '_' + ud.revisions[name]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500269
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500270 dl_dir = d.getVar("DL_DIR")
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800271 gitdir = d.getVar("GITDIR") or (dl_dir + "/git2")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500272 ud.clonedir = os.path.join(gitdir, gitsrcname)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500273 ud.localfile = ud.clonedir
274
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500275 mirrortarball = 'git2_%s.tar.gz' % gitsrcname
276 ud.fullmirror = os.path.join(dl_dir, mirrortarball)
277 ud.mirrortarballs = [mirrortarball]
278 if ud.shallow:
279 tarballname = gitsrcname
280 if ud.bareclone:
281 tarballname = "%s_bare" % tarballname
282
283 if ud.shallow_revs:
284 tarballname = "%s_%s" % (tarballname, "_".join(sorted(ud.shallow_revs)))
285
286 for name, revision in sorted(ud.revisions.items()):
287 tarballname = "%s_%s" % (tarballname, ud.revisions[name][:7])
288 depth = ud.shallow_depths[name]
289 if depth:
290 tarballname = "%s-%s" % (tarballname, depth)
291
292 shallow_refs = []
293 if not ud.nobranch:
294 shallow_refs.extend(ud.branches.values())
295 if ud.shallow_extra_refs:
296 shallow_refs.extend(r.replace('refs/heads/', '').replace('*', 'ALL') for r in ud.shallow_extra_refs)
297 if shallow_refs:
298 tarballname = "%s_%s" % (tarballname, "_".join(sorted(shallow_refs)).replace('/', '.'))
299
300 fetcher = self.__class__.__name__.lower()
301 ud.shallowtarball = '%sshallow_%s.tar.gz' % (fetcher, tarballname)
302 ud.fullshallow = os.path.join(dl_dir, ud.shallowtarball)
303 ud.mirrortarballs.insert(0, ud.shallowtarball)
304
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500305 def localpath(self, ud, d):
306 return ud.clonedir
307
308 def need_update(self, ud, d):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800309 return self.clonedir_need_update(ud, d) or self.shallow_tarball_need_update(ud) or self.tarball_need_update(ud)
310
311 def clonedir_need_update(self, ud, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500312 if not os.path.exists(ud.clonedir):
313 return True
Brad Bishop64c979e2019-11-04 13:55:29 -0500314 if ud.shallow and ud.write_shallow_tarballs and self.clonedir_need_shallow_revs(ud, d):
315 return True
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500316 for name in ud.names:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600317 if not self._contains_ref(ud, d, name, ud.clonedir):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500318 return True
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500319 return False
320
Brad Bishop64c979e2019-11-04 13:55:29 -0500321 def clonedir_need_shallow_revs(self, ud, d):
322 for rev in ud.shallow_revs:
323 try:
324 runfetchcmd('%s rev-parse -q --verify %s' % (ud.basecmd, rev), d, quiet=True, workdir=ud.clonedir)
325 except bb.fetch2.FetchError:
326 return rev
327 return None
328
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800329 def shallow_tarball_need_update(self, ud):
330 return ud.shallow and ud.write_shallow_tarballs and not os.path.exists(ud.fullshallow)
331
332 def tarball_need_update(self, ud):
333 return ud.write_tarballs and not os.path.exists(ud.fullmirror)
334
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500335 def try_premirror(self, ud, d):
336 # If we don't do this, updating an existing checkout with only premirrors
337 # is not possible
Brad Bishop19323692019-04-05 15:28:33 -0400338 if bb.utils.to_boolean(d.getVar("BB_FETCH_PREMIRRORONLY")):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500339 return True
340 if os.path.exists(ud.clonedir):
341 return False
342 return True
343
344 def download(self, ud, d):
345 """Fetch url"""
346
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500347 # A current clone is preferred to either tarball, a shallow tarball is
348 # preferred to an out of date clone, and a missing clone will use
349 # either tarball.
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800350 if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500351 ud.localpath = ud.fullshallow
352 return
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800353 elif os.path.exists(ud.fullmirror) and not os.path.exists(ud.clonedir):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500354 bb.utils.mkdirhier(ud.clonedir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500355 runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500356
357 repourl = self._get_repo_url(ud)
358
359 # If the repo still doesn't exist, fallback to cloning it
360 if not os.path.exists(ud.clonedir):
361 # We do this since git will use a "-l" option automatically for local urls where possible
362 if repourl.startswith("file://"):
363 repourl = repourl[7:]
Andrew Geissler4c19ea12020-10-27 13:52:24 -0500364 clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, shlex.quote(repourl), ud.clonedir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500365 if ud.proto.lower() != 'file':
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500366 bb.fetch2.check_network_access(d, clone_cmd, ud.url)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600367 progresshandler = GitProgressHandler(d)
368 runfetchcmd(clone_cmd, d, log=progresshandler)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500369
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500370 # Update the checkout if needed
Brad Bishop64c979e2019-11-04 13:55:29 -0500371 if self.clonedir_need_update(ud, d):
Brad Bishop6ef32652018-10-09 18:59:25 +0100372 output = runfetchcmd("%s remote" % ud.basecmd, d, quiet=True, workdir=ud.clonedir)
373 if "origin" in output:
374 runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500375
Andrew Geissler4c19ea12020-10-27 13:52:24 -0500376 runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=ud.clonedir)
377 fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, shlex.quote(repourl))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500378 if ud.proto.lower() != 'file':
379 bb.fetch2.check_network_access(d, fetch_cmd, ud.url)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600380 progresshandler = GitProgressHandler(d)
381 runfetchcmd(fetch_cmd, d, log=progresshandler, workdir=ud.clonedir)
382 runfetchcmd("%s prune-packed" % ud.basecmd, d, workdir=ud.clonedir)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400383 runfetchcmd("%s pack-refs --all" % ud.basecmd, d, workdir=ud.clonedir)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600384 runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d, workdir=ud.clonedir)
Patrick Williamsd7e96312015-09-22 08:09:05 -0500385 try:
386 os.unlink(ud.fullmirror)
387 except OSError as exc:
388 if exc.errno != errno.ENOENT:
389 raise
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800390
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500391 for name in ud.names:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600392 if not self._contains_ref(ud, d, name, ud.clonedir):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500393 raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revisions[name], ud.branches[name]))
394
Brad Bishop64c979e2019-11-04 13:55:29 -0500395 if ud.shallow and ud.write_shallow_tarballs:
396 missing_rev = self.clonedir_need_shallow_revs(ud, d)
397 if missing_rev:
398 raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev)
399
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600400 if self._contains_lfs(ud, d, ud.clonedir) and self._need_lfs(ud):
401 # Unpack temporary working copy, use it to run 'git checkout' to force pre-fetching
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000402 # of all LFS blobs needed at the srcrev.
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600403 #
404 # It would be nice to just do this inline here by running 'git-lfs fetch'
405 # on the bare clonedir, but that operation requires a working copy on some
406 # releases of Git LFS.
407 tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
408 try:
409 # Do the checkout. This implicitly involves a Git LFS fetch.
Andrew Geisslerc926e172021-05-07 16:11:35 -0500410 Git.unpack(self, ud, tmpdir, d)
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600411
412 # Scoop up a copy of any stuff that Git LFS downloaded. Merge them into
413 # the bare clonedir.
414 #
415 # As this procedure is invoked repeatedly on incremental fetches as
416 # a recipe's SRCREV is bumped throughout its lifetime, this will
417 # result in a gradual accumulation of LFS blobs in <ud.clonedir>/lfs
418 # corresponding to all the blobs reachable from the different revs
419 # fetched across time.
420 #
421 # Only do this if the unpack resulted in a .git/lfs directory being
422 # created; this only happens if at least one blob needed to be
423 # downloaded.
424 if os.path.exists(os.path.join(tmpdir, "git", ".git", "lfs")):
425 runfetchcmd("tar -cf - lfs | tar -xf - -C %s" % ud.clonedir, d, workdir="%s/git/.git" % tmpdir)
426 finally:
427 bb.utils.remove(tmpdir, recurse=True)
428
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500429 def build_mirror_data(self, ud, d):
Andrew Geissler5199d832021-09-24 16:47:35 -0500430
431 # Create as a temp file and move atomically into position to avoid races
432 @contextmanager
433 def create_atomic(filename):
434 fd, tfile = tempfile.mkstemp(dir=os.path.dirname(filename))
435 try:
436 yield tfile
437 umask = os.umask(0o666)
438 os.umask(umask)
439 os.chmod(tfile, (0o666 & ~umask))
440 os.rename(tfile, filename)
441 finally:
442 os.close(fd)
443
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500444 if ud.shallow and ud.write_shallow_tarballs:
445 if not os.path.exists(ud.fullshallow):
446 if os.path.islink(ud.fullshallow):
447 os.unlink(ud.fullshallow)
448 tempdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
449 shallowclone = os.path.join(tempdir, 'git')
450 try:
451 self.clone_shallow_local(ud, shallowclone, d)
452
453 logger.info("Creating tarball of git repository")
Andrew Geissler5199d832021-09-24 16:47:35 -0500454 with create_atomic(ud.fullshallow) as tfile:
455 runfetchcmd("tar -czf %s ." % tfile, d, workdir=shallowclone)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500456 runfetchcmd("touch %s.done" % ud.fullshallow, d)
457 finally:
458 bb.utils.remove(tempdir, recurse=True)
459 elif ud.write_tarballs and not os.path.exists(ud.fullmirror):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500460 if os.path.islink(ud.fullmirror):
461 os.unlink(ud.fullmirror)
462
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500463 logger.info("Creating tarball of git repository")
Andrew Geissler5199d832021-09-24 16:47:35 -0500464 with create_atomic(ud.fullmirror) as tfile:
465 runfetchcmd("tar -czf %s ." % tfile, d, workdir=ud.clonedir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500466 runfetchcmd("touch %s.done" % ud.fullmirror, d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500467
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500468 def clone_shallow_local(self, ud, dest, d):
469 """Clone the repo and make it shallow.
470
471 The upstream url of the new clone isn't set at this time, as it'll be
472 set correctly when unpacked."""
473 runfetchcmd("%s clone %s %s %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, dest), d)
474
475 to_parse, shallow_branches = [], []
476 for name in ud.names:
477 revision = ud.revisions[name]
478 depth = ud.shallow_depths[name]
479 if depth:
480 to_parse.append('%s~%d^{}' % (revision, depth - 1))
481
482 # For nobranch, we need a ref, otherwise the commits will be
483 # removed, and for non-nobranch, we truncate the branch to our
484 # srcrev, to avoid keeping unnecessary history beyond that.
485 branch = ud.branches[name]
486 if ud.nobranch:
487 ref = "refs/shallow/%s" % name
488 elif ud.bareclone:
489 ref = "refs/heads/%s" % branch
490 else:
491 ref = "refs/remotes/origin/%s" % branch
492
493 shallow_branches.append(ref)
494 runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest)
495
496 # Map srcrev+depths to revisions
497 parsed_depths = runfetchcmd("%s rev-parse %s" % (ud.basecmd, " ".join(to_parse)), d, workdir=dest)
498
499 # Resolve specified revisions
500 parsed_revs = runfetchcmd("%s rev-parse %s" % (ud.basecmd, " ".join('"%s^{}"' % r for r in ud.shallow_revs)), d, workdir=dest)
501 shallow_revisions = parsed_depths.splitlines() + parsed_revs.splitlines()
502
503 # Apply extra ref wildcards
504 all_refs = runfetchcmd('%s for-each-ref "--format=%%(refname)"' % ud.basecmd,
505 d, workdir=dest).splitlines()
506 for r in ud.shallow_extra_refs:
507 if not ud.bareclone:
508 r = r.replace('refs/heads/', 'refs/remotes/origin/')
509
510 if '*' in r:
511 matches = filter(lambda a: fnmatch.fnmatchcase(a, r), all_refs)
512 shallow_branches.extend(matches)
513 else:
514 shallow_branches.append(r)
515
516 # Make the repository shallow
Brad Bishop316dfdd2018-06-25 12:45:53 -0400517 shallow_cmd = [self.make_shallow_path, '-s']
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500518 for b in shallow_branches:
519 shallow_cmd.append('-r')
520 shallow_cmd.append(b)
521 shallow_cmd.extend(shallow_revisions)
522 runfetchcmd(subprocess.list2cmdline(shallow_cmd), d, workdir=dest)
523
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500524 def unpack(self, ud, destdir, d):
525 """ unpack the downloaded src to destdir"""
526
Andrew Geissler595f6302022-01-24 19:11:47 +0000527 subdir = ud.parm.get("subdir")
528 subpath = ud.parm.get("subpath")
529 readpathspec = ""
530 def_destsuffix = "git/"
531
532 if subpath:
533 readpathspec = ":%s" % subpath
534 def_destsuffix = "%s/" % os.path.basename(subpath.rstrip('/'))
535
536 if subdir:
537 # If 'subdir' param exists, create a dir and use it as destination for unpack cmd
538 if os.path.isabs(subdir):
539 if not os.path.realpath(subdir).startswith(os.path.realpath(destdir)):
540 raise bb.fetch2.UnpackError("subdir argument isn't a subdirectory of unpack root %s" % destdir, ud.url)
541 destdir = subdir
542 else:
543 destdir = os.path.join(destdir, subdir)
544 def_destsuffix = ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500545
546 destsuffix = ud.parm.get("destsuffix", def_destsuffix)
547 destdir = ud.destdir = os.path.join(destdir, destsuffix)
548 if os.path.exists(destdir):
549 bb.utils.prunedir(destdir)
550
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600551 need_lfs = self._need_lfs(ud)
Brad Bishopa34c0302019-09-23 22:34:48 -0400552
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500553 if not need_lfs:
554 ud.basecmd = "GIT_LFS_SKIP_SMUDGE=1 " + ud.basecmd
555
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800556 source_found = False
557 source_error = []
558
559 if not source_found:
560 clonedir_is_up_to_date = not self.clonedir_need_update(ud, d)
561 if clonedir_is_up_to_date:
562 runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d)
563 source_found = True
564 else:
565 source_error.append("clone directory not available or not up to date: " + ud.clonedir)
566
567 if not source_found:
568 if ud.shallow:
569 if os.path.exists(ud.fullshallow):
570 bb.utils.mkdirhier(destdir)
571 runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=destdir)
572 source_found = True
573 else:
574 source_error.append("shallow clone not available: " + ud.fullshallow)
575 else:
576 source_error.append("shallow clone not enabled")
577
578 if not source_found:
579 raise bb.fetch2.UnpackError("No up to date source found: " + "; ".join(source_error), ud.url)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500580
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500581 repourl = self._get_repo_url(ud)
Andrew Geissler4c19ea12020-10-27 13:52:24 -0500582 runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=destdir)
Brad Bishopc342db32019-05-15 21:57:59 -0400583
584 if self._contains_lfs(ud, d, destdir):
Brad Bishop00e122a2019-10-05 11:10:57 -0400585 if need_lfs and not self._find_git_lfs(d):
586 raise bb.fetch2.FetchError("Repository %s has LFS content, install git-lfs on host to download (or set lfs=0 to ignore it)" % (repourl))
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500587 elif not need_lfs:
Brad Bishopa34c0302019-09-23 22:34:48 -0400588 bb.note("Repository %s has LFS content but it is not being fetched" % (repourl))
Brad Bishopc342db32019-05-15 21:57:59 -0400589
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500590 if not ud.nocheckout:
Andrew Geissler595f6302022-01-24 19:11:47 +0000591 if subpath:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600592 runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revisions[ud.names[0]], readpathspec), d,
593 workdir=destdir)
594 runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d, workdir=destdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500595 elif not ud.nobranch:
596 branchname = ud.branches[ud.names[0]]
597 runfetchcmd("%s checkout -B %s %s" % (ud.basecmd, branchname, \
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600598 ud.revisions[ud.names[0]]), d, workdir=destdir)
Andre Rosa49271d42017-09-07 11:15:55 +0200599 runfetchcmd("%s branch %s --set-upstream-to origin/%s" % (ud.basecmd, branchname, \
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600600 branchname), d, workdir=destdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500601 else:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600602 runfetchcmd("%s checkout %s" % (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=destdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500603
604 return True
605
606 def clean(self, ud, d):
607 """ clean the git directory """
608
Brad Bishop19323692019-04-05 15:28:33 -0400609 to_remove = [ud.localpath, ud.fullmirror, ud.fullmirror + ".done"]
610 # The localpath is a symlink to clonedir when it is cloned from a
611 # mirror, so remove both of them.
612 if os.path.islink(ud.localpath):
613 clonedir = os.path.realpath(ud.localpath)
614 to_remove.append(clonedir)
615
616 for r in to_remove:
617 if os.path.exists(r):
618 bb.note('Removing %s' % r)
619 bb.utils.remove(r, True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500620
621 def supports_srcrev(self):
622 return True
623
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600624 def _contains_ref(self, ud, d, name, wd):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500625 cmd = ""
626 if ud.nobranch:
627 cmd = "%s log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % (
628 ud.basecmd, ud.revisions[name])
629 else:
630 cmd = "%s branch --contains %s --list %s 2> /dev/null | wc -l" % (
631 ud.basecmd, ud.revisions[name], ud.branches[name])
632 try:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600633 output = runfetchcmd(cmd, d, quiet=True, workdir=wd)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500634 except bb.fetch2.FetchError:
635 return False
636 if len(output.split()) > 1:
637 raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output))
638 return output.split()[0] != "0"
639
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600640 def _need_lfs(self, ud):
641 return ud.parm.get("lfs", "1") == "1"
642
Brad Bishopc342db32019-05-15 21:57:59 -0400643 def _contains_lfs(self, ud, d, wd):
644 """
645 Check if the repository has 'lfs' (large file) content
646 """
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500647
648 if not ud.nobranch:
649 branchname = ud.branches[ud.names[0]]
650 else:
651 branchname = "master"
652
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600653 # The bare clonedir doesn't use the remote names; it has the branch immediately.
654 if wd == ud.clonedir:
655 refname = ud.branches[ud.names[0]]
656 else:
657 refname = "origin/%s" % ud.branches[ud.names[0]]
658
659 cmd = "%s grep lfs %s:.gitattributes | wc -l" % (
660 ud.basecmd, refname)
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500661
Brad Bishopc342db32019-05-15 21:57:59 -0400662 try:
663 output = runfetchcmd(cmd, d, quiet=True, workdir=wd)
664 if int(output) > 0:
665 return True
666 except (bb.fetch2.FetchError,ValueError):
667 pass
668 return False
669
Brad Bishop00e122a2019-10-05 11:10:57 -0400670 def _find_git_lfs(self, d):
671 """
672 Return True if git-lfs can be found, False otherwise.
673 """
674 import shutil
675 return shutil.which("git-lfs", path=d.getVar('PATH')) is not None
676
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500677 def _get_repo_url(self, ud):
678 """
679 Return the repository URL
680 """
Andrew Geissler6ce62a22020-11-30 19:58:47 -0600681 # Note that we do not support passwords directly in the git urls. There are several
682 # reasons. SRC_URI can be written out to things like buildhistory and people don't
683 # want to leak passwords like that. Its also all too easy to share metadata without
684 # removing the password. ssh keys, ~/.netrc and ~/.ssh/config files can be used as
685 # alternatives so we will not take patches adding password support here.
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500686 if ud.user:
687 username = ud.user + '@'
688 else:
689 username = ""
690 return "%s://%s%s%s" % (ud.proto, username, ud.host, ud.path)
691
692 def _revision_key(self, ud, d, name):
693 """
694 Return a unique key for the url
695 """
Andrew Geissler82c905d2020-04-13 13:39:40 -0500696 # Collapse adjacent slashes
697 slash_re = re.compile(r"/+")
698 return "git:" + ud.host + slash_re.sub(".", ud.path) + ud.unresolvedrev[name]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500699
700 def _lsremote(self, ud, d, search):
701 """
702 Run git ls-remote with the specified search string
703 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500704 # Prevent recursion e.g. in OE if SRCPV is in PV, PV is in WORKDIR,
705 # and WORKDIR is in PATH (as a result of RSS), our call to
706 # runfetchcmd() exports PATH so this function will get called again (!)
707 # In this scenario the return call of the function isn't actually
708 # important - WORKDIR isn't needed in PATH to call git ls-remote
709 # anyway.
710 if d.getVar('_BB_GIT_IN_LSREMOTE', False):
711 return ''
712 d.setVar('_BB_GIT_IN_LSREMOTE', '1')
713 try:
714 repourl = self._get_repo_url(ud)
Andrew Geissler4c19ea12020-10-27 13:52:24 -0500715 cmd = "%s ls-remote %s %s" % \
716 (ud.basecmd, shlex.quote(repourl), search)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500717 if ud.proto.lower() != 'file':
718 bb.fetch2.check_network_access(d, cmd, repourl)
719 output = runfetchcmd(cmd, d, True)
720 if not output:
721 raise bb.fetch2.FetchError("The command %s gave empty output unexpectedly" % cmd, ud.url)
722 finally:
723 d.delVar('_BB_GIT_IN_LSREMOTE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500724 return output
725
726 def _latest_revision(self, ud, d, name):
727 """
728 Compute the HEAD revision for the url
729 """
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000730 if not d.getVar("__BBSEENSRCREV"):
731 raise bb.fetch2.FetchError("Recipe uses a floating tag/branch without a fixed SRCREV yet doesn't call bb.fetch2.get_srcrev() (use SRCPV in PV for OE).")
732
733 # Ensure we mark as not cached
734 bb.fetch2.get_autorev(d)
735
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500736 output = self._lsremote(ud, d, "")
737 # Tags of the form ^{} may not work, need to fallback to other form
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600738 if ud.unresolvedrev[name][:5] == "refs/" or ud.usehead:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500739 head = ud.unresolvedrev[name]
740 tag = ud.unresolvedrev[name]
741 else:
742 head = "refs/heads/%s" % ud.unresolvedrev[name]
743 tag = "refs/tags/%s" % ud.unresolvedrev[name]
744 for s in [head, tag + "^{}", tag]:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600745 for l in output.strip().split('\n'):
746 sha1, ref = l.split()
747 if s == ref:
748 return sha1
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500749 raise bb.fetch2.FetchError("Unable to resolve '%s' in upstream git repository in git ls-remote output for %s" % \
750 (ud.unresolvedrev[name], ud.host+ud.path))
751
752 def latest_versionstring(self, ud, d):
753 """
754 Compute the latest release name like "x.y.x" in "x.y.x+gitHASH"
755 by searching through the tags output of ls-remote, comparing
756 versions and returning the highest match.
757 """
758 pupver = ('', '')
759
Brad Bishop19323692019-04-05 15:28:33 -0400760 tagregex = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX') or r"(?P<pver>([0-9][\.|_]?)+)")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500761 try:
762 output = self._lsremote(ud, d, "refs/tags/*")
Brad Bishop316dfdd2018-06-25 12:45:53 -0400763 except (bb.fetch2.FetchError, bb.fetch2.NetworkAccess) as e:
764 bb.note("Could not list remote: %s" % str(e))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500765 return pupver
766
767 verstring = ""
768 revision = ""
769 for line in output.split("\n"):
770 if not line:
771 break
772
773 tag_head = line.split("/")[-1]
774 # Ignore non-released branches
Brad Bishop19323692019-04-05 15:28:33 -0400775 m = re.search(r"(alpha|beta|rc|final)+", tag_head)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500776 if m:
777 continue
778
779 # search for version in the line
780 tag = tagregex.search(tag_head)
Andrew Geissler82c905d2020-04-13 13:39:40 -0500781 if tag is None:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500782 continue
783
784 tag = tag.group('pver')
785 tag = tag.replace("_", ".")
786
787 if verstring and bb.utils.vercmp(("0", tag, ""), ("0", verstring, "")) < 0:
788 continue
789
790 verstring = tag
791 revision = line.split()[0]
792 pupver = (verstring, revision)
793
794 return pupver
795
796 def _build_revision(self, ud, d, name):
797 return ud.revisions[name]
798
799 def gitpkgv_revision(self, ud, d, name):
800 """
801 Return a sortable revision number by counting commits in the history
802 Based on gitpkgv.bblass in meta-openembedded
803 """
804 rev = self._build_revision(ud, d, name)
805 localpath = ud.localpath
806 rev_file = os.path.join(localpath, "oe-gitpkgv_" + rev)
807 if not os.path.exists(localpath):
808 commits = None
809 else:
810 if not os.path.exists(rev_file) or not os.path.getsize(rev_file):
811 from pipes import quote
812 commits = bb.fetch2.runfetchcmd(
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500813 "git rev-list %s -- | wc -l" % quote(rev),
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500814 d, quiet=True).strip().lstrip('0')
815 if commits:
816 open(rev_file, "w").write("%d\n" % int(commits))
817 else:
818 commits = open(rev_file, "r").readline(128).strip()
819 if commits:
820 return False, "%s+%s" % (commits, rev[:7])
821 else:
822 return True, str(rev)
823
824 def checkstatus(self, fetch, ud, d):
825 try:
826 self._lsremote(ud, d, "")
827 return True
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500828 except bb.fetch2.FetchError:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500829 return False