blob: a4527bf3645e0c5c672ab3679dce3efe690e9105 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001"""
2BitBake 'Fetch' git submodules implementation
3
4Inherits from and extends the Git fetcher to retrieve submodules of a git repository
5after cloning.
6
7SRC_URI = "gitsm://<see Git fetcher for syntax>"
8
9See the Git fetcher, git://, for usage documentation.
10
11NOTE: Switching a SRC_URI from "git://" to "gitsm://" requires a clean of your recipe.
12
13"""
14
15# Copyright (C) 2013 Richard Purdie
16#
Brad Bishopc342db32019-05-15 21:57:59 -040017# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -050018#
Patrick Williamsc124f4f2015-09-15 14:41:29 -050019
20import os
21import bb
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080022import copy
Andrew Geissler82c905d2020-04-13 13:39:40 -050023import shutil
24import tempfile
Patrick Williamsc124f4f2015-09-15 14:41:29 -050025from bb.fetch2.git import Git
26from bb.fetch2 import runfetchcmd
27from bb.fetch2 import logger
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080028from bb.fetch2 import Fetch
Patrick Williamsc124f4f2015-09-15 14:41:29 -050029
30class GitSM(Git):
31 def supports(self, ud, d):
32 """
33 Check to see if a given url can be fetched with git.
34 """
35 return ud.type in ['gitsm']
36
Brad Bishopf8caae32019-03-25 13:13:56 -040037 def process_submodules(self, ud, workdir, function, d):
38 """
39 Iterate over all of the submodules in this repository and execute
40 the 'function' for each of them.
41 """
Patrick Williamsc124f4f2015-09-15 14:41:29 -050042
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080043 submodules = []
44 paths = {}
Brad Bishopf8caae32019-03-25 13:13:56 -040045 revision = {}
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080046 uris = {}
Brad Bishopf8caae32019-03-25 13:13:56 -040047 subrevision = {}
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080048
Brad Bishopf8caae32019-03-25 13:13:56 -040049 def parse_gitmodules(gitmodules):
50 modules = {}
51 module = ""
52 for line in gitmodules.splitlines():
53 if line.startswith('[submodule'):
54 module = line.split('"')[1]
55 modules[module] = {}
56 elif module and line.strip().startswith('path'):
57 path = line.split('=')[1].strip()
58 modules[module]['path'] = path
59 elif module and line.strip().startswith('url'):
60 url = line.split('=')[1].strip()
61 modules[module]['url'] = url
62 return modules
63
64 # Collect the defined submodules, and their attributes
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080065 for name in ud.names:
66 try:
Brad Bishopf8caae32019-03-25 13:13:56 -040067 gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=workdir)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080068 except:
69 # No submodules to update
70 continue
71
Brad Bishopf8caae32019-03-25 13:13:56 -040072 for m, md in parse_gitmodules(gitmodules).items():
73 try:
74 module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=workdir)
75 except:
76 # If the command fails, we don't have a valid file to check. If it doesn't
77 # fail -- it still might be a failure, see next check...
78 module_hash = ""
79
80 if not module_hash:
Andrew Geisslerd1e89492021-02-12 15:35:20 -060081 logger.debug("submodule %s is defined, but is not initialized in the repository. Skipping", m)
Brad Bishopf8caae32019-03-25 13:13:56 -040082 continue
83
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080084 submodules.append(m)
85 paths[m] = md['path']
Brad Bishopf8caae32019-03-25 13:13:56 -040086 revision[m] = ud.revisions[name]
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080087 uris[m] = md['url']
Brad Bishopf8caae32019-03-25 13:13:56 -040088 subrevision[m] = module_hash.split()[2]
89
90 # Convert relative to absolute uri based on parent uri
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080091 if uris[m].startswith('..'):
92 newud = copy.copy(ud)
Brad Bishopf8caae32019-03-25 13:13:56 -040093 newud.path = os.path.realpath(os.path.join(newud.path, uris[m]))
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080094 uris[m] = Git._get_repo_url(self, newud)
95
96 for module in submodules:
Brad Bishopf8caae32019-03-25 13:13:56 -040097 # Translate the module url into a SRC_URI
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080098
Brad Bishopf8caae32019-03-25 13:13:56 -040099 if "://" in uris[module]:
100 # Properly formated URL already
101 proto = uris[module].split(':', 1)[0]
102 url = uris[module].replace('%s:' % proto, 'gitsm:', 1)
103 else:
104 if ":" in uris[module]:
105 # Most likely an SSH style reference
106 proto = "ssh"
107 if ":/" in uris[module]:
108 # Absolute reference, easy to convert..
109 url = "gitsm://" + uris[module].replace(':/', '/', 1)
110 else:
111 # Relative reference, no way to know if this is right!
112 logger.warning("Submodule included by %s refers to relative ssh reference %s. References may fail if not absolute." % (ud.url, uris[module]))
113 url = "gitsm://" + uris[module].replace(':', '/', 1)
114 else:
115 # This has to be a file reference
116 proto = "file"
117 url = "gitsm://" + uris[module]
118
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800119 url += ';protocol=%s' % proto
120 url += ";name=%s" % module
Brad Bishop393846f2019-05-20 12:24:11 -0400121 url += ";subpath=%s" % module
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800122
123 ld = d.createCopy()
124 # Not necessary to set SRC_URI, since we're passing the URI to
125 # Fetch.
126 #ld.setVar('SRC_URI', url)
Brad Bishopf8caae32019-03-25 13:13:56 -0400127 ld.setVar('SRCREV_%s' % module, subrevision[module])
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800128
129 # Workaround for issues with SRCPV/SRCREV_FORMAT errors
130 # error refer to 'multiple' repositories. Only the repository
131 # in the original SRC_URI actually matters...
132 ld.setVar('SRCPV', d.getVar('SRCPV'))
133 ld.setVar('SRCREV_FORMAT', module)
134
Andrew Geissler82c905d2020-04-13 13:39:40 -0500135 function(ud, url, module, paths[module], workdir, ld)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800136
Brad Bishopf8caae32019-03-25 13:13:56 -0400137 return submodules != []
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500138
Brad Bishop19323692019-04-05 15:28:33 -0400139 def need_update(self, ud, d):
140 if Git.need_update(self, ud, d):
141 return True
142
143 try:
144 # Check for the nugget dropped by the download operation
145 known_srcrevs = runfetchcmd("%s config --get-all bitbake.srcrev" % \
Andrew Geisslerd25ed322020-06-27 00:28:28 -0500146 (ud.basecmd), d, workdir=ud.clonedir)
Brad Bishop19323692019-04-05 15:28:33 -0400147
Andrew Geisslerd25ed322020-06-27 00:28:28 -0500148 if ud.revisions[ud.names[0]] in known_srcrevs.split():
149 return False
Brad Bishop19323692019-04-05 15:28:33 -0400150 except bb.fetch2.FetchError:
Andrew Geisslerd25ed322020-06-27 00:28:28 -0500151 pass
152
153 need_update_list = []
154 def need_update_submodule(ud, url, module, modpath, workdir, d):
155 url += ";bareclone=1;nobranch=1"
156
157 try:
158 newfetch = Fetch([url], d, cache=False)
159 new_ud = newfetch.ud[url]
160 if new_ud.method.need_update(new_ud, d):
161 need_update_list.append(modpath)
162 except Exception as e:
163 logger.error('gitsm: submodule update check failed: %s %s' % (type(e).__name__, str(e)))
164 need_update_result = True
165
166 # If we're using a shallow mirror tarball it needs to be unpacked
167 # temporarily so that we can examine the .gitmodules file
168 if ud.shallow and os.path.exists(ud.fullshallow) and not os.path.exists(ud.clonedir):
169 tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR"))
170 runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir)
171 self.process_submodules(ud, tmpdir, need_update_submodule, d)
172 shutil.rmtree(tmpdir)
173 else:
174 self.process_submodules(ud, ud.clonedir, need_update_submodule, d)
175 if len(need_update_list) == 0:
176 # We already have the required commits of all submodules. Drop
177 # a nugget so we don't need to check again.
178 runfetchcmd("%s config --add bitbake.srcrev %s" % \
179 (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=ud.clonedir)
180
181 if len(need_update_list) > 0:
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600182 logger.debug('gitsm: Submodules requiring update: %s' % (' '.join(need_update_list)))
Brad Bishop19323692019-04-05 15:28:33 -0400183 return True
184
185 return False
186
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500187 def download(self, ud, d):
Andrew Geissler82c905d2020-04-13 13:39:40 -0500188 def download_submodule(ud, url, module, modpath, workdir, d):
Brad Bishopf8caae32019-03-25 13:13:56 -0400189 url += ";bareclone=1;nobranch=1"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500190
Brad Bishopf8caae32019-03-25 13:13:56 -0400191 # Is the following still needed?
192 #url += ";nocheckout=1"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800193
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800194 try:
Brad Bishopf8caae32019-03-25 13:13:56 -0400195 newfetch = Fetch([url], d, cache=False)
196 newfetch.download()
197 except Exception as e:
198 logger.error('gitsm: submodule download failed: %s %s' % (type(e).__name__, str(e)))
199 raise
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800200
Brad Bishopf8caae32019-03-25 13:13:56 -0400201 Git.download(self, ud, d)
Andrew Geissler82c905d2020-04-13 13:39:40 -0500202
203 # If we're using a shallow mirror tarball it needs to be unpacked
204 # temporarily so that we can examine the .gitmodules file
205 if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d):
206 tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR"))
207 runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir)
208 self.process_submodules(ud, tmpdir, download_submodule, d)
209 shutil.rmtree(tmpdir)
210 else:
211 self.process_submodules(ud, ud.clonedir, download_submodule, d)
Andrew Geisslerd25ed322020-06-27 00:28:28 -0500212 # Drop a nugget for the srcrev we've fetched (used by need_update)
213 runfetchcmd("%s config --add bitbake.srcrev %s" % \
214 (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=ud.clonedir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500215
216 def unpack(self, ud, destdir, d):
Andrew Geissler82c905d2020-04-13 13:39:40 -0500217 def unpack_submodules(ud, url, module, modpath, workdir, d):
Brad Bishopf8caae32019-03-25 13:13:56 -0400218 url += ";bareclone=1;nobranch=1"
219
220 # Figure out where we clone over the bare submodules...
221 if ud.bareclone:
222 repo_conf = ud.destdir
223 else:
224 repo_conf = os.path.join(ud.destdir, '.git')
225
226 try:
227 newfetch = Fetch([url], d, cache=False)
Brad Bishop393846f2019-05-20 12:24:11 -0400228 newfetch.unpack(root=os.path.dirname(os.path.join(repo_conf, 'modules', module)))
Brad Bishopf8caae32019-03-25 13:13:56 -0400229 except Exception as e:
230 logger.error('gitsm: submodule unpack failed: %s %s' % (type(e).__name__, str(e)))
231 raise
232
233 local_path = newfetch.localpath(url)
234
235 # Correct the submodule references to the local download version...
236 runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_path}, d, workdir=ud.destdir)
237
238 if ud.shallow:
239 runfetchcmd("%(basecmd)s config submodule.%(module)s.shallow true" % {'basecmd': ud.basecmd, 'module': module}, d, workdir=ud.destdir)
240
241 # Ensure the submodule repository is NOT set to bare, since we're checking it out...
242 try:
Brad Bishop393846f2019-05-20 12:24:11 -0400243 runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=os.path.join(repo_conf, 'modules', module))
Brad Bishopf8caae32019-03-25 13:13:56 -0400244 except:
Brad Bishop393846f2019-05-20 12:24:11 -0400245 logger.error("Unable to set git config core.bare to false for %s" % os.path.join(repo_conf, 'modules', module))
Brad Bishopf8caae32019-03-25 13:13:56 -0400246 raise
247
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500248 Git.unpack(self, ud, destdir, d)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500249
Brad Bishopf8caae32019-03-25 13:13:56 -0400250 ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800251
Brad Bishopf8caae32019-03-25 13:13:56 -0400252 if not ud.bareclone and ret:
Brad Bishop393846f2019-05-20 12:24:11 -0400253 # All submodules should already be downloaded and configured in the tree. This simply sets
254 # up the configuration and checks out the files. The main project config should remain
255 # unmodified, and no download from the internet should occur.
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800256 runfetchcmd("%s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir)
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500257
258 def implicit_urldata(self, ud, d):
259 import shutil, subprocess, tempfile
260
261 urldata = []
262 def add_submodule(ud, url, module, modpath, workdir, d):
263 url += ";bareclone=1;nobranch=1"
264 newfetch = Fetch([url], d, cache=False)
265 urldata.extend(newfetch.expanded_urldata())
266
267 # If we're using a shallow mirror tarball it needs to be unpacked
268 # temporarily so that we can examine the .gitmodules file
269 if ud.shallow and os.path.exists(ud.fullshallow) and ud.method.need_update(ud, d):
270 tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR"))
271 subprocess.check_call("tar -xzf %s" % ud.fullshallow, cwd=tmpdir, shell=True)
272 self.process_submodules(ud, tmpdir, add_submodule, d)
273 shutil.rmtree(tmpdir)
274 else:
275 self.process_submodules(ud, ud.clonedir, add_submodule, d)
276
277 return urldata