blob: 32389130bd8944da8c8fcbc957e7b391ca97d577 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# ex:ts=4:sw=4:sts=4:et
2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
3"""
4BitBake 'Fetch' git submodules implementation
5
6Inherits from and extends the Git fetcher to retrieve submodules of a git repository
7after cloning.
8
9SRC_URI = "gitsm://<see Git fetcher for syntax>"
10
11See the Git fetcher, git://, for usage documentation.
12
13NOTE: Switching a SRC_URI from "git://" to "gitsm://" requires a clean of your recipe.
14
15"""
16
17# Copyright (C) 2013 Richard Purdie
18#
19# This program is free software; you can redistribute it and/or modify
20# it under the terms of the GNU General Public License version 2 as
21# published by the Free Software Foundation.
22#
23# This program is distributed in the hope that it will be useful,
24# but WITHOUT ANY WARRANTY; without even the implied warranty of
25# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
26# GNU General Public License for more details.
27#
28# You should have received a copy of the GNU General Public License along
29# with this program; if not, write to the Free Software Foundation, Inc.,
30# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
31
32import os
33import bb
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080034import copy
Patrick Williamsc124f4f2015-09-15 14:41:29 -050035from bb.fetch2.git import Git
36from bb.fetch2 import runfetchcmd
37from bb.fetch2 import logger
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080038from bb.fetch2 import Fetch
39from bb.fetch2 import BBFetchException
Patrick Williamsc124f4f2015-09-15 14:41:29 -050040
41class GitSM(Git):
42 def supports(self, ud, d):
43 """
44 Check to see if a given url can be fetched with git.
45 """
46 return ud.type in ['gitsm']
47
Brad Bishopf8caae32019-03-25 13:13:56 -040048 def process_submodules(self, ud, workdir, function, d):
49 """
50 Iterate over all of the submodules in this repository and execute
51 the 'function' for each of them.
52 """
Patrick Williamsc124f4f2015-09-15 14:41:29 -050053
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080054 submodules = []
55 paths = {}
Brad Bishopf8caae32019-03-25 13:13:56 -040056 revision = {}
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080057 uris = {}
Brad Bishopf8caae32019-03-25 13:13:56 -040058 subrevision = {}
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080059
Brad Bishopf8caae32019-03-25 13:13:56 -040060 def parse_gitmodules(gitmodules):
61 modules = {}
62 module = ""
63 for line in gitmodules.splitlines():
64 if line.startswith('[submodule'):
65 module = line.split('"')[1]
66 modules[module] = {}
67 elif module and line.strip().startswith('path'):
68 path = line.split('=')[1].strip()
69 modules[module]['path'] = path
70 elif module and line.strip().startswith('url'):
71 url = line.split('=')[1].strip()
72 modules[module]['url'] = url
73 return modules
74
75 # Collect the defined submodules, and their attributes
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080076 for name in ud.names:
77 try:
Brad Bishopf8caae32019-03-25 13:13:56 -040078 gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=workdir)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080079 except:
80 # No submodules to update
81 continue
82
Brad Bishopf8caae32019-03-25 13:13:56 -040083 for m, md in parse_gitmodules(gitmodules).items():
84 try:
85 module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=workdir)
86 except:
87 # If the command fails, we don't have a valid file to check. If it doesn't
88 # fail -- it still might be a failure, see next check...
89 module_hash = ""
90
91 if not module_hash:
92 logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", m)
93 continue
94
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080095 submodules.append(m)
96 paths[m] = md['path']
Brad Bishopf8caae32019-03-25 13:13:56 -040097 revision[m] = ud.revisions[name]
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080098 uris[m] = md['url']
Brad Bishopf8caae32019-03-25 13:13:56 -040099 subrevision[m] = module_hash.split()[2]
100
101 # Convert relative to absolute uri based on parent uri
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800102 if uris[m].startswith('..'):
103 newud = copy.copy(ud)
Brad Bishopf8caae32019-03-25 13:13:56 -0400104 newud.path = os.path.realpath(os.path.join(newud.path, uris[m]))
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800105 uris[m] = Git._get_repo_url(self, newud)
106
107 for module in submodules:
Brad Bishopf8caae32019-03-25 13:13:56 -0400108 # Translate the module url into a SRC_URI
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800109
Brad Bishopf8caae32019-03-25 13:13:56 -0400110 if "://" in uris[module]:
111 # Properly formated URL already
112 proto = uris[module].split(':', 1)[0]
113 url = uris[module].replace('%s:' % proto, 'gitsm:', 1)
114 else:
115 if ":" in uris[module]:
116 # Most likely an SSH style reference
117 proto = "ssh"
118 if ":/" in uris[module]:
119 # Absolute reference, easy to convert..
120 url = "gitsm://" + uris[module].replace(':/', '/', 1)
121 else:
122 # Relative reference, no way to know if this is right!
123 logger.warning("Submodule included by %s refers to relative ssh reference %s. References may fail if not absolute." % (ud.url, uris[module]))
124 url = "gitsm://" + uris[module].replace(':', '/', 1)
125 else:
126 # This has to be a file reference
127 proto = "file"
128 url = "gitsm://" + uris[module]
129
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800130 url += ';protocol=%s' % proto
131 url += ";name=%s" % module
Brad Bishopf8caae32019-03-25 13:13:56 -0400132 url += ";subpath=%s" % paths[module]
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800133
134 ld = d.createCopy()
135 # Not necessary to set SRC_URI, since we're passing the URI to
136 # Fetch.
137 #ld.setVar('SRC_URI', url)
Brad Bishopf8caae32019-03-25 13:13:56 -0400138 ld.setVar('SRCREV_%s' % module, subrevision[module])
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800139
140 # Workaround for issues with SRCPV/SRCREV_FORMAT errors
141 # error refer to 'multiple' repositories. Only the repository
142 # in the original SRC_URI actually matters...
143 ld.setVar('SRCPV', d.getVar('SRCPV'))
144 ld.setVar('SRCREV_FORMAT', module)
145
Brad Bishopf8caae32019-03-25 13:13:56 -0400146 function(ud, url, module, paths[module], ld)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800147
Brad Bishopf8caae32019-03-25 13:13:56 -0400148 return submodules != []
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500149
Brad Bishop19323692019-04-05 15:28:33 -0400150 def need_update(self, ud, d):
151 if Git.need_update(self, ud, d):
152 return True
153
154 try:
155 # Check for the nugget dropped by the download operation
156 known_srcrevs = runfetchcmd("%s config --get-all bitbake.srcrev" % \
157 (ud.basecmd), d, workdir=ud.clonedir)
158
159 if ud.revisions[ud.names[0]] not in known_srcrevs.split():
160 return True
161 except bb.fetch2.FetchError:
162 # No srcrev nuggets, so this is new and needs to be updated
163 return True
164
165 return False
166
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500167 def download(self, ud, d):
Brad Bishopf8caae32019-03-25 13:13:56 -0400168 def download_submodule(ud, url, module, modpath, d):
169 url += ";bareclone=1;nobranch=1"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500170
Brad Bishopf8caae32019-03-25 13:13:56 -0400171 # Is the following still needed?
172 #url += ";nocheckout=1"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800173
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800174 try:
Brad Bishopf8caae32019-03-25 13:13:56 -0400175 newfetch = Fetch([url], d, cache=False)
176 newfetch.download()
Brad Bishop19323692019-04-05 15:28:33 -0400177 # Drop a nugget to add each of the srcrevs we've fetched (used by need_update)
178 runfetchcmd("%s config --add bitbake.srcrev %s" % \
179 (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=ud.clonedir)
Brad Bishopf8caae32019-03-25 13:13:56 -0400180 except Exception as e:
181 logger.error('gitsm: submodule download failed: %s %s' % (type(e).__name__, str(e)))
182 raise
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800183
Brad Bishopf8caae32019-03-25 13:13:56 -0400184 Git.download(self, ud, d)
185 self.process_submodules(ud, ud.clonedir, download_submodule, d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500186
187 def unpack(self, ud, destdir, d):
Brad Bishopf8caae32019-03-25 13:13:56 -0400188 def unpack_submodules(ud, url, module, modpath, d):
189 url += ";bareclone=1;nobranch=1"
190
191 # Figure out where we clone over the bare submodules...
192 if ud.bareclone:
193 repo_conf = ud.destdir
194 else:
195 repo_conf = os.path.join(ud.destdir, '.git')
196
197 try:
198 newfetch = Fetch([url], d, cache=False)
199 newfetch.unpack(root=os.path.dirname(os.path.join(repo_conf, 'modules', modpath)))
200 except Exception as e:
201 logger.error('gitsm: submodule unpack failed: %s %s' % (type(e).__name__, str(e)))
202 raise
203
204 local_path = newfetch.localpath(url)
205
206 # Correct the submodule references to the local download version...
207 runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_path}, d, workdir=ud.destdir)
208
209 if ud.shallow:
210 runfetchcmd("%(basecmd)s config submodule.%(module)s.shallow true" % {'basecmd': ud.basecmd, 'module': module}, d, workdir=ud.destdir)
211
212 # Ensure the submodule repository is NOT set to bare, since we're checking it out...
213 try:
214 runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=os.path.join(repo_conf, 'modules', modpath))
215 except:
216 logger.error("Unable to set git config core.bare to false for %s" % os.path.join(repo_conf, 'modules', modpath))
217 raise
218
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500219 Git.unpack(self, ud, destdir, d)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500220
Brad Bishopf8caae32019-03-25 13:13:56 -0400221 ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800222
Brad Bishopf8caae32019-03-25 13:13:56 -0400223 if not ud.bareclone and ret:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800224 # Run submodule update, this sets up the directories -- without touching the config
225 runfetchcmd("%s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir)