blob: a8c4d3528f56964800ea17dbf230d8ab2b07293e [file] [log] [blame]
Andrew Geissler82c905d2020-04-13 13:39:40 -05001# Copyright (C) 2020 Savoir-Faire Linux
2#
3# SPDX-License-Identifier: GPL-2.0-only
4#
5"""
6BitBake 'Fetch' npm shrinkwrap implementation
7
8npm fetcher support the SRC_URI with format of:
9SRC_URI = "npmsw://some.registry.url;OptionA=xxx;OptionB=xxx;..."
10
11Supported SRC_URI options are:
12
13- dev
14 Set to 1 to also install devDependencies.
15
16- destsuffix
17 Specifies the directory to use to unpack the dependencies (default: ${S}).
18"""
19
20import json
21import os
22import re
23import bb
24from bb.fetch2 import Fetch
25from bb.fetch2 import FetchMethod
26from bb.fetch2 import ParameterError
Andrew Geisslereff27472021-10-29 15:35:00 -050027from bb.fetch2 import runfetchcmd
Andrew Geissler82c905d2020-04-13 13:39:40 -050028from bb.fetch2 import URI
29from bb.fetch2.npm import npm_integrity
30from bb.fetch2.npm import npm_localfile
31from bb.fetch2.npm import npm_unpack
32from bb.utils import is_semver
Andrew Geisslereff27472021-10-29 15:35:00 -050033from bb.utils import lockfile
34from bb.utils import unlockfile
Andrew Geissler82c905d2020-04-13 13:39:40 -050035
36def foreach_dependencies(shrinkwrap, callback=None, dev=False):
37 """
38 Run a callback for each dependencies of a shrinkwrap file.
39 The callback is using the format:
40 callback(name, params, deptree)
41 with:
42 name = the package name (string)
43 params = the package parameters (dictionary)
44 deptree = the package dependency tree (array of strings)
45 """
46 def _walk_deps(deps, deptree):
47 for name in deps:
48 subtree = [*deptree, name]
49 _walk_deps(deps[name].get("dependencies", {}), subtree)
50 if callback is not None:
51 if deps[name].get("dev", False) and not dev:
52 continue
53 elif deps[name].get("bundled", False):
54 continue
55 callback(name, deps[name], subtree)
56
57 _walk_deps(shrinkwrap.get("dependencies", {}), [])
58
59class NpmShrinkWrap(FetchMethod):
60 """Class to fetch all package from a shrinkwrap file"""
61
62 def supports(self, ud, d):
63 """Check if a given url can be fetched with npmsw"""
64 return ud.type in ["npmsw"]
65
66 def urldata_init(self, ud, d):
67 """Init npmsw specific variables within url data"""
68
69 # Get the 'shrinkwrap' parameter
70 ud.shrinkwrap_file = re.sub(r"^npmsw://", "", ud.url.split(";")[0])
71
72 # Get the 'dev' parameter
73 ud.dev = bb.utils.to_boolean(ud.parm.get("dev"), False)
74
75 # Resolve the dependencies
76 ud.deps = []
77
78 def _resolve_dependency(name, params, deptree):
79 url = None
80 localpath = None
81 extrapaths = []
82 destsubdirs = [os.path.join("node_modules", dep) for dep in deptree]
83 destsuffix = os.path.join(*destsubdirs)
Andrew Geisslereff27472021-10-29 15:35:00 -050084 unpack = True
Andrew Geissler82c905d2020-04-13 13:39:40 -050085
86 integrity = params.get("integrity", None)
87 resolved = params.get("resolved", None)
88 version = params.get("version", None)
89
90 # Handle registry sources
Andrew Geissler595f6302022-01-24 19:11:47 +000091 if is_semver(version) and integrity:
92 # Handle duplicate dependencies without url
93 if not resolved:
94 return
95
Andrew Geissler82c905d2020-04-13 13:39:40 -050096 localfile = npm_localfile(name, version)
97
98 uri = URI(resolved)
99 uri.params["downloadfilename"] = localfile
100
101 checksum_name, checksum_expected = npm_integrity(integrity)
102 uri.params[checksum_name] = checksum_expected
103
104 url = str(uri)
105
106 localpath = os.path.join(d.getVar("DL_DIR"), localfile)
107
108 # Create a resolve file to mimic the npm fetcher and allow
109 # re-usability of the downloaded file.
110 resolvefile = localpath + ".resolved"
111
112 bb.utils.mkdirhier(os.path.dirname(resolvefile))
113 with open(resolvefile, "w") as f:
114 f.write(url)
115
116 extrapaths.append(resolvefile)
117
118 # Handle http tarball sources
119 elif version.startswith("http") and integrity:
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000120 localfile = npm_localfile(os.path.basename(version))
Andrew Geissler82c905d2020-04-13 13:39:40 -0500121
122 uri = URI(version)
123 uri.params["downloadfilename"] = localfile
124
125 checksum_name, checksum_expected = npm_integrity(integrity)
126 uri.params[checksum_name] = checksum_expected
127
128 url = str(uri)
129
130 localpath = os.path.join(d.getVar("DL_DIR"), localfile)
131
132 # Handle git sources
133 elif version.startswith("git"):
Andrew Geissler595f6302022-01-24 19:11:47 +0000134 if version.startswith("github:"):
135 version = "git+https://github.com/" + version[len("github:"):]
Andrew Geissler82c905d2020-04-13 13:39:40 -0500136 regex = re.compile(r"""
137 ^
138 git\+
139 (?P<protocol>[a-z]+)
140 ://
141 (?P<url>[^#]+)
142 \#
143 (?P<rev>[0-9a-f]+)
144 $
145 """, re.VERBOSE)
146
147 match = regex.match(version)
148
149 if not match:
150 raise ParameterError("Invalid git url: %s" % version, ud.url)
151
152 groups = match.groupdict()
153
154 uri = URI("git://" + str(groups["url"]))
155 uri.params["protocol"] = str(groups["protocol"])
156 uri.params["rev"] = str(groups["rev"])
157 uri.params["destsuffix"] = destsuffix
158
159 url = str(uri)
160
Andrew Geisslereff27472021-10-29 15:35:00 -0500161 # Handle local tarball and link sources
162 elif version.startswith("file"):
163 localpath = version[5:]
164 if not version.endswith(".tgz"):
165 unpack = False
166
Andrew Geissler82c905d2020-04-13 13:39:40 -0500167 else:
168 raise ParameterError("Unsupported dependency: %s" % name, ud.url)
169
170 ud.deps.append({
171 "url": url,
172 "localpath": localpath,
173 "extrapaths": extrapaths,
174 "destsuffix": destsuffix,
Andrew Geisslereff27472021-10-29 15:35:00 -0500175 "unpack": unpack,
Andrew Geissler82c905d2020-04-13 13:39:40 -0500176 })
177
178 try:
179 with open(ud.shrinkwrap_file, "r") as f:
180 shrinkwrap = json.load(f)
181 except Exception as e:
182 raise ParameterError("Invalid shrinkwrap file: %s" % str(e), ud.url)
183
184 foreach_dependencies(shrinkwrap, _resolve_dependency, ud.dev)
185
186 # Avoid conflicts between the environment data and:
187 # - the proxy url revision
188 # - the proxy url checksum
189 data = bb.data.createCopy(d)
190 data.delVar("SRCREV")
191 data.delVarFlags("SRC_URI")
192
193 # This fetcher resolves multiple URIs from a shrinkwrap file and then
194 # forwards it to a proxy fetcher. The management of the donestamp file,
195 # the lockfile and the checksums are forwarded to the proxy fetcher.
Andrew Geisslereff27472021-10-29 15:35:00 -0500196 ud.proxy = Fetch([dep["url"] for dep in ud.deps if dep["url"]], data)
Andrew Geissler82c905d2020-04-13 13:39:40 -0500197 ud.needdonestamp = False
198
199 @staticmethod
200 def _foreach_proxy_method(ud, handle):
201 returns = []
202 for proxy_url in ud.proxy.urls:
203 proxy_ud = ud.proxy.ud[proxy_url]
204 proxy_d = ud.proxy.d
205 proxy_ud.setup_localpath(proxy_d)
Andrew Geisslereff27472021-10-29 15:35:00 -0500206 lf = lockfile(proxy_ud.lockfile)
Andrew Geissler82c905d2020-04-13 13:39:40 -0500207 returns.append(handle(proxy_ud.method, proxy_ud, proxy_d))
Andrew Geisslereff27472021-10-29 15:35:00 -0500208 unlockfile(lf)
Andrew Geissler82c905d2020-04-13 13:39:40 -0500209 return returns
210
211 def verify_donestamp(self, ud, d):
212 """Verify the donestamp file"""
213 def _handle(m, ud, d):
214 return m.verify_donestamp(ud, d)
215 return all(self._foreach_proxy_method(ud, _handle))
216
217 def update_donestamp(self, ud, d):
218 """Update the donestamp file"""
219 def _handle(m, ud, d):
220 m.update_donestamp(ud, d)
221 self._foreach_proxy_method(ud, _handle)
222
223 def need_update(self, ud, d):
224 """Force a fetch, even if localpath exists ?"""
225 def _handle(m, ud, d):
226 return m.need_update(ud, d)
227 return all(self._foreach_proxy_method(ud, _handle))
228
229 def try_mirrors(self, fetch, ud, d, mirrors):
230 """Try to use a mirror"""
231 def _handle(m, ud, d):
232 return m.try_mirrors(fetch, ud, d, mirrors)
233 return all(self._foreach_proxy_method(ud, _handle))
234
235 def download(self, ud, d):
236 """Fetch url"""
237 ud.proxy.download()
238
239 def unpack(self, ud, rootdir, d):
240 """Unpack the downloaded dependencies"""
241 destdir = d.getVar("S")
242 destsuffix = ud.parm.get("destsuffix")
243 if destsuffix:
244 destdir = os.path.join(rootdir, destsuffix)
245
246 bb.utils.mkdirhier(destdir)
247 bb.utils.copyfile(ud.shrinkwrap_file,
248 os.path.join(destdir, "npm-shrinkwrap.json"))
249
250 auto = [dep["url"] for dep in ud.deps if not dep["localpath"]]
251 manual = [dep for dep in ud.deps if dep["localpath"]]
252
253 if auto:
254 ud.proxy.unpack(destdir, auto)
255
256 for dep in manual:
257 depdestdir = os.path.join(destdir, dep["destsuffix"])
Andrew Geisslereff27472021-10-29 15:35:00 -0500258 if dep["url"]:
259 npm_unpack(dep["localpath"], depdestdir, d)
260 else:
261 depsrcdir= os.path.join(destdir, dep["localpath"])
262 if dep["unpack"]:
263 npm_unpack(depsrcdir, depdestdir, d)
264 else:
265 bb.utils.mkdirhier(depdestdir)
266 cmd = 'cp -fpPRH "%s/." .' % (depsrcdir)
267 runfetchcmd(cmd, d, workdir=depdestdir)
Andrew Geissler82c905d2020-04-13 13:39:40 -0500268
269 def clean(self, ud, d):
270 """Clean any existing full or partial download"""
271 ud.proxy.clean()
272
273 # Clean extra files
274 for dep in ud.deps:
275 for path in dep["extrapaths"]:
276 bb.utils.remove(path)
277
278 def done(self, ud, d):
279 """Is the download done ?"""
280 def _handle(m, ud, d):
281 return m.done(ud, d)
282 return all(self._foreach_proxy_method(ud, _handle))