blob: 408dfc3d03021c1a55b3720a4069a08601d0ddab [file] [log] [blame]
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001# ex:ts=4:sw=4:sts=4:et
2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
3"""
4BitBake 'Fetch' NPM implementation
5
6The NPM fetcher is used to retrieve files from the npmjs repository
7
8Usage in the recipe:
9
10 SRC_URI = "npm://registry.npmjs.org/;name=${PN};version=${PV}"
11 Suported SRC_URI options are:
12
13 - name
14 - version
15
Patrick Williamsc0f7c042017-02-23 20:41:17 -060016 npm://registry.npmjs.org/${PN}/-/${PN}-${PV}.tgz would become npm://registry.npmjs.org;name=${PN};version=${PV}
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050017 The fetcher all triggers off the existence of ud.localpath. If that exists and has the ".done" stamp, its assumed the fetch is good/done
18
19"""
20
21import os
22import sys
Patrick Williamsc0f7c042017-02-23 20:41:17 -060023import urllib.request, urllib.parse, urllib.error
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050024import json
25import subprocess
26import signal
27import bb
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050028from bb.fetch2 import FetchMethod
29from bb.fetch2 import FetchError
30from bb.fetch2 import ChecksumError
31from bb.fetch2 import runfetchcmd
32from bb.fetch2 import logger
33from bb.fetch2 import UnpackError
34from bb.fetch2 import ParameterError
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050035
36def subprocess_setup():
37 # Python installs a SIGPIPE handler by default. This is usually not what
38 # non-Python subprocesses expect.
39 # SIGPIPE errors are known issues with gzip/bash
40 signal.signal(signal.SIGPIPE, signal.SIG_DFL)
41
42class Npm(FetchMethod):
43
44 """Class to fetch urls via 'npm'"""
45 def init(self, d):
46 pass
47
48 def supports(self, ud, d):
49 """
50 Check to see if a given url can be fetched with npm
51 """
52 return ud.type in ['npm']
53
54 def debug(self, msg):
55 logger.debug(1, "NpmFetch: %s", msg)
56
57 def clean(self, ud, d):
58 logger.debug(2, "Calling cleanup %s" % ud.pkgname)
59 bb.utils.remove(ud.localpath, False)
60 bb.utils.remove(ud.pkgdatadir, True)
61 bb.utils.remove(ud.fullmirror, False)
62
63 def urldata_init(self, ud, d):
64 """
65 init NPM specific variable within url data
66 """
67 if 'downloadfilename' in ud.parm:
68 ud.basename = ud.parm['downloadfilename']
69 else:
70 ud.basename = os.path.basename(ud.path)
71
72 # can't call it ud.name otherwise fetcher base class will start doing sha1stuff
73 # TODO: find a way to get an sha1/sha256 manifest of pkg & all deps
74 ud.pkgname = ud.parm.get("name", None)
75 if not ud.pkgname:
76 raise ParameterError("NPM fetcher requires a name parameter", ud.url)
77 ud.version = ud.parm.get("version", None)
78 if not ud.version:
79 raise ParameterError("NPM fetcher requires a version parameter", ud.url)
80 ud.bbnpmmanifest = "%s-%s.deps.json" % (ud.pkgname, ud.version)
Brad Bishop6e60e8b2018-02-01 10:27:11 -050081 ud.bbnpmmanifest = ud.bbnpmmanifest.replace('/', '-')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050082 ud.registry = "http://%s" % (ud.url.replace('npm://', '', 1).split(';'))[0]
83 prefixdir = "npm/%s" % ud.pkgname
84 ud.pkgdatadir = d.expand("${DL_DIR}/%s" % prefixdir)
85 if not os.path.exists(ud.pkgdatadir):
86 bb.utils.mkdirhier(ud.pkgdatadir)
87 ud.localpath = d.expand("${DL_DIR}/npm/%s" % ud.bbnpmmanifest)
88
Brad Bishop6e60e8b2018-02-01 10:27:11 -050089 self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -O -t 2 -T 30 -nv --passive-ftp --no-check-certificate "
Patrick Williamsc0f7c042017-02-23 20:41:17 -060090 ud.prefixdir = prefixdir
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050091
Brad Bishop6e60e8b2018-02-01 10:27:11 -050092 ud.write_tarballs = ((d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0") != "0")
Brad Bishopd7bf8c12018-02-25 22:55:05 -050093 mirrortarball = 'npm_%s-%s.tar.xz' % (ud.pkgname, ud.version)
94 mirrortarball = mirrortarball.replace('/', '-')
95 ud.fullmirror = os.path.join(d.getVar("DL_DIR"), mirrortarball)
96 ud.mirrortarballs = [mirrortarball]
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050097
98 def need_update(self, ud, d):
99 if os.path.exists(ud.localpath):
100 return False
101 return True
102
103 def _runwget(self, ud, d, command, quiet):
104 logger.debug(2, "Fetching %s using command '%s'" % (ud.url, command))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500105 bb.fetch2.check_network_access(d, command, ud.url)
106 dldir = d.getVar("DL_DIR")
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600107 runfetchcmd(command, d, quiet, workdir=dldir)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500108
109 def _unpackdep(self, ud, pkg, data, destdir, dldir, d):
110 file = data[pkg]['tgz']
111 logger.debug(2, "file to extract is %s" % file)
112 if file.endswith('.tgz') or file.endswith('.tar.gz') or file.endswith('.tar.Z'):
113 cmd = 'tar xz --strip 1 --no-same-owner --warning=no-unknown-keyword -f %s/%s' % (dldir, file)
114 else:
115 bb.fatal("NPM package %s downloaded not a tarball!" % file)
116
117 # Change to subdir before executing command
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500118 if not os.path.exists(destdir):
119 os.makedirs(destdir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500120 path = d.getVar('PATH')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500121 if path:
122 cmd = "PATH=\"%s\" %s" % (path, cmd)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600123 bb.note("Unpacking %s to %s/" % (file, destdir))
124 ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True, cwd=destdir)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500125
126 if ret != 0:
127 raise UnpackError("Unpack command %s failed with return value %s" % (cmd, ret), ud.url)
128
129 if 'deps' not in data[pkg]:
130 return
131 for dep in data[pkg]['deps']:
132 self._unpackdep(ud, dep, data[pkg]['deps'], "%s/node_modules/%s" % (destdir, dep), dldir, d)
133
134
135 def unpack(self, ud, destdir, d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500136 dldir = d.getVar("DL_DIR")
137 with open("%s/npm/%s" % (dldir, ud.bbnpmmanifest)) as datafile:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500138 workobj = json.load(datafile)
139 dldir = "%s/%s" % (os.path.dirname(ud.localpath), ud.pkgname)
140
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600141 if 'subdir' in ud.parm:
142 unpackdir = '%s/%s' % (destdir, ud.parm.get('subdir'))
143 else:
144 unpackdir = '%s/npmpkg' % destdir
145
146 self._unpackdep(ud, ud.pkgname, workobj, unpackdir, dldir, d)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500147
148 def _parse_view(self, output):
149 '''
150 Parse the output of npm view --json; the last JSON result
151 is assumed to be the one that we're interested in.
152 '''
153 pdata = None
154 outdeps = {}
155 datalines = []
156 bracelevel = 0
157 for line in output.splitlines():
158 if bracelevel:
159 datalines.append(line)
160 elif '{' in line:
161 datalines = []
162 datalines.append(line)
163 bracelevel = bracelevel + line.count('{') - line.count('}')
164 if datalines:
165 pdata = json.loads('\n'.join(datalines))
166 return pdata
167
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600168 def _getdependencies(self, pkg, data, version, d, ud, optional=False, fetchedlist=None):
169 if fetchedlist is None:
170 fetchedlist = []
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500171 pkgfullname = pkg
172 if version != '*' and not '/' in version:
173 pkgfullname += "@'%s'" % version
174 logger.debug(2, "Calling getdeps on %s" % pkg)
175 fetchcmd = "npm view %s --json --registry %s" % (pkgfullname, ud.registry)
176 output = runfetchcmd(fetchcmd, d, True)
177 pdata = self._parse_view(output)
178 if not pdata:
179 raise FetchError("The command '%s' returned no output" % fetchcmd)
180 if optional:
181 pkg_os = pdata.get('os', None)
182 if pkg_os:
183 if not isinstance(pkg_os, list):
184 pkg_os = [pkg_os]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500185 blacklist = False
186 for item in pkg_os:
187 if item.startswith('!'):
188 blacklist = True
189 break
190 if (not blacklist and 'linux' not in pkg_os) or '!linux' in pkg_os:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500191 logger.debug(2, "Skipping %s since it's incompatible with Linux" % pkg)
192 return
193 #logger.debug(2, "Output URL is %s - %s - %s" % (ud.basepath, ud.basename, ud.localfile))
194 outputurl = pdata['dist']['tarball']
195 data[pkg] = {}
196 data[pkg]['tgz'] = os.path.basename(outputurl)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400197 if outputurl in fetchedlist:
198 return
199
200 self._runwget(ud, d, "%s --directory-prefix=%s %s" % (self.basecmd, ud.prefixdir, outputurl), False)
201 fetchedlist.append(outputurl)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500202
203 dependencies = pdata.get('dependencies', {})
204 optionalDependencies = pdata.get('optionalDependencies', {})
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500205 dependencies.update(optionalDependencies)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500206 depsfound = {}
207 optdepsfound = {}
208 data[pkg]['deps'] = {}
209 for dep in dependencies:
210 if dep in optionalDependencies:
211 optdepsfound[dep] = dependencies[dep]
212 else:
213 depsfound[dep] = dependencies[dep]
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600214 for dep, version in optdepsfound.items():
215 self._getdependencies(dep, data[pkg]['deps'], version, d, ud, optional=True, fetchedlist=fetchedlist)
216 for dep, version in depsfound.items():
217 self._getdependencies(dep, data[pkg]['deps'], version, d, ud, fetchedlist=fetchedlist)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500218
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600219 def _getshrinkeddependencies(self, pkg, data, version, d, ud, lockdown, manifest, toplevel=True):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500220 logger.debug(2, "NPM shrinkwrap file is %s" % data)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600221 if toplevel:
222 name = data.get('name', None)
223 if name and name != pkg:
224 for obj in data.get('dependencies', []):
225 if obj == pkg:
226 self._getshrinkeddependencies(obj, data['dependencies'][obj], data['dependencies'][obj]['version'], d, ud, lockdown, manifest, False)
227 return
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500228 outputurl = "invalid"
229 if ('resolved' not in data) or (not data['resolved'].startswith('http')):
230 # will be the case for ${PN}
231 fetchcmd = "npm view %s@%s dist.tarball --registry %s" % (pkg, version, ud.registry)
232 logger.debug(2, "Found this matching URL: %s" % str(fetchcmd))
233 outputurl = runfetchcmd(fetchcmd, d, True)
234 else:
235 outputurl = data['resolved']
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600236 self._runwget(ud, d, "%s --directory-prefix=%s %s" % (self.basecmd, ud.prefixdir, outputurl), False)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500237 manifest[pkg] = {}
238 manifest[pkg]['tgz'] = os.path.basename(outputurl).rstrip()
239 manifest[pkg]['deps'] = {}
240
241 if pkg in lockdown:
242 sha1_expected = lockdown[pkg][version]
243 sha1_data = bb.utils.sha1_file("npm/%s/%s" % (ud.pkgname, manifest[pkg]['tgz']))
244 if sha1_expected != sha1_data:
245 msg = "\nFile: '%s' has %s checksum %s when %s was expected" % (manifest[pkg]['tgz'], 'sha1', sha1_data, sha1_expected)
246 raise ChecksumError('Checksum mismatch!%s' % msg)
247 else:
248 logger.debug(2, "No lockdown data for %s@%s" % (pkg, version))
249
250 if 'dependencies' in data:
251 for obj in data['dependencies']:
252 logger.debug(2, "Found dep is %s" % str(obj))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600253 self._getshrinkeddependencies(obj, data['dependencies'][obj], data['dependencies'][obj]['version'], d, ud, lockdown, manifest[pkg]['deps'], False)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500254
255 def download(self, ud, d):
256 """Fetch url"""
257 jsondepobj = {}
258 shrinkobj = {}
259 lockdown = {}
260
261 if not os.listdir(ud.pkgdatadir) and os.path.exists(ud.fullmirror):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500262 dest = d.getVar("DL_DIR")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500263 bb.utils.mkdirhier(dest)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600264 runfetchcmd("tar -xJf %s" % (ud.fullmirror), d, workdir=dest)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500265 return
266
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500267 if ud.parm.get("noverify", None) != '1':
268 shwrf = d.getVar('NPM_SHRINKWRAP')
269 logger.debug(2, "NPM shrinkwrap file is %s" % shwrf)
270 if shwrf:
271 try:
272 with open(shwrf) as datafile:
273 shrinkobj = json.load(datafile)
274 except Exception as e:
275 raise FetchError('Error loading NPM_SHRINKWRAP file "%s" for %s: %s' % (shwrf, ud.pkgname, str(e)))
276 elif not ud.ignore_checksums:
277 logger.warning('Missing shrinkwrap file in NPM_SHRINKWRAP for %s, this will lead to unreliable builds!' % ud.pkgname)
278 lckdf = d.getVar('NPM_LOCKDOWN')
279 logger.debug(2, "NPM lockdown file is %s" % lckdf)
280 if lckdf:
281 try:
282 with open(lckdf) as datafile:
283 lockdown = json.load(datafile)
284 except Exception as e:
285 raise FetchError('Error loading NPM_LOCKDOWN file "%s" for %s: %s' % (lckdf, ud.pkgname, str(e)))
286 elif not ud.ignore_checksums:
287 logger.warning('Missing lockdown file in NPM_LOCKDOWN for %s, this will lead to unreproducible builds!' % ud.pkgname)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500288
289 if ('name' not in shrinkobj):
290 self._getdependencies(ud.pkgname, jsondepobj, ud.version, d, ud)
291 else:
292 self._getshrinkeddependencies(ud.pkgname, shrinkobj, ud.version, d, ud, lockdown, jsondepobj)
293
294 with open(ud.localpath, 'w') as outfile:
295 json.dump(jsondepobj, outfile)
296
297 def build_mirror_data(self, ud, d):
298 # Generate a mirror tarball if needed
299 if ud.write_tarballs and not os.path.exists(ud.fullmirror):
300 # it's possible that this symlink points to read-only filesystem with PREMIRROR
301 if os.path.islink(ud.fullmirror):
302 os.unlink(ud.fullmirror)
303
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500304 dldir = d.getVar("DL_DIR")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500305 logger.info("Creating tarball of npm data")
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600306 runfetchcmd("tar -cJf %s npm/%s npm/%s" % (ud.fullmirror, ud.bbnpmmanifest, ud.pkgname), d,
307 workdir=dldir)
308 runfetchcmd("touch %s.done" % (ud.fullmirror), d, workdir=dldir)