blob: 639f461a3a41e5c37d1d6176911218d53008624f [file] [log] [blame]
Patrick Williams92b42cb2022-09-03 06:53:57 -05001# Copyright (C) 2020 Savoir-Faire Linux
2#
3# SPDX-License-Identifier: GPL-2.0-only
4#
5# This bbclass builds and installs an npm package to the target. The package
6# sources files should be fetched in the calling recipe by using the SRC_URI
7# variable. The ${S} variable should be updated depending of your fetcher.
8#
9# Usage:
10# SRC_URI = "..."
11# inherit npm
12#
13# Optional variables:
14# NPM_ARCH:
15# Override the auto generated npm architecture.
16#
17# NPM_INSTALL_DEV:
18# Set to 1 to also install devDependencies.
19
20inherit python3native
21
22DEPENDS:prepend = "nodejs-native nodejs-oe-cache-native "
23RDEPENDS:${PN}:append:class-target = " nodejs"
24
25EXTRA_OENPM = ""
26
27NPM_INSTALL_DEV ?= "0"
28
29NPM_NODEDIR ?= "${RECIPE_SYSROOT_NATIVE}${prefix_native}"
30
31## must match mapping in nodejs.bb (openembedded-meta)
32def map_nodejs_arch(a, d):
33 import re
34
35 if re.match('i.86$', a): return 'ia32'
36 elif re.match('x86_64$', a): return 'x64'
37 elif re.match('aarch64$', a): return 'arm64'
38 elif re.match('(powerpc64|powerpc64le|ppc64le)$', a): return 'ppc64'
39 elif re.match('powerpc$', a): return 'ppc'
40 return a
41
42NPM_ARCH ?= "${@map_nodejs_arch(d.getVar("TARGET_ARCH"), d)}"
43
44NPM_PACKAGE = "${WORKDIR}/npm-package"
45NPM_CACHE = "${WORKDIR}/npm-cache"
46NPM_BUILD = "${WORKDIR}/npm-build"
47NPM_REGISTRY = "${WORKDIR}/npm-registry"
48
49def npm_global_configs(d):
50 """Get the npm global configuration"""
51 configs = []
52 # Ensure no network access is done
53 configs.append(("offline", "true"))
54 configs.append(("proxy", "http://invalid"))
55 configs.append(("fund", False))
56 configs.append(("audit", False))
57 # Configure the cache directory
58 configs.append(("cache", d.getVar("NPM_CACHE")))
59 return configs
60
61## 'npm pack' runs 'prepare' and 'prepack' scripts. Support for
62## 'ignore-scripts' which prevents this behavior has been removed
63## from nodejs 16. Use simple 'tar' instead of.
64def npm_pack(env, srcdir, workdir):
65 """Emulate 'npm pack' on a specified directory"""
66 import subprocess
67 import os
68 import json
69
70 src = os.path.join(srcdir, 'package.json')
71 with open(src) as f:
72 j = json.load(f)
73
74 # base does not really matter and is for documentation purposes
75 # only. But the 'version' part must exist because other parts of
76 # the bbclass rely on it.
77 base = j['name'].split('/')[-1]
78 tarball = os.path.join(workdir, "%s-%s.tgz" % (base, j['version']));
79
80 # TODO: real 'npm pack' does not include directories while 'tar'
81 # does. But this does not seem to matter...
82 subprocess.run(['tar', 'czf', tarball,
83 '--exclude', './node-modules',
84 '--exclude-vcs',
Andrew Geisslerfc113ea2023-03-31 09:59:46 -050085 '--transform', r's,^\./,package/,',
Patrick Williams92b42cb2022-09-03 06:53:57 -050086 '--mtime', '1985-10-26T08:15:00.000Z',
87 '.'],
88 check = True, cwd = srcdir)
89
90 return (tarball, j)
91
92python npm_do_configure() {
93 """
94 Step one: configure the npm cache and the main npm package
95
96 Every dependencies have been fetched and patched in the source directory.
97 They have to be packed (this remove unneeded files) and added to the npm
98 cache to be available for the next step.
99
100 The main package and its associated manifest file and shrinkwrap file have
101 to be configured to take into account these cached dependencies.
102 """
103 import base64
104 import copy
105 import json
106 import re
107 import shlex
108 import stat
109 import tempfile
110 from bb.fetch2.npm import NpmEnvironment
111 from bb.fetch2.npm import npm_unpack
112 from bb.fetch2.npmsw import foreach_dependencies
113 from bb.progress import OutOfProgressHandler
114 from oe.npm_registry import NpmRegistry
115
116 bb.utils.remove(d.getVar("NPM_CACHE"), recurse=True)
117 bb.utils.remove(d.getVar("NPM_PACKAGE"), recurse=True)
118
119 env = NpmEnvironment(d, configs=npm_global_configs(d))
120 registry = NpmRegistry(d.getVar('NPM_REGISTRY'), d.getVar('NPM_CACHE'))
121
122 def _npm_cache_add(tarball, pkg):
123 """Add tarball to local registry and register it in the
124 cache"""
125 registry.add_pkg(tarball, pkg)
126
127 def _npm_integrity(tarball):
128 """Return the npm integrity of a specified tarball"""
129 sha512 = bb.utils.sha512_file(tarball)
130 return "sha512-" + base64.b64encode(bytes.fromhex(sha512)).decode()
131
132 def _npmsw_dependency_dict(orig, deptree):
133 """
134 Return the sub dictionary in the 'orig' dictionary corresponding to the
135 'deptree' dependency tree. This function follows the shrinkwrap file
136 format.
137 """
138 ptr = orig
139 for dep in deptree:
140 if "dependencies" not in ptr:
141 ptr["dependencies"] = {}
142 ptr = ptr["dependencies"]
143 if dep not in ptr:
144 ptr[dep] = {}
145 ptr = ptr[dep]
146 return ptr
147
148 # Manage the manifest file and shrinkwrap files
149 orig_manifest_file = d.expand("${S}/package.json")
150 orig_shrinkwrap_file = d.expand("${S}/npm-shrinkwrap.json")
151 cached_manifest_file = d.expand("${NPM_PACKAGE}/package.json")
152 cached_shrinkwrap_file = d.expand("${NPM_PACKAGE}/npm-shrinkwrap.json")
153
154 with open(orig_manifest_file, "r") as f:
155 orig_manifest = json.load(f)
156
157 cached_manifest = copy.deepcopy(orig_manifest)
158 cached_manifest.pop("dependencies", None)
159 cached_manifest.pop("devDependencies", None)
160
161 has_shrinkwrap_file = True
162
163 try:
164 with open(orig_shrinkwrap_file, "r") as f:
165 orig_shrinkwrap = json.load(f)
166 except IOError:
167 has_shrinkwrap_file = False
168
169 if has_shrinkwrap_file:
170 cached_shrinkwrap = copy.deepcopy(orig_shrinkwrap)
171 cached_shrinkwrap.pop("dependencies", None)
172
173 # Manage the dependencies
174 progress = OutOfProgressHandler(d, r"^(\d+)/(\d+)$")
175 progress_total = 1 # also count the main package
176 progress_done = 0
177
178 def _count_dependency(name, params, deptree):
179 nonlocal progress_total
180 progress_total += 1
181
182 def _cache_dependency(name, params, deptree):
183 destsubdirs = [os.path.join("node_modules", dep) for dep in deptree]
184 destsuffix = os.path.join(*destsubdirs)
185 with tempfile.TemporaryDirectory() as tmpdir:
186 # Add the dependency to the npm cache
187 destdir = os.path.join(d.getVar("S"), destsuffix)
188 (tarball, pkg) = npm_pack(env, destdir, tmpdir)
189 _npm_cache_add(tarball, pkg)
190 # Add its signature to the cached shrinkwrap
191 dep = _npmsw_dependency_dict(cached_shrinkwrap, deptree)
192 dep["version"] = pkg['version']
193 dep["integrity"] = _npm_integrity(tarball)
194 if params.get("dev", False):
195 dep["dev"] = True
196 # Display progress
197 nonlocal progress_done
198 progress_done += 1
199 progress.write("%d/%d" % (progress_done, progress_total))
200
201 dev = bb.utils.to_boolean(d.getVar("NPM_INSTALL_DEV"), False)
202
203 if has_shrinkwrap_file:
204 foreach_dependencies(orig_shrinkwrap, _count_dependency, dev)
205 foreach_dependencies(orig_shrinkwrap, _cache_dependency, dev)
206
207 # Configure the main package
208 with tempfile.TemporaryDirectory() as tmpdir:
209 (tarball, _) = npm_pack(env, d.getVar("S"), tmpdir)
210 npm_unpack(tarball, d.getVar("NPM_PACKAGE"), d)
211
212 # Configure the cached manifest file and cached shrinkwrap file
213 def _update_manifest(depkey):
214 for name in orig_manifest.get(depkey, {}):
215 version = cached_shrinkwrap["dependencies"][name]["version"]
216 if depkey not in cached_manifest:
217 cached_manifest[depkey] = {}
218 cached_manifest[depkey][name] = version
219
220 if has_shrinkwrap_file:
221 _update_manifest("dependencies")
222
223 if dev:
224 if has_shrinkwrap_file:
225 _update_manifest("devDependencies")
226
227 os.chmod(cached_manifest_file, os.stat(cached_manifest_file).st_mode | stat.S_IWUSR)
228 with open(cached_manifest_file, "w") as f:
229 json.dump(cached_manifest, f, indent=2)
230
231 if has_shrinkwrap_file:
232 with open(cached_shrinkwrap_file, "w") as f:
233 json.dump(cached_shrinkwrap, f, indent=2)
234}
235
236python npm_do_compile() {
237 """
238 Step two: install the npm package
239
240 Use the configured main package and the cached dependencies to run the
241 installation process. The installation is done in a directory which is
242 not the destination directory yet.
243
244 A combination of 'npm pack' and 'npm install' is used to ensure that the
245 installed files are actual copies instead of symbolic links (which is the
246 default npm behavior).
247 """
248 import shlex
249 import tempfile
250 from bb.fetch2.npm import NpmEnvironment
251
252 bb.utils.remove(d.getVar("NPM_BUILD"), recurse=True)
253
254 with tempfile.TemporaryDirectory() as tmpdir:
255 args = []
256 configs = npm_global_configs(d)
257
258 if bb.utils.to_boolean(d.getVar("NPM_INSTALL_DEV"), False):
259 configs.append(("also", "development"))
260 else:
261 configs.append(("only", "production"))
262
263 # Report as many logs as possible for debugging purpose
264 configs.append(("loglevel", "silly"))
265
266 # Configure the installation to be done globally in the build directory
267 configs.append(("global", "true"))
268 configs.append(("prefix", d.getVar("NPM_BUILD")))
269
270 # Add node-gyp configuration
271 configs.append(("arch", d.getVar("NPM_ARCH")))
272 configs.append(("release", "true"))
273 configs.append(("nodedir", d.getVar("NPM_NODEDIR")))
274 configs.append(("python", d.getVar("PYTHON")))
275
276 env = NpmEnvironment(d, configs)
277
278 # Add node-pre-gyp configuration
279 args.append(("target_arch", d.getVar("NPM_ARCH")))
280 args.append(("build-from-source", "true"))
281
282 # Pack and install the main package
283 (tarball, _) = npm_pack(env, d.getVar("NPM_PACKAGE"), tmpdir)
284 cmd = "npm install %s %s" % (shlex.quote(tarball), d.getVar("EXTRA_OENPM"))
285 env.run(cmd, args=args)
286}
287
288npm_do_install() {
289 # Step three: final install
290 #
291 # The previous installation have to be filtered to remove some extra files.
292
293 rm -rf ${D}
294
295 # Copy the entire lib and bin directories
296 install -d ${D}/${nonarch_libdir}
297 cp --no-preserve=ownership --recursive ${NPM_BUILD}/lib/. ${D}/${nonarch_libdir}
298
299 if [ -d "${NPM_BUILD}/bin" ]
300 then
301 install -d ${D}/${bindir}
302 cp --no-preserve=ownership --recursive ${NPM_BUILD}/bin/. ${D}/${bindir}
303 fi
304
305 # If the package (or its dependencies) uses node-gyp to build native addons,
306 # object files, static libraries or other temporary files can be hidden in
307 # the lib directory. To reduce the package size and to avoid QA issues
308 # (staticdev with static library files) these files must be removed.
309 local GYP_REGEX=".*/build/Release/[^/]*.node"
310
311 # Remove any node-gyp directory in ${D} to remove temporary build files
312 for GYP_D_FILE in $(find ${D} -regex "${GYP_REGEX}")
313 do
314 local GYP_D_DIR=${GYP_D_FILE%/Release/*}
315
316 rm --recursive --force ${GYP_D_DIR}
317 done
318
319 # Copy only the node-gyp release files
320 for GYP_B_FILE in $(find ${NPM_BUILD} -regex "${GYP_REGEX}")
321 do
322 local GYP_D_FILE=${D}/${prefix}/${GYP_B_FILE#${NPM_BUILD}}
323
324 install -d ${GYP_D_FILE%/*}
325 install -m 755 ${GYP_B_FILE} ${GYP_D_FILE}
326 done
327
328 # Remove the shrinkwrap file which does not need to be packed
329 rm -f ${D}/${nonarch_libdir}/node_modules/*/npm-shrinkwrap.json
330 rm -f ${D}/${nonarch_libdir}/node_modules/@*/*/npm-shrinkwrap.json
331}
332
333FILES:${PN} += " \
334 ${bindir} \
335 ${nonarch_libdir} \
336"
337
338EXPORT_FUNCTIONS do_configure do_compile do_install