blob: 579b7ae48ad483fc6bad5d7bf5bbe7fb61695793 [file] [log] [blame]
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001# Copyright (C) 2016 Intel Corporation
Andrew Geissler82c905d2020-04-13 13:39:40 -05002# Copyright (C) 2020 Savoir-Faire Linux
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05003#
Brad Bishopc342db32019-05-15 21:57:59 -04004# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05005#
Andrew Geissler82c905d2020-04-13 13:39:40 -05006"""Recipe creation tool - npm module support plugin"""
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05007
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05008import json
Andrew Geissler82c905d2020-04-13 13:39:40 -05009import os
10import re
11import sys
12import tempfile
13import bb
14from bb.fetch2.npm import NpmEnvironment
15from bb.fetch2.npmsw import foreach_dependencies
16from recipetool.create import RecipeHandler
17from recipetool.create import guess_license
18from recipetool.create import split_pkg_licenses
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050019
Andrew Geissler82c905d2020-04-13 13:39:40 -050020TINFOIL = None
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050021
22def tinfoil_init(instance):
Andrew Geissler82c905d2020-04-13 13:39:40 -050023 """Initialize tinfoil"""
24 global TINFOIL
25 TINFOIL = instance
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050026
27class NpmRecipeHandler(RecipeHandler):
Andrew Geissler82c905d2020-04-13 13:39:40 -050028 """Class to handle the npm recipe creation"""
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050029
Andrew Geissler82c905d2020-04-13 13:39:40 -050030 @staticmethod
31 def _npm_name(name):
32 """Generate a Yocto friendly npm name"""
33 name = re.sub("/", "-", name)
34 name = name.lower()
35 name = re.sub(r"[^\-a-z0-9]", "", name)
36 name = name.strip("-")
37 return name
38
39 @staticmethod
40 def _get_registry(lines):
41 """Get the registry value from the 'npm://registry' url"""
42 registry = None
43
44 def _handle_registry(varname, origvalue, op, newlines):
45 nonlocal registry
46 if origvalue.startswith("npm://"):
47 registry = re.sub(r"^npm://", "http://", origvalue.split(";")[0])
48 return origvalue, None, 0, True
49
50 bb.utils.edit_metadata(lines, ["SRC_URI"], _handle_registry)
51
52 return registry
53
54 @staticmethod
55 def _ensure_npm():
56 """Check if the 'npm' command is available in the recipes"""
57 if not TINFOIL.recipes_parsed:
58 TINFOIL.parse_recipes()
59
Brad Bishopd7bf8c12018-02-25 22:55:05 -050060 try:
Andrew Geissler82c905d2020-04-13 13:39:40 -050061 d = TINFOIL.parse_recipe("nodejs-native")
Brad Bishopd7bf8c12018-02-25 22:55:05 -050062 except bb.providers.NoProvider:
Andrew Geissler82c905d2020-04-13 13:39:40 -050063 bb.error("Nothing provides 'nodejs-native' which is required for the build")
64 bb.note("You will likely need to add a layer that provides nodejs")
65 sys.exit(14)
66
67 bindir = d.getVar("STAGING_BINDIR_NATIVE")
68 npmpath = os.path.join(bindir, "npm")
69
Brad Bishopd7bf8c12018-02-25 22:55:05 -050070 if not os.path.exists(npmpath):
Andrew Geissler82c905d2020-04-13 13:39:40 -050071 TINFOIL.build_targets("nodejs-native", "addto_recipe_sysroot")
72
Brad Bishopd7bf8c12018-02-25 22:55:05 -050073 if not os.path.exists(npmpath):
Andrew Geissler82c905d2020-04-13 13:39:40 -050074 bb.error("Failed to add 'npm' to sysroot")
75 sys.exit(14)
76
Brad Bishopd7bf8c12018-02-25 22:55:05 -050077 return bindir
78
Andrew Geissler82c905d2020-04-13 13:39:40 -050079 @staticmethod
80 def _npm_global_configs(dev):
81 """Get the npm global configuration"""
82 configs = []
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050083
Andrew Geissler82c905d2020-04-13 13:39:40 -050084 if dev:
85 configs.append(("also", "development"))
86 else:
87 configs.append(("only", "production"))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050088
Andrew Geissler82c905d2020-04-13 13:39:40 -050089 configs.append(("save", "false"))
90 configs.append(("package-lock", "false"))
91 configs.append(("shrinkwrap", "false"))
92 return configs
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050093
Andrew Geissler82c905d2020-04-13 13:39:40 -050094 def _run_npm_install(self, d, srctree, registry, dev):
95 """Run the 'npm install' command without building the addons"""
96 configs = self._npm_global_configs(dev)
97 configs.append(("ignore-scripts", "true"))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050098
Andrew Geissler82c905d2020-04-13 13:39:40 -050099 if registry:
100 configs.append(("registry", registry))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500101
Andrew Geissler82c905d2020-04-13 13:39:40 -0500102 bb.utils.remove(os.path.join(srctree, "node_modules"), recurse=True)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500103
Andrew Geissler82c905d2020-04-13 13:39:40 -0500104 env = NpmEnvironment(d, configs=configs)
105 env.run("npm install", workdir=srctree)
106
107 def _generate_shrinkwrap(self, d, srctree, dev):
108 """Check and generate the 'npm-shrinkwrap.json' file if needed"""
109 configs = self._npm_global_configs(dev)
110
111 env = NpmEnvironment(d, configs=configs)
112 env.run("npm shrinkwrap", workdir=srctree)
113
114 return os.path.join(srctree, "npm-shrinkwrap.json")
115
116 def _handle_licenses(self, srctree, shrinkwrap_file, dev):
117 """Return the extra license files and the list of packages"""
118 licfiles = []
119 packages = {}
120
121 def _licfiles_append(licfile):
122 """Append 'licfile' to the license files list"""
123 licfilepath = os.path.join(srctree, licfile)
124 licmd5 = bb.utils.md5_file(licfilepath)
125 licfiles.append("file://%s;md5=%s" % (licfile, licmd5))
126
127 # Handle the parent package
128 _licfiles_append("package.json")
129 packages["${PN}"] = ""
130
131 # Handle the dependencies
132 def _handle_dependency(name, params, deptree):
133 suffix = "-".join([self._npm_name(dep) for dep in deptree])
134 destdirs = [os.path.join("node_modules", dep) for dep in deptree]
135 destdir = os.path.join(*destdirs)
136 _licfiles_append(os.path.join(destdir, "package.json"))
137 packages["${PN}-" + suffix] = destdir
138
139 with open(shrinkwrap_file, "r") as f:
140 shrinkwrap = json.load(f)
141
142 foreach_dependencies(shrinkwrap, _handle_dependency, dev)
143
144 return licfiles, packages
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600145
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500146 def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
Andrew Geissler82c905d2020-04-13 13:39:40 -0500147 """Handle the npm recipe creation"""
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500148
Andrew Geissler82c905d2020-04-13 13:39:40 -0500149 if "buildsystem" in handled:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500150 return False
151
Andrew Geissler82c905d2020-04-13 13:39:40 -0500152 files = RecipeHandler.checkfiles(srctree, ["package.json"])
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500153
Andrew Geissler82c905d2020-04-13 13:39:40 -0500154 if not files:
155 return False
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600156
Andrew Geissler82c905d2020-04-13 13:39:40 -0500157 with open(files[0], "r") as f:
158 data = json.load(f)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600159
Andrew Geissler82c905d2020-04-13 13:39:40 -0500160 if "name" not in data or "version" not in data:
161 return False
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500162
Andrew Geissler82c905d2020-04-13 13:39:40 -0500163 extravalues["PN"] = self._npm_name(data["name"])
164 extravalues["PV"] = data["version"]
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500165
Andrew Geissler82c905d2020-04-13 13:39:40 -0500166 if "description" in data:
167 extravalues["SUMMARY"] = data["description"]
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500168
Andrew Geissler82c905d2020-04-13 13:39:40 -0500169 if "homepage" in data:
170 extravalues["HOMEPAGE"] = data["homepage"]
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500171
Andrew Geissler82c905d2020-04-13 13:39:40 -0500172 dev = bb.utils.to_boolean(str(extravalues.get("NPM_INSTALL_DEV", "0")), False)
173 registry = self._get_registry(lines_before)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600174
Andrew Geissler82c905d2020-04-13 13:39:40 -0500175 bb.note("Checking if npm is available ...")
176 # The native npm is used here (and not the host one) to ensure that the
177 # npm version is high enough to ensure an efficient dependency tree
178 # resolution and avoid issue with the shrinkwrap file format.
179 # Moreover the native npm is mandatory for the build.
180 bindir = self._ensure_npm()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500181
Andrew Geissler82c905d2020-04-13 13:39:40 -0500182 d = bb.data.createCopy(TINFOIL.config_data)
183 d.prependVar("PATH", bindir + ":")
184 d.setVar("S", srctree)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500185
Andrew Geissler82c905d2020-04-13 13:39:40 -0500186 bb.note("Generating shrinkwrap file ...")
187 # To generate the shrinkwrap file the dependencies have to be installed
188 # first. During the generation process some files may be updated /
189 # deleted. By default devtool tracks the diffs in the srctree and raises
190 # errors when finishing the recipe if some diffs are found.
191 git_exclude_file = os.path.join(srctree, ".git", "info", "exclude")
192 if os.path.exists(git_exclude_file):
193 with open(git_exclude_file, "r+") as f:
194 lines = f.readlines()
195 for line in ["/node_modules/", "/npm-shrinkwrap.json"]:
196 if line not in lines:
197 f.write(line + "\n")
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600198
Andrew Geissler82c905d2020-04-13 13:39:40 -0500199 lock_file = os.path.join(srctree, "package-lock.json")
200 lock_copy = lock_file + ".copy"
201 if os.path.exists(lock_file):
202 bb.utils.copyfile(lock_file, lock_copy)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500203
Andrew Geissler82c905d2020-04-13 13:39:40 -0500204 self._run_npm_install(d, srctree, registry, dev)
205 shrinkwrap_file = self._generate_shrinkwrap(d, srctree, dev)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500206
Andrew Geissler82c905d2020-04-13 13:39:40 -0500207 if os.path.exists(lock_copy):
208 bb.utils.movefile(lock_copy, lock_file)
209
210 # Add the shrinkwrap file as 'extrafiles'
211 shrinkwrap_copy = shrinkwrap_file + ".copy"
212 bb.utils.copyfile(shrinkwrap_file, shrinkwrap_copy)
213 extravalues.setdefault("extrafiles", {})
214 extravalues["extrafiles"]["npm-shrinkwrap.json"] = shrinkwrap_copy
215
216 url_local = "npmsw://%s" % shrinkwrap_file
217 url_recipe= "npmsw://${THISDIR}/${BPN}/npm-shrinkwrap.json"
218
219 if dev:
220 url_local += ";dev=1"
221 url_recipe += ";dev=1"
222
223 # Add the npmsw url in the SRC_URI of the generated recipe
224 def _handle_srcuri(varname, origvalue, op, newlines):
225 """Update the version value and add the 'npmsw://' url"""
226 value = origvalue.replace("version=" + data["version"], "version=${PV}")
227 value = value.replace("version=latest", "version=${PV}")
228 values = [line.strip() for line in value.strip('\n').splitlines()]
229 values.append(url_recipe)
230 return values, None, 4, False
231
232 (_, newlines) = bb.utils.edit_metadata(lines_before, ["SRC_URI"], _handle_srcuri)
233 lines_before[:] = [line.rstrip('\n') for line in newlines]
234
235 # In order to generate correct licence checksums in the recipe the
236 # dependencies have to be fetched again using the npmsw url
237 bb.note("Fetching npm dependencies ...")
238 bb.utils.remove(os.path.join(srctree, "node_modules"), recurse=True)
239 fetcher = bb.fetch2.Fetch([url_local], d)
240 fetcher.download()
241 fetcher.unpack(srctree)
242
243 bb.note("Handling licences ...")
244 (licfiles, packages) = self._handle_licenses(srctree, shrinkwrap_file, dev)
245 extravalues["LIC_FILES_CHKSUM"] = licfiles
246 split_pkg_licenses(guess_license(srctree, d), packages, lines_after, [])
247
248 classes.append("npm")
249 handled.append("buildsystem")
250
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500251 return True
252
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500253def register_recipe_handlers(handlers):
Andrew Geissler82c905d2020-04-13 13:39:40 -0500254 """Register the npm handler"""
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500255 handlers.append((NpmRecipeHandler(), 60))