Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 1 | # Recipe creation tool - node.js NPM module support plugin |
| 2 | # |
| 3 | # Copyright (C) 2016 Intel Corporation |
| 4 | # |
| 5 | # This program is free software; you can redistribute it and/or modify |
| 6 | # it under the terms of the GNU General Public License version 2 as |
| 7 | # published by the Free Software Foundation. |
| 8 | # |
| 9 | # This program is distributed in the hope that it will be useful, |
| 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 12 | # GNU General Public License for more details. |
| 13 | # |
| 14 | # You should have received a copy of the GNU General Public License along |
| 15 | # with this program; if not, write to the Free Software Foundation, Inc., |
| 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. |
| 17 | |
| 18 | import os |
| 19 | import logging |
| 20 | import subprocess |
| 21 | import tempfile |
| 22 | import shutil |
| 23 | import json |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame^] | 24 | from recipetool.create import RecipeHandler, split_pkg_licenses, handle_license_vars |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 25 | |
| 26 | logger = logging.getLogger('recipetool') |
| 27 | |
| 28 | |
| 29 | tinfoil = None |
| 30 | |
| 31 | def tinfoil_init(instance): |
| 32 | global tinfoil |
| 33 | tinfoil = instance |
| 34 | |
| 35 | |
| 36 | class NpmRecipeHandler(RecipeHandler): |
| 37 | lockdownpath = None |
| 38 | |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame^] | 39 | def _ensure_npm(self, fixed_setup=False): |
| 40 | if not tinfoil.recipes_parsed: |
| 41 | tinfoil.parse_recipes() |
| 42 | try: |
| 43 | rd = tinfoil.parse_recipe('nodejs-native') |
| 44 | except bb.providers.NoProvider: |
| 45 | if fixed_setup: |
| 46 | msg = 'nodejs-native is required for npm but is not available within this SDK' |
| 47 | else: |
| 48 | msg = 'nodejs-native is required for npm but is not available - you will likely need to add a layer that provides nodejs' |
| 49 | logger.error(msg) |
| 50 | return None |
| 51 | bindir = rd.getVar('STAGING_BINDIR_NATIVE') |
| 52 | npmpath = os.path.join(bindir, 'npm') |
| 53 | if not os.path.exists(npmpath): |
| 54 | tinfoil.build_targets('nodejs-native', 'addto_recipe_sysroot') |
| 55 | if not os.path.exists(npmpath): |
| 56 | logger.error('npm required to process specified source, but nodejs-native did not seem to populate it') |
| 57 | return None |
| 58 | return bindir |
| 59 | |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 60 | def _handle_license(self, data): |
| 61 | ''' |
| 62 | Handle the license value from an npm package.json file |
| 63 | ''' |
| 64 | license = None |
| 65 | if 'license' in data: |
| 66 | license = data['license'] |
| 67 | if isinstance(license, dict): |
| 68 | license = license.get('type', None) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 69 | if license: |
| 70 | if 'OR' in license: |
| 71 | license = license.replace('OR', '|') |
| 72 | license = license.replace('AND', '&') |
| 73 | license = license.replace(' ', '_') |
| 74 | if not license[0] == '(': |
| 75 | license = '(' + license + ')' |
| 76 | print('LICENSE: {}'.format(license)) |
| 77 | else: |
| 78 | license = license.replace('AND', '&') |
| 79 | if license[0] == '(': |
| 80 | license = license[1:] |
| 81 | if license[-1] == ')': |
| 82 | license = license[:-1] |
| 83 | license = license.replace('MIT/X11', 'MIT') |
| 84 | license = license.replace('Public Domain', 'PD') |
| 85 | license = license.replace('SEE LICENSE IN EULA', |
| 86 | 'SEE-LICENSE-IN-EULA') |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 87 | return license |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 88 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 89 | def _shrinkwrap(self, srctree, localfilesdir, extravalues, lines_before, d): |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 90 | try: |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 91 | runenv = dict(os.environ, PATH=d.getVar('PATH')) |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 92 | bb.process.run('npm shrinkwrap', cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True) |
| 93 | except bb.process.ExecutionError as e: |
| 94 | logger.warn('npm shrinkwrap failed:\n%s' % e.stdout) |
| 95 | return |
| 96 | |
| 97 | tmpfile = os.path.join(localfilesdir, 'npm-shrinkwrap.json') |
| 98 | shutil.move(os.path.join(srctree, 'npm-shrinkwrap.json'), tmpfile) |
| 99 | extravalues.setdefault('extrafiles', {}) |
| 100 | extravalues['extrafiles']['npm-shrinkwrap.json'] = tmpfile |
| 101 | lines_before.append('NPM_SHRINKWRAP := "${THISDIR}/${PN}/npm-shrinkwrap.json"') |
| 102 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 103 | def _lockdown(self, srctree, localfilesdir, extravalues, lines_before, d): |
| 104 | runenv = dict(os.environ, PATH=d.getVar('PATH')) |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 105 | if not NpmRecipeHandler.lockdownpath: |
| 106 | NpmRecipeHandler.lockdownpath = tempfile.mkdtemp('recipetool-npm-lockdown') |
| 107 | bb.process.run('npm install lockdown --prefix %s' % NpmRecipeHandler.lockdownpath, |
| 108 | cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True) |
| 109 | relockbin = os.path.join(NpmRecipeHandler.lockdownpath, 'node_modules', 'lockdown', 'relock.js') |
| 110 | if not os.path.exists(relockbin): |
| 111 | logger.warn('Could not find relock.js within lockdown directory; skipping lockdown') |
| 112 | return |
| 113 | try: |
| 114 | bb.process.run('node %s' % relockbin, cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True) |
| 115 | except bb.process.ExecutionError as e: |
| 116 | logger.warn('lockdown-relock failed:\n%s' % e.stdout) |
| 117 | return |
| 118 | |
| 119 | tmpfile = os.path.join(localfilesdir, 'lockdown.json') |
| 120 | shutil.move(os.path.join(srctree, 'lockdown.json'), tmpfile) |
| 121 | extravalues.setdefault('extrafiles', {}) |
| 122 | extravalues['extrafiles']['lockdown.json'] = tmpfile |
| 123 | lines_before.append('NPM_LOCKDOWN := "${THISDIR}/${PN}/lockdown.json"') |
| 124 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 125 | def _handle_dependencies(self, d, deps, optdeps, devdeps, lines_before, srctree): |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 126 | import scriptutils |
| 127 | # If this isn't a single module we need to get the dependencies |
| 128 | # and add them to SRC_URI |
| 129 | def varfunc(varname, origvalue, op, newlines): |
| 130 | if varname == 'SRC_URI': |
| 131 | if not origvalue.startswith('npm://'): |
| 132 | src_uri = origvalue.split() |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 133 | deplist = {} |
| 134 | for dep, depver in optdeps.items(): |
| 135 | depdata = self.get_npm_data(dep, depver, d) |
| 136 | if self.check_npm_optional_dependency(depdata): |
| 137 | deplist[dep] = depdata |
| 138 | for dep, depver in devdeps.items(): |
| 139 | depdata = self.get_npm_data(dep, depver, d) |
| 140 | if self.check_npm_optional_dependency(depdata): |
| 141 | deplist[dep] = depdata |
| 142 | for dep, depver in deps.items(): |
| 143 | depdata = self.get_npm_data(dep, depver, d) |
| 144 | deplist[dep] = depdata |
| 145 | |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame^] | 146 | extra_urls = [] |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 147 | for dep, depdata in deplist.items(): |
| 148 | version = depdata.get('version', None) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 149 | if version: |
| 150 | url = 'npm://registry.npmjs.org;name=%s;version=%s;subdir=node_modules/%s' % (dep, version, dep) |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame^] | 151 | extra_urls.append(url) |
| 152 | if extra_urls: |
| 153 | scriptutils.fetch_url(tinfoil, ' '.join(extra_urls), None, srctree, logger) |
| 154 | src_uri.extend(extra_urls) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 155 | return src_uri, None, -1, True |
| 156 | return origvalue, None, 0, True |
| 157 | updated, newlines = bb.utils.edit_metadata(lines_before, ['SRC_URI'], varfunc) |
| 158 | if updated: |
| 159 | del lines_before[:] |
| 160 | for line in newlines: |
| 161 | # Hack to avoid newlines that edit_metadata inserts |
| 162 | if line.endswith('\n'): |
| 163 | line = line[:-1] |
| 164 | lines_before.append(line) |
| 165 | return updated |
| 166 | |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 167 | def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): |
| 168 | import bb.utils |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame^] | 169 | import oe.package |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 170 | from collections import OrderedDict |
| 171 | |
| 172 | if 'buildsystem' in handled: |
| 173 | return False |
| 174 | |
| 175 | def read_package_json(fn): |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 176 | with open(fn, 'r', errors='surrogateescape') as f: |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 177 | return json.loads(f.read()) |
| 178 | |
| 179 | files = RecipeHandler.checkfiles(srctree, ['package.json']) |
| 180 | if files: |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 181 | d = bb.data.createCopy(tinfoil.config_data) |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame^] | 182 | npm_bindir = self._ensure_npm() |
| 183 | if not npm_bindir: |
| 184 | sys.exit(14) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 185 | d.prependVar('PATH', '%s:' % npm_bindir) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 186 | |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 187 | data = read_package_json(files[0]) |
| 188 | if 'name' in data and 'version' in data: |
| 189 | extravalues['PN'] = data['name'] |
| 190 | extravalues['PV'] = data['version'] |
| 191 | classes.append('npm') |
| 192 | handled.append('buildsystem') |
| 193 | if 'description' in data: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 194 | extravalues['SUMMARY'] = data['description'] |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 195 | if 'homepage' in data: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 196 | extravalues['HOMEPAGE'] = data['homepage'] |
| 197 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 198 | fetchdev = extravalues['fetchdev'] or None |
| 199 | deps, optdeps, devdeps = self.get_npm_package_dependencies(data, fetchdev) |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame^] | 200 | self._handle_dependencies(d, deps, optdeps, devdeps, lines_before, srctree) |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 201 | |
| 202 | # Shrinkwrap |
| 203 | localfilesdir = tempfile.mkdtemp(prefix='recipetool-npm') |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 204 | self._shrinkwrap(srctree, localfilesdir, extravalues, lines_before, d) |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 205 | |
| 206 | # Lockdown |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 207 | self._lockdown(srctree, localfilesdir, extravalues, lines_before, d) |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 208 | |
| 209 | # Split each npm module out to is own package |
| 210 | npmpackages = oe.package.npm_split_package_dirs(srctree) |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame^] | 211 | licvalues = None |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 212 | for item in handled: |
| 213 | if isinstance(item, tuple): |
| 214 | if item[0] == 'license': |
| 215 | licvalues = item[1] |
| 216 | break |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame^] | 217 | if not licvalues: |
| 218 | licvalues = handle_license_vars(srctree, lines_before, handled, extravalues, d) |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 219 | if licvalues: |
| 220 | # Augment the license list with information we have in the packages |
| 221 | licenses = {} |
| 222 | license = self._handle_license(data) |
| 223 | if license: |
| 224 | licenses['${PN}'] = license |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 225 | for pkgname, pkgitem in npmpackages.items(): |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 226 | _, pdata = pkgitem |
| 227 | license = self._handle_license(pdata) |
| 228 | if license: |
| 229 | licenses[pkgname] = license |
| 230 | # Now write out the package-specific license values |
| 231 | # We need to strip out the json data dicts for this since split_pkg_licenses |
| 232 | # isn't expecting it |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 233 | packages = OrderedDict((x,y[0]) for x,y in npmpackages.items()) |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 234 | packages['${PN}'] = '' |
| 235 | pkglicenses = split_pkg_licenses(licvalues, packages, lines_after, licenses) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 236 | all_licenses = list(set([item.replace('_', ' ') for pkglicense in pkglicenses.values() for item in pkglicense])) |
| 237 | if '&' in all_licenses: |
| 238 | all_licenses.remove('&') |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame^] | 239 | extravalues['LICENSE'] = ' & '.join(all_licenses) |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 240 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 241 | # Need to move S setting after inherit npm |
| 242 | for i, line in enumerate(lines_before): |
| 243 | if line.startswith('S ='): |
| 244 | lines_before.pop(i) |
| 245 | lines_after.insert(0, '# Must be set after inherit npm since that itself sets S') |
| 246 | lines_after.insert(1, line) |
| 247 | break |
| 248 | |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 249 | return True |
| 250 | |
| 251 | return False |
| 252 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 253 | # FIXME this is duplicated from lib/bb/fetch2/npm.py |
| 254 | def _parse_view(self, output): |
| 255 | ''' |
| 256 | Parse the output of npm view --json; the last JSON result |
| 257 | is assumed to be the one that we're interested in. |
| 258 | ''' |
| 259 | pdata = None |
| 260 | outdeps = {} |
| 261 | datalines = [] |
| 262 | bracelevel = 0 |
| 263 | for line in output.splitlines(): |
| 264 | if bracelevel: |
| 265 | datalines.append(line) |
| 266 | elif '{' in line: |
| 267 | datalines = [] |
| 268 | datalines.append(line) |
| 269 | bracelevel = bracelevel + line.count('{') - line.count('}') |
| 270 | if datalines: |
| 271 | pdata = json.loads('\n'.join(datalines)) |
| 272 | return pdata |
| 273 | |
| 274 | # FIXME this is effectively duplicated from lib/bb/fetch2/npm.py |
| 275 | # (split out from _getdependencies()) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 276 | def get_npm_data(self, pkg, version, d): |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 277 | import bb.fetch2 |
| 278 | pkgfullname = pkg |
| 279 | if version != '*' and not '/' in version: |
| 280 | pkgfullname += "@'%s'" % version |
| 281 | logger.debug(2, "Calling getdeps on %s" % pkg) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 282 | runenv = dict(os.environ, PATH=d.getVar('PATH')) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 283 | fetchcmd = "npm view %s --json" % pkgfullname |
| 284 | output, _ = bb.process.run(fetchcmd, stderr=subprocess.STDOUT, env=runenv, shell=True) |
| 285 | data = self._parse_view(output) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 286 | return data |
| 287 | |
| 288 | # FIXME this is effectively duplicated from lib/bb/fetch2/npm.py |
| 289 | # (split out from _getdependencies()) |
| 290 | def get_npm_package_dependencies(self, pdata, fetchdev): |
| 291 | dependencies = pdata.get('dependencies', {}) |
| 292 | optionalDependencies = pdata.get('optionalDependencies', {}) |
| 293 | dependencies.update(optionalDependencies) |
| 294 | if fetchdev: |
| 295 | devDependencies = pdata.get('devDependencies', {}) |
| 296 | dependencies.update(devDependencies) |
| 297 | else: |
| 298 | devDependencies = {} |
| 299 | depsfound = {} |
| 300 | optdepsfound = {} |
| 301 | devdepsfound = {} |
| 302 | for dep in dependencies: |
| 303 | if dep in optionalDependencies: |
| 304 | optdepsfound[dep] = dependencies[dep] |
| 305 | elif dep in devDependencies: |
| 306 | devdepsfound[dep] = dependencies[dep] |
| 307 | else: |
| 308 | depsfound[dep] = dependencies[dep] |
| 309 | return depsfound, optdepsfound, devdepsfound |
| 310 | |
| 311 | # FIXME this is effectively duplicated from lib/bb/fetch2/npm.py |
| 312 | # (split out from _getdependencies()) |
| 313 | def check_npm_optional_dependency(self, pdata): |
| 314 | pkg_os = pdata.get('os', None) |
| 315 | if pkg_os: |
| 316 | if not isinstance(pkg_os, list): |
| 317 | pkg_os = [pkg_os] |
| 318 | blacklist = False |
| 319 | for item in pkg_os: |
| 320 | if item.startswith('!'): |
| 321 | blacklist = True |
| 322 | break |
| 323 | if (not blacklist and 'linux' not in pkg_os) or '!linux' in pkg_os: |
| 324 | logger.debug(2, "Skipping %s since it's incompatible with Linux" % pkg) |
| 325 | return False |
| 326 | return True |
| 327 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 328 | |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 329 | def register_recipe_handlers(handlers): |
| 330 | handlers.append((NpmRecipeHandler(), 60)) |