blob: 39429ebad3cc0561ff64b4a4aad640aee4570504 [file] [log] [blame]
# Recipe creation tool - node.js NPM module support plugin
#
# Copyright (C) 2016 Intel Corporation
#
# SPDX-License-Identifier: GPL-2.0-only
#
import os
import sys
import logging
import subprocess
import tempfile
import shutil
import json
from recipetool.create import RecipeHandler, split_pkg_licenses, handle_license_vars
logger = logging.getLogger('recipetool')
tinfoil = None
def tinfoil_init(instance):
global tinfoil
tinfoil = instance
class NpmRecipeHandler(RecipeHandler):
lockdownpath = None
def _ensure_npm(self, fixed_setup=False):
if not tinfoil.recipes_parsed:
tinfoil.parse_recipes()
try:
rd = tinfoil.parse_recipe('nodejs-native')
except bb.providers.NoProvider:
if fixed_setup:
msg = 'nodejs-native is required for npm but is not available within this SDK'
else:
msg = 'nodejs-native is required for npm but is not available - you will likely need to add a layer that provides nodejs'
logger.error(msg)
return None
bindir = rd.getVar('STAGING_BINDIR_NATIVE')
npmpath = os.path.join(bindir, 'npm')
if not os.path.exists(npmpath):
tinfoil.build_targets('nodejs-native', 'addto_recipe_sysroot')
if not os.path.exists(npmpath):
logger.error('npm required to process specified source, but nodejs-native did not seem to populate it')
return None
return bindir
def _handle_license(self, data):
'''
Handle the license value from an npm package.json file
'''
license = None
if 'license' in data:
license = data['license']
if isinstance(license, dict):
license = license.get('type', None)
if license:
if 'OR' in license:
license = license.replace('OR', '|')
license = license.replace('AND', '&')
license = license.replace(' ', '_')
if not license[0] == '(':
license = '(' + license + ')'
else:
license = license.replace('AND', '&')
if license[0] == '(':
license = license[1:]
if license[-1] == ')':
license = license[:-1]
license = license.replace('MIT/X11', 'MIT')
license = license.replace('Public Domain', 'PD')
license = license.replace('SEE LICENSE IN EULA',
'SEE-LICENSE-IN-EULA')
return license
def _shrinkwrap(self, srctree, localfilesdir, extravalues, lines_before, d):
try:
runenv = dict(os.environ, PATH=d.getVar('PATH'))
bb.process.run('npm shrinkwrap', cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
except bb.process.ExecutionError as e:
logger.warning('npm shrinkwrap failed:\n%s' % e.stdout)
return
tmpfile = os.path.join(localfilesdir, 'npm-shrinkwrap.json')
shutil.move(os.path.join(srctree, 'npm-shrinkwrap.json'), tmpfile)
extravalues.setdefault('extrafiles', {})
extravalues['extrafiles']['npm-shrinkwrap.json'] = tmpfile
lines_before.append('NPM_SHRINKWRAP := "${THISDIR}/${PN}/npm-shrinkwrap.json"')
def _lockdown(self, srctree, localfilesdir, extravalues, lines_before, d):
runenv = dict(os.environ, PATH=d.getVar('PATH'))
if not NpmRecipeHandler.lockdownpath:
NpmRecipeHandler.lockdownpath = tempfile.mkdtemp('recipetool-npm-lockdown')
bb.process.run('npm install lockdown --prefix %s' % NpmRecipeHandler.lockdownpath,
cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
relockbin = os.path.join(NpmRecipeHandler.lockdownpath, 'node_modules', 'lockdown', 'relock.js')
if not os.path.exists(relockbin):
logger.warning('Could not find relock.js within lockdown directory; skipping lockdown')
return
try:
bb.process.run('node %s' % relockbin, cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
except bb.process.ExecutionError as e:
logger.warning('lockdown-relock failed:\n%s' % e.stdout)
return
tmpfile = os.path.join(localfilesdir, 'lockdown.json')
shutil.move(os.path.join(srctree, 'lockdown.json'), tmpfile)
extravalues.setdefault('extrafiles', {})
extravalues['extrafiles']['lockdown.json'] = tmpfile
lines_before.append('NPM_LOCKDOWN := "${THISDIR}/${PN}/lockdown.json"')
def _handle_dependencies(self, d, deps, optdeps, devdeps, lines_before, srctree):
import scriptutils
# If this isn't a single module we need to get the dependencies
# and add them to SRC_URI
def varfunc(varname, origvalue, op, newlines):
if varname == 'SRC_URI':
if not origvalue.startswith('npm://'):
src_uri = origvalue.split()
deplist = {}
for dep, depver in optdeps.items():
depdata = self.get_npm_data(dep, depver, d)
if self.check_npm_optional_dependency(depdata):
deplist[dep] = depdata
for dep, depver in devdeps.items():
depdata = self.get_npm_data(dep, depver, d)
if self.check_npm_optional_dependency(depdata):
deplist[dep] = depdata
for dep, depver in deps.items():
depdata = self.get_npm_data(dep, depver, d)
deplist[dep] = depdata
extra_urls = []
for dep, depdata in deplist.items():
version = depdata.get('version', None)
if version:
url = 'npm://registry.npmjs.org;name=%s;version=%s;subdir=node_modules/%s' % (dep, version, dep)
extra_urls.append(url)
if extra_urls:
scriptutils.fetch_url(tinfoil, ' '.join(extra_urls), None, srctree, logger)
src_uri.extend(extra_urls)
return src_uri, None, -1, True
return origvalue, None, 0, True
updated, newlines = bb.utils.edit_metadata(lines_before, ['SRC_URI'], varfunc)
if updated:
del lines_before[:]
for line in newlines:
# Hack to avoid newlines that edit_metadata inserts
if line.endswith('\n'):
line = line[:-1]
lines_before.append(line)
return updated
def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
import bb.utils
import oe.package
from collections import OrderedDict
if 'buildsystem' in handled:
return False
def read_package_json(fn):
with open(fn, 'r', errors='surrogateescape') as f:
return json.loads(f.read())
files = RecipeHandler.checkfiles(srctree, ['package.json'])
if files:
d = bb.data.createCopy(tinfoil.config_data)
npm_bindir = self._ensure_npm()
if not npm_bindir:
sys.exit(14)
d.prependVar('PATH', '%s:' % npm_bindir)
data = read_package_json(files[0])
if 'name' in data and 'version' in data:
extravalues['PN'] = data['name']
extravalues['PV'] = data['version']
classes.append('npm')
handled.append('buildsystem')
if 'description' in data:
extravalues['SUMMARY'] = data['description']
if 'homepage' in data:
extravalues['HOMEPAGE'] = data['homepage']
fetchdev = extravalues['fetchdev'] or None
deps, optdeps, devdeps = self.get_npm_package_dependencies(data, fetchdev)
self._handle_dependencies(d, deps, optdeps, devdeps, lines_before, srctree)
# Shrinkwrap
localfilesdir = tempfile.mkdtemp(prefix='recipetool-npm')
self._shrinkwrap(srctree, localfilesdir, extravalues, lines_before, d)
# Lockdown
self._lockdown(srctree, localfilesdir, extravalues, lines_before, d)
# Split each npm module out to is own package
npmpackages = oe.package.npm_split_package_dirs(srctree)
licvalues = None
for item in handled:
if isinstance(item, tuple):
if item[0] == 'license':
licvalues = item[1]
break
if not licvalues:
licvalues = handle_license_vars(srctree, lines_before, handled, extravalues, d)
if licvalues:
# Augment the license list with information we have in the packages
licenses = {}
license = self._handle_license(data)
if license:
licenses['${PN}'] = license
for pkgname, pkgitem in npmpackages.items():
_, pdata = pkgitem
license = self._handle_license(pdata)
if license:
licenses[pkgname] = license
# Now write out the package-specific license values
# We need to strip out the json data dicts for this since split_pkg_licenses
# isn't expecting it
packages = OrderedDict((x,y[0]) for x,y in npmpackages.items())
packages['${PN}'] = ''
pkglicenses = split_pkg_licenses(licvalues, packages, lines_after, licenses)
all_licenses = list(set([item.replace('_', ' ') for pkglicense in pkglicenses.values() for item in pkglicense]))
if '&' in all_licenses:
all_licenses.remove('&')
extravalues['LICENSE'] = ' & '.join(all_licenses)
# Need to move S setting after inherit npm
for i, line in enumerate(lines_before):
if line.startswith('S ='):
lines_before.pop(i)
lines_after.insert(0, '# Must be set after inherit npm since that itself sets S')
lines_after.insert(1, line)
break
return True
return False
# FIXME this is duplicated from lib/bb/fetch2/npm.py
def _parse_view(self, output):
'''
Parse the output of npm view --json; the last JSON result
is assumed to be the one that we're interested in.
'''
pdata = None
outdeps = {}
datalines = []
bracelevel = 0
for line in output.splitlines():
if bracelevel:
datalines.append(line)
elif '{' in line:
datalines = []
datalines.append(line)
bracelevel = bracelevel + line.count('{') - line.count('}')
if datalines:
pdata = json.loads('\n'.join(datalines))
return pdata
# FIXME this is effectively duplicated from lib/bb/fetch2/npm.py
# (split out from _getdependencies())
def get_npm_data(self, pkg, version, d):
import bb.fetch2
pkgfullname = pkg
if version != '*' and not '/' in version:
pkgfullname += "@'%s'" % version
logger.debug(2, "Calling getdeps on %s" % pkg)
runenv = dict(os.environ, PATH=d.getVar('PATH'))
fetchcmd = "npm view %s --json" % pkgfullname
output, _ = bb.process.run(fetchcmd, stderr=subprocess.STDOUT, env=runenv, shell=True)
data = self._parse_view(output)
return data
# FIXME this is effectively duplicated from lib/bb/fetch2/npm.py
# (split out from _getdependencies())
def get_npm_package_dependencies(self, pdata, fetchdev):
dependencies = pdata.get('dependencies', {})
optionalDependencies = pdata.get('optionalDependencies', {})
dependencies.update(optionalDependencies)
if fetchdev:
devDependencies = pdata.get('devDependencies', {})
dependencies.update(devDependencies)
else:
devDependencies = {}
depsfound = {}
optdepsfound = {}
devdepsfound = {}
for dep in dependencies:
if dep in optionalDependencies:
optdepsfound[dep] = dependencies[dep]
elif dep in devDependencies:
devdepsfound[dep] = dependencies[dep]
else:
depsfound[dep] = dependencies[dep]
return depsfound, optdepsfound, devdepsfound
# FIXME this is effectively duplicated from lib/bb/fetch2/npm.py
# (split out from _getdependencies())
def check_npm_optional_dependency(self, pdata):
pkg_os = pdata.get('os', None)
if pkg_os:
if not isinstance(pkg_os, list):
pkg_os = [pkg_os]
blacklist = False
for item in pkg_os:
if item.startswith('!'):
blacklist = True
break
if (not blacklist and 'linux' not in pkg_os) or '!linux' in pkg_os:
pkg = pdata.get('name', 'Unnamed package')
logger.debug(2, "Skipping %s since it's incompatible with Linux" % pkg)
return False
return True
def register_recipe_handlers(handlers):
handlers.append((NpmRecipeHandler(), 60))