poky/scripts/lib/recipetool/create_npm.py - openbmc/openbmc - Gitiles

 # Copyright (C) 2016 Intel Corporation
 # Copyright (C) 2020 Savoir-Faire Linux
 #
 # SPDX-License-Identifier: GPL-2.0-only
 #
 """Recipe creation tool - npm module support plugin"""

 import json
 import logging
 import os
 import re
 import sys
 import tempfile
 import bb
 from bb.fetch2.npm import NpmEnvironment
 from bb.fetch2.npmsw import foreach_dependencies
 from recipetool.create import RecipeHandler
 from recipetool.create import get_license_md5sums
 from recipetool.create import guess_license
 from recipetool.create import split_pkg_licenses
 logger = logging.getLogger('recipetool')

 TINFOIL = None

 def tinfoil_init(instance):
     """Initialize tinfoil"""
     global TINFOIL
     TINFOIL = instance

 class NpmRecipeHandler(RecipeHandler):
     """Class to handle the npm recipe creation"""

     @staticmethod
     def _npm_name(name):
         """Generate a Yocto friendly npm name"""
         name = re.sub("/", "-", name)
         name = name.lower()
         name = re.sub(r"[^\-a-z0-9]", "", name)
         name = name.strip("-")
         return name

     @staticmethod
     def _get_registry(lines):
         """Get the registry value from the 'npm://registry' url"""
         registry = None

         def _handle_registry(varname, origvalue, op, newlines):
             nonlocal registry
             if origvalue.startswith("npm://"):
                 registry = re.sub(r"^npm://", "http://", origvalue.split(";")[0])
             return origvalue, None, 0, True

         bb.utils.edit_metadata(lines, ["SRC_URI"], _handle_registry)

         return registry

     @staticmethod
     def _ensure_npm():
         """Check if the 'npm' command is available in the recipes"""
         if not TINFOIL.recipes_parsed:
             TINFOIL.parse_recipes()

         try:
             d = TINFOIL.parse_recipe("nodejs-native")
         except bb.providers.NoProvider:
             bb.error("Nothing provides 'nodejs-native' which is required for the build")
             bb.note("You will likely need to add a layer that provides nodejs")
             sys.exit(14)

         bindir = d.getVar("STAGING_BINDIR_NATIVE")
         npmpath = os.path.join(bindir, "npm")

         if not os.path.exists(npmpath):
             TINFOIL.build_targets("nodejs-native", "addto_recipe_sysroot")

             if not os.path.exists(npmpath):
                 bb.error("Failed to add 'npm' to sysroot")
                 sys.exit(14)

         return bindir

     @staticmethod
     def _npm_global_configs(dev):
         """Get the npm global configuration"""
         configs = []

         if dev:
             configs.append(("also", "development"))
         else:
             configs.append(("only", "production"))

         configs.append(("save", "false"))
         configs.append(("package-lock", "false"))
         configs.append(("shrinkwrap", "false"))
         return configs

     def _run_npm_install(self, d, srctree, registry, dev):
         """Run the 'npm install' command without building the addons"""
         configs = self._npm_global_configs(dev)
         configs.append(("ignore-scripts", "true"))

         if registry:
             configs.append(("registry", registry))

         bb.utils.remove(os.path.join(srctree, "node_modules"), recurse=True)

         env = NpmEnvironment(d, configs=configs)
         env.run("npm install", workdir=srctree)

     def _generate_shrinkwrap(self, d, srctree, dev):
         """Check and generate the 'npm-shrinkwrap.json' file if needed"""
         configs = self._npm_global_configs(dev)

         env = NpmEnvironment(d, configs=configs)
         env.run("npm shrinkwrap", workdir=srctree)

         return os.path.join(srctree, "npm-shrinkwrap.json")

     def _handle_licenses(self, srctree, shrinkwrap_file, dev):
         """Return the extra license files and the list of packages"""
         licfiles = []
         packages = {}

         # Handle the parent package
         packages["${PN}"] = ""

         def _licfiles_append_fallback_readme_files(destdir):
             """Append README files as fallback to license files if a license files is missing"""

             fallback = True
             readmes = []
             basedir = os.path.join(srctree, destdir)
             for fn in os.listdir(basedir):
                 upper = fn.upper()
                 if upper.startswith("README"):
                     fullpath = os.path.join(basedir, fn)
                     readmes.append(fullpath)
                 if upper.startswith("COPYING") or "LICENCE" in upper or "LICENSE" in upper:
                     fallback = False
             if fallback:
                 for readme in readmes:
                     licfiles.append(os.path.relpath(readme, srctree))

         # Handle the dependencies
         def _handle_dependency(name, params, deptree):
             suffix = "-".join([self._npm_name(dep) for dep in deptree])
             destdirs = [os.path.join("node_modules", dep) for dep in deptree]
             destdir = os.path.join(*destdirs)
             packages["${PN}-" + suffix] = destdir
             _licfiles_append_fallback_readme_files(destdir)

         with open(shrinkwrap_file, "r") as f:
             shrinkwrap = json.load(f)

         foreach_dependencies(shrinkwrap, _handle_dependency, dev)

         return licfiles, packages

     def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
         """Handle the npm recipe creation"""

         if "buildsystem" in handled:
             return False

         files = RecipeHandler.checkfiles(srctree, ["package.json"])

         if not files:
             return False

         with open(files[0], "r") as f:
             data = json.load(f)

         if "name" not in data or "version" not in data:
             return False

         extravalues["PN"] = self._npm_name(data["name"])
         extravalues["PV"] = data["version"]

         if "description" in data:
             extravalues["SUMMARY"] = data["description"]

         if "homepage" in data:
             extravalues["HOMEPAGE"] = data["homepage"]

         dev = bb.utils.to_boolean(str(extravalues.get("NPM_INSTALL_DEV", "0")), False)
         registry = self._get_registry(lines_before)

         bb.note("Checking if npm is available ...")
         # The native npm is used here (and not the host one) to ensure that the
         # npm version is high enough to ensure an efficient dependency tree
         # resolution and avoid issue with the shrinkwrap file format.
         # Moreover the native npm is mandatory for the build.
         bindir = self._ensure_npm()

         d = bb.data.createCopy(TINFOIL.config_data)
         d.prependVar("PATH", bindir + ":")
         d.setVar("S", srctree)

         bb.note("Generating shrinkwrap file ...")
         # To generate the shrinkwrap file the dependencies have to be installed
         # first. During the generation process some files may be updated /
         # deleted. By default devtool tracks the diffs in the srctree and raises
         # errors when finishing the recipe if some diffs are found.
         git_exclude_file = os.path.join(srctree, ".git", "info", "exclude")
         if os.path.exists(git_exclude_file):
             with open(git_exclude_file, "r+") as f:
                 lines = f.readlines()
                 for line in ["/node_modules/", "/npm-shrinkwrap.json"]:
                     if line not in lines:
                         f.write(line + "\n")

         lock_file = os.path.join(srctree, "package-lock.json")
         lock_copy = lock_file + ".copy"
         if os.path.exists(lock_file):
             bb.utils.copyfile(lock_file, lock_copy)

         self._run_npm_install(d, srctree, registry, dev)
         shrinkwrap_file = self._generate_shrinkwrap(d, srctree, dev)

         with open(shrinkwrap_file, "r") as f:
             shrinkwrap = json.load(f)

         if os.path.exists(lock_copy):
             bb.utils.movefile(lock_copy, lock_file)

         # Add the shrinkwrap file as 'extrafiles'
         shrinkwrap_copy = shrinkwrap_file + ".copy"
         bb.utils.copyfile(shrinkwrap_file, shrinkwrap_copy)
         extravalues.setdefault("extrafiles", {})
         extravalues["extrafiles"]["npm-shrinkwrap.json"] = shrinkwrap_copy

         url_local = "npmsw://%s" % shrinkwrap_file
         url_recipe= "npmsw://${THISDIR}/${BPN}/npm-shrinkwrap.json"

         if dev:
             url_local += ";dev=1"
             url_recipe += ";dev=1"

         # Add the npmsw url in the SRC_URI of the generated recipe
         def _handle_srcuri(varname, origvalue, op, newlines):
             """Update the version value and add the 'npmsw://' url"""
             value = origvalue.replace("version=" + data["version"], "version=${PV}")
             value = value.replace("version=latest", "version=${PV}")
             values = [line.strip() for line in value.strip('\n').splitlines()]
             if "dependencies" in shrinkwrap:
                 values.append(url_recipe)
             return values, None, 4, False

         (_, newlines) = bb.utils.edit_metadata(lines_before, ["SRC_URI"], _handle_srcuri)
         lines_before[:] = [line.rstrip('\n') for line in newlines]

         # In order to generate correct licence checksums in the recipe the
         # dependencies have to be fetched again using the npmsw url
         bb.note("Fetching npm dependencies ...")
         bb.utils.remove(os.path.join(srctree, "node_modules"), recurse=True)
         fetcher = bb.fetch2.Fetch([url_local], d)
         fetcher.download()
         fetcher.unpack(srctree)

         bb.note("Handling licences ...")
         (licfiles, packages) = self._handle_licenses(srctree, shrinkwrap_file, dev)

         def _guess_odd_license(licfiles):
             import bb

             md5sums = get_license_md5sums(d, linenumbers=True)

             chksums = []
             licenses = []
             for licfile in licfiles:
                 f = os.path.join(srctree, licfile)
                 md5value = bb.utils.md5_file(f)
                 (license, beginline, endline, md5) = md5sums.get(md5value,
                     (None, "", "", ""))
                 if not license:
                     license = "Unknown"
                     logger.info("Please add the following line for '%s' to a "
                         "'lib/recipetool/licenses.csv' and replace `Unknown`, "
                         "`X`, `Y` and `MD5` with the license, begin line, "
                         "end line and partial MD5 checksum:\n" \
                         "%s,Unknown,X,Y,MD5" % (licfile, md5value))
                 chksums.append("file://%s%s%s;md5=%s" % (licfile,
                     ";beginline=%s" % (beginline) if beginline else "",
                     ";endline=%s" % (endline) if endline else "",
                     md5 if md5 else md5value))
                 licenses.append((license, licfile, md5value))
             return (licenses, chksums)

         (licenses, extravalues["LIC_FILES_CHKSUM"]) = _guess_odd_license(licfiles)
         split_pkg_licenses([*licenses, *guess_license(srctree, d)], packages, lines_after)

         classes.append("npm")
         handled.append("buildsystem")

         return True

 def register_recipe_handlers(handlers):
     """Register the npm handler"""
     handlers.append((NpmRecipeHandler(), 60))
	# Copyright (C) 2016 Intel Corporation
	# Copyright (C) 2020 Savoir-Faire Linux
	#
	# SPDX-License-Identifier: GPL-2.0-only
	#
	"""Recipe creation tool - npm module support plugin"""

	import json
	import logging
	import os
	import re
	import sys
	import tempfile
	import bb
	from bb.fetch2.npm import NpmEnvironment
	from bb.fetch2.npmsw import foreach_dependencies
	from recipetool.create import RecipeHandler
	from recipetool.create import get_license_md5sums
	from recipetool.create import guess_license
	from recipetool.create import split_pkg_licenses
	logger = logging.getLogger('recipetool')

	TINFOIL = None

	def tinfoil_init(instance):
	"""Initialize tinfoil"""
	global TINFOIL
	TINFOIL = instance

	class NpmRecipeHandler(RecipeHandler):
	"""Class to handle the npm recipe creation"""

	@staticmethod
	def _npm_name(name):
	"""Generate a Yocto friendly npm name"""
	name = re.sub("/", "-", name)
	name = name.lower()
	name = re.sub(r"[^\-a-z0-9]", "", name)
	name = name.strip("-")
	return name

	@staticmethod
	def _get_registry(lines):
	"""Get the registry value from the 'npm://registry' url"""
	registry = None

	def _handle_registry(varname, origvalue, op, newlines):
	nonlocal registry
	if origvalue.startswith("npm://"):
	registry = re.sub(r"^npm://", "http://", origvalue.split(";")[0])
	return origvalue, None, 0, True

	bb.utils.edit_metadata(lines, ["SRC_URI"], _handle_registry)

	return registry

	@staticmethod
	def _ensure_npm():
	"""Check if the 'npm' command is available in the recipes"""
	if not TINFOIL.recipes_parsed:
	TINFOIL.parse_recipes()

	try:
	d = TINFOIL.parse_recipe("nodejs-native")
	except bb.providers.NoProvider:
	bb.error("Nothing provides 'nodejs-native' which is required for the build")
	bb.note("You will likely need to add a layer that provides nodejs")
	sys.exit(14)

	bindir = d.getVar("STAGING_BINDIR_NATIVE")
	npmpath = os.path.join(bindir, "npm")

	if not os.path.exists(npmpath):
	TINFOIL.build_targets("nodejs-native", "addto_recipe_sysroot")

	if not os.path.exists(npmpath):
	bb.error("Failed to add 'npm' to sysroot")
	sys.exit(14)

	return bindir

	@staticmethod
	def _npm_global_configs(dev):
	"""Get the npm global configuration"""
	configs = []

	if dev:
	configs.append(("also", "development"))
	else:
	configs.append(("only", "production"))

	configs.append(("save", "false"))
	configs.append(("package-lock", "false"))
	configs.append(("shrinkwrap", "false"))
	return configs

	def _run_npm_install(self, d, srctree, registry, dev):
	"""Run the 'npm install' command without building the addons"""
	configs = self._npm_global_configs(dev)
	configs.append(("ignore-scripts", "true"))

	if registry:
	configs.append(("registry", registry))

	bb.utils.remove(os.path.join(srctree, "node_modules"), recurse=True)

	env = NpmEnvironment(d, configs=configs)
	env.run("npm install", workdir=srctree)

	def _generate_shrinkwrap(self, d, srctree, dev):
	"""Check and generate the 'npm-shrinkwrap.json' file if needed"""
	configs = self._npm_global_configs(dev)

	env = NpmEnvironment(d, configs=configs)
	env.run("npm shrinkwrap", workdir=srctree)

	return os.path.join(srctree, "npm-shrinkwrap.json")

	def _handle_licenses(self, srctree, shrinkwrap_file, dev):
	"""Return the extra license files and the list of packages"""
	licfiles = []
	packages = {}

	# Handle the parent package
	packages["${PN}"] = ""

	def _licfiles_append_fallback_readme_files(destdir):
	"""Append README files as fallback to license files if a license files is missing"""

	fallback = True
	readmes = []
	basedir = os.path.join(srctree, destdir)
	for fn in os.listdir(basedir):
	upper = fn.upper()
	if upper.startswith("README"):
	fullpath = os.path.join(basedir, fn)
	readmes.append(fullpath)
	if upper.startswith("COPYING") or "LICENCE" in upper or "LICENSE" in upper:
	fallback = False
	if fallback:
	for readme in readmes:
	licfiles.append(os.path.relpath(readme, srctree))

	# Handle the dependencies
	def _handle_dependency(name, params, deptree):
	suffix = "-".join([self._npm_name(dep) for dep in deptree])
	destdirs = [os.path.join("node_modules", dep) for dep in deptree]
	destdir = os.path.join(*destdirs)
	packages["${PN}-" + suffix] = destdir
	_licfiles_append_fallback_readme_files(destdir)

	with open(shrinkwrap_file, "r") as f:
	shrinkwrap = json.load(f)

	foreach_dependencies(shrinkwrap, _handle_dependency, dev)

	return licfiles, packages

	def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
	"""Handle the npm recipe creation"""

	if "buildsystem" in handled:
	return False

	files = RecipeHandler.checkfiles(srctree, ["package.json"])

	if not files:
	return False

	with open(files[0], "r") as f:
	data = json.load(f)

	if "name" not in data or "version" not in data:
	return False

	extravalues["PN"] = self._npm_name(data["name"])
	extravalues["PV"] = data["version"]

	if "description" in data:
	extravalues["SUMMARY"] = data["description"]

	if "homepage" in data:
	extravalues["HOMEPAGE"] = data["homepage"]

	dev = bb.utils.to_boolean(str(extravalues.get("NPM_INSTALL_DEV", "0")), False)
	registry = self._get_registry(lines_before)

	bb.note("Checking if npm is available ...")
	# The native npm is used here (and not the host one) to ensure that the
	# npm version is high enough to ensure an efficient dependency tree
	# resolution and avoid issue with the shrinkwrap file format.
	# Moreover the native npm is mandatory for the build.
	bindir = self._ensure_npm()

	d = bb.data.createCopy(TINFOIL.config_data)
	d.prependVar("PATH", bindir + ":")
	d.setVar("S", srctree)

	bb.note("Generating shrinkwrap file ...")
	# To generate the shrinkwrap file the dependencies have to be installed
	# first. During the generation process some files may be updated /
	# deleted. By default devtool tracks the diffs in the srctree and raises
	# errors when finishing the recipe if some diffs are found.
	git_exclude_file = os.path.join(srctree, ".git", "info", "exclude")
	if os.path.exists(git_exclude_file):
	with open(git_exclude_file, "r+") as f:
	lines = f.readlines()
	for line in ["/node_modules/", "/npm-shrinkwrap.json"]:
	if line not in lines:
	f.write(line + "\n")

	lock_file = os.path.join(srctree, "package-lock.json")
	lock_copy = lock_file + ".copy"
	if os.path.exists(lock_file):
	bb.utils.copyfile(lock_file, lock_copy)

	self._run_npm_install(d, srctree, registry, dev)
	shrinkwrap_file = self._generate_shrinkwrap(d, srctree, dev)

	with open(shrinkwrap_file, "r") as f:
	shrinkwrap = json.load(f)

	if os.path.exists(lock_copy):
	bb.utils.movefile(lock_copy, lock_file)

	# Add the shrinkwrap file as 'extrafiles'
	shrinkwrap_copy = shrinkwrap_file + ".copy"
	bb.utils.copyfile(shrinkwrap_file, shrinkwrap_copy)
	extravalues.setdefault("extrafiles", {})
	extravalues["extrafiles"]["npm-shrinkwrap.json"] = shrinkwrap_copy

	url_local = "npmsw://%s" % shrinkwrap_file
	url_recipe= "npmsw://${THISDIR}/${BPN}/npm-shrinkwrap.json"

	if dev:
	url_local += ";dev=1"
	url_recipe += ";dev=1"

	# Add the npmsw url in the SRC_URI of the generated recipe
	def _handle_srcuri(varname, origvalue, op, newlines):
	"""Update the version value and add the 'npmsw://' url"""
	value = origvalue.replace("version=" + data["version"], "version=${PV}")
	value = value.replace("version=latest", "version=${PV}")
	values = [line.strip() for line in value.strip('\n').splitlines()]
	if "dependencies" in shrinkwrap:
	values.append(url_recipe)
	return values, None, 4, False

	(_, newlines) = bb.utils.edit_metadata(lines_before, ["SRC_URI"], _handle_srcuri)
	lines_before[:] = [line.rstrip('\n') for line in newlines]

	# In order to generate correct licence checksums in the recipe the
	# dependencies have to be fetched again using the npmsw url
	bb.note("Fetching npm dependencies ...")
	bb.utils.remove(os.path.join(srctree, "node_modules"), recurse=True)
	fetcher = bb.fetch2.Fetch([url_local], d)
	fetcher.download()
	fetcher.unpack(srctree)

	bb.note("Handling licences ...")
	(licfiles, packages) = self._handle_licenses(srctree, shrinkwrap_file, dev)

	def _guess_odd_license(licfiles):
	import bb

	md5sums = get_license_md5sums(d, linenumbers=True)

	chksums = []
	licenses = []
	for licfile in licfiles:
	f = os.path.join(srctree, licfile)
	md5value = bb.utils.md5_file(f)
	(license, beginline, endline, md5) = md5sums.get(md5value,
	(None, "", "", ""))
	if not license:
	license = "Unknown"
	logger.info("Please add the following line for '%s' to a "
	"'lib/recipetool/licenses.csv' and replace `Unknown`, "
	"`X`, `Y` and `MD5` with the license, begin line, "
	"end line and partial MD5 checksum:\n" \
	"%s,Unknown,X,Y,MD5" % (licfile, md5value))
	chksums.append("file://%s%s%s;md5=%s" % (licfile,
	";beginline=%s" % (beginline) if beginline else "",
	";endline=%s" % (endline) if endline else "",
	md5 if md5 else md5value))
	licenses.append((license, licfile, md5value))
	return (licenses, chksums)

	(licenses, extravalues["LIC_FILES_CHKSUM"]) = _guess_odd_license(licfiles)
	split_pkg_licenses([licenses, guess_license(srctree, d)], packages, lines_after)

	classes.append("npm")
	handled.append("buildsystem")

	return True

	def register_recipe_handlers(handlers):
	"""Register the npm handler"""
	handlers.append((NpmRecipeHandler(), 60))