Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 1 | # Recipe creation tool - node.js NPM module support plugin |
| 2 | # |
| 3 | # Copyright (C) 2016 Intel Corporation |
| 4 | # |
| 5 | # This program is free software; you can redistribute it and/or modify |
| 6 | # it under the terms of the GNU General Public License version 2 as |
| 7 | # published by the Free Software Foundation. |
| 8 | # |
| 9 | # This program is distributed in the hope that it will be useful, |
| 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 12 | # GNU General Public License for more details. |
| 13 | # |
| 14 | # You should have received a copy of the GNU General Public License along |
| 15 | # with this program; if not, write to the Free Software Foundation, Inc., |
| 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. |
| 17 | |
| 18 | import os |
| 19 | import logging |
| 20 | import subprocess |
| 21 | import tempfile |
| 22 | import shutil |
| 23 | import json |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 24 | from recipetool.create import RecipeHandler, split_pkg_licenses, handle_license_vars, check_npm |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 25 | |
| 26 | logger = logging.getLogger('recipetool') |
| 27 | |
| 28 | |
| 29 | tinfoil = None |
| 30 | |
| 31 | def tinfoil_init(instance): |
| 32 | global tinfoil |
| 33 | tinfoil = instance |
| 34 | |
| 35 | |
| 36 | class NpmRecipeHandler(RecipeHandler): |
| 37 | lockdownpath = None |
| 38 | |
| 39 | def _handle_license(self, data): |
| 40 | ''' |
| 41 | Handle the license value from an npm package.json file |
| 42 | ''' |
| 43 | license = None |
| 44 | if 'license' in data: |
| 45 | license = data['license'] |
| 46 | if isinstance(license, dict): |
| 47 | license = license.get('type', None) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 48 | if license: |
| 49 | if 'OR' in license: |
| 50 | license = license.replace('OR', '|') |
| 51 | license = license.replace('AND', '&') |
| 52 | license = license.replace(' ', '_') |
| 53 | if not license[0] == '(': |
| 54 | license = '(' + license + ')' |
| 55 | print('LICENSE: {}'.format(license)) |
| 56 | else: |
| 57 | license = license.replace('AND', '&') |
| 58 | if license[0] == '(': |
| 59 | license = license[1:] |
| 60 | if license[-1] == ')': |
| 61 | license = license[:-1] |
| 62 | license = license.replace('MIT/X11', 'MIT') |
| 63 | license = license.replace('Public Domain', 'PD') |
| 64 | license = license.replace('SEE LICENSE IN EULA', |
| 65 | 'SEE-LICENSE-IN-EULA') |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 66 | return license |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 67 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 68 | def _shrinkwrap(self, srctree, localfilesdir, extravalues, lines_before, d): |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 69 | try: |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 70 | runenv = dict(os.environ, PATH=d.getVar('PATH')) |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 71 | bb.process.run('npm shrinkwrap', cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True) |
| 72 | except bb.process.ExecutionError as e: |
| 73 | logger.warn('npm shrinkwrap failed:\n%s' % e.stdout) |
| 74 | return |
| 75 | |
| 76 | tmpfile = os.path.join(localfilesdir, 'npm-shrinkwrap.json') |
| 77 | shutil.move(os.path.join(srctree, 'npm-shrinkwrap.json'), tmpfile) |
| 78 | extravalues.setdefault('extrafiles', {}) |
| 79 | extravalues['extrafiles']['npm-shrinkwrap.json'] = tmpfile |
| 80 | lines_before.append('NPM_SHRINKWRAP := "${THISDIR}/${PN}/npm-shrinkwrap.json"') |
| 81 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 82 | def _lockdown(self, srctree, localfilesdir, extravalues, lines_before, d): |
| 83 | runenv = dict(os.environ, PATH=d.getVar('PATH')) |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 84 | if not NpmRecipeHandler.lockdownpath: |
| 85 | NpmRecipeHandler.lockdownpath = tempfile.mkdtemp('recipetool-npm-lockdown') |
| 86 | bb.process.run('npm install lockdown --prefix %s' % NpmRecipeHandler.lockdownpath, |
| 87 | cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True) |
| 88 | relockbin = os.path.join(NpmRecipeHandler.lockdownpath, 'node_modules', 'lockdown', 'relock.js') |
| 89 | if not os.path.exists(relockbin): |
| 90 | logger.warn('Could not find relock.js within lockdown directory; skipping lockdown') |
| 91 | return |
| 92 | try: |
| 93 | bb.process.run('node %s' % relockbin, cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True) |
| 94 | except bb.process.ExecutionError as e: |
| 95 | logger.warn('lockdown-relock failed:\n%s' % e.stdout) |
| 96 | return |
| 97 | |
| 98 | tmpfile = os.path.join(localfilesdir, 'lockdown.json') |
| 99 | shutil.move(os.path.join(srctree, 'lockdown.json'), tmpfile) |
| 100 | extravalues.setdefault('extrafiles', {}) |
| 101 | extravalues['extrafiles']['lockdown.json'] = tmpfile |
| 102 | lines_before.append('NPM_LOCKDOWN := "${THISDIR}/${PN}/lockdown.json"') |
| 103 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 104 | def _handle_dependencies(self, d, deps, optdeps, devdeps, lines_before, srctree): |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 105 | import scriptutils |
| 106 | # If this isn't a single module we need to get the dependencies |
| 107 | # and add them to SRC_URI |
| 108 | def varfunc(varname, origvalue, op, newlines): |
| 109 | if varname == 'SRC_URI': |
| 110 | if not origvalue.startswith('npm://'): |
| 111 | src_uri = origvalue.split() |
| 112 | changed = False |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 113 | deplist = {} |
| 114 | for dep, depver in optdeps.items(): |
| 115 | depdata = self.get_npm_data(dep, depver, d) |
| 116 | if self.check_npm_optional_dependency(depdata): |
| 117 | deplist[dep] = depdata |
| 118 | for dep, depver in devdeps.items(): |
| 119 | depdata = self.get_npm_data(dep, depver, d) |
| 120 | if self.check_npm_optional_dependency(depdata): |
| 121 | deplist[dep] = depdata |
| 122 | for dep, depver in deps.items(): |
| 123 | depdata = self.get_npm_data(dep, depver, d) |
| 124 | deplist[dep] = depdata |
| 125 | |
| 126 | for dep, depdata in deplist.items(): |
| 127 | version = depdata.get('version', None) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 128 | if version: |
| 129 | url = 'npm://registry.npmjs.org;name=%s;version=%s;subdir=node_modules/%s' % (dep, version, dep) |
| 130 | scriptutils.fetch_uri(d, url, srctree) |
| 131 | src_uri.append(url) |
| 132 | changed = True |
| 133 | if changed: |
| 134 | return src_uri, None, -1, True |
| 135 | return origvalue, None, 0, True |
| 136 | updated, newlines = bb.utils.edit_metadata(lines_before, ['SRC_URI'], varfunc) |
| 137 | if updated: |
| 138 | del lines_before[:] |
| 139 | for line in newlines: |
| 140 | # Hack to avoid newlines that edit_metadata inserts |
| 141 | if line.endswith('\n'): |
| 142 | line = line[:-1] |
| 143 | lines_before.append(line) |
| 144 | return updated |
| 145 | |
| 146 | def _replace_license_vars(self, srctree, lines_before, handled, extravalues, d): |
| 147 | for item in handled: |
| 148 | if isinstance(item, tuple): |
| 149 | if item[0] == 'license': |
| 150 | del item |
| 151 | break |
| 152 | |
| 153 | calledvars = [] |
| 154 | def varfunc(varname, origvalue, op, newlines): |
| 155 | if varname in ['LICENSE', 'LIC_FILES_CHKSUM']: |
| 156 | for i, e in enumerate(reversed(newlines)): |
| 157 | if not e.startswith('#'): |
| 158 | stop = i |
| 159 | while stop > 0: |
| 160 | newlines.pop() |
| 161 | stop -= 1 |
| 162 | break |
| 163 | calledvars.append(varname) |
| 164 | if len(calledvars) > 1: |
| 165 | # The second time around, put the new license text in |
| 166 | insertpos = len(newlines) |
| 167 | handle_license_vars(srctree, newlines, handled, extravalues, d) |
| 168 | return None, None, 0, True |
| 169 | return origvalue, None, 0, True |
| 170 | updated, newlines = bb.utils.edit_metadata(lines_before, ['LICENSE', 'LIC_FILES_CHKSUM'], varfunc) |
| 171 | if updated: |
| 172 | del lines_before[:] |
| 173 | lines_before.extend(newlines) |
| 174 | else: |
| 175 | raise Exception('Did not find license variables') |
| 176 | |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 177 | def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): |
| 178 | import bb.utils |
| 179 | import oe |
| 180 | from collections import OrderedDict |
| 181 | |
| 182 | if 'buildsystem' in handled: |
| 183 | return False |
| 184 | |
| 185 | def read_package_json(fn): |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 186 | with open(fn, 'r', errors='surrogateescape') as f: |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 187 | return json.loads(f.read()) |
| 188 | |
| 189 | files = RecipeHandler.checkfiles(srctree, ['package.json']) |
| 190 | if files: |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 191 | d = bb.data.createCopy(tinfoil.config_data) |
| 192 | npm_bindir = check_npm(tinfoil, self._devtool) |
| 193 | d.prependVar('PATH', '%s:' % npm_bindir) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 194 | |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 195 | data = read_package_json(files[0]) |
| 196 | if 'name' in data and 'version' in data: |
| 197 | extravalues['PN'] = data['name'] |
| 198 | extravalues['PV'] = data['version'] |
| 199 | classes.append('npm') |
| 200 | handled.append('buildsystem') |
| 201 | if 'description' in data: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 202 | extravalues['SUMMARY'] = data['description'] |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 203 | if 'homepage' in data: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 204 | extravalues['HOMEPAGE'] = data['homepage'] |
| 205 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 206 | fetchdev = extravalues['fetchdev'] or None |
| 207 | deps, optdeps, devdeps = self.get_npm_package_dependencies(data, fetchdev) |
| 208 | updated = self._handle_dependencies(d, deps, optdeps, devdeps, lines_before, srctree) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 209 | if updated: |
| 210 | # We need to redo the license stuff |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 211 | self._replace_license_vars(srctree, lines_before, handled, extravalues, d) |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 212 | |
| 213 | # Shrinkwrap |
| 214 | localfilesdir = tempfile.mkdtemp(prefix='recipetool-npm') |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 215 | self._shrinkwrap(srctree, localfilesdir, extravalues, lines_before, d) |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 216 | |
| 217 | # Lockdown |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 218 | self._lockdown(srctree, localfilesdir, extravalues, lines_before, d) |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 219 | |
| 220 | # Split each npm module out to is own package |
| 221 | npmpackages = oe.package.npm_split_package_dirs(srctree) |
| 222 | for item in handled: |
| 223 | if isinstance(item, tuple): |
| 224 | if item[0] == 'license': |
| 225 | licvalues = item[1] |
| 226 | break |
| 227 | if licvalues: |
| 228 | # Augment the license list with information we have in the packages |
| 229 | licenses = {} |
| 230 | license = self._handle_license(data) |
| 231 | if license: |
| 232 | licenses['${PN}'] = license |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 233 | for pkgname, pkgitem in npmpackages.items(): |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 234 | _, pdata = pkgitem |
| 235 | license = self._handle_license(pdata) |
| 236 | if license: |
| 237 | licenses[pkgname] = license |
| 238 | # Now write out the package-specific license values |
| 239 | # We need to strip out the json data dicts for this since split_pkg_licenses |
| 240 | # isn't expecting it |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 241 | packages = OrderedDict((x,y[0]) for x,y in npmpackages.items()) |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 242 | packages['${PN}'] = '' |
| 243 | pkglicenses = split_pkg_licenses(licvalues, packages, lines_after, licenses) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 244 | all_licenses = list(set([item.replace('_', ' ') for pkglicense in pkglicenses.values() for item in pkglicense])) |
| 245 | if '&' in all_licenses: |
| 246 | all_licenses.remove('&') |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 247 | # Go back and update the LICENSE value since we have a bit more |
| 248 | # information than when that was written out (and we know all apply |
| 249 | # vs. there being a choice, so we can join them with &) |
| 250 | for i, line in enumerate(lines_before): |
| 251 | if line.startswith('LICENSE = '): |
| 252 | lines_before[i] = 'LICENSE = "%s"' % ' & '.join(all_licenses) |
| 253 | break |
| 254 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 255 | # Need to move S setting after inherit npm |
| 256 | for i, line in enumerate(lines_before): |
| 257 | if line.startswith('S ='): |
| 258 | lines_before.pop(i) |
| 259 | lines_after.insert(0, '# Must be set after inherit npm since that itself sets S') |
| 260 | lines_after.insert(1, line) |
| 261 | break |
| 262 | |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 263 | return True |
| 264 | |
| 265 | return False |
| 266 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 267 | # FIXME this is duplicated from lib/bb/fetch2/npm.py |
| 268 | def _parse_view(self, output): |
| 269 | ''' |
| 270 | Parse the output of npm view --json; the last JSON result |
| 271 | is assumed to be the one that we're interested in. |
| 272 | ''' |
| 273 | pdata = None |
| 274 | outdeps = {} |
| 275 | datalines = [] |
| 276 | bracelevel = 0 |
| 277 | for line in output.splitlines(): |
| 278 | if bracelevel: |
| 279 | datalines.append(line) |
| 280 | elif '{' in line: |
| 281 | datalines = [] |
| 282 | datalines.append(line) |
| 283 | bracelevel = bracelevel + line.count('{') - line.count('}') |
| 284 | if datalines: |
| 285 | pdata = json.loads('\n'.join(datalines)) |
| 286 | return pdata |
| 287 | |
| 288 | # FIXME this is effectively duplicated from lib/bb/fetch2/npm.py |
| 289 | # (split out from _getdependencies()) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 290 | def get_npm_data(self, pkg, version, d): |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 291 | import bb.fetch2 |
| 292 | pkgfullname = pkg |
| 293 | if version != '*' and not '/' in version: |
| 294 | pkgfullname += "@'%s'" % version |
| 295 | logger.debug(2, "Calling getdeps on %s" % pkg) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 296 | runenv = dict(os.environ, PATH=d.getVar('PATH')) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 297 | fetchcmd = "npm view %s --json" % pkgfullname |
| 298 | output, _ = bb.process.run(fetchcmd, stderr=subprocess.STDOUT, env=runenv, shell=True) |
| 299 | data = self._parse_view(output) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 300 | return data |
| 301 | |
| 302 | # FIXME this is effectively duplicated from lib/bb/fetch2/npm.py |
| 303 | # (split out from _getdependencies()) |
| 304 | def get_npm_package_dependencies(self, pdata, fetchdev): |
| 305 | dependencies = pdata.get('dependencies', {}) |
| 306 | optionalDependencies = pdata.get('optionalDependencies', {}) |
| 307 | dependencies.update(optionalDependencies) |
| 308 | if fetchdev: |
| 309 | devDependencies = pdata.get('devDependencies', {}) |
| 310 | dependencies.update(devDependencies) |
| 311 | else: |
| 312 | devDependencies = {} |
| 313 | depsfound = {} |
| 314 | optdepsfound = {} |
| 315 | devdepsfound = {} |
| 316 | for dep in dependencies: |
| 317 | if dep in optionalDependencies: |
| 318 | optdepsfound[dep] = dependencies[dep] |
| 319 | elif dep in devDependencies: |
| 320 | devdepsfound[dep] = dependencies[dep] |
| 321 | else: |
| 322 | depsfound[dep] = dependencies[dep] |
| 323 | return depsfound, optdepsfound, devdepsfound |
| 324 | |
| 325 | # FIXME this is effectively duplicated from lib/bb/fetch2/npm.py |
| 326 | # (split out from _getdependencies()) |
| 327 | def check_npm_optional_dependency(self, pdata): |
| 328 | pkg_os = pdata.get('os', None) |
| 329 | if pkg_os: |
| 330 | if not isinstance(pkg_os, list): |
| 331 | pkg_os = [pkg_os] |
| 332 | blacklist = False |
| 333 | for item in pkg_os: |
| 334 | if item.startswith('!'): |
| 335 | blacklist = True |
| 336 | break |
| 337 | if (not blacklist and 'linux' not in pkg_os) or '!linux' in pkg_os: |
| 338 | logger.debug(2, "Skipping %s since it's incompatible with Linux" % pkg) |
| 339 | return False |
| 340 | return True |
| 341 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 342 | |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 343 | def register_recipe_handlers(handlers): |
| 344 | handlers.append((NpmRecipeHandler(), 60)) |