Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 1 | # Recipe creation tool - node.js NPM module support plugin |
| 2 | # |
| 3 | # Copyright (C) 2016 Intel Corporation |
| 4 | # |
| 5 | # This program is free software; you can redistribute it and/or modify |
| 6 | # it under the terms of the GNU General Public License version 2 as |
| 7 | # published by the Free Software Foundation. |
| 8 | # |
| 9 | # This program is distributed in the hope that it will be useful, |
| 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 12 | # GNU General Public License for more details. |
| 13 | # |
| 14 | # You should have received a copy of the GNU General Public License along |
| 15 | # with this program; if not, write to the Free Software Foundation, Inc., |
| 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. |
| 17 | |
| 18 | import os |
| 19 | import logging |
| 20 | import subprocess |
| 21 | import tempfile |
| 22 | import shutil |
| 23 | import json |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame^] | 24 | from recipetool.create import RecipeHandler, split_pkg_licenses, handle_license_vars, check_npm |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 25 | |
| 26 | logger = logging.getLogger('recipetool') |
| 27 | |
| 28 | |
| 29 | tinfoil = None |
| 30 | |
| 31 | def tinfoil_init(instance): |
| 32 | global tinfoil |
| 33 | tinfoil = instance |
| 34 | |
| 35 | |
| 36 | class NpmRecipeHandler(RecipeHandler): |
| 37 | lockdownpath = None |
| 38 | |
| 39 | def _handle_license(self, data): |
| 40 | ''' |
| 41 | Handle the license value from an npm package.json file |
| 42 | ''' |
| 43 | license = None |
| 44 | if 'license' in data: |
| 45 | license = data['license'] |
| 46 | if isinstance(license, dict): |
| 47 | license = license.get('type', None) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame^] | 48 | return license |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 49 | |
| 50 | def _shrinkwrap(self, srctree, localfilesdir, extravalues, lines_before): |
| 51 | try: |
| 52 | runenv = dict(os.environ, PATH=tinfoil.config_data.getVar('PATH', True)) |
| 53 | bb.process.run('npm shrinkwrap', cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True) |
| 54 | except bb.process.ExecutionError as e: |
| 55 | logger.warn('npm shrinkwrap failed:\n%s' % e.stdout) |
| 56 | return |
| 57 | |
| 58 | tmpfile = os.path.join(localfilesdir, 'npm-shrinkwrap.json') |
| 59 | shutil.move(os.path.join(srctree, 'npm-shrinkwrap.json'), tmpfile) |
| 60 | extravalues.setdefault('extrafiles', {}) |
| 61 | extravalues['extrafiles']['npm-shrinkwrap.json'] = tmpfile |
| 62 | lines_before.append('NPM_SHRINKWRAP := "${THISDIR}/${PN}/npm-shrinkwrap.json"') |
| 63 | |
| 64 | def _lockdown(self, srctree, localfilesdir, extravalues, lines_before): |
| 65 | runenv = dict(os.environ, PATH=tinfoil.config_data.getVar('PATH', True)) |
| 66 | if not NpmRecipeHandler.lockdownpath: |
| 67 | NpmRecipeHandler.lockdownpath = tempfile.mkdtemp('recipetool-npm-lockdown') |
| 68 | bb.process.run('npm install lockdown --prefix %s' % NpmRecipeHandler.lockdownpath, |
| 69 | cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True) |
| 70 | relockbin = os.path.join(NpmRecipeHandler.lockdownpath, 'node_modules', 'lockdown', 'relock.js') |
| 71 | if not os.path.exists(relockbin): |
| 72 | logger.warn('Could not find relock.js within lockdown directory; skipping lockdown') |
| 73 | return |
| 74 | try: |
| 75 | bb.process.run('node %s' % relockbin, cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True) |
| 76 | except bb.process.ExecutionError as e: |
| 77 | logger.warn('lockdown-relock failed:\n%s' % e.stdout) |
| 78 | return |
| 79 | |
| 80 | tmpfile = os.path.join(localfilesdir, 'lockdown.json') |
| 81 | shutil.move(os.path.join(srctree, 'lockdown.json'), tmpfile) |
| 82 | extravalues.setdefault('extrafiles', {}) |
| 83 | extravalues['extrafiles']['lockdown.json'] = tmpfile |
| 84 | lines_before.append('NPM_LOCKDOWN := "${THISDIR}/${PN}/lockdown.json"') |
| 85 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame^] | 86 | def _handle_dependencies(self, d, deps, lines_before, srctree): |
| 87 | import scriptutils |
| 88 | # If this isn't a single module we need to get the dependencies |
| 89 | # and add them to SRC_URI |
| 90 | def varfunc(varname, origvalue, op, newlines): |
| 91 | if varname == 'SRC_URI': |
| 92 | if not origvalue.startswith('npm://'): |
| 93 | src_uri = origvalue.split() |
| 94 | changed = False |
| 95 | for dep, depdata in deps.items(): |
| 96 | version = self.get_node_version(dep, depdata, d) |
| 97 | if version: |
| 98 | url = 'npm://registry.npmjs.org;name=%s;version=%s;subdir=node_modules/%s' % (dep, version, dep) |
| 99 | scriptutils.fetch_uri(d, url, srctree) |
| 100 | src_uri.append(url) |
| 101 | changed = True |
| 102 | if changed: |
| 103 | return src_uri, None, -1, True |
| 104 | return origvalue, None, 0, True |
| 105 | updated, newlines = bb.utils.edit_metadata(lines_before, ['SRC_URI'], varfunc) |
| 106 | if updated: |
| 107 | del lines_before[:] |
| 108 | for line in newlines: |
| 109 | # Hack to avoid newlines that edit_metadata inserts |
| 110 | if line.endswith('\n'): |
| 111 | line = line[:-1] |
| 112 | lines_before.append(line) |
| 113 | return updated |
| 114 | |
| 115 | def _replace_license_vars(self, srctree, lines_before, handled, extravalues, d): |
| 116 | for item in handled: |
| 117 | if isinstance(item, tuple): |
| 118 | if item[0] == 'license': |
| 119 | del item |
| 120 | break |
| 121 | |
| 122 | calledvars = [] |
| 123 | def varfunc(varname, origvalue, op, newlines): |
| 124 | if varname in ['LICENSE', 'LIC_FILES_CHKSUM']: |
| 125 | for i, e in enumerate(reversed(newlines)): |
| 126 | if not e.startswith('#'): |
| 127 | stop = i |
| 128 | while stop > 0: |
| 129 | newlines.pop() |
| 130 | stop -= 1 |
| 131 | break |
| 132 | calledvars.append(varname) |
| 133 | if len(calledvars) > 1: |
| 134 | # The second time around, put the new license text in |
| 135 | insertpos = len(newlines) |
| 136 | handle_license_vars(srctree, newlines, handled, extravalues, d) |
| 137 | return None, None, 0, True |
| 138 | return origvalue, None, 0, True |
| 139 | updated, newlines = bb.utils.edit_metadata(lines_before, ['LICENSE', 'LIC_FILES_CHKSUM'], varfunc) |
| 140 | if updated: |
| 141 | del lines_before[:] |
| 142 | lines_before.extend(newlines) |
| 143 | else: |
| 144 | raise Exception('Did not find license variables') |
| 145 | |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 146 | def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): |
| 147 | import bb.utils |
| 148 | import oe |
| 149 | from collections import OrderedDict |
| 150 | |
| 151 | if 'buildsystem' in handled: |
| 152 | return False |
| 153 | |
| 154 | def read_package_json(fn): |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame^] | 155 | with open(fn, 'r', errors='surrogateescape') as f: |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 156 | return json.loads(f.read()) |
| 157 | |
| 158 | files = RecipeHandler.checkfiles(srctree, ['package.json']) |
| 159 | if files: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame^] | 160 | check_npm(tinfoil.config_data) |
| 161 | |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 162 | data = read_package_json(files[0]) |
| 163 | if 'name' in data and 'version' in data: |
| 164 | extravalues['PN'] = data['name'] |
| 165 | extravalues['PV'] = data['version'] |
| 166 | classes.append('npm') |
| 167 | handled.append('buildsystem') |
| 168 | if 'description' in data: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame^] | 169 | extravalues['SUMMARY'] = data['description'] |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 170 | if 'homepage' in data: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame^] | 171 | extravalues['HOMEPAGE'] = data['homepage'] |
| 172 | |
| 173 | deps = data.get('dependencies', {}) |
| 174 | updated = self._handle_dependencies(tinfoil.config_data, deps, lines_before, srctree) |
| 175 | if updated: |
| 176 | # We need to redo the license stuff |
| 177 | self._replace_license_vars(srctree, lines_before, handled, extravalues, tinfoil.config_data) |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 178 | |
| 179 | # Shrinkwrap |
| 180 | localfilesdir = tempfile.mkdtemp(prefix='recipetool-npm') |
| 181 | self._shrinkwrap(srctree, localfilesdir, extravalues, lines_before) |
| 182 | |
| 183 | # Lockdown |
| 184 | self._lockdown(srctree, localfilesdir, extravalues, lines_before) |
| 185 | |
| 186 | # Split each npm module out to is own package |
| 187 | npmpackages = oe.package.npm_split_package_dirs(srctree) |
| 188 | for item in handled: |
| 189 | if isinstance(item, tuple): |
| 190 | if item[0] == 'license': |
| 191 | licvalues = item[1] |
| 192 | break |
| 193 | if licvalues: |
| 194 | # Augment the license list with information we have in the packages |
| 195 | licenses = {} |
| 196 | license = self._handle_license(data) |
| 197 | if license: |
| 198 | licenses['${PN}'] = license |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame^] | 199 | for pkgname, pkgitem in npmpackages.items(): |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 200 | _, pdata = pkgitem |
| 201 | license = self._handle_license(pdata) |
| 202 | if license: |
| 203 | licenses[pkgname] = license |
| 204 | # Now write out the package-specific license values |
| 205 | # We need to strip out the json data dicts for this since split_pkg_licenses |
| 206 | # isn't expecting it |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame^] | 207 | packages = OrderedDict((x,y[0]) for x,y in npmpackages.items()) |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 208 | packages['${PN}'] = '' |
| 209 | pkglicenses = split_pkg_licenses(licvalues, packages, lines_after, licenses) |
| 210 | all_licenses = list(set([item for pkglicense in pkglicenses.values() for item in pkglicense])) |
| 211 | # Go back and update the LICENSE value since we have a bit more |
| 212 | # information than when that was written out (and we know all apply |
| 213 | # vs. there being a choice, so we can join them with &) |
| 214 | for i, line in enumerate(lines_before): |
| 215 | if line.startswith('LICENSE = '): |
| 216 | lines_before[i] = 'LICENSE = "%s"' % ' & '.join(all_licenses) |
| 217 | break |
| 218 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame^] | 219 | # Need to move S setting after inherit npm |
| 220 | for i, line in enumerate(lines_before): |
| 221 | if line.startswith('S ='): |
| 222 | lines_before.pop(i) |
| 223 | lines_after.insert(0, '# Must be set after inherit npm since that itself sets S') |
| 224 | lines_after.insert(1, line) |
| 225 | break |
| 226 | |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 227 | return True |
| 228 | |
| 229 | return False |
| 230 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame^] | 231 | # FIXME this is duplicated from lib/bb/fetch2/npm.py |
| 232 | def _parse_view(self, output): |
| 233 | ''' |
| 234 | Parse the output of npm view --json; the last JSON result |
| 235 | is assumed to be the one that we're interested in. |
| 236 | ''' |
| 237 | pdata = None |
| 238 | outdeps = {} |
| 239 | datalines = [] |
| 240 | bracelevel = 0 |
| 241 | for line in output.splitlines(): |
| 242 | if bracelevel: |
| 243 | datalines.append(line) |
| 244 | elif '{' in line: |
| 245 | datalines = [] |
| 246 | datalines.append(line) |
| 247 | bracelevel = bracelevel + line.count('{') - line.count('}') |
| 248 | if datalines: |
| 249 | pdata = json.loads('\n'.join(datalines)) |
| 250 | return pdata |
| 251 | |
| 252 | # FIXME this is effectively duplicated from lib/bb/fetch2/npm.py |
| 253 | # (split out from _getdependencies()) |
| 254 | def get_node_version(self, pkg, version, d): |
| 255 | import bb.fetch2 |
| 256 | pkgfullname = pkg |
| 257 | if version != '*' and not '/' in version: |
| 258 | pkgfullname += "@'%s'" % version |
| 259 | logger.debug(2, "Calling getdeps on %s" % pkg) |
| 260 | runenv = dict(os.environ, PATH=d.getVar('PATH', True)) |
| 261 | fetchcmd = "npm view %s --json" % pkgfullname |
| 262 | output, _ = bb.process.run(fetchcmd, stderr=subprocess.STDOUT, env=runenv, shell=True) |
| 263 | data = self._parse_view(output) |
| 264 | return data.get('version', None) |
| 265 | |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 266 | def register_recipe_handlers(handlers): |
| 267 | handlers.append((NpmRecipeHandler(), 60)) |