Yocto 2.3
Move OpenBMC to Yocto 2.3(pyro).
Tested: Built and verified Witherspoon and Palmetto images
Change-Id: I50744030e771f4850afc2a93a10d3507e76d36bc
Signed-off-by: Brad Bishop <bradleyb@fuzziesquirrel.com>
Resolves: openbmc/openbmc#2461
diff --git a/import-layers/yocto-poky/scripts/lib/recipetool/append.py b/import-layers/yocto-poky/scripts/lib/recipetool/append.py
index 1e0fc1e..69c8bb7 100644
--- a/import-layers/yocto-poky/scripts/lib/recipetool/append.py
+++ b/import-layers/yocto-poky/scripts/lib/recipetool/append.py
@@ -48,7 +48,7 @@
"""Find the recipe installing the specified target path, optionally limited to a select list of packages"""
import json
- pkgdata_dir = d.getVar('PKGDATA_DIR', True)
+ pkgdata_dir = d.getVar('PKGDATA_DIR')
# The mix between /etc and ${sysconfdir} here may look odd, but it is just
# being consistent with usage elsewhere
@@ -97,25 +97,12 @@
recipes[targetpath].append('!%s' % pn)
return recipes
-def _get_recipe_file(cooker, pn):
- import oe.recipeutils
- recipefile = oe.recipeutils.pn_to_recipe(cooker, pn)
- if not recipefile:
- skipreasons = oe.recipeutils.get_unavailable_reasons(cooker, pn)
- if skipreasons:
- logger.error('\n'.join(skipreasons))
- else:
- logger.error("Unable to find any recipe file matching %s" % pn)
- return recipefile
-
def _parse_recipe(pn, tinfoil):
- import oe.recipeutils
- recipefile = _get_recipe_file(tinfoil.cooker, pn)
- if not recipefile:
- # Error already logged
+ try:
+ rd = tinfoil.parse_recipe(pn)
+ except bb.providers.NoProvider as e:
+ logger.error(str(e))
return None
- append_files = tinfoil.cooker.collection.get_file_appends(recipefile)
- rd = oe.recipeutils.parse_recipe(tinfoil.cooker, recipefile, append_files)
return rd
def determine_file_source(targetpath, rd):
@@ -123,8 +110,8 @@
import oe.recipeutils
# See if it's in do_install for the recipe
- workdir = rd.getVar('WORKDIR', True)
- src_uri = rd.getVar('SRC_URI', True)
+ workdir = rd.getVar('WORKDIR')
+ src_uri = rd.getVar('SRC_URI')
srcfile = ''
modpatches = []
elements = check_do_install(rd, targetpath)
@@ -134,7 +121,7 @@
logger.debug('source path: %s' % srcpath)
if not srcpath.startswith('/'):
# Handle non-absolute path
- srcpath = os.path.abspath(os.path.join(rd.getVarFlag('do_install', 'dirs', True).split()[-1], srcpath))
+ srcpath = os.path.abspath(os.path.join(rd.getVarFlag('do_install', 'dirs').split()[-1], srcpath))
if srcpath.startswith(workdir):
# OK, now we have the source file name, look for it in SRC_URI
workdirfile = os.path.relpath(srcpath, workdir)
@@ -203,22 +190,22 @@
def get_func_deps(func, d):
"""Find the function dependencies of a shell function"""
- deps = bb.codeparser.ShellParser(func, logger).parse_shell(d.getVar(func, True))
- deps |= set((d.getVarFlag(func, "vardeps", True) or "").split())
+ deps = bb.codeparser.ShellParser(func, logger).parse_shell(d.getVar(func))
+ deps |= set((d.getVarFlag(func, "vardeps") or "").split())
funcdeps = []
for dep in deps:
- if d.getVarFlag(dep, 'func', True):
+ if d.getVarFlag(dep, 'func'):
funcdeps.append(dep)
return funcdeps
def check_do_install(rd, targetpath):
"""Look at do_install for a command that installs/copies the specified target path"""
- instpath = os.path.abspath(os.path.join(rd.getVar('D', True), targetpath.lstrip('/')))
- do_install = rd.getVar('do_install', True)
+ instpath = os.path.abspath(os.path.join(rd.getVar('D'), targetpath.lstrip('/')))
+ do_install = rd.getVar('do_install')
# Handle where do_install calls other functions (somewhat crudely, but good enough for this purpose)
deps = get_func_deps('do_install', rd)
for dep in deps:
- do_install = do_install.replace(dep, rd.getVar(dep, True))
+ do_install = do_install.replace(dep, rd.getVar(dep))
# Look backwards through do_install as we want to catch where a later line (perhaps
# from a bbappend) is writing over the top
@@ -335,12 +322,12 @@
def appendsrc(args, files, rd, extralines=None):
import oe.recipeutils
- srcdir = rd.getVar('S', True)
- workdir = rd.getVar('WORKDIR', True)
+ srcdir = rd.getVar('S')
+ workdir = rd.getVar('WORKDIR')
import bb.fetch
simplified = {}
- src_uri = rd.getVar('SRC_URI', True).split()
+ src_uri = rd.getVar('SRC_URI').split()
for uri in src_uri:
if uri.endswith(';'):
uri = uri[:-1]
@@ -353,7 +340,7 @@
for newfile, srcfile in files.items():
src_destdir = os.path.dirname(srcfile)
if not args.use_workdir:
- if rd.getVar('S', True) == rd.getVar('STAGING_KERNEL_DIR', True):
+ if rd.getVar('S') == rd.getVar('STAGING_KERNEL_DIR'):
srcdir = os.path.join(workdir, 'git')
if not bb.data.inherits_class('kernel-yocto', rd):
logger.warn('S == STAGING_KERNEL_DIR and non-kernel-yocto, unable to determine path to srcdir, defaulting to ${WORKDIR}/git')
diff --git a/import-layers/yocto-poky/scripts/lib/recipetool/create.py b/import-layers/yocto-poky/scripts/lib/recipetool/create.py
index d427d32..4de52fc 100644
--- a/import-layers/yocto-poky/scripts/lib/recipetool/create.py
+++ b/import-layers/yocto-poky/scripts/lib/recipetool/create.py
@@ -26,12 +26,24 @@
import scriptutils
from urllib.parse import urlparse, urldefrag, urlsplit
import hashlib
-
+import bb.fetch2
logger = logging.getLogger('recipetool')
tinfoil = None
plugins = None
+def log_error_cond(message, debugonly):
+ if debugonly:
+ logger.debug(message)
+ else:
+ logger.error(message)
+
+def log_info_cond(message, debugonly):
+ if debugonly:
+ logger.debug(message)
+ else:
+ logger.info(message)
+
def plugin_init(pluginlist):
# Take a reference to the list so we can use it later
global plugins
@@ -47,6 +59,9 @@
recipecmakefilemap = {}
recipebinmap = {}
+ def __init__(self):
+ self._devtool = False
+
@staticmethod
def load_libmap(d):
'''Load library->recipe mapping'''
@@ -56,8 +71,8 @@
return
# First build up library->package mapping
shlib_providers = oe.package.read_shlib_providers(d)
- libdir = d.getVar('libdir', True)
- base_libdir = d.getVar('base_libdir', True)
+ libdir = d.getVar('libdir')
+ base_libdir = d.getVar('base_libdir')
libpaths = list(set([base_libdir, libdir]))
libname_re = re.compile('^lib(.+)\.so.*$')
pkglibmap = {}
@@ -73,7 +88,7 @@
logger.debug('unable to extract library name from %s' % lib)
# Now turn it into a library->recipe mapping
- pkgdata_dir = d.getVar('PKGDATA_DIR', True)
+ pkgdata_dir = d.getVar('PKGDATA_DIR')
for libname, pkg in pkglibmap.items():
try:
with open(os.path.join(pkgdata_dir, 'runtime', pkg)) as f:
@@ -97,9 +112,9 @@
'''Build up development file->recipe mapping'''
if RecipeHandler.recipeheadermap:
return
- pkgdata_dir = d.getVar('PKGDATA_DIR', True)
- includedir = d.getVar('includedir', True)
- cmakedir = os.path.join(d.getVar('libdir', True), 'cmake')
+ pkgdata_dir = d.getVar('PKGDATA_DIR')
+ includedir = d.getVar('includedir')
+ cmakedir = os.path.join(d.getVar('libdir'), 'cmake')
for pkg in glob.glob(os.path.join(pkgdata_dir, 'runtime', '*-dev')):
with open(os.path.join(pkgdata_dir, 'runtime', pkg)) as f:
pn = None
@@ -128,9 +143,9 @@
'''Build up native binary->recipe mapping'''
if RecipeHandler.recipebinmap:
return
- sstate_manifests = d.getVar('SSTATE_MANIFESTS', True)
- staging_bindir_native = d.getVar('STAGING_BINDIR_NATIVE', True)
- build_arch = d.getVar('BUILD_ARCH', True)
+ sstate_manifests = d.getVar('SSTATE_MANIFESTS')
+ staging_bindir_native = d.getVar('STAGING_BINDIR_NATIVE')
+ build_arch = d.getVar('BUILD_ARCH')
fileprefix = 'manifest-%s-' % build_arch
for fn in glob.glob(os.path.join(sstate_manifests, '%s*-native.populate_sysroot' % fileprefix)):
with open(fn, 'r') as f:
@@ -222,7 +237,8 @@
if deps:
values['DEPENDS'] = ' '.join(deps)
- def genfunction(self, outlines, funcname, content, python=False, forcespace=False):
+ @staticmethod
+ def genfunction(outlines, funcname, content, python=False, forcespace=False):
if python:
prefix = 'python '
else:
@@ -323,7 +339,7 @@
pn = res.group(1).strip().replace('_', '-')
pv = res.group(2).strip().replace('_', '.')
- if not pn and not pv:
+ if not pn and not pv and parseres.scheme not in ['git', 'gitsm', 'svn', 'hg']:
srcfile = os.path.basename(parseres.path.rstrip('/'))
pn, pv = determine_from_filename(srcfile)
@@ -335,7 +351,6 @@
# This is a bit sad, but if you don't have this set there can be some
# odd interactions with the urldata cache which lead to errors
localdata.setVar('SRCREV', '${AUTOREV}')
- bb.data.update_data(localdata)
try:
fetcher = bb.fetch2.Fetch([uri], localdata)
urldata = fetcher.ud
@@ -353,14 +368,31 @@
'''Convert any http[s]://....git URI into git://...;protocol=http[s]'''
checkuri = uri.split(';', 1)[0]
if checkuri.endswith('.git') or '/git/' in checkuri or re.match('https?://github.com/[^/]+/[^/]+/?$', checkuri):
- res = re.match('(http|https|ssh)://([^;]+(\.git)?)(;.*)?$', uri)
- if res:
- # Need to switch the URI around so that the git fetcher is used
- return 'git://%s;protocol=%s%s' % (res.group(2), res.group(1), res.group(4) or '')
- elif '@' in checkuri:
- # Catch e.g. git@git.example.com:repo.git
- return 'git://%s;protocol=ssh' % checkuri.replace(':', '/', 1)
- return uri
+ # Appends scheme if the scheme is missing
+ if not '://' in uri:
+ uri = 'git://' + uri
+ scheme, host, path, user, pswd, parms = bb.fetch2.decodeurl(uri)
+ # Detection mechanism, this is required due to certain URL are formatter with ":" rather than "/"
+ # which causes decodeurl to fail getting the right host and path
+ if len(host.split(':')) > 1:
+ splitslash = host.split(':')
+ host = splitslash[0]
+ path = '/' + splitslash[1] + path
+ #Algorithm:
+ # if user is defined, append protocol=ssh or if a protocol is defined, then honor the user-defined protocol
+ # if no user & password is defined, check for scheme type and append the protocol with the scheme type
+ # finally if protocols or if the url is well-formed, do nothing and rejoin everything back to normal
+ # Need to repackage the arguments for encodeurl, the format is: (scheme, host, path, user, password, OrderedDict([('key', 'value')]))
+ if user:
+ if not 'protocol' in parms:
+ parms.update({('protocol', 'ssh')})
+ elif (scheme == "http" or scheme == 'https' or scheme == 'ssh') and not ('protocol' in parms):
+ parms.update({('protocol', scheme)})
+ # Always append 'git://'
+ fUrl = bb.fetch2.encodeurl(('git', host, path, user, pswd, parms))
+ return fUrl
+ else:
+ return uri
def is_package(url):
'''Check if a URL points to a package'''
@@ -404,12 +436,14 @@
srcuri = rev_re.sub('', srcuri)
tempsrc = tempfile.mkdtemp(prefix='recipetool-')
srctree = tempsrc
+ d = bb.data.createCopy(tinfoil.config_data)
if fetchuri.startswith('npm://'):
# Check if npm is available
- check_npm(tinfoil.config_data)
+ npm_bindir = check_npm(tinfoil, args.devtool)
+ d.prependVar('PATH', '%s:' % npm_bindir)
logger.info('Fetching %s...' % srcuri)
try:
- checksums = scriptutils.fetch_uri(tinfoil.config_data, fetchuri, srctree, srcrev)
+ checksums = scriptutils.fetch_uri(d, fetchuri, srctree, srcrev)
except bb.fetch2.BBFetchException as e:
logger.error(str(e).rstrip())
sys.exit(1)
@@ -448,8 +482,8 @@
if pkgfile:
if pkgfile.endswith(('.deb', '.ipk')):
- stdout, _ = bb.process.run('ar x %s control.tar.gz' % pkgfile, cwd=tmpfdir)
- stdout, _ = bb.process.run('tar xf control.tar.gz ./control', cwd=tmpfdir)
+ stdout, _ = bb.process.run('ar x %s' % pkgfile, cwd=tmpfdir)
+ stdout, _ = bb.process.run('tar xf control.tar.gz', cwd=tmpfdir)
values = convert_debian(tmpfdir)
extravalues.update(values)
elif pkgfile.endswith(('.rpm', '.srpm')):
@@ -554,7 +588,6 @@
if name_pv and not realpv:
realpv = name_pv
-
if not srcuri:
lines_before.append('# No information for SRC_URI yet (only an external source tree was specified)')
lines_before.append('SRC_URI = "%s"' % srcuri)
@@ -588,6 +621,11 @@
lines_after.append('INSANE_SKIP_${PN} += "already-stripped"')
lines_after.append('')
+ if args.fetch_dev:
+ extravalues['fetchdev'] = True
+ else:
+ extravalues['fetchdev'] = None
+
# Find all plugins that want to register handlers
logger.debug('Loading recipe handlers')
raw_handlers = []
@@ -604,6 +642,7 @@
handlers.sort(key=lambda item: (item[1], -item[2]), reverse=True)
for handler, priority, _ in handlers:
logger.debug('Handler: %s (priority %d)' % (handler.__class__.__name__, priority))
+ setattr(handler, '_devtool', args.devtool)
handlers = [item[0] for item in handlers]
# Apply the handlers
@@ -640,7 +679,7 @@
if not outfile:
if not pn:
- logger.error('Unable to determine short program name from source tree - please specify name with -N/--name or output file name with -o/--outfile')
+ log_error_cond('Unable to determine short program name from source tree - please specify name with -N/--name or output file name with -o/--outfile', args.devtool)
# devtool looks for this specific exit code, so don't change it
sys.exit(15)
else:
@@ -710,6 +749,15 @@
if not bbclassextend:
lines_after.append('BBCLASSEXTEND = "native"')
+ postinst = ("postinst", extravalues.pop('postinst', None))
+ postrm = ("postrm", extravalues.pop('postrm', None))
+ preinst = ("preinst", extravalues.pop('preinst', None))
+ prerm = ("prerm", extravalues.pop('prerm', None))
+ funcs = [postinst, postrm, preinst, prerm]
+ for func in funcs:
+ if func[1]:
+ RecipeHandler.genfunction(lines_after, 'pkg_%s_${PN}' % func[0], func[1])
+
outlines = []
outlines.extend(lines_before)
if classes:
@@ -736,7 +784,7 @@
shutil.move(srctree, args.extract_to)
if tempsrc == srctree:
tempsrc = None
- logger.info('Source extracted to %s' % args.extract_to)
+ log_info_cond('Source extracted to %s' % args.extract_to, args.devtool)
if outfile == '-':
sys.stdout.write('\n'.join(outlines) + '\n')
@@ -749,7 +797,7 @@
continue
f.write('%s\n' % line)
lastline = line
- logger.info('Recipe %s has been created; further editing may be required to make it fully functional' % outfile)
+ log_info_cond('Recipe %s has been created; further editing may be required to make it fully functional' % outfile, args.devtool)
if tempsrc:
if args.keep_temp:
@@ -775,10 +823,12 @@
lines_before.append('# your responsibility to verify that the values are complete and correct.')
if len(licvalues) > 1:
lines_before.append('#')
- lines_before.append('# NOTE: multiple licenses have been detected; if that is correct you should separate')
- lines_before.append('# these in the LICENSE value using & if the multiple licenses all apply, or | if there')
- lines_before.append('# is a choice between the multiple licenses. If in doubt, check the accompanying')
- lines_before.append('# documentation to determine which situation is applicable.')
+ lines_before.append('# NOTE: multiple licenses have been detected; they have been separated with &')
+ lines_before.append('# in the LICENSE value for now since it is a reasonable assumption that all')
+ lines_before.append('# of the licenses apply. If instead there is a choice between the multiple')
+ lines_before.append('# licenses then you should change the value to separate the licenses with |')
+ lines_before.append('# instead of &. If there is any doubt, check the accompanying documentation')
+ lines_before.append('# to determine which situation is applicable.')
if lic_unknown:
lines_before.append('#')
lines_before.append('# The following license files were not able to be identified and are')
@@ -802,7 +852,7 @@
licenses = [pkg_license]
else:
lines_before.append('# NOTE: Original package metadata indicates license is: %s' % pkg_license)
- lines_before.append('LICENSE = "%s"' % ' '.join(licenses))
+ lines_before.append('LICENSE = "%s"' % ' & '.join(licenses))
lines_before.append('LIC_FILES_CHKSUM = "%s"' % ' \\\n '.join(lic_files_chksum))
lines_before.append('')
handled.append(('license', licvalues))
@@ -813,7 +863,7 @@
md5sums = {}
if not static_only:
# Gather md5sums of license files in common license dir
- commonlicdir = d.getVar('COMMON_LICENSE_DIR', True)
+ commonlicdir = d.getVar('COMMON_LICENSE_DIR')
for fn in os.listdir(commonlicdir):
md5value = bb.utils.md5_file(os.path.join(commonlicdir, fn))
md5sums[md5value] = fn
@@ -983,7 +1033,7 @@
return outlicenses
def read_pkgconfig_provides(d):
- pkgdatadir = d.getVar('PKGDATA_DIR', True)
+ pkgdatadir = d.getVar('PKGDATA_DIR')
pkgmap = {}
for fn in glob.glob(os.path.join(pkgdatadir, 'shlibs2', '*.pclist')):
with open(fn, 'r') as f:
@@ -1044,6 +1094,25 @@
varname = value_map.get(key, None)
if varname:
values[varname] = value
+ postinst = os.path.join(debpath, 'postinst')
+ postrm = os.path.join(debpath, 'postrm')
+ preinst = os.path.join(debpath, 'preinst')
+ prerm = os.path.join(debpath, 'prerm')
+ sfiles = [postinst, postrm, preinst, prerm]
+ for sfile in sfiles:
+ if os.path.isfile(sfile):
+ logger.info("Converting %s file to recipe function..." %
+ os.path.basename(sfile).upper())
+ content = []
+ with open(sfile) as f:
+ for line in f:
+ if "#!/" in line:
+ continue
+ line = line.rstrip("\n")
+ if line.strip():
+ content.append(line)
+ if content:
+ values[os.path.basename(f.name)] = content
#if depends:
# values['DEPENDS'] = ' '.join(depends)
@@ -1073,10 +1142,21 @@
return values
-def check_npm(d):
- if not os.path.exists(os.path.join(d.getVar('STAGING_BINDIR_NATIVE', True), 'npm')):
- logger.error('npm required to process specified source, but npm is not available - you need to build nodejs-native first')
+def check_npm(tinfoil, debugonly=False):
+ try:
+ rd = tinfoil.parse_recipe('nodejs-native')
+ except bb.providers.NoProvider:
+ # We still conditionally show the message and exit with the special
+ # return code, otherwise we can't show the proper message for eSDK
+ # users
+ log_error_cond('nodejs-native is required for npm but is not available - you will likely need to add a layer that provides nodejs', debugonly)
sys.exit(14)
+ bindir = rd.getVar('STAGING_BINDIR_NATIVE')
+ npmpath = os.path.join(bindir, 'npm')
+ if not os.path.exists(npmpath):
+ log_error_cond('npm required to process specified source, but npm is not available - you need to run bitbake -c addto_recipe_sysroot nodejs-native first', debugonly)
+ sys.exit(14)
+ return bindir
def register_commands(subparsers):
parser_create = subparsers.add_parser('create',
@@ -1093,5 +1173,10 @@
parser_create.add_argument('--src-subdir', help='Specify subdirectory within source tree to use', metavar='SUBDIR')
parser_create.add_argument('-a', '--autorev', help='When fetching from a git repository, set SRCREV in the recipe to a floating revision instead of fixed', action="store_true")
parser_create.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
- parser_create.set_defaults(func=create_recipe)
+ parser_create.add_argument('--fetch-dev', action="store_true", help='For npm, also fetch devDependencies')
+ parser_create.add_argument('--devtool', action="store_true", help=argparse.SUPPRESS)
+ # FIXME I really hate having to set parserecipes for this, but given we may need
+ # to call into npm (and we don't know in advance if we will or not) and in order
+ # to do so we need to know npm's recipe sysroot path, there's not much alternative
+ parser_create.set_defaults(func=create_recipe, parserecipes=True)
diff --git a/import-layers/yocto-poky/scripts/lib/recipetool/create_buildsys_python.py b/import-layers/yocto-poky/scripts/lib/recipetool/create_buildsys_python.py
index 82a2be1..ec5449b 100644
--- a/import-layers/yocto-poky/scripts/lib/recipetool/create_buildsys_python.py
+++ b/import-layers/yocto-poky/scripts/lib/recipetool/create_buildsys_python.py
@@ -532,11 +532,11 @@
def parse_pkgdata_for_python_packages(self):
suffixes = [t[0] for t in imp.get_suffixes()]
- pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR', True)
+ pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR')
ldata = tinfoil.config_data.createCopy()
bb.parse.handle('classes/python-dir.bbclass', ldata, True)
- python_sitedir = ldata.getVar('PYTHON_SITEPACKAGES_DIR', True)
+ python_sitedir = ldata.getVar('PYTHON_SITEPACKAGES_DIR')
dynload_dir = os.path.join(os.path.dirname(python_sitedir), 'lib-dynload')
python_dirs = [python_sitedir + os.sep,
diff --git a/import-layers/yocto-poky/scripts/lib/recipetool/create_kernel.py b/import-layers/yocto-poky/scripts/lib/recipetool/create_kernel.py
index 7dac59f..ca4996c 100644
--- a/import-layers/yocto-poky/scripts/lib/recipetool/create_kernel.py
+++ b/import-layers/yocto-poky/scripts/lib/recipetool/create_kernel.py
@@ -41,7 +41,7 @@
handled.append('buildsystem')
del lines_after[:]
del classes[:]
- template = os.path.join(tinfoil.config_data.getVar('COREBASE', True), 'meta-skeleton', 'recipes-kernel', 'linux', 'linux-yocto-custom.bb')
+ template = os.path.join(tinfoil.config_data.getVar('COREBASE'), 'meta-skeleton', 'recipes-kernel', 'linux', 'linux-yocto-custom.bb')
def handle_var(varname, origvalue, op, newlines):
if varname in ['SRCREV', 'SRCREV_machine']:
while newlines[-1].startswith('#'):
@@ -85,7 +85,7 @@
elif varname == 'COMPATIBLE_MACHINE':
while newlines[-1].startswith('#'):
del newlines[-1]
- machine = tinfoil.config_data.getVar('MACHINE', True)
+ machine = tinfoil.config_data.getVar('MACHINE')
return machine, op, 0, True
return origvalue, op, 0, True
with open(template, 'r') as f:
diff --git a/import-layers/yocto-poky/scripts/lib/recipetool/create_npm.py b/import-layers/yocto-poky/scripts/lib/recipetool/create_npm.py
index 7bb844c..cb8f338 100644
--- a/import-layers/yocto-poky/scripts/lib/recipetool/create_npm.py
+++ b/import-layers/yocto-poky/scripts/lib/recipetool/create_npm.py
@@ -45,11 +45,29 @@
license = data['license']
if isinstance(license, dict):
license = license.get('type', None)
+ if license:
+ if 'OR' in license:
+ license = license.replace('OR', '|')
+ license = license.replace('AND', '&')
+ license = license.replace(' ', '_')
+ if not license[0] == '(':
+ license = '(' + license + ')'
+ print('LICENSE: {}'.format(license))
+ else:
+ license = license.replace('AND', '&')
+ if license[0] == '(':
+ license = license[1:]
+ if license[-1] == ')':
+ license = license[:-1]
+ license = license.replace('MIT/X11', 'MIT')
+ license = license.replace('Public Domain', 'PD')
+ license = license.replace('SEE LICENSE IN EULA',
+ 'SEE-LICENSE-IN-EULA')
return license
- def _shrinkwrap(self, srctree, localfilesdir, extravalues, lines_before):
+ def _shrinkwrap(self, srctree, localfilesdir, extravalues, lines_before, d):
try:
- runenv = dict(os.environ, PATH=tinfoil.config_data.getVar('PATH', True))
+ runenv = dict(os.environ, PATH=d.getVar('PATH'))
bb.process.run('npm shrinkwrap', cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
except bb.process.ExecutionError as e:
logger.warn('npm shrinkwrap failed:\n%s' % e.stdout)
@@ -61,8 +79,8 @@
extravalues['extrafiles']['npm-shrinkwrap.json'] = tmpfile
lines_before.append('NPM_SHRINKWRAP := "${THISDIR}/${PN}/npm-shrinkwrap.json"')
- def _lockdown(self, srctree, localfilesdir, extravalues, lines_before):
- runenv = dict(os.environ, PATH=tinfoil.config_data.getVar('PATH', True))
+ def _lockdown(self, srctree, localfilesdir, extravalues, lines_before, d):
+ runenv = dict(os.environ, PATH=d.getVar('PATH'))
if not NpmRecipeHandler.lockdownpath:
NpmRecipeHandler.lockdownpath = tempfile.mkdtemp('recipetool-npm-lockdown')
bb.process.run('npm install lockdown --prefix %s' % NpmRecipeHandler.lockdownpath,
@@ -83,7 +101,7 @@
extravalues['extrafiles']['lockdown.json'] = tmpfile
lines_before.append('NPM_LOCKDOWN := "${THISDIR}/${PN}/lockdown.json"')
- def _handle_dependencies(self, d, deps, lines_before, srctree):
+ def _handle_dependencies(self, d, deps, optdeps, devdeps, lines_before, srctree):
import scriptutils
# If this isn't a single module we need to get the dependencies
# and add them to SRC_URI
@@ -92,8 +110,21 @@
if not origvalue.startswith('npm://'):
src_uri = origvalue.split()
changed = False
- for dep, depdata in deps.items():
- version = self.get_node_version(dep, depdata, d)
+ deplist = {}
+ for dep, depver in optdeps.items():
+ depdata = self.get_npm_data(dep, depver, d)
+ if self.check_npm_optional_dependency(depdata):
+ deplist[dep] = depdata
+ for dep, depver in devdeps.items():
+ depdata = self.get_npm_data(dep, depver, d)
+ if self.check_npm_optional_dependency(depdata):
+ deplist[dep] = depdata
+ for dep, depver in deps.items():
+ depdata = self.get_npm_data(dep, depver, d)
+ deplist[dep] = depdata
+
+ for dep, depdata in deplist.items():
+ version = depdata.get('version', None)
if version:
url = 'npm://registry.npmjs.org;name=%s;version=%s;subdir=node_modules/%s' % (dep, version, dep)
scriptutils.fetch_uri(d, url, srctree)
@@ -157,7 +188,9 @@
files = RecipeHandler.checkfiles(srctree, ['package.json'])
if files:
- check_npm(tinfoil.config_data)
+ d = bb.data.createCopy(tinfoil.config_data)
+ npm_bindir = check_npm(tinfoil, self._devtool)
+ d.prependVar('PATH', '%s:' % npm_bindir)
data = read_package_json(files[0])
if 'name' in data and 'version' in data:
@@ -170,18 +203,19 @@
if 'homepage' in data:
extravalues['HOMEPAGE'] = data['homepage']
- deps = data.get('dependencies', {})
- updated = self._handle_dependencies(tinfoil.config_data, deps, lines_before, srctree)
+ fetchdev = extravalues['fetchdev'] or None
+ deps, optdeps, devdeps = self.get_npm_package_dependencies(data, fetchdev)
+ updated = self._handle_dependencies(d, deps, optdeps, devdeps, lines_before, srctree)
if updated:
# We need to redo the license stuff
- self._replace_license_vars(srctree, lines_before, handled, extravalues, tinfoil.config_data)
+ self._replace_license_vars(srctree, lines_before, handled, extravalues, d)
# Shrinkwrap
localfilesdir = tempfile.mkdtemp(prefix='recipetool-npm')
- self._shrinkwrap(srctree, localfilesdir, extravalues, lines_before)
+ self._shrinkwrap(srctree, localfilesdir, extravalues, lines_before, d)
# Lockdown
- self._lockdown(srctree, localfilesdir, extravalues, lines_before)
+ self._lockdown(srctree, localfilesdir, extravalues, lines_before, d)
# Split each npm module out to is own package
npmpackages = oe.package.npm_split_package_dirs(srctree)
@@ -207,7 +241,9 @@
packages = OrderedDict((x,y[0]) for x,y in npmpackages.items())
packages['${PN}'] = ''
pkglicenses = split_pkg_licenses(licvalues, packages, lines_after, licenses)
- all_licenses = list(set([item for pkglicense in pkglicenses.values() for item in pkglicense]))
+ all_licenses = list(set([item.replace('_', ' ') for pkglicense in pkglicenses.values() for item in pkglicense]))
+ if '&' in all_licenses:
+ all_licenses.remove('&')
# Go back and update the LICENSE value since we have a bit more
# information than when that was written out (and we know all apply
# vs. there being a choice, so we can join them with &)
@@ -251,17 +287,58 @@
# FIXME this is effectively duplicated from lib/bb/fetch2/npm.py
# (split out from _getdependencies())
- def get_node_version(self, pkg, version, d):
+ def get_npm_data(self, pkg, version, d):
import bb.fetch2
pkgfullname = pkg
if version != '*' and not '/' in version:
pkgfullname += "@'%s'" % version
logger.debug(2, "Calling getdeps on %s" % pkg)
- runenv = dict(os.environ, PATH=d.getVar('PATH', True))
+ runenv = dict(os.environ, PATH=d.getVar('PATH'))
fetchcmd = "npm view %s --json" % pkgfullname
output, _ = bb.process.run(fetchcmd, stderr=subprocess.STDOUT, env=runenv, shell=True)
data = self._parse_view(output)
- return data.get('version', None)
+ return data
+
+ # FIXME this is effectively duplicated from lib/bb/fetch2/npm.py
+ # (split out from _getdependencies())
+ def get_npm_package_dependencies(self, pdata, fetchdev):
+ dependencies = pdata.get('dependencies', {})
+ optionalDependencies = pdata.get('optionalDependencies', {})
+ dependencies.update(optionalDependencies)
+ if fetchdev:
+ devDependencies = pdata.get('devDependencies', {})
+ dependencies.update(devDependencies)
+ else:
+ devDependencies = {}
+ depsfound = {}
+ optdepsfound = {}
+ devdepsfound = {}
+ for dep in dependencies:
+ if dep in optionalDependencies:
+ optdepsfound[dep] = dependencies[dep]
+ elif dep in devDependencies:
+ devdepsfound[dep] = dependencies[dep]
+ else:
+ depsfound[dep] = dependencies[dep]
+ return depsfound, optdepsfound, devdepsfound
+
+ # FIXME this is effectively duplicated from lib/bb/fetch2/npm.py
+ # (split out from _getdependencies())
+ def check_npm_optional_dependency(self, pdata):
+ pkg_os = pdata.get('os', None)
+ if pkg_os:
+ if not isinstance(pkg_os, list):
+ pkg_os = [pkg_os]
+ blacklist = False
+ for item in pkg_os:
+ if item.startswith('!'):
+ blacklist = True
+ break
+ if (not blacklist and 'linux' not in pkg_os) or '!linux' in pkg_os:
+ logger.debug(2, "Skipping %s since it's incompatible with Linux" % pkg)
+ return False
+ return True
+
def register_recipe_handlers(handlers):
handlers.append((NpmRecipeHandler(), 60))
diff --git a/import-layers/yocto-poky/scripts/lib/recipetool/newappend.py b/import-layers/yocto-poky/scripts/lib/recipetool/newappend.py
index fbdd7bc..0b63759 100644
--- a/import-layers/yocto-poky/scripts/lib/recipetool/newappend.py
+++ b/import-layers/yocto-poky/scripts/lib/recipetool/newappend.py
@@ -39,18 +39,6 @@
tinfoil = instance
-def _get_recipe_file(cooker, pn):
- import oe.recipeutils
- recipefile = oe.recipeutils.pn_to_recipe(cooker, pn)
- if not recipefile:
- skipreasons = oe.recipeutils.get_unavailable_reasons(cooker, pn)
- if skipreasons:
- logger.error('\n'.join(skipreasons))
- else:
- logger.error("Unable to find any recipe file matching %s" % pn)
- return recipefile
-
-
def layer(layerpath):
if not os.path.exists(os.path.join(layerpath, 'conf', 'layer.conf')):
raise argparse.ArgumentTypeError('{0!r} must be a path to a valid layer'.format(layerpath))
@@ -60,7 +48,7 @@
def newappend(args):
import oe.recipeutils
- recipe_path = _get_recipe_file(tinfoil.cooker, args.target)
+ recipe_path = tinfoil.get_recipe_file(args.target)
rd = tinfoil.config_data.createCopy()
rd.setVar('FILE', recipe_path)
@@ -72,7 +60,7 @@
if not path_ok:
logger.warn('Unable to determine correct subdirectory path for bbappend file - check that what %s adds to BBFILES also matches .bbappend files. Using %s for now, but until you fix this the bbappend will not be applied.', os.path.join(args.destlayer, 'conf', 'layer.conf'), os.path.dirname(append_path))
- layerdirs = [os.path.abspath(layerdir) for layerdir in rd.getVar('BBLAYERS', True).split()]
+ layerdirs = [os.path.abspath(layerdir) for layerdir in rd.getVar('BBLAYERS').split()]
if not os.path.abspath(args.destlayer) in layerdirs:
logger.warn('Specified layer is not currently enabled in bblayers.conf, you will need to add it before this bbappend will be active')
diff --git a/import-layers/yocto-poky/scripts/lib/recipetool/setvar.py b/import-layers/yocto-poky/scripts/lib/recipetool/setvar.py
index 85701c0..9de315a 100644
--- a/import-layers/yocto-poky/scripts/lib/recipetool/setvar.py
+++ b/import-layers/yocto-poky/scripts/lib/recipetool/setvar.py
@@ -51,7 +51,7 @@
if args.recipe_only:
patches = [oe.recipeutils.patch_recipe_file(args.recipefile, varvalues, patch=args.patch)]
else:
- rd = oe.recipeutils.parse_recipe(tinfoil.cooker, args.recipefile, None)
+ rd = tinfoil.parse_recipe_file(args.recipefile, False)
if not rd:
return 1
patches = oe.recipeutils.patch_recipe(rd, args.recipefile, varvalues, patch=args.patch)