| # Recipe creation tool - create command plugin |
| # |
| # Copyright (C) 2014-2016 Intel Corporation |
| # |
| # This program is free software; you can redistribute it and/or modify |
| # it under the terms of the GNU General Public License version 2 as |
| # published by the Free Software Foundation. |
| # |
| # This program is distributed in the hope that it will be useful, |
| # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| # GNU General Public License for more details. |
| # |
| # You should have received a copy of the GNU General Public License along |
| # with this program; if not, write to the Free Software Foundation, Inc., |
| # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. |
| |
| import sys |
| import os |
| import argparse |
| import glob |
| import fnmatch |
| import re |
| import json |
| import logging |
| import scriptutils |
| from urllib.parse import urlparse, urldefrag, urlsplit |
| import hashlib |
| |
| logger = logging.getLogger('recipetool') |
| |
| tinfoil = None |
| plugins = None |
| |
| def plugin_init(pluginlist): |
| # Take a reference to the list so we can use it later |
| global plugins |
| plugins = pluginlist |
| |
| def tinfoil_init(instance): |
| global tinfoil |
| tinfoil = instance |
| |
| class RecipeHandler(object): |
| recipelibmap = {} |
| recipeheadermap = {} |
| recipecmakefilemap = {} |
| recipebinmap = {} |
| |
| @staticmethod |
| def load_libmap(d): |
| '''Load library->recipe mapping''' |
| import oe.package |
| |
| if RecipeHandler.recipelibmap: |
| return |
| # First build up library->package mapping |
| shlib_providers = oe.package.read_shlib_providers(d) |
| libdir = d.getVar('libdir', True) |
| base_libdir = d.getVar('base_libdir', True) |
| libpaths = list(set([base_libdir, libdir])) |
| libname_re = re.compile('^lib(.+)\.so.*$') |
| pkglibmap = {} |
| for lib, item in shlib_providers.items(): |
| for path, pkg in item.items(): |
| if path in libpaths: |
| res = libname_re.match(lib) |
| if res: |
| libname = res.group(1) |
| if not libname in pkglibmap: |
| pkglibmap[libname] = pkg[0] |
| else: |
| logger.debug('unable to extract library name from %s' % lib) |
| |
| # Now turn it into a library->recipe mapping |
| pkgdata_dir = d.getVar('PKGDATA_DIR', True) |
| for libname, pkg in pkglibmap.items(): |
| try: |
| with open(os.path.join(pkgdata_dir, 'runtime', pkg)) as f: |
| for line in f: |
| if line.startswith('PN:'): |
| RecipeHandler.recipelibmap[libname] = line.split(':', 1)[-1].strip() |
| break |
| except IOError as ioe: |
| if ioe.errno == 2: |
| logger.warn('unable to find a pkgdata file for package %s' % pkg) |
| else: |
| raise |
| |
| # Some overrides - these should be mapped to the virtual |
| RecipeHandler.recipelibmap['GL'] = 'virtual/libgl' |
| RecipeHandler.recipelibmap['EGL'] = 'virtual/egl' |
| RecipeHandler.recipelibmap['GLESv2'] = 'virtual/libgles2' |
| |
| @staticmethod |
| def load_devel_filemap(d): |
| '''Build up development file->recipe mapping''' |
| if RecipeHandler.recipeheadermap: |
| return |
| pkgdata_dir = d.getVar('PKGDATA_DIR', True) |
| includedir = d.getVar('includedir', True) |
| cmakedir = os.path.join(d.getVar('libdir', True), 'cmake') |
| for pkg in glob.glob(os.path.join(pkgdata_dir, 'runtime', '*-dev')): |
| with open(os.path.join(pkgdata_dir, 'runtime', pkg)) as f: |
| pn = None |
| headers = [] |
| cmakefiles = [] |
| for line in f: |
| if line.startswith('PN:'): |
| pn = line.split(':', 1)[-1].strip() |
| elif line.startswith('FILES_INFO:'): |
| val = line.split(':', 1)[1].strip() |
| dictval = json.loads(val) |
| for fullpth in sorted(dictval): |
| if fullpth.startswith(includedir) and fullpth.endswith('.h'): |
| headers.append(os.path.relpath(fullpth, includedir)) |
| elif fullpth.startswith(cmakedir) and fullpth.endswith('.cmake'): |
| cmakefiles.append(os.path.relpath(fullpth, cmakedir)) |
| if pn and headers: |
| for header in headers: |
| RecipeHandler.recipeheadermap[header] = pn |
| if pn and cmakefiles: |
| for fn in cmakefiles: |
| RecipeHandler.recipecmakefilemap[fn] = pn |
| |
| @staticmethod |
| def load_binmap(d): |
| '''Build up native binary->recipe mapping''' |
| if RecipeHandler.recipebinmap: |
| return |
| sstate_manifests = d.getVar('SSTATE_MANIFESTS', True) |
| staging_bindir_native = d.getVar('STAGING_BINDIR_NATIVE', True) |
| build_arch = d.getVar('BUILD_ARCH', True) |
| fileprefix = 'manifest-%s-' % build_arch |
| for fn in glob.glob(os.path.join(sstate_manifests, '%s*-native.populate_sysroot' % fileprefix)): |
| with open(fn, 'r') as f: |
| pn = os.path.basename(fn).rsplit('.', 1)[0][len(fileprefix):] |
| for line in f: |
| if line.startswith(staging_bindir_native): |
| prog = os.path.basename(line.rstrip()) |
| RecipeHandler.recipebinmap[prog] = pn |
| |
| @staticmethod |
| def checkfiles(path, speclist, recursive=False): |
| results = [] |
| if recursive: |
| for root, _, files in os.walk(path): |
| for fn in files: |
| for spec in speclist: |
| if fnmatch.fnmatch(fn, spec): |
| results.append(os.path.join(root, fn)) |
| else: |
| for spec in speclist: |
| results.extend(glob.glob(os.path.join(path, spec))) |
| return results |
| |
| @staticmethod |
| def handle_depends(libdeps, pcdeps, deps, outlines, values, d): |
| if pcdeps: |
| recipemap = read_pkgconfig_provides(d) |
| if libdeps: |
| RecipeHandler.load_libmap(d) |
| |
| ignorelibs = ['socket'] |
| ignoredeps = ['gcc-runtime', 'glibc', 'uclibc', 'musl', 'tar-native', 'binutils-native', 'coreutils-native'] |
| |
| unmappedpc = [] |
| pcdeps = list(set(pcdeps)) |
| for pcdep in pcdeps: |
| if isinstance(pcdep, str): |
| recipe = recipemap.get(pcdep, None) |
| if recipe: |
| deps.append(recipe) |
| else: |
| if not pcdep.startswith('$'): |
| unmappedpc.append(pcdep) |
| else: |
| for item in pcdep: |
| recipe = recipemap.get(pcdep, None) |
| if recipe: |
| deps.append(recipe) |
| break |
| else: |
| unmappedpc.append('(%s)' % ' or '.join(pcdep)) |
| |
| unmappedlibs = [] |
| for libdep in libdeps: |
| if isinstance(libdep, tuple): |
| lib, header = libdep |
| else: |
| lib = libdep |
| header = None |
| |
| if lib in ignorelibs: |
| logger.debug('Ignoring library dependency %s' % lib) |
| continue |
| |
| recipe = RecipeHandler.recipelibmap.get(lib, None) |
| if recipe: |
| deps.append(recipe) |
| elif recipe is None: |
| if header: |
| RecipeHandler.load_devel_filemap(d) |
| recipe = RecipeHandler.recipeheadermap.get(header, None) |
| if recipe: |
| deps.append(recipe) |
| elif recipe is None: |
| unmappedlibs.append(lib) |
| else: |
| unmappedlibs.append(lib) |
| |
| deps = set(deps).difference(set(ignoredeps)) |
| |
| if unmappedpc: |
| outlines.append('# NOTE: unable to map the following pkg-config dependencies: %s' % ' '.join(unmappedpc)) |
| outlines.append('# (this is based on recipes that have previously been built and packaged)') |
| |
| if unmappedlibs: |
| outlines.append('# NOTE: the following library dependencies are unknown, ignoring: %s' % ' '.join(list(set(unmappedlibs)))) |
| outlines.append('# (this is based on recipes that have previously been built and packaged)') |
| |
| if deps: |
| values['DEPENDS'] = ' '.join(deps) |
| |
| def genfunction(self, outlines, funcname, content, python=False, forcespace=False): |
| if python: |
| prefix = 'python ' |
| else: |
| prefix = '' |
| outlines.append('%s%s () {' % (prefix, funcname)) |
| if python or forcespace: |
| indent = ' ' |
| else: |
| indent = '\t' |
| addnoop = not python |
| for line in content: |
| outlines.append('%s%s' % (indent, line)) |
| if addnoop: |
| strippedline = line.lstrip() |
| if strippedline and not strippedline.startswith('#'): |
| addnoop = False |
| if addnoop: |
| # Without this there'll be a syntax error |
| outlines.append('%s:' % indent) |
| outlines.append('}') |
| outlines.append('') |
| |
| def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): |
| return False |
| |
| |
| def validate_pv(pv): |
| if not pv or '_version' in pv.lower() or pv[0] not in '0123456789': |
| return False |
| return True |
| |
| def determine_from_filename(srcfile): |
| """Determine name and version from a filename""" |
| if is_package(srcfile): |
| # Force getting the value from the package metadata |
| return None, None |
| |
| if '.tar.' in srcfile: |
| namepart = srcfile.split('.tar.')[0] |
| else: |
| namepart = os.path.splitext(srcfile)[0] |
| namepart = namepart.lower().replace('_', '-') |
| if namepart.endswith('.src'): |
| namepart = namepart[:-4] |
| if namepart.endswith('.orig'): |
| namepart = namepart[:-5] |
| splitval = namepart.split('-') |
| logger.debug('determine_from_filename: split name %s into: %s' % (srcfile, splitval)) |
| |
| ver_re = re.compile('^v?[0-9]') |
| |
| pv = None |
| pn = None |
| if len(splitval) == 1: |
| # Try to split the version out if there is no separator (or a .) |
| res = re.match('^([^0-9]+)([0-9.]+.*)$', namepart) |
| if res: |
| if len(res.group(1)) > 1 and len(res.group(2)) > 1: |
| pn = res.group(1).rstrip('.') |
| pv = res.group(2) |
| else: |
| pn = namepart |
| else: |
| if splitval[-1] in ['source', 'src']: |
| splitval.pop() |
| if len(splitval) > 2 and re.match('^(alpha|beta|stable|release|rc[0-9]|pre[0-9]|p[0-9]|[0-9]{8})', splitval[-1]) and ver_re.match(splitval[-2]): |
| pv = '-'.join(splitval[-2:]) |
| if pv.endswith('-release'): |
| pv = pv[:-8] |
| splitval = splitval[:-2] |
| elif ver_re.match(splitval[-1]): |
| pv = splitval.pop() |
| pn = '-'.join(splitval) |
| if pv and pv.startswith('v'): |
| pv = pv[1:] |
| logger.debug('determine_from_filename: name = "%s" version = "%s"' % (pn, pv)) |
| return (pn, pv) |
| |
| def determine_from_url(srcuri): |
| """Determine name and version from a URL""" |
| pn = None |
| pv = None |
| parseres = urlparse(srcuri.lower().split(';', 1)[0]) |
| if parseres.path: |
| if 'github.com' in parseres.netloc: |
| res = re.search(r'.*/(.*?)/archive/(.*)-final\.(tar|zip)', parseres.path) |
| if res: |
| pn = res.group(1).strip().replace('_', '-') |
| pv = res.group(2).strip().replace('_', '.') |
| else: |
| res = re.search(r'.*/(.*?)/archive/v?(.*)\.(tar|zip)', parseres.path) |
| if res: |
| pn = res.group(1).strip().replace('_', '-') |
| pv = res.group(2).strip().replace('_', '.') |
| elif 'bitbucket.org' in parseres.netloc: |
| res = re.search(r'.*/(.*?)/get/[a-zA-Z_-]*([0-9][0-9a-zA-Z_.]*)\.(tar|zip)', parseres.path) |
| if res: |
| pn = res.group(1).strip().replace('_', '-') |
| pv = res.group(2).strip().replace('_', '.') |
| |
| if not pn and not pv: |
| srcfile = os.path.basename(parseres.path.rstrip('/')) |
| pn, pv = determine_from_filename(srcfile) |
| |
| logger.debug('Determined from source URL: name = "%s", version = "%s"' % (pn, pv)) |
| return (pn, pv) |
| |
| def supports_srcrev(uri): |
| localdata = bb.data.createCopy(tinfoil.config_data) |
| # This is a bit sad, but if you don't have this set there can be some |
| # odd interactions with the urldata cache which lead to errors |
| localdata.setVar('SRCREV', '${AUTOREV}') |
| bb.data.update_data(localdata) |
| try: |
| fetcher = bb.fetch2.Fetch([uri], localdata) |
| urldata = fetcher.ud |
| for u in urldata: |
| if urldata[u].method.supports_srcrev(): |
| return True |
| except bb.fetch2.FetchError as e: |
| logger.debug('FetchError in supports_srcrev: %s' % str(e)) |
| # Fall back to basic check |
| if uri.startswith(('git://', 'gitsm://')): |
| return True |
| return False |
| |
| def reformat_git_uri(uri): |
| '''Convert any http[s]://....git URI into git://...;protocol=http[s]''' |
| checkuri = uri.split(';', 1)[0] |
| if checkuri.endswith('.git') or '/git/' in checkuri or re.match('https?://github.com/[^/]+/[^/]+/?$', checkuri): |
| res = re.match('(http|https|ssh)://([^;]+(\.git)?)(;.*)?$', uri) |
| if res: |
| # Need to switch the URI around so that the git fetcher is used |
| return 'git://%s;protocol=%s%s' % (res.group(2), res.group(1), res.group(4) or '') |
| elif '@' in checkuri: |
| # Catch e.g. git@git.example.com:repo.git |
| return 'git://%s;protocol=ssh' % checkuri.replace(':', '/', 1) |
| return uri |
| |
| def is_package(url): |
| '''Check if a URL points to a package''' |
| checkurl = url.split(';', 1)[0] |
| if checkurl.endswith(('.deb', '.ipk', '.rpm', '.srpm')): |
| return True |
| return False |
| |
| def create_recipe(args): |
| import bb.process |
| import tempfile |
| import shutil |
| import oe.recipeutils |
| |
| pkgarch = "" |
| if args.machine: |
| pkgarch = "${MACHINE_ARCH}" |
| |
| extravalues = {} |
| checksums = (None, None) |
| tempsrc = '' |
| source = args.source |
| srcsubdir = '' |
| srcrev = '${AUTOREV}' |
| |
| if os.path.isfile(source): |
| source = 'file://%s' % os.path.abspath(source) |
| |
| if scriptutils.is_src_url(source): |
| # Fetch a URL |
| fetchuri = reformat_git_uri(urldefrag(source)[0]) |
| if args.binary: |
| # Assume the archive contains the directory structure verbatim |
| # so we need to extract to a subdirectory |
| fetchuri += ';subdir=${BP}' |
| srcuri = fetchuri |
| rev_re = re.compile(';rev=([^;]+)') |
| res = rev_re.search(srcuri) |
| if res: |
| srcrev = res.group(1) |
| srcuri = rev_re.sub('', srcuri) |
| tempsrc = tempfile.mkdtemp(prefix='recipetool-') |
| srctree = tempsrc |
| if fetchuri.startswith('npm://'): |
| # Check if npm is available |
| check_npm(tinfoil.config_data) |
| logger.info('Fetching %s...' % srcuri) |
| try: |
| checksums = scriptutils.fetch_uri(tinfoil.config_data, fetchuri, srctree, srcrev) |
| except bb.fetch2.BBFetchException as e: |
| logger.error(str(e).rstrip()) |
| sys.exit(1) |
| dirlist = os.listdir(srctree) |
| if 'git.indirectionsymlink' in dirlist: |
| dirlist.remove('git.indirectionsymlink') |
| if len(dirlist) == 1: |
| singleitem = os.path.join(srctree, dirlist[0]) |
| if os.path.isdir(singleitem): |
| # We unpacked a single directory, so we should use that |
| srcsubdir = dirlist[0] |
| srctree = os.path.join(srctree, srcsubdir) |
| else: |
| with open(singleitem, 'r', errors='surrogateescape') as f: |
| if '<html' in f.read(100).lower(): |
| logger.error('Fetching "%s" returned a single HTML page - check the URL is correct and functional' % fetchuri) |
| sys.exit(1) |
| if os.path.exists(os.path.join(srctree, '.gitmodules')) and srcuri.startswith('git://'): |
| srcuri = 'gitsm://' + srcuri[6:] |
| logger.info('Fetching submodules...') |
| bb.process.run('git submodule update --init --recursive', cwd=srctree) |
| |
| if is_package(fetchuri): |
| tmpfdir = tempfile.mkdtemp(prefix='recipetool-') |
| try: |
| pkgfile = None |
| try: |
| fileuri = fetchuri + ';unpack=0' |
| scriptutils.fetch_uri(tinfoil.config_data, fileuri, tmpfdir, srcrev) |
| for root, _, files in os.walk(tmpfdir): |
| for f in files: |
| pkgfile = os.path.join(root, f) |
| break |
| except bb.fetch2.BBFetchException as e: |
| logger.warn('Second fetch to get metadata failed: %s' % str(e).rstrip()) |
| |
| if pkgfile: |
| if pkgfile.endswith(('.deb', '.ipk')): |
| stdout, _ = bb.process.run('ar x %s control.tar.gz' % pkgfile, cwd=tmpfdir) |
| stdout, _ = bb.process.run('tar xf control.tar.gz ./control', cwd=tmpfdir) |
| values = convert_debian(tmpfdir) |
| extravalues.update(values) |
| elif pkgfile.endswith(('.rpm', '.srpm')): |
| stdout, _ = bb.process.run('rpm -qp --xml %s > pkginfo.xml' % pkgfile, cwd=tmpfdir) |
| values = convert_rpm_xml(os.path.join(tmpfdir, 'pkginfo.xml')) |
| extravalues.update(values) |
| finally: |
| shutil.rmtree(tmpfdir) |
| else: |
| # Assume we're pointing to an existing source tree |
| if args.extract_to: |
| logger.error('--extract-to cannot be specified if source is a directory') |
| sys.exit(1) |
| if not os.path.isdir(source): |
| logger.error('Invalid source directory %s' % source) |
| sys.exit(1) |
| srctree = source |
| srcuri = '' |
| if os.path.exists(os.path.join(srctree, '.git')): |
| # Try to get upstream repo location from origin remote |
| try: |
| stdout, _ = bb.process.run('git remote -v', cwd=srctree, shell=True) |
| except bb.process.ExecutionError as e: |
| stdout = None |
| if stdout: |
| for line in stdout.splitlines(): |
| splitline = line.split() |
| if len(splitline) > 1: |
| if splitline[0] == 'origin' and scriptutils.is_src_url(splitline[1]): |
| srcuri = reformat_git_uri(splitline[1]) |
| srcsubdir = 'git' |
| break |
| |
| if args.src_subdir: |
| srcsubdir = os.path.join(srcsubdir, args.src_subdir) |
| srctree_use = os.path.join(srctree, args.src_subdir) |
| else: |
| srctree_use = srctree |
| |
| if args.outfile and os.path.isdir(args.outfile): |
| outfile = None |
| outdir = args.outfile |
| else: |
| outfile = args.outfile |
| outdir = None |
| if outfile and outfile != '-': |
| if os.path.exists(outfile): |
| logger.error('Output file %s already exists' % outfile) |
| sys.exit(1) |
| |
| lines_before = [] |
| lines_after = [] |
| |
| lines_before.append('# Recipe created by %s' % os.path.basename(sys.argv[0])) |
| lines_before.append('# This is the basis of a recipe and may need further editing in order to be fully functional.') |
| lines_before.append('# (Feel free to remove these comments when editing.)') |
| # We need a blank line here so that patch_recipe_lines can rewind before the LICENSE comments |
| lines_before.append('') |
| |
| handled = [] |
| licvalues = handle_license_vars(srctree_use, lines_before, handled, extravalues, tinfoil.config_data) |
| |
| classes = [] |
| |
| # FIXME This is kind of a hack, we probably ought to be using bitbake to do this |
| pn = None |
| pv = None |
| if outfile: |
| recipefn = os.path.splitext(os.path.basename(outfile))[0] |
| fnsplit = recipefn.split('_') |
| if len(fnsplit) > 1: |
| pn = fnsplit[0] |
| pv = fnsplit[1] |
| else: |
| pn = recipefn |
| |
| if args.version: |
| pv = args.version |
| |
| if args.name: |
| pn = args.name |
| if args.name.endswith('-native'): |
| if args.also_native: |
| logger.error('--also-native cannot be specified for a recipe named *-native (*-native denotes a recipe that is already only for native) - either remove the -native suffix from the name or drop --also-native') |
| sys.exit(1) |
| classes.append('native') |
| elif args.name.startswith('nativesdk-'): |
| if args.also_native: |
| logger.error('--also-native cannot be specified for a recipe named nativesdk-* (nativesdk-* denotes a recipe that is already only for nativesdk)') |
| sys.exit(1) |
| classes.append('nativesdk') |
| |
| if pv and pv not in 'git svn hg'.split(): |
| realpv = pv |
| else: |
| realpv = None |
| |
| if srcuri and not realpv or not pn: |
| name_pn, name_pv = determine_from_url(srcuri) |
| if name_pn and not pn: |
| pn = name_pn |
| if name_pv and not realpv: |
| realpv = name_pv |
| |
| |
| if not srcuri: |
| lines_before.append('# No information for SRC_URI yet (only an external source tree was specified)') |
| lines_before.append('SRC_URI = "%s"' % srcuri) |
| (md5value, sha256value) = checksums |
| if md5value: |
| lines_before.append('SRC_URI[md5sum] = "%s"' % md5value) |
| if sha256value: |
| lines_before.append('SRC_URI[sha256sum] = "%s"' % sha256value) |
| if srcuri and supports_srcrev(srcuri): |
| lines_before.append('') |
| lines_before.append('# Modify these as desired') |
| lines_before.append('PV = "%s+git${SRCPV}"' % (realpv or '1.0')) |
| if not args.autorev and srcrev == '${AUTOREV}': |
| if os.path.exists(os.path.join(srctree, '.git')): |
| (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree) |
| srcrev = stdout.rstrip() |
| lines_before.append('SRCREV = "%s"' % srcrev) |
| lines_before.append('') |
| |
| if srcsubdir and not args.binary: |
| # (for binary packages we explicitly specify subdir= when fetching to |
| # match the default value of S, so we don't need to set it in that case) |
| lines_before.append('S = "${WORKDIR}/%s"' % srcsubdir) |
| lines_before.append('') |
| |
| if pkgarch: |
| lines_after.append('PACKAGE_ARCH = "%s"' % pkgarch) |
| lines_after.append('') |
| |
| if args.binary: |
| lines_after.append('INSANE_SKIP_${PN} += "already-stripped"') |
| lines_after.append('') |
| |
| # Find all plugins that want to register handlers |
| logger.debug('Loading recipe handlers') |
| raw_handlers = [] |
| for plugin in plugins: |
| if hasattr(plugin, 'register_recipe_handlers'): |
| plugin.register_recipe_handlers(raw_handlers) |
| # Sort handlers by priority |
| handlers = [] |
| for i, handler in enumerate(raw_handlers): |
| if isinstance(handler, tuple): |
| handlers.append((handler[0], handler[1], i)) |
| else: |
| handlers.append((handler, 0, i)) |
| handlers.sort(key=lambda item: (item[1], -item[2]), reverse=True) |
| for handler, priority, _ in handlers: |
| logger.debug('Handler: %s (priority %d)' % (handler.__class__.__name__, priority)) |
| handlers = [item[0] for item in handlers] |
| |
| # Apply the handlers |
| if args.binary: |
| classes.append('bin_package') |
| handled.append('buildsystem') |
| |
| for handler in handlers: |
| handler.process(srctree_use, classes, lines_before, lines_after, handled, extravalues) |
| |
| extrafiles = extravalues.pop('extrafiles', {}) |
| extra_pn = extravalues.pop('PN', None) |
| extra_pv = extravalues.pop('PV', None) |
| |
| if extra_pv and not realpv: |
| realpv = extra_pv |
| if not validate_pv(realpv): |
| realpv = None |
| else: |
| realpv = realpv.lower().split()[0] |
| if '_' in realpv: |
| realpv = realpv.replace('_', '-') |
| if extra_pn and not pn: |
| pn = extra_pn |
| if pn.startswith('GNU '): |
| pn = pn[4:] |
| if ' ' in pn: |
| # Probably a descriptive identifier rather than a proper name |
| pn = None |
| else: |
| pn = pn.lower() |
| if '_' in pn: |
| pn = pn.replace('_', '-') |
| |
| if not outfile: |
| if not pn: |
| logger.error('Unable to determine short program name from source tree - please specify name with -N/--name or output file name with -o/--outfile') |
| # devtool looks for this specific exit code, so don't change it |
| sys.exit(15) |
| else: |
| if srcuri and srcuri.startswith(('gitsm://', 'git://', 'hg://', 'svn://')): |
| suffix = srcuri.split(':', 1)[0] |
| if suffix == 'gitsm': |
| suffix = 'git' |
| outfile = '%s_%s.bb' % (pn, suffix) |
| elif realpv: |
| outfile = '%s_%s.bb' % (pn, realpv) |
| else: |
| outfile = '%s.bb' % pn |
| if outdir: |
| outfile = os.path.join(outdir, outfile) |
| # We need to check this again |
| if os.path.exists(outfile): |
| logger.error('Output file %s already exists' % outfile) |
| sys.exit(1) |
| |
| # Move any extra files the plugins created to a directory next to the recipe |
| if extrafiles: |
| if outfile == '-': |
| extraoutdir = pn |
| else: |
| extraoutdir = os.path.join(os.path.dirname(outfile), pn) |
| bb.utils.mkdirhier(extraoutdir) |
| for destfn, extrafile in extrafiles.items(): |
| shutil.move(extrafile, os.path.join(extraoutdir, destfn)) |
| |
| lines = lines_before |
| lines_before = [] |
| skipblank = True |
| for line in lines: |
| if skipblank: |
| skipblank = False |
| if not line: |
| continue |
| if line.startswith('S = '): |
| if realpv and pv not in 'git svn hg'.split(): |
| line = line.replace(realpv, '${PV}') |
| if pn: |
| line = line.replace(pn, '${BPN}') |
| if line == 'S = "${WORKDIR}/${BPN}-${PV}"': |
| skipblank = True |
| continue |
| elif line.startswith('SRC_URI = '): |
| if realpv: |
| line = line.replace(realpv, '${PV}') |
| elif line.startswith('PV = '): |
| if realpv: |
| line = re.sub('"[^+]*\+', '"%s+' % realpv, line) |
| lines_before.append(line) |
| |
| if args.also_native: |
| lines = lines_after |
| lines_after = [] |
| bbclassextend = None |
| for line in lines: |
| if line.startswith('BBCLASSEXTEND ='): |
| splitval = line.split('"') |
| if len(splitval) > 1: |
| bbclassextend = splitval[1].split() |
| if not 'native' in bbclassextend: |
| bbclassextend.insert(0, 'native') |
| line = 'BBCLASSEXTEND = "%s"' % ' '.join(bbclassextend) |
| lines_after.append(line) |
| if not bbclassextend: |
| lines_after.append('BBCLASSEXTEND = "native"') |
| |
| outlines = [] |
| outlines.extend(lines_before) |
| if classes: |
| if outlines[-1] and not outlines[-1].startswith('#'): |
| outlines.append('') |
| outlines.append('inherit %s' % ' '.join(classes)) |
| outlines.append('') |
| outlines.extend(lines_after) |
| |
| if extravalues: |
| if 'LICENSE' in extravalues and not licvalues: |
| # Don't blow away 'CLOSED' value that comments say we set |
| del extravalues['LICENSE'] |
| _, outlines = oe.recipeutils.patch_recipe_lines(outlines, extravalues, trailing_newline=False) |
| |
| if args.extract_to: |
| scriptutils.git_convert_standalone_clone(srctree) |
| if os.path.isdir(args.extract_to): |
| # If the directory exists we'll move the temp dir into it instead of |
| # its contents - of course, we could try to always move its contents |
| # but that is a pain if there are symlinks; the simplest solution is |
| # to just remove it first |
| os.rmdir(args.extract_to) |
| shutil.move(srctree, args.extract_to) |
| if tempsrc == srctree: |
| tempsrc = None |
| logger.info('Source extracted to %s' % args.extract_to) |
| |
| if outfile == '-': |
| sys.stdout.write('\n'.join(outlines) + '\n') |
| else: |
| with open(outfile, 'w') as f: |
| lastline = None |
| for line in outlines: |
| if not lastline and not line: |
| # Skip extra blank lines |
| continue |
| f.write('%s\n' % line) |
| lastline = line |
| logger.info('Recipe %s has been created; further editing may be required to make it fully functional' % outfile) |
| |
| if tempsrc: |
| if args.keep_temp: |
| logger.info('Preserving temporary directory %s' % tempsrc) |
| else: |
| shutil.rmtree(tempsrc) |
| |
| return 0 |
| |
| def handle_license_vars(srctree, lines_before, handled, extravalues, d): |
| licvalues = guess_license(srctree, d) |
| lic_files_chksum = [] |
| lic_unknown = [] |
| if licvalues: |
| licenses = [] |
| for licvalue in licvalues: |
| if not licvalue[0] in licenses: |
| licenses.append(licvalue[0]) |
| lic_files_chksum.append('file://%s;md5=%s' % (licvalue[1], licvalue[2])) |
| if licvalue[0] == 'Unknown': |
| lic_unknown.append(licvalue[1]) |
| lines_before.append('# WARNING: the following LICENSE and LIC_FILES_CHKSUM values are best guesses - it is') |
| lines_before.append('# your responsibility to verify that the values are complete and correct.') |
| if len(licvalues) > 1: |
| lines_before.append('#') |
| lines_before.append('# NOTE: multiple licenses have been detected; if that is correct you should separate') |
| lines_before.append('# these in the LICENSE value using & if the multiple licenses all apply, or | if there') |
| lines_before.append('# is a choice between the multiple licenses. If in doubt, check the accompanying') |
| lines_before.append('# documentation to determine which situation is applicable.') |
| if lic_unknown: |
| lines_before.append('#') |
| lines_before.append('# The following license files were not able to be identified and are') |
| lines_before.append('# represented as "Unknown" below, you will need to check them yourself:') |
| for licfile in lic_unknown: |
| lines_before.append('# %s' % licfile) |
| lines_before.append('#') |
| else: |
| lines_before.append('# Unable to find any files that looked like license statements. Check the accompanying') |
| lines_before.append('# documentation and source headers and set LICENSE and LIC_FILES_CHKSUM accordingly.') |
| lines_before.append('#') |
| lines_before.append('# NOTE: LICENSE is being set to "CLOSED" to allow you to at least start building - if') |
| lines_before.append('# this is not accurate with respect to the licensing of the software being built (it') |
| lines_before.append('# will not be in most cases) you must specify the correct value before using this') |
| lines_before.append('# recipe for anything other than initial testing/development!') |
| licenses = ['CLOSED'] |
| pkg_license = extravalues.pop('LICENSE', None) |
| if pkg_license: |
| if licenses == ['Unknown']: |
| lines_before.append('# NOTE: The following LICENSE value was determined from the original package metadata') |
| licenses = [pkg_license] |
| else: |
| lines_before.append('# NOTE: Original package metadata indicates license is: %s' % pkg_license) |
| lines_before.append('LICENSE = "%s"' % ' '.join(licenses)) |
| lines_before.append('LIC_FILES_CHKSUM = "%s"' % ' \\\n '.join(lic_files_chksum)) |
| lines_before.append('') |
| handled.append(('license', licvalues)) |
| return licvalues |
| |
| def get_license_md5sums(d, static_only=False): |
| import bb.utils |
| md5sums = {} |
| if not static_only: |
| # Gather md5sums of license files in common license dir |
| commonlicdir = d.getVar('COMMON_LICENSE_DIR', True) |
| for fn in os.listdir(commonlicdir): |
| md5value = bb.utils.md5_file(os.path.join(commonlicdir, fn)) |
| md5sums[md5value] = fn |
| # The following were extracted from common values in various recipes |
| # (double checking the license against the license file itself, not just |
| # the LICENSE value in the recipe) |
| md5sums['94d55d512a9ba36caa9b7df079bae19f'] = 'GPLv2' |
| md5sums['b234ee4d69f5fce4486a80fdaf4a4263'] = 'GPLv2' |
| md5sums['59530bdf33659b29e73d4adb9f9f6552'] = 'GPLv2' |
| md5sums['0636e73ff0215e8d672dc4c32c317bb3'] = 'GPLv2' |
| md5sums['eb723b61539feef013de476e68b5c50a'] = 'GPLv2' |
| md5sums['751419260aa954499f7abaabaa882bbe'] = 'GPLv2' |
| md5sums['393a5ca445f6965873eca0259a17f833'] = 'GPLv2' |
| md5sums['12f884d2ae1ff87c09e5b7ccc2c4ca7e'] = 'GPLv2' |
| md5sums['8ca43cbc842c2336e835926c2166c28b'] = 'GPLv2' |
| md5sums['ebb5c50ab7cab4baeffba14977030c07'] = 'GPLv2' |
| md5sums['c93c0550bd3173f4504b2cbd8991e50b'] = 'GPLv2' |
| md5sums['9ac2e7cff1ddaf48b6eab6028f23ef88'] = 'GPLv2' |
| md5sums['4325afd396febcb659c36b49533135d4'] = 'GPLv2' |
| md5sums['18810669f13b87348459e611d31ab760'] = 'GPLv2' |
| md5sums['d7810fab7487fb0aad327b76f1be7cd7'] = 'GPLv2' # the Linux kernel's COPYING file |
| md5sums['bbb461211a33b134d42ed5ee802b37ff'] = 'LGPLv2.1' |
| md5sums['7fbc338309ac38fefcd64b04bb903e34'] = 'LGPLv2.1' |
| md5sums['4fbd65380cdd255951079008b364516c'] = 'LGPLv2.1' |
| md5sums['2d5025d4aa3495befef8f17206a5b0a1'] = 'LGPLv2.1' |
| md5sums['fbc093901857fcd118f065f900982c24'] = 'LGPLv2.1' |
| md5sums['a6f89e2100d9b6cdffcea4f398e37343'] = 'LGPLv2.1' |
| md5sums['d8045f3b8f929c1cb29a1e3fd737b499'] = 'LGPLv2.1' |
| md5sums['fad9b3332be894bab9bc501572864b29'] = 'LGPLv2.1' |
| md5sums['3bf50002aefd002f49e7bb854063f7e7'] = 'LGPLv2' |
| md5sums['9f604d8a4f8e74f4f5140845a21b6674'] = 'LGPLv2' |
| md5sums['5f30f0716dfdd0d91eb439ebec522ec2'] = 'LGPLv2' |
| md5sums['55ca817ccb7d5b5b66355690e9abc605'] = 'LGPLv2' |
| md5sums['252890d9eee26aab7b432e8b8a616475'] = 'LGPLv2' |
| md5sums['3214f080875748938ba060314b4f727d'] = 'LGPLv2' |
| md5sums['db979804f025cf55aabec7129cb671ed'] = 'LGPLv2' |
| md5sums['d32239bcb673463ab874e80d47fae504'] = 'GPLv3' |
| md5sums['f27defe1e96c2e1ecd4e0c9be8967949'] = 'GPLv3' |
| md5sums['6a6a8e020838b23406c81b19c1d46df6'] = 'LGPLv3' |
| md5sums['3b83ef96387f14655fc854ddc3c6bd57'] = 'Apache-2.0' |
| md5sums['385c55653886acac3821999a3ccd17b3'] = 'Artistic-1.0 | GPL-2.0' # some perl modules |
| md5sums['54c7042be62e169199200bc6477f04d1'] = 'BSD-3-Clause' |
| return md5sums |
| |
| def crunch_license(licfile): |
| ''' |
| Remove non-material text from a license file and then check |
| its md5sum against a known list. This works well for licenses |
| which contain a copyright statement, but is also a useful way |
| to handle people's insistence upon reformatting the license text |
| slightly (with no material difference to the text of the |
| license). |
| ''' |
| |
| import oe.utils |
| |
| # Note: these are carefully constructed! |
| license_title_re = re.compile('^\(?(#+ *)?(The )?.{1,10} [Ll]icen[sc]e( \(.{1,10}\))?\)?:?$') |
| license_statement_re = re.compile('^(This (project|software) is( free software)? (released|licen[sc]ed)|(Released|Licen[cs]ed)) under the .{1,10} [Ll]icen[sc]e:?$') |
| copyright_re = re.compile('^(#+)? *Copyright .*$') |
| |
| crunched_md5sums = {} |
| # The following two were gleaned from the "forever" npm package |
| crunched_md5sums['0a97f8e4cbaf889d6fa51f84b89a79f6'] = 'ISC' |
| crunched_md5sums['eecf6429523cbc9693547cf2db790b5c'] = 'MIT' |
| # https://github.com/vasi/pixz/blob/master/LICENSE |
| crunched_md5sums['2f03392b40bbe663597b5bd3cc5ebdb9'] = 'BSD-2-Clause' |
| # https://github.com/waffle-gl/waffle/blob/master/LICENSE.txt |
| crunched_md5sums['e72e5dfef0b1a4ca8a3d26a60587db66'] = 'BSD-2-Clause' |
| # https://github.com/spigwitmer/fakeds1963s/blob/master/LICENSE |
| crunched_md5sums['8be76ac6d191671f347ee4916baa637e'] = 'GPLv2' |
| # https://github.com/datto/dattobd/blob/master/COPYING |
| # http://git.savannah.gnu.org/cgit/freetype/freetype2.git/tree/docs/GPLv2.TXT |
| crunched_md5sums['1d65c5ad4bf6489f85f4812bf08ae73d'] = 'GPLv2' |
| # http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt |
| # http://git.neil.brown.name/?p=mdadm.git;a=blob;f=COPYING;h=d159169d1050894d3ea3b98e1c965c4058208fe1;hb=HEAD |
| crunched_md5sums['fb530f66a7a89ce920f0e912b5b66d4b'] = 'GPLv2' |
| # https://github.com/gkos/nrf24/blob/master/COPYING |
| crunched_md5sums['7b6aaa4daeafdfa6ed5443fd2684581b'] = 'GPLv2' |
| # https://github.com/josch09/resetusb/blob/master/COPYING |
| crunched_md5sums['8b8ac1d631a4d220342e83bcf1a1fbc3'] = 'GPLv3' |
| # https://github.com/FFmpeg/FFmpeg/blob/master/COPYING.LGPLv2.1 |
| crunched_md5sums['2ea316ed973ae176e502e2297b574bb3'] = 'LGPLv2.1' |
| # unixODBC-2.3.4 COPYING |
| crunched_md5sums['1daebd9491d1e8426900b4fa5a422814'] = 'LGPLv2.1' |
| # https://github.com/FFmpeg/FFmpeg/blob/master/COPYING.LGPLv3 |
| crunched_md5sums['2ebfb3bb49b9a48a075cc1425e7f4129'] = 'LGPLv3' |
| lictext = [] |
| with open(licfile, 'r', errors='surrogateescape') as f: |
| for line in f: |
| # Drop opening statements |
| if copyright_re.match(line): |
| continue |
| elif license_title_re.match(line): |
| continue |
| elif license_statement_re.match(line): |
| continue |
| # Squash spaces, and replace smart quotes, double quotes |
| # and backticks with single quotes |
| line = oe.utils.squashspaces(line.strip()) |
| line = line.replace(u"\u2018", "'").replace(u"\u2019", "'").replace(u"\u201c","'").replace(u"\u201d", "'").replace('"', '\'').replace('`', '\'') |
| if line: |
| lictext.append(line) |
| |
| m = hashlib.md5() |
| try: |
| m.update(' '.join(lictext).encode('utf-8')) |
| md5val = m.hexdigest() |
| except UnicodeEncodeError: |
| md5val = None |
| lictext = '' |
| license = crunched_md5sums.get(md5val, None) |
| return license, md5val, lictext |
| |
| def guess_license(srctree, d): |
| import bb |
| md5sums = get_license_md5sums(d) |
| |
| licenses = [] |
| licspecs = ['*LICEN[CS]E*', 'COPYING*', '*[Ll]icense*', 'LEGAL*', '[Ll]egal*', '*GPL*', 'README.lic*', 'COPYRIGHT*', '[Cc]opyright*'] |
| licfiles = [] |
| for root, dirs, files in os.walk(srctree): |
| for fn in files: |
| for spec in licspecs: |
| if fnmatch.fnmatch(fn, spec): |
| fullpath = os.path.join(root, fn) |
| if not fullpath in licfiles: |
| licfiles.append(fullpath) |
| for licfile in licfiles: |
| md5value = bb.utils.md5_file(licfile) |
| license = md5sums.get(md5value, None) |
| if not license: |
| license, crunched_md5, lictext = crunch_license(licfile) |
| if not license: |
| license = 'Unknown' |
| licenses.append((license, os.path.relpath(licfile, srctree), md5value)) |
| |
| # FIXME should we grab at least one source file with a license header and add that too? |
| |
| return licenses |
| |
| def split_pkg_licenses(licvalues, packages, outlines, fallback_licenses=None, pn='${PN}'): |
| """ |
| Given a list of (license, path, md5sum) as returned by guess_license(), |
| a dict of package name to path mappings, write out a set of |
| package-specific LICENSE values. |
| """ |
| pkglicenses = {pn: []} |
| for license, licpath, _ in licvalues: |
| for pkgname, pkgpath in packages.items(): |
| if licpath.startswith(pkgpath + '/'): |
| if pkgname in pkglicenses: |
| pkglicenses[pkgname].append(license) |
| else: |
| pkglicenses[pkgname] = [license] |
| break |
| else: |
| # Accumulate on the main package |
| pkglicenses[pn].append(license) |
| outlicenses = {} |
| for pkgname in packages: |
| license = ' '.join(list(set(pkglicenses.get(pkgname, ['Unknown'])))) or 'Unknown' |
| if license == 'Unknown' and pkgname in fallback_licenses: |
| license = fallback_licenses[pkgname] |
| outlines.append('LICENSE_%s = "%s"' % (pkgname, license)) |
| outlicenses[pkgname] = license.split() |
| return outlicenses |
| |
| def read_pkgconfig_provides(d): |
| pkgdatadir = d.getVar('PKGDATA_DIR', True) |
| pkgmap = {} |
| for fn in glob.glob(os.path.join(pkgdatadir, 'shlibs2', '*.pclist')): |
| with open(fn, 'r') as f: |
| for line in f: |
| pkgmap[os.path.basename(line.rstrip())] = os.path.splitext(os.path.basename(fn))[0] |
| recipemap = {} |
| for pc, pkg in pkgmap.items(): |
| pkgdatafile = os.path.join(pkgdatadir, 'runtime', pkg) |
| if os.path.exists(pkgdatafile): |
| with open(pkgdatafile, 'r') as f: |
| for line in f: |
| if line.startswith('PN: '): |
| recipemap[pc] = line.split(':', 1)[1].strip() |
| return recipemap |
| |
| def convert_debian(debpath): |
| value_map = {'Package': 'PN', |
| 'Version': 'PV', |
| 'Section': 'SECTION', |
| 'License': 'LICENSE', |
| 'Homepage': 'HOMEPAGE'} |
| |
| # FIXME extend this mapping - perhaps use distro_alias.inc? |
| depmap = {'libz-dev': 'zlib'} |
| |
| values = {} |
| depends = [] |
| with open(os.path.join(debpath, 'control'), 'r', errors='surrogateescape') as f: |
| indesc = False |
| for line in f: |
| if indesc: |
| if line.startswith(' '): |
| if line.startswith(' This package contains'): |
| indesc = False |
| else: |
| if 'DESCRIPTION' in values: |
| values['DESCRIPTION'] += ' ' + line.strip() |
| else: |
| values['DESCRIPTION'] = line.strip() |
| else: |
| indesc = False |
| if not indesc: |
| splitline = line.split(':', 1) |
| if len(splitline) < 2: |
| continue |
| key = splitline[0] |
| value = splitline[1].strip() |
| if key == 'Build-Depends': |
| for dep in value.split(','): |
| dep = dep.split()[0] |
| mapped = depmap.get(dep, '') |
| if mapped: |
| depends.append(mapped) |
| elif key == 'Description': |
| values['SUMMARY'] = value |
| indesc = True |
| else: |
| varname = value_map.get(key, None) |
| if varname: |
| values[varname] = value |
| |
| #if depends: |
| # values['DEPENDS'] = ' '.join(depends) |
| |
| return values |
| |
| def convert_rpm_xml(xmlfile): |
| '''Converts the output from rpm -qp --xml to a set of variable values''' |
| import xml.etree.ElementTree as ElementTree |
| rpmtag_map = {'Name': 'PN', |
| 'Version': 'PV', |
| 'Summary': 'SUMMARY', |
| 'Description': 'DESCRIPTION', |
| 'License': 'LICENSE', |
| 'Url': 'HOMEPAGE'} |
| |
| values = {} |
| tree = ElementTree.parse(xmlfile) |
| root = tree.getroot() |
| for child in root: |
| if child.tag == 'rpmTag': |
| name = child.attrib.get('name', None) |
| if name: |
| varname = rpmtag_map.get(name, None) |
| if varname: |
| values[varname] = child[0].text |
| return values |
| |
| |
| def check_npm(d): |
| if not os.path.exists(os.path.join(d.getVar('STAGING_BINDIR_NATIVE', True), 'npm')): |
| logger.error('npm required to process specified source, but npm is not available - you need to build nodejs-native first') |
| sys.exit(14) |
| |
| def register_commands(subparsers): |
| parser_create = subparsers.add_parser('create', |
| help='Create a new recipe', |
| description='Creates a new recipe from a source tree') |
| parser_create.add_argument('source', help='Path or URL to source') |
| parser_create.add_argument('-o', '--outfile', help='Specify filename for recipe to create') |
| parser_create.add_argument('-m', '--machine', help='Make recipe machine-specific as opposed to architecture-specific', action='store_true') |
| parser_create.add_argument('-x', '--extract-to', metavar='EXTRACTPATH', help='Assuming source is a URL, fetch it and extract it to the directory specified as %(metavar)s') |
| parser_create.add_argument('-N', '--name', help='Name to use within recipe (PN)') |
| parser_create.add_argument('-V', '--version', help='Version to use within recipe (PV)') |
| parser_create.add_argument('-b', '--binary', help='Treat the source tree as something that should be installed verbatim (no compilation, same directory structure)', action='store_true') |
| parser_create.add_argument('--also-native', help='Also add native variant (i.e. support building recipe for the build host as well as the target machine)', action='store_true') |
| parser_create.add_argument('--src-subdir', help='Specify subdirectory within source tree to use', metavar='SUBDIR') |
| parser_create.add_argument('-a', '--autorev', help='When fetching from a git repository, set SRCREV in the recipe to a floating revision instead of fixed', action="store_true") |
| parser_create.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') |
| parser_create.set_defaults(func=create_recipe) |
| |