blob: 4de52fc30f8b1e77d92d7a7f9d083e813ba10dc2 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# Recipe creation tool - create command plugin
2#
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05003# Copyright (C) 2014-2016 Intel Corporation
Patrick Williamsc124f4f2015-09-15 14:41:29 -05004#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License version 2 as
7# published by the Free Software Foundation.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License along
15# with this program; if not, write to the Free Software Foundation, Inc.,
16# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17
18import sys
19import os
20import argparse
21import glob
22import fnmatch
23import re
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050024import json
Patrick Williamsc124f4f2015-09-15 14:41:29 -050025import logging
26import scriptutils
Patrick Williamsc0f7c042017-02-23 20:41:17 -060027from urllib.parse import urlparse, urldefrag, urlsplit
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050028import hashlib
Brad Bishop6e60e8b2018-02-01 10:27:11 -050029import bb.fetch2
Patrick Williamsc124f4f2015-09-15 14:41:29 -050030logger = logging.getLogger('recipetool')
31
32tinfoil = None
33plugins = None
34
Brad Bishop6e60e8b2018-02-01 10:27:11 -050035def log_error_cond(message, debugonly):
36 if debugonly:
37 logger.debug(message)
38 else:
39 logger.error(message)
40
41def log_info_cond(message, debugonly):
42 if debugonly:
43 logger.debug(message)
44 else:
45 logger.info(message)
46
Patrick Williamsc124f4f2015-09-15 14:41:29 -050047def plugin_init(pluginlist):
48 # Take a reference to the list so we can use it later
49 global plugins
50 plugins = pluginlist
51
52def tinfoil_init(instance):
53 global tinfoil
54 tinfoil = instance
55
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050056class RecipeHandler(object):
57 recipelibmap = {}
58 recipeheadermap = {}
59 recipecmakefilemap = {}
60 recipebinmap = {}
61
Brad Bishop6e60e8b2018-02-01 10:27:11 -050062 def __init__(self):
63 self._devtool = False
64
Patrick Williamsc124f4f2015-09-15 14:41:29 -050065 @staticmethod
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050066 def load_libmap(d):
67 '''Load library->recipe mapping'''
68 import oe.package
69
70 if RecipeHandler.recipelibmap:
71 return
72 # First build up library->package mapping
73 shlib_providers = oe.package.read_shlib_providers(d)
Brad Bishop6e60e8b2018-02-01 10:27:11 -050074 libdir = d.getVar('libdir')
75 base_libdir = d.getVar('base_libdir')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050076 libpaths = list(set([base_libdir, libdir]))
77 libname_re = re.compile('^lib(.+)\.so.*$')
78 pkglibmap = {}
Patrick Williamsc0f7c042017-02-23 20:41:17 -060079 for lib, item in shlib_providers.items():
80 for path, pkg in item.items():
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050081 if path in libpaths:
82 res = libname_re.match(lib)
83 if res:
84 libname = res.group(1)
85 if not libname in pkglibmap:
86 pkglibmap[libname] = pkg[0]
87 else:
88 logger.debug('unable to extract library name from %s' % lib)
89
90 # Now turn it into a library->recipe mapping
Brad Bishop6e60e8b2018-02-01 10:27:11 -050091 pkgdata_dir = d.getVar('PKGDATA_DIR')
Patrick Williamsc0f7c042017-02-23 20:41:17 -060092 for libname, pkg in pkglibmap.items():
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050093 try:
94 with open(os.path.join(pkgdata_dir, 'runtime', pkg)) as f:
95 for line in f:
96 if line.startswith('PN:'):
97 RecipeHandler.recipelibmap[libname] = line.split(':', 1)[-1].strip()
98 break
99 except IOError as ioe:
100 if ioe.errno == 2:
101 logger.warn('unable to find a pkgdata file for package %s' % pkg)
102 else:
103 raise
104
105 # Some overrides - these should be mapped to the virtual
106 RecipeHandler.recipelibmap['GL'] = 'virtual/libgl'
107 RecipeHandler.recipelibmap['EGL'] = 'virtual/egl'
108 RecipeHandler.recipelibmap['GLESv2'] = 'virtual/libgles2'
109
110 @staticmethod
111 def load_devel_filemap(d):
112 '''Build up development file->recipe mapping'''
113 if RecipeHandler.recipeheadermap:
114 return
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500115 pkgdata_dir = d.getVar('PKGDATA_DIR')
116 includedir = d.getVar('includedir')
117 cmakedir = os.path.join(d.getVar('libdir'), 'cmake')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500118 for pkg in glob.glob(os.path.join(pkgdata_dir, 'runtime', '*-dev')):
119 with open(os.path.join(pkgdata_dir, 'runtime', pkg)) as f:
120 pn = None
121 headers = []
122 cmakefiles = []
123 for line in f:
124 if line.startswith('PN:'):
125 pn = line.split(':', 1)[-1].strip()
126 elif line.startswith('FILES_INFO:'):
127 val = line.split(':', 1)[1].strip()
128 dictval = json.loads(val)
129 for fullpth in sorted(dictval):
130 if fullpth.startswith(includedir) and fullpth.endswith('.h'):
131 headers.append(os.path.relpath(fullpth, includedir))
132 elif fullpth.startswith(cmakedir) and fullpth.endswith('.cmake'):
133 cmakefiles.append(os.path.relpath(fullpth, cmakedir))
134 if pn and headers:
135 for header in headers:
136 RecipeHandler.recipeheadermap[header] = pn
137 if pn and cmakefiles:
138 for fn in cmakefiles:
139 RecipeHandler.recipecmakefilemap[fn] = pn
140
141 @staticmethod
142 def load_binmap(d):
143 '''Build up native binary->recipe mapping'''
144 if RecipeHandler.recipebinmap:
145 return
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500146 sstate_manifests = d.getVar('SSTATE_MANIFESTS')
147 staging_bindir_native = d.getVar('STAGING_BINDIR_NATIVE')
148 build_arch = d.getVar('BUILD_ARCH')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500149 fileprefix = 'manifest-%s-' % build_arch
150 for fn in glob.glob(os.path.join(sstate_manifests, '%s*-native.populate_sysroot' % fileprefix)):
151 with open(fn, 'r') as f:
152 pn = os.path.basename(fn).rsplit('.', 1)[0][len(fileprefix):]
153 for line in f:
154 if line.startswith(staging_bindir_native):
155 prog = os.path.basename(line.rstrip())
156 RecipeHandler.recipebinmap[prog] = pn
157
158 @staticmethod
159 def checkfiles(path, speclist, recursive=False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500160 results = []
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500161 if recursive:
162 for root, _, files in os.walk(path):
163 for fn in files:
164 for spec in speclist:
165 if fnmatch.fnmatch(fn, spec):
166 results.append(os.path.join(root, fn))
167 else:
168 for spec in speclist:
169 results.extend(glob.glob(os.path.join(path, spec)))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500170 return results
171
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500172 @staticmethod
173 def handle_depends(libdeps, pcdeps, deps, outlines, values, d):
174 if pcdeps:
175 recipemap = read_pkgconfig_provides(d)
176 if libdeps:
177 RecipeHandler.load_libmap(d)
178
179 ignorelibs = ['socket']
180 ignoredeps = ['gcc-runtime', 'glibc', 'uclibc', 'musl', 'tar-native', 'binutils-native', 'coreutils-native']
181
182 unmappedpc = []
183 pcdeps = list(set(pcdeps))
184 for pcdep in pcdeps:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600185 if isinstance(pcdep, str):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500186 recipe = recipemap.get(pcdep, None)
187 if recipe:
188 deps.append(recipe)
189 else:
190 if not pcdep.startswith('$'):
191 unmappedpc.append(pcdep)
192 else:
193 for item in pcdep:
194 recipe = recipemap.get(pcdep, None)
195 if recipe:
196 deps.append(recipe)
197 break
198 else:
199 unmappedpc.append('(%s)' % ' or '.join(pcdep))
200
201 unmappedlibs = []
202 for libdep in libdeps:
203 if isinstance(libdep, tuple):
204 lib, header = libdep
205 else:
206 lib = libdep
207 header = None
208
209 if lib in ignorelibs:
210 logger.debug('Ignoring library dependency %s' % lib)
211 continue
212
213 recipe = RecipeHandler.recipelibmap.get(lib, None)
214 if recipe:
215 deps.append(recipe)
216 elif recipe is None:
217 if header:
218 RecipeHandler.load_devel_filemap(d)
219 recipe = RecipeHandler.recipeheadermap.get(header, None)
220 if recipe:
221 deps.append(recipe)
222 elif recipe is None:
223 unmappedlibs.append(lib)
224 else:
225 unmappedlibs.append(lib)
226
227 deps = set(deps).difference(set(ignoredeps))
228
229 if unmappedpc:
230 outlines.append('# NOTE: unable to map the following pkg-config dependencies: %s' % ' '.join(unmappedpc))
231 outlines.append('# (this is based on recipes that have previously been built and packaged)')
232
233 if unmappedlibs:
234 outlines.append('# NOTE: the following library dependencies are unknown, ignoring: %s' % ' '.join(list(set(unmappedlibs))))
235 outlines.append('# (this is based on recipes that have previously been built and packaged)')
236
237 if deps:
238 values['DEPENDS'] = ' '.join(deps)
239
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500240 @staticmethod
241 def genfunction(outlines, funcname, content, python=False, forcespace=False):
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500242 if python:
243 prefix = 'python '
244 else:
245 prefix = ''
246 outlines.append('%s%s () {' % (prefix, funcname))
247 if python or forcespace:
248 indent = ' '
249 else:
250 indent = '\t'
251 addnoop = not python
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500252 for line in content:
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500253 outlines.append('%s%s' % (indent, line))
254 if addnoop:
255 strippedline = line.lstrip()
256 if strippedline and not strippedline.startswith('#'):
257 addnoop = False
258 if addnoop:
259 # Without this there'll be a syntax error
260 outlines.append('%s:' % indent)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500261 outlines.append('}')
262 outlines.append('')
263
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500264 def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500265 return False
266
267
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500268def validate_pv(pv):
269 if not pv or '_version' in pv.lower() or pv[0] not in '0123456789':
270 return False
271 return True
272
273def determine_from_filename(srcfile):
274 """Determine name and version from a filename"""
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600275 if is_package(srcfile):
276 # Force getting the value from the package metadata
277 return None, None
278
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500279 if '.tar.' in srcfile:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600280 namepart = srcfile.split('.tar.')[0]
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500281 else:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600282 namepart = os.path.splitext(srcfile)[0]
283 namepart = namepart.lower().replace('_', '-')
284 if namepart.endswith('.src'):
285 namepart = namepart[:-4]
286 if namepart.endswith('.orig'):
287 namepart = namepart[:-5]
288 splitval = namepart.split('-')
289 logger.debug('determine_from_filename: split name %s into: %s' % (srcfile, splitval))
290
291 ver_re = re.compile('^v?[0-9]')
292
293 pv = None
294 pn = None
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500295 if len(splitval) == 1:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600296 # Try to split the version out if there is no separator (or a .)
297 res = re.match('^([^0-9]+)([0-9.]+.*)$', namepart)
298 if res:
299 if len(res.group(1)) > 1 and len(res.group(2)) > 1:
300 pn = res.group(1).rstrip('.')
301 pv = res.group(2)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500302 else:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600303 pn = namepart
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500304 else:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600305 if splitval[-1] in ['source', 'src']:
306 splitval.pop()
307 if len(splitval) > 2 and re.match('^(alpha|beta|stable|release|rc[0-9]|pre[0-9]|p[0-9]|[0-9]{8})', splitval[-1]) and ver_re.match(splitval[-2]):
308 pv = '-'.join(splitval[-2:])
309 if pv.endswith('-release'):
310 pv = pv[:-8]
311 splitval = splitval[:-2]
312 elif ver_re.match(splitval[-1]):
313 pv = splitval.pop()
314 pn = '-'.join(splitval)
315 if pv and pv.startswith('v'):
316 pv = pv[1:]
317 logger.debug('determine_from_filename: name = "%s" version = "%s"' % (pn, pv))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500318 return (pn, pv)
319
320def determine_from_url(srcuri):
321 """Determine name and version from a URL"""
322 pn = None
323 pv = None
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600324 parseres = urlparse(srcuri.lower().split(';', 1)[0])
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500325 if parseres.path:
326 if 'github.com' in parseres.netloc:
327 res = re.search(r'.*/(.*?)/archive/(.*)-final\.(tar|zip)', parseres.path)
328 if res:
329 pn = res.group(1).strip().replace('_', '-')
330 pv = res.group(2).strip().replace('_', '.')
331 else:
332 res = re.search(r'.*/(.*?)/archive/v?(.*)\.(tar|zip)', parseres.path)
333 if res:
334 pn = res.group(1).strip().replace('_', '-')
335 pv = res.group(2).strip().replace('_', '.')
336 elif 'bitbucket.org' in parseres.netloc:
337 res = re.search(r'.*/(.*?)/get/[a-zA-Z_-]*([0-9][0-9a-zA-Z_.]*)\.(tar|zip)', parseres.path)
338 if res:
339 pn = res.group(1).strip().replace('_', '-')
340 pv = res.group(2).strip().replace('_', '.')
341
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500342 if not pn and not pv and parseres.scheme not in ['git', 'gitsm', 'svn', 'hg']:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500343 srcfile = os.path.basename(parseres.path.rstrip('/'))
344 pn, pv = determine_from_filename(srcfile)
345
346 logger.debug('Determined from source URL: name = "%s", version = "%s"' % (pn, pv))
347 return (pn, pv)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500348
349def supports_srcrev(uri):
350 localdata = bb.data.createCopy(tinfoil.config_data)
351 # This is a bit sad, but if you don't have this set there can be some
352 # odd interactions with the urldata cache which lead to errors
353 localdata.setVar('SRCREV', '${AUTOREV}')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600354 try:
355 fetcher = bb.fetch2.Fetch([uri], localdata)
356 urldata = fetcher.ud
357 for u in urldata:
358 if urldata[u].method.supports_srcrev():
359 return True
360 except bb.fetch2.FetchError as e:
361 logger.debug('FetchError in supports_srcrev: %s' % str(e))
362 # Fall back to basic check
363 if uri.startswith(('git://', 'gitsm://')):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500364 return True
365 return False
366
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500367def reformat_git_uri(uri):
368 '''Convert any http[s]://....git URI into git://...;protocol=http[s]'''
369 checkuri = uri.split(';', 1)[0]
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600370 if checkuri.endswith('.git') or '/git/' in checkuri or re.match('https?://github.com/[^/]+/[^/]+/?$', checkuri):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500371 # Appends scheme if the scheme is missing
372 if not '://' in uri:
373 uri = 'git://' + uri
374 scheme, host, path, user, pswd, parms = bb.fetch2.decodeurl(uri)
375 # Detection mechanism, this is required due to certain URL are formatter with ":" rather than "/"
376 # which causes decodeurl to fail getting the right host and path
377 if len(host.split(':')) > 1:
378 splitslash = host.split(':')
379 host = splitslash[0]
380 path = '/' + splitslash[1] + path
381 #Algorithm:
382 # if user is defined, append protocol=ssh or if a protocol is defined, then honor the user-defined protocol
383 # if no user & password is defined, check for scheme type and append the protocol with the scheme type
384 # finally if protocols or if the url is well-formed, do nothing and rejoin everything back to normal
385 # Need to repackage the arguments for encodeurl, the format is: (scheme, host, path, user, password, OrderedDict([('key', 'value')]))
386 if user:
387 if not 'protocol' in parms:
388 parms.update({('protocol', 'ssh')})
389 elif (scheme == "http" or scheme == 'https' or scheme == 'ssh') and not ('protocol' in parms):
390 parms.update({('protocol', scheme)})
391 # Always append 'git://'
392 fUrl = bb.fetch2.encodeurl(('git', host, path, user, pswd, parms))
393 return fUrl
394 else:
395 return uri
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500396
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600397def is_package(url):
398 '''Check if a URL points to a package'''
399 checkurl = url.split(';', 1)[0]
400 if checkurl.endswith(('.deb', '.ipk', '.rpm', '.srpm')):
401 return True
402 return False
403
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500404def create_recipe(args):
405 import bb.process
406 import tempfile
407 import shutil
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600408 import oe.recipeutils
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500409
410 pkgarch = ""
411 if args.machine:
412 pkgarch = "${MACHINE_ARCH}"
413
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600414 extravalues = {}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500415 checksums = (None, None)
416 tempsrc = ''
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600417 source = args.source
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500418 srcsubdir = ''
419 srcrev = '${AUTOREV}'
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600420
421 if os.path.isfile(source):
422 source = 'file://%s' % os.path.abspath(source)
423
424 if scriptutils.is_src_url(source):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500425 # Fetch a URL
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600426 fetchuri = reformat_git_uri(urldefrag(source)[0])
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500427 if args.binary:
428 # Assume the archive contains the directory structure verbatim
429 # so we need to extract to a subdirectory
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600430 fetchuri += ';subdir=${BP}'
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500431 srcuri = fetchuri
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500432 rev_re = re.compile(';rev=([^;]+)')
433 res = rev_re.search(srcuri)
434 if res:
435 srcrev = res.group(1)
436 srcuri = rev_re.sub('', srcuri)
437 tempsrc = tempfile.mkdtemp(prefix='recipetool-')
438 srctree = tempsrc
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500439 d = bb.data.createCopy(tinfoil.config_data)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500440 if fetchuri.startswith('npm://'):
441 # Check if npm is available
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500442 npm_bindir = check_npm(tinfoil, args.devtool)
443 d.prependVar('PATH', '%s:' % npm_bindir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500444 logger.info('Fetching %s...' % srcuri)
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500445 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500446 checksums = scriptutils.fetch_uri(d, fetchuri, srctree, srcrev)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500447 except bb.fetch2.BBFetchException as e:
448 logger.error(str(e).rstrip())
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500449 sys.exit(1)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500450 dirlist = os.listdir(srctree)
451 if 'git.indirectionsymlink' in dirlist:
452 dirlist.remove('git.indirectionsymlink')
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500453 if len(dirlist) == 1:
454 singleitem = os.path.join(srctree, dirlist[0])
455 if os.path.isdir(singleitem):
456 # We unpacked a single directory, so we should use that
457 srcsubdir = dirlist[0]
458 srctree = os.path.join(srctree, srcsubdir)
459 else:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600460 with open(singleitem, 'r', errors='surrogateescape') as f:
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500461 if '<html' in f.read(100).lower():
462 logger.error('Fetching "%s" returned a single HTML page - check the URL is correct and functional' % fetchuri)
463 sys.exit(1)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600464 if os.path.exists(os.path.join(srctree, '.gitmodules')) and srcuri.startswith('git://'):
465 srcuri = 'gitsm://' + srcuri[6:]
466 logger.info('Fetching submodules...')
467 bb.process.run('git submodule update --init --recursive', cwd=srctree)
468
469 if is_package(fetchuri):
470 tmpfdir = tempfile.mkdtemp(prefix='recipetool-')
471 try:
472 pkgfile = None
473 try:
474 fileuri = fetchuri + ';unpack=0'
475 scriptutils.fetch_uri(tinfoil.config_data, fileuri, tmpfdir, srcrev)
476 for root, _, files in os.walk(tmpfdir):
477 for f in files:
478 pkgfile = os.path.join(root, f)
479 break
480 except bb.fetch2.BBFetchException as e:
481 logger.warn('Second fetch to get metadata failed: %s' % str(e).rstrip())
482
483 if pkgfile:
484 if pkgfile.endswith(('.deb', '.ipk')):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500485 stdout, _ = bb.process.run('ar x %s' % pkgfile, cwd=tmpfdir)
486 stdout, _ = bb.process.run('tar xf control.tar.gz', cwd=tmpfdir)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600487 values = convert_debian(tmpfdir)
488 extravalues.update(values)
489 elif pkgfile.endswith(('.rpm', '.srpm')):
490 stdout, _ = bb.process.run('rpm -qp --xml %s > pkginfo.xml' % pkgfile, cwd=tmpfdir)
491 values = convert_rpm_xml(os.path.join(tmpfdir, 'pkginfo.xml'))
492 extravalues.update(values)
493 finally:
494 shutil.rmtree(tmpfdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500495 else:
496 # Assume we're pointing to an existing source tree
497 if args.extract_to:
498 logger.error('--extract-to cannot be specified if source is a directory')
499 sys.exit(1)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600500 if not os.path.isdir(source):
501 logger.error('Invalid source directory %s' % source)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500502 sys.exit(1)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600503 srctree = source
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500504 srcuri = ''
505 if os.path.exists(os.path.join(srctree, '.git')):
506 # Try to get upstream repo location from origin remote
507 try:
508 stdout, _ = bb.process.run('git remote -v', cwd=srctree, shell=True)
509 except bb.process.ExecutionError as e:
510 stdout = None
511 if stdout:
512 for line in stdout.splitlines():
513 splitline = line.split()
514 if len(splitline) > 1:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600515 if splitline[0] == 'origin' and scriptutils.is_src_url(splitline[1]):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500516 srcuri = reformat_git_uri(splitline[1])
517 srcsubdir = 'git'
518 break
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500519
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500520 if args.src_subdir:
521 srcsubdir = os.path.join(srcsubdir, args.src_subdir)
522 srctree_use = os.path.join(srctree, args.src_subdir)
523 else:
524 srctree_use = srctree
525
526 if args.outfile and os.path.isdir(args.outfile):
527 outfile = None
528 outdir = args.outfile
529 else:
530 outfile = args.outfile
531 outdir = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500532 if outfile and outfile != '-':
533 if os.path.exists(outfile):
534 logger.error('Output file %s already exists' % outfile)
535 sys.exit(1)
536
537 lines_before = []
538 lines_after = []
539
540 lines_before.append('# Recipe created by %s' % os.path.basename(sys.argv[0]))
541 lines_before.append('# This is the basis of a recipe and may need further editing in order to be fully functional.')
542 lines_before.append('# (Feel free to remove these comments when editing.)')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600543 # We need a blank line here so that patch_recipe_lines can rewind before the LICENSE comments
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500544 lines_before.append('')
545
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600546 handled = []
547 licvalues = handle_license_vars(srctree_use, lines_before, handled, extravalues, tinfoil.config_data)
548
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500549 classes = []
550
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500551 # FIXME This is kind of a hack, we probably ought to be using bitbake to do this
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500552 pn = None
553 pv = None
554 if outfile:
555 recipefn = os.path.splitext(os.path.basename(outfile))[0]
556 fnsplit = recipefn.split('_')
557 if len(fnsplit) > 1:
558 pn = fnsplit[0]
559 pv = fnsplit[1]
560 else:
561 pn = recipefn
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500562
563 if args.version:
564 pv = args.version
565
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500566 if args.name:
567 pn = args.name
568 if args.name.endswith('-native'):
569 if args.also_native:
570 logger.error('--also-native cannot be specified for a recipe named *-native (*-native denotes a recipe that is already only for native) - either remove the -native suffix from the name or drop --also-native')
571 sys.exit(1)
572 classes.append('native')
573 elif args.name.startswith('nativesdk-'):
574 if args.also_native:
575 logger.error('--also-native cannot be specified for a recipe named nativesdk-* (nativesdk-* denotes a recipe that is already only for nativesdk)')
576 sys.exit(1)
577 classes.append('nativesdk')
578
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500579 if pv and pv not in 'git svn hg'.split():
580 realpv = pv
581 else:
582 realpv = None
583
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500584 if srcuri and not realpv or not pn:
585 name_pn, name_pv = determine_from_url(srcuri)
586 if name_pn and not pn:
587 pn = name_pn
588 if name_pv and not realpv:
589 realpv = name_pv
590
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500591 if not srcuri:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500592 lines_before.append('# No information for SRC_URI yet (only an external source tree was specified)')
593 lines_before.append('SRC_URI = "%s"' % srcuri)
594 (md5value, sha256value) = checksums
595 if md5value:
596 lines_before.append('SRC_URI[md5sum] = "%s"' % md5value)
597 if sha256value:
598 lines_before.append('SRC_URI[sha256sum] = "%s"' % sha256value)
599 if srcuri and supports_srcrev(srcuri):
600 lines_before.append('')
601 lines_before.append('# Modify these as desired')
602 lines_before.append('PV = "%s+git${SRCPV}"' % (realpv or '1.0'))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600603 if not args.autorev and srcrev == '${AUTOREV}':
604 if os.path.exists(os.path.join(srctree, '.git')):
605 (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree)
606 srcrev = stdout.rstrip()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500607 lines_before.append('SRCREV = "%s"' % srcrev)
608 lines_before.append('')
609
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600610 if srcsubdir and not args.binary:
611 # (for binary packages we explicitly specify subdir= when fetching to
612 # match the default value of S, so we don't need to set it in that case)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500613 lines_before.append('S = "${WORKDIR}/%s"' % srcsubdir)
614 lines_before.append('')
615
616 if pkgarch:
617 lines_after.append('PACKAGE_ARCH = "%s"' % pkgarch)
618 lines_after.append('')
619
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500620 if args.binary:
621 lines_after.append('INSANE_SKIP_${PN} += "already-stripped"')
622 lines_after.append('')
623
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500624 if args.fetch_dev:
625 extravalues['fetchdev'] = True
626 else:
627 extravalues['fetchdev'] = None
628
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500629 # Find all plugins that want to register handlers
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500630 logger.debug('Loading recipe handlers')
631 raw_handlers = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500632 for plugin in plugins:
633 if hasattr(plugin, 'register_recipe_handlers'):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500634 plugin.register_recipe_handlers(raw_handlers)
635 # Sort handlers by priority
636 handlers = []
637 for i, handler in enumerate(raw_handlers):
638 if isinstance(handler, tuple):
639 handlers.append((handler[0], handler[1], i))
640 else:
641 handlers.append((handler, 0, i))
642 handlers.sort(key=lambda item: (item[1], -item[2]), reverse=True)
643 for handler, priority, _ in handlers:
644 logger.debug('Handler: %s (priority %d)' % (handler.__class__.__name__, priority))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500645 setattr(handler, '_devtool', args.devtool)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500646 handlers = [item[0] for item in handlers]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500647
648 # Apply the handlers
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500649 if args.binary:
650 classes.append('bin_package')
651 handled.append('buildsystem')
652
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500653 for handler in handlers:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500654 handler.process(srctree_use, classes, lines_before, lines_after, handled, extravalues)
655
656 extrafiles = extravalues.pop('extrafiles', {})
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600657 extra_pn = extravalues.pop('PN', None)
658 extra_pv = extravalues.pop('PV', None)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500659
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600660 if extra_pv and not realpv:
661 realpv = extra_pv
662 if not validate_pv(realpv):
663 realpv = None
664 else:
665 realpv = realpv.lower().split()[0]
666 if '_' in realpv:
667 realpv = realpv.replace('_', '-')
668 if extra_pn and not pn:
669 pn = extra_pn
670 if pn.startswith('GNU '):
671 pn = pn[4:]
672 if ' ' in pn:
673 # Probably a descriptive identifier rather than a proper name
674 pn = None
675 else:
676 pn = pn.lower()
677 if '_' in pn:
678 pn = pn.replace('_', '-')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500679
680 if not outfile:
681 if not pn:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500682 log_error_cond('Unable to determine short program name from source tree - please specify name with -N/--name or output file name with -o/--outfile', args.devtool)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500683 # devtool looks for this specific exit code, so don't change it
684 sys.exit(15)
685 else:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600686 if srcuri and srcuri.startswith(('gitsm://', 'git://', 'hg://', 'svn://')):
687 suffix = srcuri.split(':', 1)[0]
688 if suffix == 'gitsm':
689 suffix = 'git'
690 outfile = '%s_%s.bb' % (pn, suffix)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500691 elif realpv:
692 outfile = '%s_%s.bb' % (pn, realpv)
693 else:
694 outfile = '%s.bb' % pn
695 if outdir:
696 outfile = os.path.join(outdir, outfile)
697 # We need to check this again
698 if os.path.exists(outfile):
699 logger.error('Output file %s already exists' % outfile)
700 sys.exit(1)
701
702 # Move any extra files the plugins created to a directory next to the recipe
703 if extrafiles:
704 if outfile == '-':
705 extraoutdir = pn
706 else:
707 extraoutdir = os.path.join(os.path.dirname(outfile), pn)
708 bb.utils.mkdirhier(extraoutdir)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600709 for destfn, extrafile in extrafiles.items():
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500710 shutil.move(extrafile, os.path.join(extraoutdir, destfn))
711
712 lines = lines_before
713 lines_before = []
714 skipblank = True
715 for line in lines:
716 if skipblank:
717 skipblank = False
718 if not line:
719 continue
720 if line.startswith('S = '):
721 if realpv and pv not in 'git svn hg'.split():
722 line = line.replace(realpv, '${PV}')
723 if pn:
724 line = line.replace(pn, '${BPN}')
725 if line == 'S = "${WORKDIR}/${BPN}-${PV}"':
726 skipblank = True
727 continue
728 elif line.startswith('SRC_URI = '):
729 if realpv:
730 line = line.replace(realpv, '${PV}')
731 elif line.startswith('PV = '):
732 if realpv:
733 line = re.sub('"[^+]*\+', '"%s+' % realpv, line)
734 lines_before.append(line)
735
736 if args.also_native:
737 lines = lines_after
738 lines_after = []
739 bbclassextend = None
740 for line in lines:
741 if line.startswith('BBCLASSEXTEND ='):
742 splitval = line.split('"')
743 if len(splitval) > 1:
744 bbclassextend = splitval[1].split()
745 if not 'native' in bbclassextend:
746 bbclassextend.insert(0, 'native')
747 line = 'BBCLASSEXTEND = "%s"' % ' '.join(bbclassextend)
748 lines_after.append(line)
749 if not bbclassextend:
750 lines_after.append('BBCLASSEXTEND = "native"')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500751
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500752 postinst = ("postinst", extravalues.pop('postinst', None))
753 postrm = ("postrm", extravalues.pop('postrm', None))
754 preinst = ("preinst", extravalues.pop('preinst', None))
755 prerm = ("prerm", extravalues.pop('prerm', None))
756 funcs = [postinst, postrm, preinst, prerm]
757 for func in funcs:
758 if func[1]:
759 RecipeHandler.genfunction(lines_after, 'pkg_%s_${PN}' % func[0], func[1])
760
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500761 outlines = []
762 outlines.extend(lines_before)
763 if classes:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500764 if outlines[-1] and not outlines[-1].startswith('#'):
765 outlines.append('')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500766 outlines.append('inherit %s' % ' '.join(classes))
767 outlines.append('')
768 outlines.extend(lines_after)
769
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600770 if extravalues:
771 if 'LICENSE' in extravalues and not licvalues:
772 # Don't blow away 'CLOSED' value that comments say we set
773 del extravalues['LICENSE']
774 _, outlines = oe.recipeutils.patch_recipe_lines(outlines, extravalues, trailing_newline=False)
775
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500776 if args.extract_to:
777 scriptutils.git_convert_standalone_clone(srctree)
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500778 if os.path.isdir(args.extract_to):
779 # If the directory exists we'll move the temp dir into it instead of
780 # its contents - of course, we could try to always move its contents
781 # but that is a pain if there are symlinks; the simplest solution is
782 # to just remove it first
783 os.rmdir(args.extract_to)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500784 shutil.move(srctree, args.extract_to)
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500785 if tempsrc == srctree:
786 tempsrc = None
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500787 log_info_cond('Source extracted to %s' % args.extract_to, args.devtool)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500788
789 if outfile == '-':
790 sys.stdout.write('\n'.join(outlines) + '\n')
791 else:
792 with open(outfile, 'w') as f:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600793 lastline = None
794 for line in outlines:
795 if not lastline and not line:
796 # Skip extra blank lines
797 continue
798 f.write('%s\n' % line)
799 lastline = line
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500800 log_info_cond('Recipe %s has been created; further editing may be required to make it fully functional' % outfile, args.devtool)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500801
802 if tempsrc:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600803 if args.keep_temp:
804 logger.info('Preserving temporary directory %s' % tempsrc)
805 else:
806 shutil.rmtree(tempsrc)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500807
808 return 0
809
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600810def handle_license_vars(srctree, lines_before, handled, extravalues, d):
811 licvalues = guess_license(srctree, d)
812 lic_files_chksum = []
813 lic_unknown = []
814 if licvalues:
815 licenses = []
816 for licvalue in licvalues:
817 if not licvalue[0] in licenses:
818 licenses.append(licvalue[0])
819 lic_files_chksum.append('file://%s;md5=%s' % (licvalue[1], licvalue[2]))
820 if licvalue[0] == 'Unknown':
821 lic_unknown.append(licvalue[1])
822 lines_before.append('# WARNING: the following LICENSE and LIC_FILES_CHKSUM values are best guesses - it is')
823 lines_before.append('# your responsibility to verify that the values are complete and correct.')
824 if len(licvalues) > 1:
825 lines_before.append('#')
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500826 lines_before.append('# NOTE: multiple licenses have been detected; they have been separated with &')
827 lines_before.append('# in the LICENSE value for now since it is a reasonable assumption that all')
828 lines_before.append('# of the licenses apply. If instead there is a choice between the multiple')
829 lines_before.append('# licenses then you should change the value to separate the licenses with |')
830 lines_before.append('# instead of &. If there is any doubt, check the accompanying documentation')
831 lines_before.append('# to determine which situation is applicable.')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600832 if lic_unknown:
833 lines_before.append('#')
834 lines_before.append('# The following license files were not able to be identified and are')
835 lines_before.append('# represented as "Unknown" below, you will need to check them yourself:')
836 for licfile in lic_unknown:
837 lines_before.append('# %s' % licfile)
838 lines_before.append('#')
839 else:
840 lines_before.append('# Unable to find any files that looked like license statements. Check the accompanying')
841 lines_before.append('# documentation and source headers and set LICENSE and LIC_FILES_CHKSUM accordingly.')
842 lines_before.append('#')
843 lines_before.append('# NOTE: LICENSE is being set to "CLOSED" to allow you to at least start building - if')
844 lines_before.append('# this is not accurate with respect to the licensing of the software being built (it')
845 lines_before.append('# will not be in most cases) you must specify the correct value before using this')
846 lines_before.append('# recipe for anything other than initial testing/development!')
847 licenses = ['CLOSED']
848 pkg_license = extravalues.pop('LICENSE', None)
849 if pkg_license:
850 if licenses == ['Unknown']:
851 lines_before.append('# NOTE: The following LICENSE value was determined from the original package metadata')
852 licenses = [pkg_license]
853 else:
854 lines_before.append('# NOTE: Original package metadata indicates license is: %s' % pkg_license)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500855 lines_before.append('LICENSE = "%s"' % ' & '.join(licenses))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600856 lines_before.append('LIC_FILES_CHKSUM = "%s"' % ' \\\n '.join(lic_files_chksum))
857 lines_before.append('')
858 handled.append(('license', licvalues))
859 return licvalues
860
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500861def get_license_md5sums(d, static_only=False):
862 import bb.utils
863 md5sums = {}
864 if not static_only:
865 # Gather md5sums of license files in common license dir
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500866 commonlicdir = d.getVar('COMMON_LICENSE_DIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500867 for fn in os.listdir(commonlicdir):
868 md5value = bb.utils.md5_file(os.path.join(commonlicdir, fn))
869 md5sums[md5value] = fn
870 # The following were extracted from common values in various recipes
871 # (double checking the license against the license file itself, not just
872 # the LICENSE value in the recipe)
873 md5sums['94d55d512a9ba36caa9b7df079bae19f'] = 'GPLv2'
874 md5sums['b234ee4d69f5fce4486a80fdaf4a4263'] = 'GPLv2'
875 md5sums['59530bdf33659b29e73d4adb9f9f6552'] = 'GPLv2'
876 md5sums['0636e73ff0215e8d672dc4c32c317bb3'] = 'GPLv2'
877 md5sums['eb723b61539feef013de476e68b5c50a'] = 'GPLv2'
878 md5sums['751419260aa954499f7abaabaa882bbe'] = 'GPLv2'
879 md5sums['393a5ca445f6965873eca0259a17f833'] = 'GPLv2'
880 md5sums['12f884d2ae1ff87c09e5b7ccc2c4ca7e'] = 'GPLv2'
881 md5sums['8ca43cbc842c2336e835926c2166c28b'] = 'GPLv2'
882 md5sums['ebb5c50ab7cab4baeffba14977030c07'] = 'GPLv2'
883 md5sums['c93c0550bd3173f4504b2cbd8991e50b'] = 'GPLv2'
884 md5sums['9ac2e7cff1ddaf48b6eab6028f23ef88'] = 'GPLv2'
885 md5sums['4325afd396febcb659c36b49533135d4'] = 'GPLv2'
886 md5sums['18810669f13b87348459e611d31ab760'] = 'GPLv2'
887 md5sums['d7810fab7487fb0aad327b76f1be7cd7'] = 'GPLv2' # the Linux kernel's COPYING file
888 md5sums['bbb461211a33b134d42ed5ee802b37ff'] = 'LGPLv2.1'
889 md5sums['7fbc338309ac38fefcd64b04bb903e34'] = 'LGPLv2.1'
890 md5sums['4fbd65380cdd255951079008b364516c'] = 'LGPLv2.1'
891 md5sums['2d5025d4aa3495befef8f17206a5b0a1'] = 'LGPLv2.1'
892 md5sums['fbc093901857fcd118f065f900982c24'] = 'LGPLv2.1'
893 md5sums['a6f89e2100d9b6cdffcea4f398e37343'] = 'LGPLv2.1'
894 md5sums['d8045f3b8f929c1cb29a1e3fd737b499'] = 'LGPLv2.1'
895 md5sums['fad9b3332be894bab9bc501572864b29'] = 'LGPLv2.1'
896 md5sums['3bf50002aefd002f49e7bb854063f7e7'] = 'LGPLv2'
897 md5sums['9f604d8a4f8e74f4f5140845a21b6674'] = 'LGPLv2'
898 md5sums['5f30f0716dfdd0d91eb439ebec522ec2'] = 'LGPLv2'
899 md5sums['55ca817ccb7d5b5b66355690e9abc605'] = 'LGPLv2'
900 md5sums['252890d9eee26aab7b432e8b8a616475'] = 'LGPLv2'
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500901 md5sums['3214f080875748938ba060314b4f727d'] = 'LGPLv2'
902 md5sums['db979804f025cf55aabec7129cb671ed'] = 'LGPLv2'
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500903 md5sums['d32239bcb673463ab874e80d47fae504'] = 'GPLv3'
904 md5sums['f27defe1e96c2e1ecd4e0c9be8967949'] = 'GPLv3'
905 md5sums['6a6a8e020838b23406c81b19c1d46df6'] = 'LGPLv3'
906 md5sums['3b83ef96387f14655fc854ddc3c6bd57'] = 'Apache-2.0'
907 md5sums['385c55653886acac3821999a3ccd17b3'] = 'Artistic-1.0 | GPL-2.0' # some perl modules
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500908 md5sums['54c7042be62e169199200bc6477f04d1'] = 'BSD-3-Clause'
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500909 return md5sums
910
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500911def crunch_license(licfile):
912 '''
913 Remove non-material text from a license file and then check
914 its md5sum against a known list. This works well for licenses
915 which contain a copyright statement, but is also a useful way
916 to handle people's insistence upon reformatting the license text
917 slightly (with no material difference to the text of the
918 license).
919 '''
920
921 import oe.utils
922
923 # Note: these are carefully constructed!
924 license_title_re = re.compile('^\(?(#+ *)?(The )?.{1,10} [Ll]icen[sc]e( \(.{1,10}\))?\)?:?$')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600925 license_statement_re = re.compile('^(This (project|software) is( free software)? (released|licen[sc]ed)|(Released|Licen[cs]ed)) under the .{1,10} [Ll]icen[sc]e:?$')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500926 copyright_re = re.compile('^(#+)? *Copyright .*$')
927
928 crunched_md5sums = {}
929 # The following two were gleaned from the "forever" npm package
930 crunched_md5sums['0a97f8e4cbaf889d6fa51f84b89a79f6'] = 'ISC'
931 crunched_md5sums['eecf6429523cbc9693547cf2db790b5c'] = 'MIT'
932 # https://github.com/vasi/pixz/blob/master/LICENSE
933 crunched_md5sums['2f03392b40bbe663597b5bd3cc5ebdb9'] = 'BSD-2-Clause'
934 # https://github.com/waffle-gl/waffle/blob/master/LICENSE.txt
935 crunched_md5sums['e72e5dfef0b1a4ca8a3d26a60587db66'] = 'BSD-2-Clause'
936 # https://github.com/spigwitmer/fakeds1963s/blob/master/LICENSE
937 crunched_md5sums['8be76ac6d191671f347ee4916baa637e'] = 'GPLv2'
938 # https://github.com/datto/dattobd/blob/master/COPYING
939 # http://git.savannah.gnu.org/cgit/freetype/freetype2.git/tree/docs/GPLv2.TXT
940 crunched_md5sums['1d65c5ad4bf6489f85f4812bf08ae73d'] = 'GPLv2'
941 # http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt
942 # http://git.neil.brown.name/?p=mdadm.git;a=blob;f=COPYING;h=d159169d1050894d3ea3b98e1c965c4058208fe1;hb=HEAD
943 crunched_md5sums['fb530f66a7a89ce920f0e912b5b66d4b'] = 'GPLv2'
944 # https://github.com/gkos/nrf24/blob/master/COPYING
945 crunched_md5sums['7b6aaa4daeafdfa6ed5443fd2684581b'] = 'GPLv2'
946 # https://github.com/josch09/resetusb/blob/master/COPYING
947 crunched_md5sums['8b8ac1d631a4d220342e83bcf1a1fbc3'] = 'GPLv3'
948 # https://github.com/FFmpeg/FFmpeg/blob/master/COPYING.LGPLv2.1
949 crunched_md5sums['2ea316ed973ae176e502e2297b574bb3'] = 'LGPLv2.1'
950 # unixODBC-2.3.4 COPYING
951 crunched_md5sums['1daebd9491d1e8426900b4fa5a422814'] = 'LGPLv2.1'
952 # https://github.com/FFmpeg/FFmpeg/blob/master/COPYING.LGPLv3
953 crunched_md5sums['2ebfb3bb49b9a48a075cc1425e7f4129'] = 'LGPLv3'
954 lictext = []
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600955 with open(licfile, 'r', errors='surrogateescape') as f:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500956 for line in f:
957 # Drop opening statements
958 if copyright_re.match(line):
959 continue
960 elif license_title_re.match(line):
961 continue
962 elif license_statement_re.match(line):
963 continue
964 # Squash spaces, and replace smart quotes, double quotes
965 # and backticks with single quotes
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600966 line = oe.utils.squashspaces(line.strip())
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500967 line = line.replace(u"\u2018", "'").replace(u"\u2019", "'").replace(u"\u201c","'").replace(u"\u201d", "'").replace('"', '\'').replace('`', '\'')
968 if line:
969 lictext.append(line)
970
971 m = hashlib.md5()
972 try:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600973 m.update(' '.join(lictext).encode('utf-8'))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500974 md5val = m.hexdigest()
975 except UnicodeEncodeError:
976 md5val = None
977 lictext = ''
978 license = crunched_md5sums.get(md5val, None)
979 return license, md5val, lictext
980
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600981def guess_license(srctree, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500982 import bb
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600983 md5sums = get_license_md5sums(d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500984
985 licenses = []
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500986 licspecs = ['*LICEN[CS]E*', 'COPYING*', '*[Ll]icense*', 'LEGAL*', '[Ll]egal*', '*GPL*', 'README.lic*', 'COPYRIGHT*', '[Cc]opyright*']
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500987 licfiles = []
988 for root, dirs, files in os.walk(srctree):
989 for fn in files:
990 for spec in licspecs:
991 if fnmatch.fnmatch(fn, spec):
992 fullpath = os.path.join(root, fn)
993 if not fullpath in licfiles:
994 licfiles.append(fullpath)
995 for licfile in licfiles:
996 md5value = bb.utils.md5_file(licfile)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500997 license = md5sums.get(md5value, None)
998 if not license:
999 license, crunched_md5, lictext = crunch_license(licfile)
1000 if not license:
1001 license = 'Unknown'
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001002 licenses.append((license, os.path.relpath(licfile, srctree), md5value))
1003
1004 # FIXME should we grab at least one source file with a license header and add that too?
1005
1006 return licenses
1007
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001008def split_pkg_licenses(licvalues, packages, outlines, fallback_licenses=None, pn='${PN}'):
1009 """
1010 Given a list of (license, path, md5sum) as returned by guess_license(),
1011 a dict of package name to path mappings, write out a set of
1012 package-specific LICENSE values.
1013 """
1014 pkglicenses = {pn: []}
1015 for license, licpath, _ in licvalues:
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001016 for pkgname, pkgpath in packages.items():
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001017 if licpath.startswith(pkgpath + '/'):
1018 if pkgname in pkglicenses:
1019 pkglicenses[pkgname].append(license)
1020 else:
1021 pkglicenses[pkgname] = [license]
1022 break
1023 else:
1024 # Accumulate on the main package
1025 pkglicenses[pn].append(license)
1026 outlicenses = {}
1027 for pkgname in packages:
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001028 license = ' '.join(list(set(pkglicenses.get(pkgname, ['Unknown'])))) or 'Unknown'
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001029 if license == 'Unknown' and pkgname in fallback_licenses:
1030 license = fallback_licenses[pkgname]
1031 outlines.append('LICENSE_%s = "%s"' % (pkgname, license))
1032 outlicenses[pkgname] = license.split()
1033 return outlicenses
1034
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001035def read_pkgconfig_provides(d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001036 pkgdatadir = d.getVar('PKGDATA_DIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001037 pkgmap = {}
1038 for fn in glob.glob(os.path.join(pkgdatadir, 'shlibs2', '*.pclist')):
1039 with open(fn, 'r') as f:
1040 for line in f:
1041 pkgmap[os.path.basename(line.rstrip())] = os.path.splitext(os.path.basename(fn))[0]
1042 recipemap = {}
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001043 for pc, pkg in pkgmap.items():
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001044 pkgdatafile = os.path.join(pkgdatadir, 'runtime', pkg)
1045 if os.path.exists(pkgdatafile):
1046 with open(pkgdatafile, 'r') as f:
1047 for line in f:
1048 if line.startswith('PN: '):
1049 recipemap[pc] = line.split(':', 1)[1].strip()
1050 return recipemap
1051
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001052def convert_debian(debpath):
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001053 value_map = {'Package': 'PN',
1054 'Version': 'PV',
1055 'Section': 'SECTION',
1056 'License': 'LICENSE',
1057 'Homepage': 'HOMEPAGE'}
1058
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001059 # FIXME extend this mapping - perhaps use distro_alias.inc?
1060 depmap = {'libz-dev': 'zlib'}
1061
1062 values = {}
1063 depends = []
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001064 with open(os.path.join(debpath, 'control'), 'r', errors='surrogateescape') as f:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001065 indesc = False
1066 for line in f:
1067 if indesc:
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001068 if line.startswith(' '):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001069 if line.startswith(' This package contains'):
1070 indesc = False
1071 else:
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001072 if 'DESCRIPTION' in values:
1073 values['DESCRIPTION'] += ' ' + line.strip()
1074 else:
1075 values['DESCRIPTION'] = line.strip()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001076 else:
1077 indesc = False
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001078 if not indesc:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001079 splitline = line.split(':', 1)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001080 if len(splitline) < 2:
1081 continue
1082 key = splitline[0]
1083 value = splitline[1].strip()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001084 if key == 'Build-Depends':
1085 for dep in value.split(','):
1086 dep = dep.split()[0]
1087 mapped = depmap.get(dep, '')
1088 if mapped:
1089 depends.append(mapped)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001090 elif key == 'Description':
1091 values['SUMMARY'] = value
1092 indesc = True
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001093 else:
1094 varname = value_map.get(key, None)
1095 if varname:
1096 values[varname] = value
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001097 postinst = os.path.join(debpath, 'postinst')
1098 postrm = os.path.join(debpath, 'postrm')
1099 preinst = os.path.join(debpath, 'preinst')
1100 prerm = os.path.join(debpath, 'prerm')
1101 sfiles = [postinst, postrm, preinst, prerm]
1102 for sfile in sfiles:
1103 if os.path.isfile(sfile):
1104 logger.info("Converting %s file to recipe function..." %
1105 os.path.basename(sfile).upper())
1106 content = []
1107 with open(sfile) as f:
1108 for line in f:
1109 if "#!/" in line:
1110 continue
1111 line = line.rstrip("\n")
1112 if line.strip():
1113 content.append(line)
1114 if content:
1115 values[os.path.basename(f.name)] = content
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001116
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001117 #if depends:
1118 # values['DEPENDS'] = ' '.join(depends)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001119
1120 return values
1121
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001122def convert_rpm_xml(xmlfile):
1123 '''Converts the output from rpm -qp --xml to a set of variable values'''
1124 import xml.etree.ElementTree as ElementTree
1125 rpmtag_map = {'Name': 'PN',
1126 'Version': 'PV',
1127 'Summary': 'SUMMARY',
1128 'Description': 'DESCRIPTION',
1129 'License': 'LICENSE',
1130 'Url': 'HOMEPAGE'}
1131
1132 values = {}
1133 tree = ElementTree.parse(xmlfile)
1134 root = tree.getroot()
1135 for child in root:
1136 if child.tag == 'rpmTag':
1137 name = child.attrib.get('name', None)
1138 if name:
1139 varname = rpmtag_map.get(name, None)
1140 if varname:
1141 values[varname] = child[0].text
1142 return values
1143
1144
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001145def check_npm(tinfoil, debugonly=False):
1146 try:
1147 rd = tinfoil.parse_recipe('nodejs-native')
1148 except bb.providers.NoProvider:
1149 # We still conditionally show the message and exit with the special
1150 # return code, otherwise we can't show the proper message for eSDK
1151 # users
1152 log_error_cond('nodejs-native is required for npm but is not available - you will likely need to add a layer that provides nodejs', debugonly)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001153 sys.exit(14)
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001154 bindir = rd.getVar('STAGING_BINDIR_NATIVE')
1155 npmpath = os.path.join(bindir, 'npm')
1156 if not os.path.exists(npmpath):
1157 log_error_cond('npm required to process specified source, but npm is not available - you need to run bitbake -c addto_recipe_sysroot nodejs-native first', debugonly)
1158 sys.exit(14)
1159 return bindir
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001160
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001161def register_commands(subparsers):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001162 parser_create = subparsers.add_parser('create',
1163 help='Create a new recipe',
1164 description='Creates a new recipe from a source tree')
1165 parser_create.add_argument('source', help='Path or URL to source')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001166 parser_create.add_argument('-o', '--outfile', help='Specify filename for recipe to create')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001167 parser_create.add_argument('-m', '--machine', help='Make recipe machine-specific as opposed to architecture-specific', action='store_true')
1168 parser_create.add_argument('-x', '--extract-to', metavar='EXTRACTPATH', help='Assuming source is a URL, fetch it and extract it to the directory specified as %(metavar)s')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001169 parser_create.add_argument('-N', '--name', help='Name to use within recipe (PN)')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001170 parser_create.add_argument('-V', '--version', help='Version to use within recipe (PV)')
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001171 parser_create.add_argument('-b', '--binary', help='Treat the source tree as something that should be installed verbatim (no compilation, same directory structure)', action='store_true')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001172 parser_create.add_argument('--also-native', help='Also add native variant (i.e. support building recipe for the build host as well as the target machine)', action='store_true')
1173 parser_create.add_argument('--src-subdir', help='Specify subdirectory within source tree to use', metavar='SUBDIR')
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001174 parser_create.add_argument('-a', '--autorev', help='When fetching from a git repository, set SRCREV in the recipe to a floating revision instead of fixed', action="store_true")
1175 parser_create.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001176 parser_create.add_argument('--fetch-dev', action="store_true", help='For npm, also fetch devDependencies')
1177 parser_create.add_argument('--devtool', action="store_true", help=argparse.SUPPRESS)
1178 # FIXME I really hate having to set parserecipes for this, but given we may need
1179 # to call into npm (and we don't know in advance if we will or not) and in order
1180 # to do so we need to know npm's recipe sysroot path, there's not much alternative
1181 parser_create.set_defaults(func=create_recipe, parserecipes=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001182