blob: 69c8bb77a045b3e25577780f926ced9e7229e96f [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# Recipe creation tool - append plugin
2#
3# Copyright (C) 2015 Intel Corporation
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License version 2 as
7# published by the Free Software Foundation.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License along
15# with this program; if not, write to the Free Software Foundation, Inc.,
16# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17
18import sys
19import os
20import argparse
21import glob
22import fnmatch
23import re
24import subprocess
25import logging
26import stat
27import shutil
28import scriptutils
29import errno
30from collections import defaultdict
31
32logger = logging.getLogger('recipetool')
33
34tinfoil = None
35
36def tinfoil_init(instance):
37 global tinfoil
38 tinfoil = instance
39
40
41# FIXME guessing when we don't have pkgdata?
42# FIXME mode to create patch rather than directly substitute
43
44class InvalidTargetFileError(Exception):
45 pass
46
47def find_target_file(targetpath, d, pkglist=None):
48 """Find the recipe installing the specified target path, optionally limited to a select list of packages"""
49 import json
50
Brad Bishop6e60e8b2018-02-01 10:27:11 -050051 pkgdata_dir = d.getVar('PKGDATA_DIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -050052
53 # The mix between /etc and ${sysconfdir} here may look odd, but it is just
54 # being consistent with usage elsewhere
55 invalidtargets = {'${sysconfdir}/version': '${sysconfdir}/version is written out at image creation time',
56 '/etc/timestamp': '/etc/timestamp is written out at image creation time',
57 '/dev/*': '/dev is handled by udev (or equivalent) and the kernel (devtmpfs)',
58 '/etc/passwd': '/etc/passwd should be managed through the useradd and extrausers classes',
59 '/etc/group': '/etc/group should be managed through the useradd and extrausers classes',
60 '/etc/shadow': '/etc/shadow should be managed through the useradd and extrausers classes',
61 '/etc/gshadow': '/etc/gshadow should be managed through the useradd and extrausers classes',
62 '${sysconfdir}/hostname': '${sysconfdir}/hostname contents should be set by setting hostname_pn-base-files = "value" in configuration',}
63
Patrick Williamsc0f7c042017-02-23 20:41:17 -060064 for pthspec, message in invalidtargets.items():
Patrick Williamsc124f4f2015-09-15 14:41:29 -050065 if fnmatch.fnmatchcase(targetpath, d.expand(pthspec)):
66 raise InvalidTargetFileError(d.expand(message))
67
68 targetpath_re = re.compile(r'\s+(\$D)?%s(\s|$)' % targetpath)
69
70 recipes = defaultdict(list)
71 for root, dirs, files in os.walk(os.path.join(pkgdata_dir, 'runtime')):
72 if pkglist:
73 filelist = pkglist
74 else:
75 filelist = files
76 for fn in filelist:
77 pkgdatafile = os.path.join(root, fn)
78 if pkglist and not os.path.exists(pkgdatafile):
79 continue
80 with open(pkgdatafile, 'r') as f:
81 pn = ''
82 # This does assume that PN comes before other values, but that's a fairly safe assumption
83 for line in f:
84 if line.startswith('PN:'):
85 pn = line.split(':', 1)[1].strip()
86 elif line.startswith('FILES_INFO:'):
87 val = line.split(':', 1)[1].strip()
88 dictval = json.loads(val)
89 for fullpth in dictval.keys():
90 if fnmatch.fnmatchcase(fullpth, targetpath):
91 recipes[targetpath].append(pn)
92 elif line.startswith('pkg_preinst_') or line.startswith('pkg_postinst_'):
Patrick Williamsc0f7c042017-02-23 20:41:17 -060093 scriptval = line.split(':', 1)[1].strip().encode('utf-8').decode('unicode_escape')
Patrick Williamsc124f4f2015-09-15 14:41:29 -050094 if 'update-alternatives --install %s ' % targetpath in scriptval:
95 recipes[targetpath].append('?%s' % pn)
96 elif targetpath_re.search(scriptval):
97 recipes[targetpath].append('!%s' % pn)
98 return recipes
99
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500100def _parse_recipe(pn, tinfoil):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500101 try:
102 rd = tinfoil.parse_recipe(pn)
103 except bb.providers.NoProvider as e:
104 logger.error(str(e))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500105 return None
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500106 return rd
107
108def determine_file_source(targetpath, rd):
109 """Assuming we know a file came from a specific recipe, figure out exactly where it came from"""
110 import oe.recipeutils
111
112 # See if it's in do_install for the recipe
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500113 workdir = rd.getVar('WORKDIR')
114 src_uri = rd.getVar('SRC_URI')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500115 srcfile = ''
116 modpatches = []
117 elements = check_do_install(rd, targetpath)
118 if elements:
119 logger.debug('do_install line:\n%s' % ' '.join(elements))
120 srcpath = get_source_path(elements)
121 logger.debug('source path: %s' % srcpath)
122 if not srcpath.startswith('/'):
123 # Handle non-absolute path
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500124 srcpath = os.path.abspath(os.path.join(rd.getVarFlag('do_install', 'dirs').split()[-1], srcpath))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500125 if srcpath.startswith(workdir):
126 # OK, now we have the source file name, look for it in SRC_URI
127 workdirfile = os.path.relpath(srcpath, workdir)
128 # FIXME this is where we ought to have some code in the fetcher, because this is naive
129 for item in src_uri.split():
130 localpath = bb.fetch2.localpath(item, rd)
131 # Source path specified in do_install might be a glob
132 if fnmatch.fnmatch(os.path.basename(localpath), workdirfile):
133 srcfile = 'file://%s' % localpath
134 elif '/' in workdirfile:
135 if item == 'file://%s' % workdirfile:
136 srcfile = 'file://%s' % localpath
137
138 # Check patches
139 srcpatches = []
140 patchedfiles = oe.recipeutils.get_recipe_patched_files(rd)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600141 for patch, filelist in patchedfiles.items():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500142 for fileitem in filelist:
143 if fileitem[0] == srcpath:
144 srcpatches.append((patch, fileitem[1]))
145 if srcpatches:
146 addpatch = None
147 for patch in srcpatches:
148 if patch[1] == 'A':
149 addpatch = patch[0]
150 else:
151 modpatches.append(patch[0])
152 if addpatch:
153 srcfile = 'patch://%s' % addpatch
154
155 return (srcfile, elements, modpatches)
156
157def get_source_path(cmdelements):
158 """Find the source path specified within a command"""
159 command = cmdelements[0]
160 if command in ['install', 'cp']:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600161 helptext = subprocess.check_output('LC_ALL=C %s --help' % command, shell=True).decode('utf-8')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500162 argopts = ''
163 argopt_line_re = re.compile('^-([a-zA-Z0-9]), --[a-z-]+=')
164 for line in helptext.splitlines():
165 line = line.lstrip()
166 res = argopt_line_re.search(line)
167 if res:
168 argopts += res.group(1)
169 if not argopts:
170 # Fallback
171 if command == 'install':
172 argopts = 'gmoSt'
173 elif command == 'cp':
174 argopts = 't'
175 else:
176 raise Exception('No fallback arguments for command %s' % command)
177
178 skipnext = False
179 for elem in cmdelements[1:-1]:
180 if elem.startswith('-'):
181 if len(elem) > 1 and elem[1] in argopts:
182 skipnext = True
183 continue
184 if skipnext:
185 skipnext = False
186 continue
187 return elem
188 else:
189 raise Exception('get_source_path: no handling for command "%s"')
190
191def get_func_deps(func, d):
192 """Find the function dependencies of a shell function"""
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500193 deps = bb.codeparser.ShellParser(func, logger).parse_shell(d.getVar(func))
194 deps |= set((d.getVarFlag(func, "vardeps") or "").split())
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500195 funcdeps = []
196 for dep in deps:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500197 if d.getVarFlag(dep, 'func'):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500198 funcdeps.append(dep)
199 return funcdeps
200
201def check_do_install(rd, targetpath):
202 """Look at do_install for a command that installs/copies the specified target path"""
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500203 instpath = os.path.abspath(os.path.join(rd.getVar('D'), targetpath.lstrip('/')))
204 do_install = rd.getVar('do_install')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500205 # Handle where do_install calls other functions (somewhat crudely, but good enough for this purpose)
206 deps = get_func_deps('do_install', rd)
207 for dep in deps:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500208 do_install = do_install.replace(dep, rd.getVar(dep))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500209
210 # Look backwards through do_install as we want to catch where a later line (perhaps
211 # from a bbappend) is writing over the top
212 for line in reversed(do_install.splitlines()):
213 line = line.strip()
214 if (line.startswith('install ') and ' -m' in line) or line.startswith('cp '):
215 elements = line.split()
216 destpath = os.path.abspath(elements[-1])
217 if destpath == instpath:
218 return elements
219 elif destpath.rstrip('/') == os.path.dirname(instpath):
220 # FIXME this doesn't take recursive copy into account; unsure if it's practical to do so
221 srcpath = get_source_path(elements)
222 if fnmatch.fnmatchcase(os.path.basename(instpath), os.path.basename(srcpath)):
223 return elements
224 return None
225
226
227def appendfile(args):
228 import oe.recipeutils
229
230 stdout = ''
231 try:
232 (stdout, _) = bb.process.run('LANG=C file -b %s' % args.newfile, shell=True)
233 if 'cannot open' in stdout:
234 raise bb.process.ExecutionError(stdout)
235 except bb.process.ExecutionError as err:
236 logger.debug('file command returned error: %s' % err)
237 stdout = ''
238 if stdout:
239 logger.debug('file command output: %s' % stdout.rstrip())
240 if ('executable' in stdout and not 'shell script' in stdout) or 'shared object' in stdout:
241 logger.warn('This file looks like it is a binary or otherwise the output of compilation. If it is, you should consider building it properly instead of substituting a binary file directly.')
242
243 if args.recipe:
244 recipes = {args.targetpath: [args.recipe],}
245 else:
246 try:
247 recipes = find_target_file(args.targetpath, tinfoil.config_data)
248 except InvalidTargetFileError as e:
249 logger.error('%s cannot be handled by this tool: %s' % (args.targetpath, e))
250 return 1
251 if not recipes:
252 logger.error('Unable to find any package producing path %s - this may be because the recipe packaging it has not been built yet' % args.targetpath)
253 return 1
254
255 alternative_pns = []
256 postinst_pns = []
257
258 selectpn = None
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600259 for targetpath, pnlist in recipes.items():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500260 for pn in pnlist:
261 if pn.startswith('?'):
262 alternative_pns.append(pn[1:])
263 elif pn.startswith('!'):
264 postinst_pns.append(pn[1:])
265 elif selectpn:
266 # hit here with multilibs
267 continue
268 else:
269 selectpn = pn
270
271 if not selectpn and len(alternative_pns) == 1:
272 selectpn = alternative_pns[0]
273 logger.error('File %s is an alternative possibly provided by recipe %s but seemingly no other, selecting it by default - you should double check other recipes' % (args.targetpath, selectpn))
274
275 if selectpn:
276 logger.debug('Selecting recipe %s for file %s' % (selectpn, args.targetpath))
277 if postinst_pns:
278 logger.warn('%s be modified by postinstall scripts for the following recipes:\n %s\nThis may or may not be an issue depending on what modifications these postinstall scripts make.' % (args.targetpath, '\n '.join(postinst_pns)))
279 rd = _parse_recipe(selectpn, tinfoil)
280 if not rd:
281 # Error message already shown
282 return 1
283 sourcefile, instelements, modpatches = determine_file_source(args.targetpath, rd)
284 sourcepath = None
285 if sourcefile:
286 sourcetype, sourcepath = sourcefile.split('://', 1)
287 logger.debug('Original source file is %s (%s)' % (sourcepath, sourcetype))
288 if sourcetype == 'patch':
289 logger.warn('File %s is added by the patch %s - you may need to remove or replace this patch in order to replace the file.' % (args.targetpath, sourcepath))
290 sourcepath = None
291 else:
292 logger.debug('Unable to determine source file, proceeding anyway')
293 if modpatches:
294 logger.warn('File %s is modified by the following patches:\n %s' % (args.targetpath, '\n '.join(modpatches)))
295
296 if instelements and sourcepath:
297 install = None
298 else:
299 # Auto-determine permissions
300 # Check destination
301 binpaths = '${bindir}:${sbindir}:${base_bindir}:${base_sbindir}:${libexecdir}:${sysconfdir}/init.d'
302 perms = '0644'
303 if os.path.abspath(os.path.dirname(args.targetpath)) in rd.expand(binpaths).split(':'):
304 # File is going into a directory normally reserved for executables, so it should be executable
305 perms = '0755'
306 else:
307 # Check source
308 st = os.stat(args.newfile)
309 if st.st_mode & stat.S_IXUSR:
310 perms = '0755'
311 install = {args.newfile: (args.targetpath, perms)}
312 oe.recipeutils.bbappend_recipe(rd, args.destlayer, {args.newfile: sourcepath}, install, wildcardver=args.wildcard_version, machine=args.machine)
313 return 0
314 else:
315 if alternative_pns:
316 logger.error('File %s is an alternative possibly provided by the following recipes:\n %s\nPlease select recipe with -r/--recipe' % (targetpath, '\n '.join(alternative_pns)))
317 elif postinst_pns:
318 logger.error('File %s may be written out in a pre/postinstall script of the following recipes:\n %s\nPlease select recipe with -r/--recipe' % (targetpath, '\n '.join(postinst_pns)))
319 return 3
320
321
322def appendsrc(args, files, rd, extralines=None):
323 import oe.recipeutils
324
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500325 srcdir = rd.getVar('S')
326 workdir = rd.getVar('WORKDIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500327
328 import bb.fetch
329 simplified = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500330 src_uri = rd.getVar('SRC_URI').split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500331 for uri in src_uri:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500332 if uri.endswith(';'):
333 uri = uri[:-1]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500334 simple_uri = bb.fetch.URI(uri)
335 simple_uri.params = {}
336 simplified[str(simple_uri)] = uri
337
338 copyfiles = {}
339 extralines = extralines or []
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600340 for newfile, srcfile in files.items():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500341 src_destdir = os.path.dirname(srcfile)
342 if not args.use_workdir:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500343 if rd.getVar('S') == rd.getVar('STAGING_KERNEL_DIR'):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500344 srcdir = os.path.join(workdir, 'git')
345 if not bb.data.inherits_class('kernel-yocto', rd):
346 logger.warn('S == STAGING_KERNEL_DIR and non-kernel-yocto, unable to determine path to srcdir, defaulting to ${WORKDIR}/git')
347 src_destdir = os.path.join(os.path.relpath(srcdir, workdir), src_destdir)
348 src_destdir = os.path.normpath(src_destdir)
349
350 source_uri = 'file://{0}'.format(os.path.basename(srcfile))
351 if src_destdir and src_destdir != '.':
352 source_uri += ';subdir={0}'.format(src_destdir)
353
354 simple = bb.fetch.URI(source_uri)
355 simple.params = {}
356 simple_str = str(simple)
357 if simple_str in simplified:
358 existing = simplified[simple_str]
359 if source_uri != existing:
360 logger.warn('{0!r} is already in SRC_URI, with different parameters: {1!r}, not adding'.format(source_uri, existing))
361 else:
362 logger.warn('{0!r} is already in SRC_URI, not adding'.format(source_uri))
363 else:
364 extralines.append('SRC_URI += {0}'.format(source_uri))
365 copyfiles[newfile] = srcfile
366
367 oe.recipeutils.bbappend_recipe(rd, args.destlayer, copyfiles, None, wildcardver=args.wildcard_version, machine=args.machine, extralines=extralines)
368
369
370def appendsrcfiles(parser, args):
371 recipedata = _parse_recipe(args.recipe, tinfoil)
372 if not recipedata:
373 parser.error('RECIPE must be a valid recipe name')
374
375 files = dict((f, os.path.join(args.destdir, os.path.basename(f)))
376 for f in args.files)
377 return appendsrc(args, files, recipedata)
378
379
380def appendsrcfile(parser, args):
381 recipedata = _parse_recipe(args.recipe, tinfoil)
382 if not recipedata:
383 parser.error('RECIPE must be a valid recipe name')
384
385 if not args.destfile:
386 args.destfile = os.path.basename(args.file)
387 elif args.destfile.endswith('/'):
388 args.destfile = os.path.join(args.destfile, os.path.basename(args.file))
389
390 return appendsrc(args, {args.file: args.destfile}, recipedata)
391
392
393def layer(layerpath):
394 if not os.path.exists(os.path.join(layerpath, 'conf', 'layer.conf')):
395 raise argparse.ArgumentTypeError('{0!r} must be a path to a valid layer'.format(layerpath))
396 return layerpath
397
398
399def existing_path(filepath):
400 if not os.path.exists(filepath):
401 raise argparse.ArgumentTypeError('{0!r} must be an existing path'.format(filepath))
402 return filepath
403
404
405def existing_file(filepath):
406 filepath = existing_path(filepath)
407 if os.path.isdir(filepath):
408 raise argparse.ArgumentTypeError('{0!r} must be a file, not a directory'.format(filepath))
409 return filepath
410
411
412def destination_path(destpath):
413 if os.path.isabs(destpath):
414 raise argparse.ArgumentTypeError('{0!r} must be a relative path, not absolute'.format(destpath))
415 return destpath
416
417
418def target_path(targetpath):
419 if not os.path.isabs(targetpath):
420 raise argparse.ArgumentTypeError('{0!r} must be an absolute path, not relative'.format(targetpath))
421 return targetpath
422
423
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500424def register_commands(subparsers):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500425 common = argparse.ArgumentParser(add_help=False)
426 common.add_argument('-m', '--machine', help='Make bbappend changes specific to a machine only', metavar='MACHINE')
427 common.add_argument('-w', '--wildcard-version', help='Use wildcard to make the bbappend apply to any recipe version', action='store_true')
428 common.add_argument('destlayer', metavar='DESTLAYER', help='Base directory of the destination layer to write the bbappend to', type=layer)
429
430 parser_appendfile = subparsers.add_parser('appendfile',
431 parents=[common],
432 help='Create/update a bbappend to replace a target file',
433 description='Creates a bbappend (or updates an existing one) to replace the specified file that appears in the target system, determining the recipe that packages the file and the required path and name for the bbappend automatically. Note that the ability to determine the recipe packaging a particular file depends upon the recipe\'s do_packagedata task having already run prior to running this command (which it will have when the recipe has been built successfully, which in turn will have happened if one or more of the recipe\'s packages is included in an image that has been built successfully).')
434 parser_appendfile.add_argument('targetpath', help='Path to the file to be replaced (as it would appear within the target image, e.g. /etc/motd)', type=target_path)
435 parser_appendfile.add_argument('newfile', help='Custom file to replace the target file with', type=existing_file)
436 parser_appendfile.add_argument('-r', '--recipe', help='Override recipe to apply to (default is to find which recipe already packages the file)')
437 parser_appendfile.set_defaults(func=appendfile, parserecipes=True)
438
439 common_src = argparse.ArgumentParser(add_help=False, parents=[common])
440 common_src.add_argument('-W', '--workdir', help='Unpack file into WORKDIR rather than S', dest='use_workdir', action='store_true')
441 common_src.add_argument('recipe', metavar='RECIPE', help='Override recipe to apply to')
442
443 parser = subparsers.add_parser('appendsrcfiles',
444 parents=[common_src],
445 help='Create/update a bbappend to add or replace source files',
446 description='Creates a bbappend (or updates an existing one) to add or replace the specified file in the recipe sources, either those in WORKDIR or those in the source tree. This command lets you specify multiple files with a destination directory, so cannot specify the destination filename. See the `appendsrcfile` command for the other behavior.')
447 parser.add_argument('-D', '--destdir', help='Destination directory (relative to S or WORKDIR, defaults to ".")', default='', type=destination_path)
448 parser.add_argument('files', nargs='+', metavar='FILE', help='File(s) to be added to the recipe sources (WORKDIR or S)', type=existing_path)
449 parser.set_defaults(func=lambda a: appendsrcfiles(parser, a), parserecipes=True)
450
451 parser = subparsers.add_parser('appendsrcfile',
452 parents=[common_src],
453 help='Create/update a bbappend to add or replace a source file',
454 description='Creates a bbappend (or updates an existing one) to add or replace the specified files in the recipe sources, either those in WORKDIR or those in the source tree. This command lets you specify the destination filename, not just destination directory, but only works for one file. See the `appendsrcfiles` command for the other behavior.')
455 parser.add_argument('file', metavar='FILE', help='File to be added to the recipe sources (WORKDIR or S)', type=existing_path)
456 parser.add_argument('destfile', metavar='DESTFILE', nargs='?', help='Destination path (relative to S or WORKDIR, optional)', type=destination_path)
457 parser.set_defaults(func=lambda a: appendsrcfile(parser, a), parserecipes=True)