blob: 5f629c07d8e9c81f2c0adbb29b18c09f95bd1925 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# Recipe creation tool - append plugin
2#
3# Copyright (C) 2015 Intel Corporation
4#
Brad Bishopc342db32019-05-15 21:57:59 -04005# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -05006#
Patrick Williamsc124f4f2015-09-15 14:41:29 -05007
8import sys
9import os
10import argparse
11import glob
12import fnmatch
13import re
14import subprocess
15import logging
16import stat
17import shutil
18import scriptutils
19import errno
20from collections import defaultdict
21
22logger = logging.getLogger('recipetool')
23
24tinfoil = None
25
26def tinfoil_init(instance):
27 global tinfoil
28 tinfoil = instance
29
30
31# FIXME guessing when we don't have pkgdata?
32# FIXME mode to create patch rather than directly substitute
33
34class InvalidTargetFileError(Exception):
35 pass
36
37def find_target_file(targetpath, d, pkglist=None):
38 """Find the recipe installing the specified target path, optionally limited to a select list of packages"""
39 import json
40
Brad Bishop6e60e8b2018-02-01 10:27:11 -050041 pkgdata_dir = d.getVar('PKGDATA_DIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -050042
43 # The mix between /etc and ${sysconfdir} here may look odd, but it is just
44 # being consistent with usage elsewhere
45 invalidtargets = {'${sysconfdir}/version': '${sysconfdir}/version is written out at image creation time',
46 '/etc/timestamp': '/etc/timestamp is written out at image creation time',
47 '/dev/*': '/dev is handled by udev (or equivalent) and the kernel (devtmpfs)',
48 '/etc/passwd': '/etc/passwd should be managed through the useradd and extrausers classes',
49 '/etc/group': '/etc/group should be managed through the useradd and extrausers classes',
50 '/etc/shadow': '/etc/shadow should be managed through the useradd and extrausers classes',
51 '/etc/gshadow': '/etc/gshadow should be managed through the useradd and extrausers classes',
Patrick Williams213cb262021-08-07 19:21:33 -050052 '${sysconfdir}/hostname': '${sysconfdir}/hostname contents should be set by setting hostname:pn-base-files = "value" in configuration',}
Patrick Williamsc124f4f2015-09-15 14:41:29 -050053
Patrick Williamsc0f7c042017-02-23 20:41:17 -060054 for pthspec, message in invalidtargets.items():
Patrick Williamsc124f4f2015-09-15 14:41:29 -050055 if fnmatch.fnmatchcase(targetpath, d.expand(pthspec)):
56 raise InvalidTargetFileError(d.expand(message))
57
58 targetpath_re = re.compile(r'\s+(\$D)?%s(\s|$)' % targetpath)
59
60 recipes = defaultdict(list)
61 for root, dirs, files in os.walk(os.path.join(pkgdata_dir, 'runtime')):
62 if pkglist:
63 filelist = pkglist
64 else:
65 filelist = files
66 for fn in filelist:
67 pkgdatafile = os.path.join(root, fn)
68 if pkglist and not os.path.exists(pkgdatafile):
69 continue
70 with open(pkgdatafile, 'r') as f:
71 pn = ''
72 # This does assume that PN comes before other values, but that's a fairly safe assumption
73 for line in f:
74 if line.startswith('PN:'):
75 pn = line.split(':', 1)[1].strip()
76 elif line.startswith('FILES_INFO:'):
77 val = line.split(':', 1)[1].strip()
78 dictval = json.loads(val)
79 for fullpth in dictval.keys():
80 if fnmatch.fnmatchcase(fullpth, targetpath):
81 recipes[targetpath].append(pn)
Patrick Williams213cb262021-08-07 19:21:33 -050082 elif line.startswith('pkg_preinst:') or line.startswith('pkg_postinst:'):
Patrick Williamsc0f7c042017-02-23 20:41:17 -060083 scriptval = line.split(':', 1)[1].strip().encode('utf-8').decode('unicode_escape')
Patrick Williamsc124f4f2015-09-15 14:41:29 -050084 if 'update-alternatives --install %s ' % targetpath in scriptval:
85 recipes[targetpath].append('?%s' % pn)
86 elif targetpath_re.search(scriptval):
87 recipes[targetpath].append('!%s' % pn)
88 return recipes
89
Patrick Williamsc124f4f2015-09-15 14:41:29 -050090def _parse_recipe(pn, tinfoil):
Brad Bishop6e60e8b2018-02-01 10:27:11 -050091 try:
92 rd = tinfoil.parse_recipe(pn)
93 except bb.providers.NoProvider as e:
94 logger.error(str(e))
Patrick Williamsc124f4f2015-09-15 14:41:29 -050095 return None
Patrick Williamsc124f4f2015-09-15 14:41:29 -050096 return rd
97
98def determine_file_source(targetpath, rd):
99 """Assuming we know a file came from a specific recipe, figure out exactly where it came from"""
100 import oe.recipeutils
101
102 # See if it's in do_install for the recipe
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500103 workdir = rd.getVar('WORKDIR')
104 src_uri = rd.getVar('SRC_URI')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500105 srcfile = ''
106 modpatches = []
107 elements = check_do_install(rd, targetpath)
108 if elements:
109 logger.debug('do_install line:\n%s' % ' '.join(elements))
110 srcpath = get_source_path(elements)
111 logger.debug('source path: %s' % srcpath)
112 if not srcpath.startswith('/'):
113 # Handle non-absolute path
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500114 srcpath = os.path.abspath(os.path.join(rd.getVarFlag('do_install', 'dirs').split()[-1], srcpath))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500115 if srcpath.startswith(workdir):
116 # OK, now we have the source file name, look for it in SRC_URI
117 workdirfile = os.path.relpath(srcpath, workdir)
118 # FIXME this is where we ought to have some code in the fetcher, because this is naive
119 for item in src_uri.split():
120 localpath = bb.fetch2.localpath(item, rd)
121 # Source path specified in do_install might be a glob
122 if fnmatch.fnmatch(os.path.basename(localpath), workdirfile):
123 srcfile = 'file://%s' % localpath
124 elif '/' in workdirfile:
125 if item == 'file://%s' % workdirfile:
126 srcfile = 'file://%s' % localpath
127
128 # Check patches
129 srcpatches = []
130 patchedfiles = oe.recipeutils.get_recipe_patched_files(rd)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600131 for patch, filelist in patchedfiles.items():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500132 for fileitem in filelist:
133 if fileitem[0] == srcpath:
134 srcpatches.append((patch, fileitem[1]))
135 if srcpatches:
136 addpatch = None
137 for patch in srcpatches:
138 if patch[1] == 'A':
139 addpatch = patch[0]
140 else:
141 modpatches.append(patch[0])
142 if addpatch:
143 srcfile = 'patch://%s' % addpatch
144
145 return (srcfile, elements, modpatches)
146
147def get_source_path(cmdelements):
148 """Find the source path specified within a command"""
149 command = cmdelements[0]
150 if command in ['install', 'cp']:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600151 helptext = subprocess.check_output('LC_ALL=C %s --help' % command, shell=True).decode('utf-8')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500152 argopts = ''
153 argopt_line_re = re.compile('^-([a-zA-Z0-9]), --[a-z-]+=')
154 for line in helptext.splitlines():
155 line = line.lstrip()
156 res = argopt_line_re.search(line)
157 if res:
158 argopts += res.group(1)
159 if not argopts:
160 # Fallback
161 if command == 'install':
162 argopts = 'gmoSt'
163 elif command == 'cp':
164 argopts = 't'
165 else:
166 raise Exception('No fallback arguments for command %s' % command)
167
168 skipnext = False
169 for elem in cmdelements[1:-1]:
170 if elem.startswith('-'):
171 if len(elem) > 1 and elem[1] in argopts:
172 skipnext = True
173 continue
174 if skipnext:
175 skipnext = False
176 continue
177 return elem
178 else:
179 raise Exception('get_source_path: no handling for command "%s"')
180
181def get_func_deps(func, d):
182 """Find the function dependencies of a shell function"""
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500183 deps = bb.codeparser.ShellParser(func, logger).parse_shell(d.getVar(func))
184 deps |= set((d.getVarFlag(func, "vardeps") or "").split())
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500185 funcdeps = []
186 for dep in deps:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500187 if d.getVarFlag(dep, 'func'):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500188 funcdeps.append(dep)
189 return funcdeps
190
191def check_do_install(rd, targetpath):
192 """Look at do_install for a command that installs/copies the specified target path"""
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500193 instpath = os.path.abspath(os.path.join(rd.getVar('D'), targetpath.lstrip('/')))
194 do_install = rd.getVar('do_install')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500195 # Handle where do_install calls other functions (somewhat crudely, but good enough for this purpose)
196 deps = get_func_deps('do_install', rd)
197 for dep in deps:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500198 do_install = do_install.replace(dep, rd.getVar(dep))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500199
200 # Look backwards through do_install as we want to catch where a later line (perhaps
201 # from a bbappend) is writing over the top
202 for line in reversed(do_install.splitlines()):
203 line = line.strip()
204 if (line.startswith('install ') and ' -m' in line) or line.startswith('cp '):
205 elements = line.split()
206 destpath = os.path.abspath(elements[-1])
207 if destpath == instpath:
208 return elements
209 elif destpath.rstrip('/') == os.path.dirname(instpath):
210 # FIXME this doesn't take recursive copy into account; unsure if it's practical to do so
211 srcpath = get_source_path(elements)
212 if fnmatch.fnmatchcase(os.path.basename(instpath), os.path.basename(srcpath)):
213 return elements
214 return None
215
216
217def appendfile(args):
218 import oe.recipeutils
219
220 stdout = ''
221 try:
222 (stdout, _) = bb.process.run('LANG=C file -b %s' % args.newfile, shell=True)
223 if 'cannot open' in stdout:
224 raise bb.process.ExecutionError(stdout)
225 except bb.process.ExecutionError as err:
226 logger.debug('file command returned error: %s' % err)
227 stdout = ''
228 if stdout:
229 logger.debug('file command output: %s' % stdout.rstrip())
230 if ('executable' in stdout and not 'shell script' in stdout) or 'shared object' in stdout:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800231 logger.warning('This file looks like it is a binary or otherwise the output of compilation. If it is, you should consider building it properly instead of substituting a binary file directly.')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500232
233 if args.recipe:
234 recipes = {args.targetpath: [args.recipe],}
235 else:
236 try:
237 recipes = find_target_file(args.targetpath, tinfoil.config_data)
238 except InvalidTargetFileError as e:
239 logger.error('%s cannot be handled by this tool: %s' % (args.targetpath, e))
240 return 1
241 if not recipes:
242 logger.error('Unable to find any package producing path %s - this may be because the recipe packaging it has not been built yet' % args.targetpath)
243 return 1
244
245 alternative_pns = []
246 postinst_pns = []
247
248 selectpn = None
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600249 for targetpath, pnlist in recipes.items():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500250 for pn in pnlist:
251 if pn.startswith('?'):
252 alternative_pns.append(pn[1:])
253 elif pn.startswith('!'):
254 postinst_pns.append(pn[1:])
255 elif selectpn:
256 # hit here with multilibs
257 continue
258 else:
259 selectpn = pn
260
261 if not selectpn and len(alternative_pns) == 1:
262 selectpn = alternative_pns[0]
263 logger.error('File %s is an alternative possibly provided by recipe %s but seemingly no other, selecting it by default - you should double check other recipes' % (args.targetpath, selectpn))
264
265 if selectpn:
266 logger.debug('Selecting recipe %s for file %s' % (selectpn, args.targetpath))
267 if postinst_pns:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800268 logger.warning('%s be modified by postinstall scripts for the following recipes:\n %s\nThis may or may not be an issue depending on what modifications these postinstall scripts make.' % (args.targetpath, '\n '.join(postinst_pns)))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500269 rd = _parse_recipe(selectpn, tinfoil)
270 if not rd:
271 # Error message already shown
272 return 1
273 sourcefile, instelements, modpatches = determine_file_source(args.targetpath, rd)
274 sourcepath = None
275 if sourcefile:
276 sourcetype, sourcepath = sourcefile.split('://', 1)
277 logger.debug('Original source file is %s (%s)' % (sourcepath, sourcetype))
278 if sourcetype == 'patch':
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800279 logger.warning('File %s is added by the patch %s - you may need to remove or replace this patch in order to replace the file.' % (args.targetpath, sourcepath))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500280 sourcepath = None
281 else:
282 logger.debug('Unable to determine source file, proceeding anyway')
283 if modpatches:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800284 logger.warning('File %s is modified by the following patches:\n %s' % (args.targetpath, '\n '.join(modpatches)))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500285
286 if instelements and sourcepath:
287 install = None
288 else:
289 # Auto-determine permissions
290 # Check destination
291 binpaths = '${bindir}:${sbindir}:${base_bindir}:${base_sbindir}:${libexecdir}:${sysconfdir}/init.d'
292 perms = '0644'
293 if os.path.abspath(os.path.dirname(args.targetpath)) in rd.expand(binpaths).split(':'):
294 # File is going into a directory normally reserved for executables, so it should be executable
295 perms = '0755'
296 else:
297 # Check source
298 st = os.stat(args.newfile)
299 if st.st_mode & stat.S_IXUSR:
300 perms = '0755'
301 install = {args.newfile: (args.targetpath, perms)}
302 oe.recipeutils.bbappend_recipe(rd, args.destlayer, {args.newfile: sourcepath}, install, wildcardver=args.wildcard_version, machine=args.machine)
303 return 0
304 else:
305 if alternative_pns:
306 logger.error('File %s is an alternative possibly provided by the following recipes:\n %s\nPlease select recipe with -r/--recipe' % (targetpath, '\n '.join(alternative_pns)))
307 elif postinst_pns:
308 logger.error('File %s may be written out in a pre/postinstall script of the following recipes:\n %s\nPlease select recipe with -r/--recipe' % (targetpath, '\n '.join(postinst_pns)))
309 return 3
310
311
312def appendsrc(args, files, rd, extralines=None):
313 import oe.recipeutils
314
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500315 srcdir = rd.getVar('S')
316 workdir = rd.getVar('WORKDIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500317
318 import bb.fetch
319 simplified = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500320 src_uri = rd.getVar('SRC_URI').split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500321 for uri in src_uri:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500322 if uri.endswith(';'):
323 uri = uri[:-1]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500324 simple_uri = bb.fetch.URI(uri)
325 simple_uri.params = {}
326 simplified[str(simple_uri)] = uri
327
328 copyfiles = {}
329 extralines = extralines or []
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600330 for newfile, srcfile in files.items():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500331 src_destdir = os.path.dirname(srcfile)
332 if not args.use_workdir:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500333 if rd.getVar('S') == rd.getVar('STAGING_KERNEL_DIR'):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500334 srcdir = os.path.join(workdir, 'git')
335 if not bb.data.inherits_class('kernel-yocto', rd):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800336 logger.warning('S == STAGING_KERNEL_DIR and non-kernel-yocto, unable to determine path to srcdir, defaulting to ${WORKDIR}/git')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500337 src_destdir = os.path.join(os.path.relpath(srcdir, workdir), src_destdir)
338 src_destdir = os.path.normpath(src_destdir)
339
340 source_uri = 'file://{0}'.format(os.path.basename(srcfile))
341 if src_destdir and src_destdir != '.':
342 source_uri += ';subdir={0}'.format(src_destdir)
343
344 simple = bb.fetch.URI(source_uri)
345 simple.params = {}
346 simple_str = str(simple)
347 if simple_str in simplified:
348 existing = simplified[simple_str]
349 if source_uri != existing:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800350 logger.warning('{0!r} is already in SRC_URI, with different parameters: {1!r}, not adding'.format(source_uri, existing))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500351 else:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800352 logger.warning('{0!r} is already in SRC_URI, not adding'.format(source_uri))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500353 else:
354 extralines.append('SRC_URI += {0}'.format(source_uri))
355 copyfiles[newfile] = srcfile
356
357 oe.recipeutils.bbappend_recipe(rd, args.destlayer, copyfiles, None, wildcardver=args.wildcard_version, machine=args.machine, extralines=extralines)
358
359
360def appendsrcfiles(parser, args):
361 recipedata = _parse_recipe(args.recipe, tinfoil)
362 if not recipedata:
363 parser.error('RECIPE must be a valid recipe name')
364
365 files = dict((f, os.path.join(args.destdir, os.path.basename(f)))
366 for f in args.files)
367 return appendsrc(args, files, recipedata)
368
369
370def appendsrcfile(parser, args):
371 recipedata = _parse_recipe(args.recipe, tinfoil)
372 if not recipedata:
373 parser.error('RECIPE must be a valid recipe name')
374
375 if not args.destfile:
376 args.destfile = os.path.basename(args.file)
377 elif args.destfile.endswith('/'):
378 args.destfile = os.path.join(args.destfile, os.path.basename(args.file))
379
380 return appendsrc(args, {args.file: args.destfile}, recipedata)
381
382
383def layer(layerpath):
384 if not os.path.exists(os.path.join(layerpath, 'conf', 'layer.conf')):
385 raise argparse.ArgumentTypeError('{0!r} must be a path to a valid layer'.format(layerpath))
386 return layerpath
387
388
389def existing_path(filepath):
390 if not os.path.exists(filepath):
391 raise argparse.ArgumentTypeError('{0!r} must be an existing path'.format(filepath))
392 return filepath
393
394
395def existing_file(filepath):
396 filepath = existing_path(filepath)
397 if os.path.isdir(filepath):
398 raise argparse.ArgumentTypeError('{0!r} must be a file, not a directory'.format(filepath))
399 return filepath
400
401
402def destination_path(destpath):
403 if os.path.isabs(destpath):
404 raise argparse.ArgumentTypeError('{0!r} must be a relative path, not absolute'.format(destpath))
405 return destpath
406
407
408def target_path(targetpath):
409 if not os.path.isabs(targetpath):
410 raise argparse.ArgumentTypeError('{0!r} must be an absolute path, not relative'.format(targetpath))
411 return targetpath
412
413
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500414def register_commands(subparsers):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500415 common = argparse.ArgumentParser(add_help=False)
416 common.add_argument('-m', '--machine', help='Make bbappend changes specific to a machine only', metavar='MACHINE')
417 common.add_argument('-w', '--wildcard-version', help='Use wildcard to make the bbappend apply to any recipe version', action='store_true')
418 common.add_argument('destlayer', metavar='DESTLAYER', help='Base directory of the destination layer to write the bbappend to', type=layer)
419
420 parser_appendfile = subparsers.add_parser('appendfile',
421 parents=[common],
422 help='Create/update a bbappend to replace a target file',
423 description='Creates a bbappend (or updates an existing one) to replace the specified file that appears in the target system, determining the recipe that packages the file and the required path and name for the bbappend automatically. Note that the ability to determine the recipe packaging a particular file depends upon the recipe\'s do_packagedata task having already run prior to running this command (which it will have when the recipe has been built successfully, which in turn will have happened if one or more of the recipe\'s packages is included in an image that has been built successfully).')
424 parser_appendfile.add_argument('targetpath', help='Path to the file to be replaced (as it would appear within the target image, e.g. /etc/motd)', type=target_path)
425 parser_appendfile.add_argument('newfile', help='Custom file to replace the target file with', type=existing_file)
426 parser_appendfile.add_argument('-r', '--recipe', help='Override recipe to apply to (default is to find which recipe already packages the file)')
427 parser_appendfile.set_defaults(func=appendfile, parserecipes=True)
428
429 common_src = argparse.ArgumentParser(add_help=False, parents=[common])
430 common_src.add_argument('-W', '--workdir', help='Unpack file into WORKDIR rather than S', dest='use_workdir', action='store_true')
431 common_src.add_argument('recipe', metavar='RECIPE', help='Override recipe to apply to')
432
433 parser = subparsers.add_parser('appendsrcfiles',
434 parents=[common_src],
435 help='Create/update a bbappend to add or replace source files',
436 description='Creates a bbappend (or updates an existing one) to add or replace the specified file in the recipe sources, either those in WORKDIR or those in the source tree. This command lets you specify multiple files with a destination directory, so cannot specify the destination filename. See the `appendsrcfile` command for the other behavior.')
437 parser.add_argument('-D', '--destdir', help='Destination directory (relative to S or WORKDIR, defaults to ".")', default='', type=destination_path)
438 parser.add_argument('files', nargs='+', metavar='FILE', help='File(s) to be added to the recipe sources (WORKDIR or S)', type=existing_path)
439 parser.set_defaults(func=lambda a: appendsrcfiles(parser, a), parserecipes=True)
440
441 parser = subparsers.add_parser('appendsrcfile',
442 parents=[common_src],
443 help='Create/update a bbappend to add or replace a source file',
444 description='Creates a bbappend (or updates an existing one) to add or replace the specified files in the recipe sources, either those in WORKDIR or those in the source tree. This command lets you specify the destination filename, not just destination directory, but only works for one file. See the `appendsrcfiles` command for the other behavior.')
445 parser.add_argument('file', metavar='FILE', help='File to be added to the recipe sources (WORKDIR or S)', type=existing_path)
446 parser.add_argument('destfile', metavar='DESTFILE', nargs='?', help='Destination path (relative to S or WORKDIR, optional)', type=destination_path)
447 parser.set_defaults(func=lambda a: appendsrcfile(parser, a), parserecipes=True)