blob: 558fd25ac5648f9123cd43b00068100d51afd50f [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# Recipe creation tool - append plugin
2#
3# Copyright (C) 2015 Intel Corporation
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License version 2 as
7# published by the Free Software Foundation.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License along
15# with this program; if not, write to the Free Software Foundation, Inc.,
16# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17
18import sys
19import os
20import argparse
21import glob
22import fnmatch
23import re
24import subprocess
25import logging
26import stat
27import shutil
28import scriptutils
29import errno
30from collections import defaultdict
31
32logger = logging.getLogger('recipetool')
33
34tinfoil = None
35
36def tinfoil_init(instance):
37 global tinfoil
38 tinfoil = instance
39
40
41# FIXME guessing when we don't have pkgdata?
42# FIXME mode to create patch rather than directly substitute
43
44class InvalidTargetFileError(Exception):
45 pass
46
47def find_target_file(targetpath, d, pkglist=None):
48 """Find the recipe installing the specified target path, optionally limited to a select list of packages"""
49 import json
50
51 pkgdata_dir = d.getVar('PKGDATA_DIR', True)
52
53 # The mix between /etc and ${sysconfdir} here may look odd, but it is just
54 # being consistent with usage elsewhere
55 invalidtargets = {'${sysconfdir}/version': '${sysconfdir}/version is written out at image creation time',
56 '/etc/timestamp': '/etc/timestamp is written out at image creation time',
57 '/dev/*': '/dev is handled by udev (or equivalent) and the kernel (devtmpfs)',
58 '/etc/passwd': '/etc/passwd should be managed through the useradd and extrausers classes',
59 '/etc/group': '/etc/group should be managed through the useradd and extrausers classes',
60 '/etc/shadow': '/etc/shadow should be managed through the useradd and extrausers classes',
61 '/etc/gshadow': '/etc/gshadow should be managed through the useradd and extrausers classes',
62 '${sysconfdir}/hostname': '${sysconfdir}/hostname contents should be set by setting hostname_pn-base-files = "value" in configuration',}
63
64 for pthspec, message in invalidtargets.iteritems():
65 if fnmatch.fnmatchcase(targetpath, d.expand(pthspec)):
66 raise InvalidTargetFileError(d.expand(message))
67
68 targetpath_re = re.compile(r'\s+(\$D)?%s(\s|$)' % targetpath)
69
70 recipes = defaultdict(list)
71 for root, dirs, files in os.walk(os.path.join(pkgdata_dir, 'runtime')):
72 if pkglist:
73 filelist = pkglist
74 else:
75 filelist = files
76 for fn in filelist:
77 pkgdatafile = os.path.join(root, fn)
78 if pkglist and not os.path.exists(pkgdatafile):
79 continue
80 with open(pkgdatafile, 'r') as f:
81 pn = ''
82 # This does assume that PN comes before other values, but that's a fairly safe assumption
83 for line in f:
84 if line.startswith('PN:'):
85 pn = line.split(':', 1)[1].strip()
86 elif line.startswith('FILES_INFO:'):
87 val = line.split(':', 1)[1].strip()
88 dictval = json.loads(val)
89 for fullpth in dictval.keys():
90 if fnmatch.fnmatchcase(fullpth, targetpath):
91 recipes[targetpath].append(pn)
92 elif line.startswith('pkg_preinst_') or line.startswith('pkg_postinst_'):
93 scriptval = line.split(':', 1)[1].strip().decode('string_escape')
94 if 'update-alternatives --install %s ' % targetpath in scriptval:
95 recipes[targetpath].append('?%s' % pn)
96 elif targetpath_re.search(scriptval):
97 recipes[targetpath].append('!%s' % pn)
98 return recipes
99
100def _get_recipe_file(cooker, pn):
101 import oe.recipeutils
102 recipefile = oe.recipeutils.pn_to_recipe(cooker, pn)
103 if not recipefile:
104 skipreasons = oe.recipeutils.get_unavailable_reasons(cooker, pn)
105 if skipreasons:
106 logger.error('\n'.join(skipreasons))
107 else:
108 logger.error("Unable to find any recipe file matching %s" % pn)
109 return recipefile
110
111def _parse_recipe(pn, tinfoil):
112 import oe.recipeutils
113 recipefile = _get_recipe_file(tinfoil.cooker, pn)
114 if not recipefile:
115 # Error already logged
116 return None
117 append_files = tinfoil.cooker.collection.get_file_appends(recipefile)
118 rd = oe.recipeutils.parse_recipe(recipefile, append_files,
119 tinfoil.config_data)
120 return rd
121
122def determine_file_source(targetpath, rd):
123 """Assuming we know a file came from a specific recipe, figure out exactly where it came from"""
124 import oe.recipeutils
125
126 # See if it's in do_install for the recipe
127 workdir = rd.getVar('WORKDIR', True)
128 src_uri = rd.getVar('SRC_URI', True)
129 srcfile = ''
130 modpatches = []
131 elements = check_do_install(rd, targetpath)
132 if elements:
133 logger.debug('do_install line:\n%s' % ' '.join(elements))
134 srcpath = get_source_path(elements)
135 logger.debug('source path: %s' % srcpath)
136 if not srcpath.startswith('/'):
137 # Handle non-absolute path
138 srcpath = os.path.abspath(os.path.join(rd.getVarFlag('do_install', 'dirs', True).split()[-1], srcpath))
139 if srcpath.startswith(workdir):
140 # OK, now we have the source file name, look for it in SRC_URI
141 workdirfile = os.path.relpath(srcpath, workdir)
142 # FIXME this is where we ought to have some code in the fetcher, because this is naive
143 for item in src_uri.split():
144 localpath = bb.fetch2.localpath(item, rd)
145 # Source path specified in do_install might be a glob
146 if fnmatch.fnmatch(os.path.basename(localpath), workdirfile):
147 srcfile = 'file://%s' % localpath
148 elif '/' in workdirfile:
149 if item == 'file://%s' % workdirfile:
150 srcfile = 'file://%s' % localpath
151
152 # Check patches
153 srcpatches = []
154 patchedfiles = oe.recipeutils.get_recipe_patched_files(rd)
155 for patch, filelist in patchedfiles.iteritems():
156 for fileitem in filelist:
157 if fileitem[0] == srcpath:
158 srcpatches.append((patch, fileitem[1]))
159 if srcpatches:
160 addpatch = None
161 for patch in srcpatches:
162 if patch[1] == 'A':
163 addpatch = patch[0]
164 else:
165 modpatches.append(patch[0])
166 if addpatch:
167 srcfile = 'patch://%s' % addpatch
168
169 return (srcfile, elements, modpatches)
170
171def get_source_path(cmdelements):
172 """Find the source path specified within a command"""
173 command = cmdelements[0]
174 if command in ['install', 'cp']:
175 helptext = subprocess.check_output('LC_ALL=C %s --help' % command, shell=True)
176 argopts = ''
177 argopt_line_re = re.compile('^-([a-zA-Z0-9]), --[a-z-]+=')
178 for line in helptext.splitlines():
179 line = line.lstrip()
180 res = argopt_line_re.search(line)
181 if res:
182 argopts += res.group(1)
183 if not argopts:
184 # Fallback
185 if command == 'install':
186 argopts = 'gmoSt'
187 elif command == 'cp':
188 argopts = 't'
189 else:
190 raise Exception('No fallback arguments for command %s' % command)
191
192 skipnext = False
193 for elem in cmdelements[1:-1]:
194 if elem.startswith('-'):
195 if len(elem) > 1 and elem[1] in argopts:
196 skipnext = True
197 continue
198 if skipnext:
199 skipnext = False
200 continue
201 return elem
202 else:
203 raise Exception('get_source_path: no handling for command "%s"')
204
205def get_func_deps(func, d):
206 """Find the function dependencies of a shell function"""
207 deps = bb.codeparser.ShellParser(func, logger).parse_shell(d.getVar(func, True))
208 deps |= set((d.getVarFlag(func, "vardeps", True) or "").split())
209 funcdeps = []
210 for dep in deps:
211 if d.getVarFlag(dep, 'func', True):
212 funcdeps.append(dep)
213 return funcdeps
214
215def check_do_install(rd, targetpath):
216 """Look at do_install for a command that installs/copies the specified target path"""
217 instpath = os.path.abspath(os.path.join(rd.getVar('D', True), targetpath.lstrip('/')))
218 do_install = rd.getVar('do_install', True)
219 # Handle where do_install calls other functions (somewhat crudely, but good enough for this purpose)
220 deps = get_func_deps('do_install', rd)
221 for dep in deps:
222 do_install = do_install.replace(dep, rd.getVar(dep, True))
223
224 # Look backwards through do_install as we want to catch where a later line (perhaps
225 # from a bbappend) is writing over the top
226 for line in reversed(do_install.splitlines()):
227 line = line.strip()
228 if (line.startswith('install ') and ' -m' in line) or line.startswith('cp '):
229 elements = line.split()
230 destpath = os.path.abspath(elements[-1])
231 if destpath == instpath:
232 return elements
233 elif destpath.rstrip('/') == os.path.dirname(instpath):
234 # FIXME this doesn't take recursive copy into account; unsure if it's practical to do so
235 srcpath = get_source_path(elements)
236 if fnmatch.fnmatchcase(os.path.basename(instpath), os.path.basename(srcpath)):
237 return elements
238 return None
239
240
241def appendfile(args):
242 import oe.recipeutils
243
244 stdout = ''
245 try:
246 (stdout, _) = bb.process.run('LANG=C file -b %s' % args.newfile, shell=True)
247 if 'cannot open' in stdout:
248 raise bb.process.ExecutionError(stdout)
249 except bb.process.ExecutionError as err:
250 logger.debug('file command returned error: %s' % err)
251 stdout = ''
252 if stdout:
253 logger.debug('file command output: %s' % stdout.rstrip())
254 if ('executable' in stdout and not 'shell script' in stdout) or 'shared object' in stdout:
255 logger.warn('This file looks like it is a binary or otherwise the output of compilation. If it is, you should consider building it properly instead of substituting a binary file directly.')
256
257 if args.recipe:
258 recipes = {args.targetpath: [args.recipe],}
259 else:
260 try:
261 recipes = find_target_file(args.targetpath, tinfoil.config_data)
262 except InvalidTargetFileError as e:
263 logger.error('%s cannot be handled by this tool: %s' % (args.targetpath, e))
264 return 1
265 if not recipes:
266 logger.error('Unable to find any package producing path %s - this may be because the recipe packaging it has not been built yet' % args.targetpath)
267 return 1
268
269 alternative_pns = []
270 postinst_pns = []
271
272 selectpn = None
273 for targetpath, pnlist in recipes.iteritems():
274 for pn in pnlist:
275 if pn.startswith('?'):
276 alternative_pns.append(pn[1:])
277 elif pn.startswith('!'):
278 postinst_pns.append(pn[1:])
279 elif selectpn:
280 # hit here with multilibs
281 continue
282 else:
283 selectpn = pn
284
285 if not selectpn and len(alternative_pns) == 1:
286 selectpn = alternative_pns[0]
287 logger.error('File %s is an alternative possibly provided by recipe %s but seemingly no other, selecting it by default - you should double check other recipes' % (args.targetpath, selectpn))
288
289 if selectpn:
290 logger.debug('Selecting recipe %s for file %s' % (selectpn, args.targetpath))
291 if postinst_pns:
292 logger.warn('%s be modified by postinstall scripts for the following recipes:\n %s\nThis may or may not be an issue depending on what modifications these postinstall scripts make.' % (args.targetpath, '\n '.join(postinst_pns)))
293 rd = _parse_recipe(selectpn, tinfoil)
294 if not rd:
295 # Error message already shown
296 return 1
297 sourcefile, instelements, modpatches = determine_file_source(args.targetpath, rd)
298 sourcepath = None
299 if sourcefile:
300 sourcetype, sourcepath = sourcefile.split('://', 1)
301 logger.debug('Original source file is %s (%s)' % (sourcepath, sourcetype))
302 if sourcetype == 'patch':
303 logger.warn('File %s is added by the patch %s - you may need to remove or replace this patch in order to replace the file.' % (args.targetpath, sourcepath))
304 sourcepath = None
305 else:
306 logger.debug('Unable to determine source file, proceeding anyway')
307 if modpatches:
308 logger.warn('File %s is modified by the following patches:\n %s' % (args.targetpath, '\n '.join(modpatches)))
309
310 if instelements and sourcepath:
311 install = None
312 else:
313 # Auto-determine permissions
314 # Check destination
315 binpaths = '${bindir}:${sbindir}:${base_bindir}:${base_sbindir}:${libexecdir}:${sysconfdir}/init.d'
316 perms = '0644'
317 if os.path.abspath(os.path.dirname(args.targetpath)) in rd.expand(binpaths).split(':'):
318 # File is going into a directory normally reserved for executables, so it should be executable
319 perms = '0755'
320 else:
321 # Check source
322 st = os.stat(args.newfile)
323 if st.st_mode & stat.S_IXUSR:
324 perms = '0755'
325 install = {args.newfile: (args.targetpath, perms)}
326 oe.recipeutils.bbappend_recipe(rd, args.destlayer, {args.newfile: sourcepath}, install, wildcardver=args.wildcard_version, machine=args.machine)
327 return 0
328 else:
329 if alternative_pns:
330 logger.error('File %s is an alternative possibly provided by the following recipes:\n %s\nPlease select recipe with -r/--recipe' % (targetpath, '\n '.join(alternative_pns)))
331 elif postinst_pns:
332 logger.error('File %s may be written out in a pre/postinstall script of the following recipes:\n %s\nPlease select recipe with -r/--recipe' % (targetpath, '\n '.join(postinst_pns)))
333 return 3
334
335
336def appendsrc(args, files, rd, extralines=None):
337 import oe.recipeutils
338
339 srcdir = rd.getVar('S', True)
340 workdir = rd.getVar('WORKDIR', True)
341
342 import bb.fetch
343 simplified = {}
344 src_uri = rd.getVar('SRC_URI', True).split()
345 for uri in src_uri:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500346 if uri.endswith(';'):
347 uri = uri[:-1]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500348 simple_uri = bb.fetch.URI(uri)
349 simple_uri.params = {}
350 simplified[str(simple_uri)] = uri
351
352 copyfiles = {}
353 extralines = extralines or []
354 for newfile, srcfile in files.iteritems():
355 src_destdir = os.path.dirname(srcfile)
356 if not args.use_workdir:
357 if rd.getVar('S', True) == rd.getVar('STAGING_KERNEL_DIR', True):
358 srcdir = os.path.join(workdir, 'git')
359 if not bb.data.inherits_class('kernel-yocto', rd):
360 logger.warn('S == STAGING_KERNEL_DIR and non-kernel-yocto, unable to determine path to srcdir, defaulting to ${WORKDIR}/git')
361 src_destdir = os.path.join(os.path.relpath(srcdir, workdir), src_destdir)
362 src_destdir = os.path.normpath(src_destdir)
363
364 source_uri = 'file://{0}'.format(os.path.basename(srcfile))
365 if src_destdir and src_destdir != '.':
366 source_uri += ';subdir={0}'.format(src_destdir)
367
368 simple = bb.fetch.URI(source_uri)
369 simple.params = {}
370 simple_str = str(simple)
371 if simple_str in simplified:
372 existing = simplified[simple_str]
373 if source_uri != existing:
374 logger.warn('{0!r} is already in SRC_URI, with different parameters: {1!r}, not adding'.format(source_uri, existing))
375 else:
376 logger.warn('{0!r} is already in SRC_URI, not adding'.format(source_uri))
377 else:
378 extralines.append('SRC_URI += {0}'.format(source_uri))
379 copyfiles[newfile] = srcfile
380
381 oe.recipeutils.bbappend_recipe(rd, args.destlayer, copyfiles, None, wildcardver=args.wildcard_version, machine=args.machine, extralines=extralines)
382
383
384def appendsrcfiles(parser, args):
385 recipedata = _parse_recipe(args.recipe, tinfoil)
386 if not recipedata:
387 parser.error('RECIPE must be a valid recipe name')
388
389 files = dict((f, os.path.join(args.destdir, os.path.basename(f)))
390 for f in args.files)
391 return appendsrc(args, files, recipedata)
392
393
394def appendsrcfile(parser, args):
395 recipedata = _parse_recipe(args.recipe, tinfoil)
396 if not recipedata:
397 parser.error('RECIPE must be a valid recipe name')
398
399 if not args.destfile:
400 args.destfile = os.path.basename(args.file)
401 elif args.destfile.endswith('/'):
402 args.destfile = os.path.join(args.destfile, os.path.basename(args.file))
403
404 return appendsrc(args, {args.file: args.destfile}, recipedata)
405
406
407def layer(layerpath):
408 if not os.path.exists(os.path.join(layerpath, 'conf', 'layer.conf')):
409 raise argparse.ArgumentTypeError('{0!r} must be a path to a valid layer'.format(layerpath))
410 return layerpath
411
412
413def existing_path(filepath):
414 if not os.path.exists(filepath):
415 raise argparse.ArgumentTypeError('{0!r} must be an existing path'.format(filepath))
416 return filepath
417
418
419def existing_file(filepath):
420 filepath = existing_path(filepath)
421 if os.path.isdir(filepath):
422 raise argparse.ArgumentTypeError('{0!r} must be a file, not a directory'.format(filepath))
423 return filepath
424
425
426def destination_path(destpath):
427 if os.path.isabs(destpath):
428 raise argparse.ArgumentTypeError('{0!r} must be a relative path, not absolute'.format(destpath))
429 return destpath
430
431
432def target_path(targetpath):
433 if not os.path.isabs(targetpath):
434 raise argparse.ArgumentTypeError('{0!r} must be an absolute path, not relative'.format(targetpath))
435 return targetpath
436
437
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500438def register_commands(subparsers):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500439 common = argparse.ArgumentParser(add_help=False)
440 common.add_argument('-m', '--machine', help='Make bbappend changes specific to a machine only', metavar='MACHINE')
441 common.add_argument('-w', '--wildcard-version', help='Use wildcard to make the bbappend apply to any recipe version', action='store_true')
442 common.add_argument('destlayer', metavar='DESTLAYER', help='Base directory of the destination layer to write the bbappend to', type=layer)
443
444 parser_appendfile = subparsers.add_parser('appendfile',
445 parents=[common],
446 help='Create/update a bbappend to replace a target file',
447 description='Creates a bbappend (or updates an existing one) to replace the specified file that appears in the target system, determining the recipe that packages the file and the required path and name for the bbappend automatically. Note that the ability to determine the recipe packaging a particular file depends upon the recipe\'s do_packagedata task having already run prior to running this command (which it will have when the recipe has been built successfully, which in turn will have happened if one or more of the recipe\'s packages is included in an image that has been built successfully).')
448 parser_appendfile.add_argument('targetpath', help='Path to the file to be replaced (as it would appear within the target image, e.g. /etc/motd)', type=target_path)
449 parser_appendfile.add_argument('newfile', help='Custom file to replace the target file with', type=existing_file)
450 parser_appendfile.add_argument('-r', '--recipe', help='Override recipe to apply to (default is to find which recipe already packages the file)')
451 parser_appendfile.set_defaults(func=appendfile, parserecipes=True)
452
453 common_src = argparse.ArgumentParser(add_help=False, parents=[common])
454 common_src.add_argument('-W', '--workdir', help='Unpack file into WORKDIR rather than S', dest='use_workdir', action='store_true')
455 common_src.add_argument('recipe', metavar='RECIPE', help='Override recipe to apply to')
456
457 parser = subparsers.add_parser('appendsrcfiles',
458 parents=[common_src],
459 help='Create/update a bbappend to add or replace source files',
460 description='Creates a bbappend (or updates an existing one) to add or replace the specified file in the recipe sources, either those in WORKDIR or those in the source tree. This command lets you specify multiple files with a destination directory, so cannot specify the destination filename. See the `appendsrcfile` command for the other behavior.')
461 parser.add_argument('-D', '--destdir', help='Destination directory (relative to S or WORKDIR, defaults to ".")', default='', type=destination_path)
462 parser.add_argument('files', nargs='+', metavar='FILE', help='File(s) to be added to the recipe sources (WORKDIR or S)', type=existing_path)
463 parser.set_defaults(func=lambda a: appendsrcfiles(parser, a), parserecipes=True)
464
465 parser = subparsers.add_parser('appendsrcfile',
466 parents=[common_src],
467 help='Create/update a bbappend to add or replace a source file',
468 description='Creates a bbappend (or updates an existing one) to add or replace the specified files in the recipe sources, either those in WORKDIR or those in the source tree. This command lets you specify the destination filename, not just destination directory, but only works for one file. See the `appendsrcfiles` command for the other behavior.')
469 parser.add_argument('file', metavar='FILE', help='File to be added to the recipe sources (WORKDIR or S)', type=existing_path)
470 parser.add_argument('destfile', metavar='DESTFILE', nargs='?', help='Destination path (relative to S or WORKDIR, optional)', type=destination_path)
471 parser.set_defaults(func=lambda a: appendsrcfile(parser, a), parserecipes=True)