blob: 25b159bc1bcff90c37edaa44698692ad91f8d39e [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# Utility functions for reading and modifying recipes
2#
3# Some code borrowed from the OE layer index
4#
Brad Bishopd7bf8c12018-02-25 22:55:05 -05005# Copyright (C) 2013-2017 Intel Corporation
Patrick Williamsc124f4f2015-09-15 14:41:29 -05006#
Brad Bishopc342db32019-05-15 21:57:59 -04007# SPDX-License-Identifier: GPL-2.0-only
8#
Patrick Williamsc124f4f2015-09-15 14:41:29 -05009
10import sys
11import os
12import os.path
13import tempfile
14import textwrap
15import difflib
Patrick Williamsc0f7c042017-02-23 20:41:17 -060016from . import utils
Patrick Williamsc124f4f2015-09-15 14:41:29 -050017import shutil
18import re
19import fnmatch
Patrick Williamsc0f7c042017-02-23 20:41:17 -060020import glob
Brad Bishop19323692019-04-05 15:28:33 -040021import bb.tinfoil
Patrick Williamsc124f4f2015-09-15 14:41:29 -050022
Brad Bishop19323692019-04-05 15:28:33 -040023from collections import OrderedDict, defaultdict
24from bb.utils import vercmp_string
Patrick Williamsc124f4f2015-09-15 14:41:29 -050025
26# Help us to find places to insert values
Andrew Geissler5082cc72023-09-11 08:41:39 -040027recipe_progression = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION', 'LICENSE', 'LICENSE_FLAGS', 'LIC_FILES_CHKSUM', 'PROVIDES', 'DEPENDS', 'PR', 'PV', 'SRCREV', 'SRC_URI', 'S', 'do_fetch()', 'do_unpack()', 'do_patch()', 'EXTRA_OECONF', 'EXTRA_OECMAKE', 'EXTRA_OESCONS', 'do_configure()', 'EXTRA_OEMAKE', 'do_compile()', 'do_install()', 'do_populate_sysroot()', 'INITSCRIPT', 'USERADD', 'GROUPADD', 'PACKAGES', 'FILES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RPROVIDES', 'RREPLACES', 'RCONFLICTS', 'ALLOW_EMPTY', 'populate_packages()', 'do_package()', 'do_deploy()', 'BBCLASSEXTEND']
Patrick Williamsc124f4f2015-09-15 14:41:29 -050028# Variables that sometimes are a bit long but shouldn't be wrapped
Brad Bishop19323692019-04-05 15:28:33 -040029nowrap_vars = ['SUMMARY', 'HOMEPAGE', 'BUGTRACKER', r'SRC_URI\[(.+\.)?md5sum\]', r'SRC_URI\[(.+\.)?sha256sum\]']
Patrick Williamsc124f4f2015-09-15 14:41:29 -050030list_vars = ['SRC_URI', 'LIC_FILES_CHKSUM']
31meta_vars = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION']
32
33
Brad Bishop19323692019-04-05 15:28:33 -040034def simplify_history(history, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050035 """
Brad Bishop19323692019-04-05 15:28:33 -040036 Eliminate any irrelevant events from a variable history
Patrick Williamsc124f4f2015-09-15 14:41:29 -050037 """
Brad Bishop19323692019-04-05 15:28:33 -040038 ret_history = []
39 has_set = False
40 # Go backwards through the history and remove any immediate operations
41 # before the most recent set
42 for event in reversed(history):
43 if 'flag' in event or not 'file' in event:
44 continue
45 if event['op'] == 'set':
46 if has_set:
47 continue
48 has_set = True
49 elif event['op'] in ('append', 'prepend', 'postdot', 'predot'):
Patrick Williams213cb262021-08-07 19:21:33 -050050 # Reminder: "append" and "prepend" mean += and =+ respectively, NOT :append / :prepend
Brad Bishop19323692019-04-05 15:28:33 -040051 if has_set:
52 continue
53 ret_history.insert(0, event)
54 return ret_history
Patrick Williamsc124f4f2015-09-15 14:41:29 -050055
56
Patrick Williamsc124f4f2015-09-15 14:41:29 -050057def get_var_files(fn, varlist, d):
58 """Find the file in which each of a list of variables is set.
59 Note: requires variable history to be enabled when parsing.
60 """
61 varfiles = {}
62 for v in varlist:
Patrick Williamsc124f4f2015-09-15 14:41:29 -050063 files = []
Brad Bishop19323692019-04-05 15:28:33 -040064 if '[' in v:
65 varsplit = v.split('[')
66 varflag = varsplit[1].split(']')[0]
67 history = d.varhistory.variable(varsplit[0])
68 for event in history:
69 if 'file' in event and event.get('flag', '') == varflag:
70 files.append(event['file'])
71 else:
72 history = d.varhistory.variable(v)
73 for event in history:
74 if 'file' in event and not 'flag' in event:
75 files.append(event['file'])
Patrick Williamsc124f4f2015-09-15 14:41:29 -050076 if files:
77 actualfile = files[-1]
78 else:
79 actualfile = None
80 varfiles[v] = actualfile
81
82 return varfiles
83
84
Patrick Williamsf1e5d692016-03-30 15:21:19 -050085def split_var_value(value, assignment=True):
86 """
87 Split a space-separated variable's value into a list of items,
88 taking into account that some of the items might be made up of
89 expressions containing spaces that should not be split.
90 Parameters:
91 value:
92 The string value to split
93 assignment:
94 True to assume that the value represents an assignment
95 statement, False otherwise. If True, and an assignment
96 statement is passed in the first item in
97 the returned list will be the part of the assignment
98 statement up to and including the opening quote character,
99 and the last item will be the closing quote.
100 """
101 inexpr = 0
102 lastchar = None
103 out = []
104 buf = ''
105 for char in value:
106 if char == '{':
107 if lastchar == '$':
108 inexpr += 1
109 elif char == '}':
110 inexpr -= 1
111 elif assignment and char in '"\'' and inexpr == 0:
112 if buf:
113 out.append(buf)
114 out.append(char)
115 char = ''
116 buf = ''
117 elif char.isspace() and inexpr == 0:
118 char = ''
119 if buf:
120 out.append(buf)
121 buf = ''
122 buf += char
123 lastchar = char
124 if buf:
125 out.append(buf)
126
127 # Join together assignment statement and opening quote
128 outlist = out
129 if assignment:
130 assigfound = False
131 for idx, item in enumerate(out):
132 if '=' in item:
133 assigfound = True
134 if assigfound:
135 if '"' in item or "'" in item:
136 outlist = [' '.join(out[:idx+1])]
137 outlist.extend(out[idx+1:])
138 break
139 return outlist
140
141
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600142def patch_recipe_lines(fromlines, values, trailing_newline=True):
143 """Update or insert variable values into lines from a recipe.
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500144 Note that some manual inspection/intervention may be required
145 since this cannot handle all situations.
146 """
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500147
148 import bb.utils
149
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600150 if trailing_newline:
151 newline = '\n'
152 else:
153 newline = ''
154
Brad Bishop316dfdd2018-06-25 12:45:53 -0400155 nowrap_vars_res = []
156 for item in nowrap_vars:
157 nowrap_vars_res.append(re.compile('^%s$' % item))
158
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500159 recipe_progression_res = []
160 recipe_progression_restrs = []
161 for item in recipe_progression:
162 if item.endswith('()'):
163 key = item[:-2]
164 else:
165 key = item
Brad Bishop19323692019-04-05 15:28:33 -0400166 restr = r'%s(_[a-zA-Z0-9-_$(){}]+|\[[^\]]*\])?' % key
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500167 if item.endswith('()'):
168 recipe_progression_restrs.append(restr + '()')
169 else:
170 recipe_progression_restrs.append(restr)
171 recipe_progression_res.append(re.compile('^%s$' % restr))
172
173 def get_recipe_pos(variable):
174 for i, p in enumerate(recipe_progression_res):
175 if p.match(variable):
176 return i
177 return -1
178
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500179 remainingnames = {}
180 for k in values.keys():
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500181 remainingnames[k] = get_recipe_pos(k)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600182 remainingnames = OrderedDict(sorted(remainingnames.items(), key=lambda x: x[1]))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500183
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500184 modifying = False
185
186 def outputvalue(name, lines, rewindcomments=False):
187 if values[name] is None:
188 return
Brad Bishop19323692019-04-05 15:28:33 -0400189 if isinstance(values[name], tuple):
190 op, value = values[name]
191 if op == '+=' and value.strip() == '':
192 return
193 else:
194 value = values[name]
195 op = '='
196 rawtext = '%s %s "%s"%s' % (name, op, value, newline)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500197 addlines = []
Brad Bishop316dfdd2018-06-25 12:45:53 -0400198 nowrap = False
199 for nowrap_re in nowrap_vars_res:
200 if nowrap_re.match(name):
201 nowrap = True
202 break
203 if nowrap:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500204 addlines.append(rawtext)
205 elif name in list_vars:
Brad Bishop19323692019-04-05 15:28:33 -0400206 splitvalue = split_var_value(value, assignment=False)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500207 if len(splitvalue) > 1:
208 linesplit = ' \\\n' + (' ' * (len(name) + 4))
Brad Bishop19323692019-04-05 15:28:33 -0400209 addlines.append('%s %s "%s%s"%s' % (name, op, linesplit.join(splitvalue), linesplit, newline))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500210 else:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500211 addlines.append(rawtext)
212 else:
213 wrapped = textwrap.wrap(rawtext)
214 for wrapline in wrapped[:-1]:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600215 addlines.append('%s \\%s' % (wrapline, newline))
216 addlines.append('%s%s' % (wrapped[-1], newline))
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500217
218 # Split on newlines - this isn't strictly necessary if you are only
219 # going to write the output to disk, but if you want to compare it
220 # (as patch_recipe_file() will do if patch=True) then it's important.
221 addlines = [line for l in addlines for line in l.splitlines(True)]
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500222 if rewindcomments:
223 # Ensure we insert the lines before any leading comments
224 # (that we'd want to ensure remain leading the next value)
225 for i, ln in reversed(list(enumerate(lines))):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600226 if not ln.startswith('#'):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500227 lines[i+1:i+1] = addlines
228 break
229 else:
230 lines.extend(addlines)
231 else:
232 lines.extend(addlines)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500233
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500234 existingnames = []
235 def patch_recipe_varfunc(varname, origvalue, op, newlines):
236 if modifying:
237 # Insert anything that should come before this variable
238 pos = get_recipe_pos(varname)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600239 for k in list(remainingnames):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500240 if remainingnames[k] > -1 and pos >= remainingnames[k] and not k in existingnames:
241 outputvalue(k, newlines, rewindcomments=True)
242 del remainingnames[k]
243 # Now change this variable, if it needs to be changed
244 if varname in existingnames and op in ['+=', '=', '=+']:
245 if varname in remainingnames:
246 outputvalue(varname, newlines)
247 del remainingnames[varname]
248 return None, None, 0, True
249 else:
250 if varname in values:
251 existingnames.append(varname)
252 return origvalue, None, 0, True
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500253
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500254 # First run - establish which values we want to set are already in the file
255 varlist = [re.escape(item) for item in values.keys()]
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600256 bb.utils.edit_metadata(fromlines, varlist, patch_recipe_varfunc)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500257 # Second run - actually set everything
258 modifying = True
259 varlist.extend(recipe_progression_restrs)
260 changed, tolines = bb.utils.edit_metadata(fromlines, varlist, patch_recipe_varfunc, match_overrides=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500261
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500262 if remainingnames:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600263 if tolines and tolines[-1].strip() != '':
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500264 tolines.append('\n')
265 for k in remainingnames.keys():
266 outputvalue(k, tolines)
267
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600268 return changed, tolines
269
270
Brad Bishop316dfdd2018-06-25 12:45:53 -0400271def patch_recipe_file(fn, values, patch=False, relpath='', redirect_output=None):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600272 """Update or insert variable values into a recipe file (assuming you
273 have already identified the exact file you want to update.)
274 Note that some manual inspection/intervention may be required
275 since this cannot handle all situations.
276 """
277
278 with open(fn, 'r') as f:
279 fromlines = f.readlines()
280
281 _, tolines = patch_recipe_lines(fromlines, values)
282
Brad Bishop316dfdd2018-06-25 12:45:53 -0400283 if redirect_output:
284 with open(os.path.join(redirect_output, os.path.basename(fn)), 'w') as f:
285 f.writelines(tolines)
286 return None
287 elif patch:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500288 relfn = os.path.relpath(fn, relpath)
289 diff = difflib.unified_diff(fromlines, tolines, 'a/%s' % relfn, 'b/%s' % relfn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500290 return diff
291 else:
292 with open(fn, 'w') as f:
293 f.writelines(tolines)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500294 return None
295
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500296
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500297def localise_file_vars(fn, varfiles, varlist):
298 """Given a list of variables and variable history (fetched with get_var_files())
299 find where each variable should be set/changed. This handles for example where a
300 recipe includes an inc file where variables might be changed - in most cases
301 we want to update the inc file when changing the variable value rather than adding
302 it to the recipe itself.
303 """
304 fndir = os.path.dirname(fn) + os.sep
305
306 first_meta_file = None
307 for v in meta_vars:
308 f = varfiles.get(v, None)
309 if f:
310 actualdir = os.path.dirname(f) + os.sep
311 if actualdir.startswith(fndir):
312 first_meta_file = f
313 break
314
315 filevars = defaultdict(list)
316 for v in varlist:
317 f = varfiles[v]
318 # Only return files that are in the same directory as the recipe or in some directory below there
319 # (this excludes bbclass files and common inc files that wouldn't be appropriate to set the variable
320 # in if we were going to set a value specific to this recipe)
321 if f:
322 actualfile = f
323 else:
324 # Variable isn't in a file, if it's one of the "meta" vars, use the first file with a meta var in it
325 if first_meta_file:
326 actualfile = first_meta_file
327 else:
328 actualfile = fn
329
330 actualdir = os.path.dirname(actualfile) + os.sep
331 if not actualdir.startswith(fndir):
332 actualfile = fn
333 filevars[actualfile].append(v)
334
335 return filevars
336
Brad Bishop316dfdd2018-06-25 12:45:53 -0400337def patch_recipe(d, fn, varvalues, patch=False, relpath='', redirect_output=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500338 """Modify a list of variable values in the specified recipe. Handles inc files if
339 used by the recipe.
340 """
Brad Bishop19323692019-04-05 15:28:33 -0400341 overrides = d.getVar('OVERRIDES').split(':')
342 def override_applicable(hevent):
343 op = hevent['op']
344 if '[' in op:
Patrick Williams213cb262021-08-07 19:21:33 -0500345 opoverrides = op.split('[')[1].split(']')[0].split(':')
Brad Bishop19323692019-04-05 15:28:33 -0400346 for opoverride in opoverrides:
347 if not opoverride in overrides:
348 return False
349 return True
350
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500351 varlist = varvalues.keys()
Brad Bishop19323692019-04-05 15:28:33 -0400352 fn = os.path.abspath(fn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500353 varfiles = get_var_files(fn, varlist, d)
354 locs = localise_file_vars(fn, varfiles, varlist)
355 patches = []
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600356 for f,v in locs.items():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500357 vals = {k: varvalues[k] for k in v}
Brad Bishop19323692019-04-05 15:28:33 -0400358 f = os.path.abspath(f)
359 if f == fn:
360 extravals = {}
361 for var, value in vals.items():
362 if var in list_vars:
363 history = simplify_history(d.varhistory.variable(var), d)
364 recipe_set = False
365 for event in history:
366 if os.path.abspath(event['file']) == fn:
367 if event['op'] == 'set':
368 recipe_set = True
369 if not recipe_set:
370 for event in history:
Patrick Williams213cb262021-08-07 19:21:33 -0500371 if event['op'].startswith(':remove'):
Brad Bishop19323692019-04-05 15:28:33 -0400372 continue
373 if not override_applicable(event):
374 continue
375 newvalue = value.replace(event['detail'], '')
Patrick Williams213cb262021-08-07 19:21:33 -0500376 if newvalue == value and os.path.abspath(event['file']) == fn and event['op'].startswith(':'):
377 op = event['op'].replace('[', ':').replace(']', '')
Brad Bishop19323692019-04-05 15:28:33 -0400378 extravals[var + op] = None
379 value = newvalue
380 vals[var] = ('+=', value)
381 vals.update(extravals)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400382 patchdata = patch_recipe_file(f, vals, patch, relpath, redirect_output)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500383 if patch:
384 patches.append(patchdata)
385
386 if patch:
387 return patches
388 else:
389 return None
390
391
392
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500393def copy_recipe_files(d, tgt_dir, whole_dir=False, download=True, all_variants=False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500394 """Copy (local) recipe files, including both files included via include/require,
395 and files referred to in the SRC_URI variable."""
396 import bb.fetch2
397 import oe.path
398
399 # FIXME need a warning if the unexpanded SRC_URI value contains variable references
400
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500401 uri_values = []
402 localpaths = []
403 def fetch_urls(rdata):
404 # Collect the local paths from SRC_URI
405 srcuri = rdata.getVar('SRC_URI') or ""
406 if srcuri not in uri_values:
407 fetch = bb.fetch2.Fetch(srcuri.split(), rdata)
408 if download:
409 fetch.download()
410 for pth in fetch.localpaths():
411 if pth not in localpaths:
Andrew Geissler9b4d8b02021-02-19 12:26:16 -0600412 localpaths.append(os.path.abspath(pth))
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500413 uri_values.append(srcuri)
414
415 fetch_urls(d)
416 if all_variants:
Patrick Williams213cb262021-08-07 19:21:33 -0500417 # Get files for other variants e.g. in the case of a SRC_URI:append
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500418 localdata = bb.data.createCopy(d)
419 variants = (localdata.getVar('BBCLASSEXTEND') or '').split()
420 if variants:
421 # Ensure we handle class-target if we're dealing with one of the variants
422 variants.append('target')
423 for variant in variants:
424 localdata.setVar('CLASSOVERRIDE', 'class-%s' % variant)
425 fetch_urls(localdata)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500426
427 # Copy local files to target directory and gather any remote files
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500428 bb_dir = os.path.abspath(os.path.dirname(d.getVar('FILE'))) + os.sep
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500429 remotes = []
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600430 copied = []
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500431 # Need to do this in two steps since we want to check against the absolute path
432 includes = [os.path.abspath(path) for path in d.getVar('BBINCLUDED').split() if os.path.exists(path)]
433 # We also check this below, but we don't want any items in this list being considered remotes
434 includes = [path for path in includes if path.startswith(bb_dir)]
435 for path in localpaths + includes:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500436 # Only import files that are under the meta directory
437 if path.startswith(bb_dir):
438 if not whole_dir:
439 relpath = os.path.relpath(path, bb_dir)
440 subdir = os.path.join(tgt_dir, os.path.dirname(relpath))
441 if not os.path.exists(subdir):
442 os.makedirs(subdir)
443 shutil.copy2(path, os.path.join(tgt_dir, relpath))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600444 copied.append(relpath)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500445 else:
446 remotes.append(path)
447 # Simply copy whole meta dir, if requested
448 if whole_dir:
449 shutil.copytree(bb_dir, tgt_dir)
450
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600451 return copied, remotes
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500452
453
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500454def get_recipe_local_files(d, patches=False, archives=False):
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500455 """Get a list of local files in SRC_URI within a recipe."""
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500456 import oe.patch
457 uris = (d.getVar('SRC_URI') or "").split()
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500458 fetch = bb.fetch2.Fetch(uris, d)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500459 # FIXME this list should be factored out somewhere else (such as the
460 # fetcher) though note that this only encompasses actual container formats
461 # i.e. that can contain multiple files as opposed to those that only
462 # contain a compressed stream (i.e. .tar.gz as opposed to just .gz)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400463 archive_exts = ['.tar', '.tgz', '.tar.gz', '.tar.Z', '.tbz', '.tbz2', '.tar.bz2', '.txz', '.tar.xz', '.tar.lz', '.zip', '.jar', '.rpm', '.srpm', '.deb', '.ipk', '.tar.7z', '.7z']
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500464 ret = {}
465 for uri in uris:
466 if fetch.ud[uri].type == 'file':
467 if (not patches and
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500468 oe.patch.patch_path(uri, fetch, '', expand=False)):
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500469 continue
470 # Skip files that are referenced by absolute path
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600471 fname = fetch.ud[uri].basepath
472 if os.path.isabs(fname):
473 continue
474 # Handle subdir=
475 subdir = fetch.ud[uri].parm.get('subdir', '')
476 if subdir:
477 if os.path.isabs(subdir):
478 continue
479 fname = os.path.join(subdir, fname)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500480 localpath = fetch.localpath(uri)
481 if not archives:
482 # Ignore archives that will be unpacked
483 if localpath.endswith(tuple(archive_exts)):
484 unpack = fetch.ud[uri].parm.get('unpack', True)
485 if unpack:
486 continue
Brad Bishop19323692019-04-05 15:28:33 -0400487 if os.path.isdir(localpath):
488 for root, dirs, files in os.walk(localpath):
489 for fname in files:
490 fileabspath = os.path.join(root,fname)
491 srcdir = os.path.dirname(localpath)
492 ret[os.path.relpath(fileabspath,srcdir)] = fileabspath
493 else:
494 ret[fname] = localpath
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500495 return ret
496
497
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500498def get_recipe_patches(d):
499 """Get a list of the patches included in SRC_URI within a recipe."""
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500500 import oe.patch
501 patches = oe.patch.src_patches(d, expand=False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500502 patchfiles = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500503 for patch in patches:
504 _, _, local, _, _, parm = bb.fetch.decodeurl(patch)
505 patchfiles.append(local)
506 return patchfiles
507
508
509def get_recipe_patched_files(d):
510 """
511 Get the list of patches for a recipe along with the files each patch modifies.
512 Params:
513 d: the datastore for the recipe
514 Returns:
515 a dict mapping patch file path to a list of tuples of changed files and
516 change mode ('A' for add, 'D' for delete or 'M' for modify)
517 """
518 import oe.patch
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500519 patches = oe.patch.src_patches(d, expand=False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500520 patchedfiles = {}
521 for patch in patches:
522 _, _, patchfile, _, _, parm = bb.fetch.decodeurl(patch)
523 striplevel = int(parm['striplevel'])
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500524 patchedfiles[patchfile] = oe.patch.PatchSet.getPatchedFiles(patchfile, striplevel, os.path.join(d.getVar('S'), parm.get('patchdir', '')))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500525 return patchedfiles
526
527
528def validate_pn(pn):
529 """Perform validation on a recipe name (PN) for a new recipe."""
530 reserved_names = ['forcevariable', 'append', 'prepend', 'remove']
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600531 if not re.match('^[0-9a-z-.+]+$', pn):
532 return 'Recipe name "%s" is invalid: only characters 0-9, a-z, -, + and . are allowed' % pn
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500533 elif pn in reserved_names:
534 return 'Recipe name "%s" is invalid: is a reserved keyword' % pn
535 elif pn.startswith('pn-'):
536 return 'Recipe name "%s" is invalid: names starting with "pn-" are reserved' % pn
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500537 elif pn.endswith(('.bb', '.bbappend', '.bbclass', '.inc', '.conf')):
538 return 'Recipe name "%s" is invalid: should be just a name, not a file name' % pn
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500539 return ''
540
541
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600542def get_bbfile_path(d, destdir, extrapathhint=None):
543 """
544 Determine the correct path for a recipe within a layer
545 Parameters:
546 d: Recipe-specific datastore
547 destdir: destination directory. Can be the path to the base of the layer or a
548 partial path somewhere within the layer.
549 extrapathhint: a path relative to the base of the layer to try
550 """
551 import bb.cookerdata
552
553 destdir = os.path.abspath(destdir)
554 destlayerdir = find_layerdir(destdir)
555
556 # Parse the specified layer's layer.conf file directly, in case the layer isn't in bblayers.conf
557 confdata = d.createCopy()
558 confdata.setVar('BBFILES', '')
559 confdata.setVar('LAYERDIR', destlayerdir)
560 destlayerconf = os.path.join(destlayerdir, "conf", "layer.conf")
561 confdata = bb.cookerdata.parse_config_file(destlayerconf, confdata)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500562 pn = d.getVar('PN')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600563
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500564 # Parse BBFILES_DYNAMIC and append to BBFILES
565 bbfiles_dynamic = (confdata.getVar('BBFILES_DYNAMIC') or "").split()
566 collections = (confdata.getVar('BBFILE_COLLECTIONS') or "").split()
567 invalid = []
568 for entry in bbfiles_dynamic:
569 parts = entry.split(":", 1)
570 if len(parts) != 2:
571 invalid.append(entry)
572 continue
573 l, f = parts
574 invert = l[0] == "!"
575 if invert:
576 l = l[1:]
577 if (l in collections and not invert) or (l not in collections and invert):
578 confdata.appendVar("BBFILES", " " + f)
579 if invalid:
580 return None
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500581 bbfilespecs = (confdata.getVar('BBFILES') or '').split()
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600582 if destdir == destlayerdir:
583 for bbfilespec in bbfilespecs:
584 if not bbfilespec.endswith('.bbappend'):
585 for match in glob.glob(bbfilespec):
586 splitext = os.path.splitext(os.path.basename(match))
587 if splitext[1] == '.bb':
588 mpn = splitext[0].split('_')[0]
589 if mpn == pn:
590 return os.path.dirname(match)
591
592 # Try to make up a path that matches BBFILES
593 # this is a little crude, but better than nothing
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500594 bpn = d.getVar('BPN')
595 recipefn = os.path.basename(d.getVar('FILE'))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600596 pathoptions = [destdir]
597 if extrapathhint:
598 pathoptions.append(os.path.join(destdir, extrapathhint))
599 if destdir == destlayerdir:
600 pathoptions.append(os.path.join(destdir, 'recipes-%s' % bpn, bpn))
601 pathoptions.append(os.path.join(destdir, 'recipes', bpn))
602 pathoptions.append(os.path.join(destdir, bpn))
603 elif not destdir.endswith(('/' + pn, '/' + bpn)):
604 pathoptions.append(os.path.join(destdir, bpn))
605 closepath = ''
606 for pathoption in pathoptions:
607 bbfilepath = os.path.join(pathoption, 'test.bb')
608 for bbfilespec in bbfilespecs:
609 if fnmatch.fnmatchcase(bbfilepath, bbfilespec):
610 return pathoption
611 return None
612
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500613def get_bbappend_path(d, destlayerdir, wildcardver=False):
614 """Determine how a bbappend for a recipe should be named and located within another layer"""
615
616 import bb.cookerdata
617
618 destlayerdir = os.path.abspath(destlayerdir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500619 recipefile = d.getVar('FILE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500620 recipefn = os.path.splitext(os.path.basename(recipefile))[0]
621 if wildcardver and '_' in recipefn:
622 recipefn = recipefn.split('_', 1)[0] + '_%'
623 appendfn = recipefn + '.bbappend'
624
625 # Parse the specified layer's layer.conf file directly, in case the layer isn't in bblayers.conf
626 confdata = d.createCopy()
627 confdata.setVar('BBFILES', '')
628 confdata.setVar('LAYERDIR', destlayerdir)
629 destlayerconf = os.path.join(destlayerdir, "conf", "layer.conf")
630 confdata = bb.cookerdata.parse_config_file(destlayerconf, confdata)
631
632 origlayerdir = find_layerdir(recipefile)
633 if not origlayerdir:
634 return (None, False)
635 # Now join this to the path where the bbappend is going and check if it is covered by BBFILES
636 appendpath = os.path.join(destlayerdir, os.path.relpath(os.path.dirname(recipefile), origlayerdir), appendfn)
637 closepath = ''
638 pathok = True
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500639 for bbfilespec in confdata.getVar('BBFILES').split():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500640 if fnmatch.fnmatchcase(appendpath, bbfilespec):
641 # Our append path works, we're done
642 break
643 elif bbfilespec.startswith(destlayerdir) and fnmatch.fnmatchcase('test.bbappend', os.path.basename(bbfilespec)):
644 # Try to find the longest matching path
645 if len(bbfilespec) > len(closepath):
646 closepath = bbfilespec
647 else:
648 # Unfortunately the bbappend layer and the original recipe's layer don't have the same structure
649 if closepath:
650 # bbappend layer's layer.conf at least has a spec that picks up .bbappend files
651 # Now we just need to substitute out any wildcards
652 appendsubdir = os.path.relpath(os.path.dirname(closepath), destlayerdir)
653 if 'recipes-*' in appendsubdir:
654 # Try to copy this part from the original recipe path
655 res = re.search('/recipes-[^/]+/', recipefile)
656 if res:
657 appendsubdir = appendsubdir.replace('/recipes-*/', res.group(0))
658 # This is crude, but we have to do something
659 appendsubdir = appendsubdir.replace('*', recipefn.split('_')[0])
660 appendsubdir = appendsubdir.replace('?', 'a')
661 appendpath = os.path.join(destlayerdir, appendsubdir, appendfn)
662 else:
663 pathok = False
664 return (appendpath, pathok)
665
666
Andrew Geissler615f2f12022-07-15 14:00:58 -0500667def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, machine=None, extralines=None, removevalues=None, redirect_output=None, params=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500668 """
669 Writes a bbappend file for a recipe
670 Parameters:
671 rd: data dictionary for the recipe
672 destlayerdir: base directory of the layer to place the bbappend in
673 (subdirectory path from there will be determined automatically)
674 srcfiles: dict of source files to add to SRC_URI, where the value
Patrick Williamsda295312023-12-05 16:48:56 -0600675 is the full path to the file to be added, and the value is a
676 dict with 'path' key containing the original filename as it
677 would appear in SRC_URI or None if it isn't already present.
678 You may pass None for this parameter if you simply want to specify
679 your own content via the extralines parameter.
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500680 install: dict mapping entries in srcfiles to a tuple of two elements:
681 install path (*without* ${D} prefix) and permission value (as a
682 string, e.g. '0644').
683 wildcardver: True to use a % wildcard in the bbappend filename, or
684 False to make the bbappend specific to the recipe version.
685 machine:
686 If specified, make the changes in the bbappend specific to this
687 machine. This will also cause PACKAGE_ARCH = "${MACHINE_ARCH}"
688 to be added to the bbappend.
689 extralines:
690 Extra lines to add to the bbappend. This may be a dict of name
691 value pairs, or simply a list of the lines.
692 removevalues:
693 Variable values to remove - a dict of names/values.
Brad Bishop316dfdd2018-06-25 12:45:53 -0400694 redirect_output:
695 If specified, redirects writing the output file to the
696 specified directory (for dry-run purposes)
Andrew Geissler615f2f12022-07-15 14:00:58 -0500697 params:
698 Parameters to use when adding entries to SRC_URI. If specified,
699 should be a list of dicts with the same length as srcfiles.
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500700 """
701
702 if not removevalues:
703 removevalues = {}
704
705 # Determine how the bbappend should be named
706 appendpath, pathok = get_bbappend_path(rd, destlayerdir, wildcardver)
707 if not appendpath:
708 bb.error('Unable to determine layer directory containing %s' % recipefile)
709 return (None, None)
710 if not pathok:
711 bb.warn('Unable to determine correct subdirectory path for bbappend file - check that what %s adds to BBFILES also matches .bbappend files. Using %s for now, but until you fix this the bbappend will not be applied.' % (os.path.join(destlayerdir, 'conf', 'layer.conf'), os.path.dirname(appendpath)))
712
713 appenddir = os.path.dirname(appendpath)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400714 if not redirect_output:
715 bb.utils.mkdirhier(appenddir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500716
717 # FIXME check if the bbappend doesn't get overridden by a higher priority layer?
718
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500719 layerdirs = [os.path.abspath(layerdir) for layerdir in rd.getVar('BBLAYERS').split()]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500720 if not os.path.abspath(destlayerdir) in layerdirs:
721 bb.warn('Specified layer is not currently enabled in bblayers.conf, you will need to add it before this bbappend will be active')
722
723 bbappendlines = []
724 if extralines:
725 if isinstance(extralines, dict):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600726 for name, value in extralines.items():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500727 bbappendlines.append((name, '=', value))
728 else:
729 # Do our best to split it
730 for line in extralines:
731 if line[-1] == '\n':
732 line = line[:-1]
733 splitline = line.split(None, 2)
734 if len(splitline) == 3:
735 bbappendlines.append(tuple(splitline))
736 else:
737 raise Exception('Invalid extralines value passed')
738
739 def popline(varname):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600740 for i in range(0, len(bbappendlines)):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500741 if bbappendlines[i][0] == varname:
742 line = bbappendlines.pop(i)
743 return line
744 return None
745
746 def appendline(varname, op, value):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600747 for i in range(0, len(bbappendlines)):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500748 item = bbappendlines[i]
749 if item[0] == varname:
750 bbappendlines[i] = (item[0], item[1], item[2] + ' ' + value)
751 break
752 else:
753 bbappendlines.append((varname, op, value))
754
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500755 destsubdir = rd.getVar('PN')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500756 if srcfiles:
Patrick Williams213cb262021-08-07 19:21:33 -0500757 bbappendlines.append(('FILESEXTRAPATHS:prepend', ':=', '${THISDIR}/${PN}:'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500758
759 appendoverride = ''
760 if machine:
761 bbappendlines.append(('PACKAGE_ARCH', '=', '${MACHINE_ARCH}'))
Patrick Williams213cb262021-08-07 19:21:33 -0500762 appendoverride = ':%s' % machine
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500763 copyfiles = {}
764 if srcfiles:
765 instfunclines = []
Patrick Williamsda295312023-12-05 16:48:56 -0600766 for i, (newfile, param) in enumerate(srcfiles.items()):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500767 srcurientry = None
Patrick Williamsda295312023-12-05 16:48:56 -0600768 if not 'path' in param or not param['path']:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500769 srcfile = os.path.basename(newfile)
770 srcurientry = 'file://%s' % srcfile
Andrew Geissler615f2f12022-07-15 14:00:58 -0500771 if params and params[i]:
772 srcurientry = '%s;%s' % (srcurientry, ';'.join('%s=%s' % (k,v) for k,v in params[i].items()))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500773 # Double-check it's not there already
774 # FIXME do we care if the entry is added by another bbappend that might go away?
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500775 if not srcurientry in rd.getVar('SRC_URI').split():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500776 if machine:
Patrick Williams213cb262021-08-07 19:21:33 -0500777 appendline('SRC_URI:append%s' % appendoverride, '=', ' ' + srcurientry)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500778 else:
779 appendline('SRC_URI', '+=', srcurientry)
Patrick Williamsda295312023-12-05 16:48:56 -0600780 param['path'] = srcfile
781 else:
782 srcfile = param['path']
783 copyfiles[newfile] = param
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500784 if install:
785 institem = install.pop(newfile, None)
786 if institem:
787 (destpath, perms) = institem
788 instdestpath = replace_dir_vars(destpath, rd)
789 instdirline = 'install -d ${D}%s' % os.path.dirname(instdestpath)
790 if not instdirline in instfunclines:
791 instfunclines.append(instdirline)
792 instfunclines.append('install -m %s ${WORKDIR}/%s ${D}%s' % (perms, os.path.basename(srcfile), instdestpath))
793 if instfunclines:
Patrick Williams213cb262021-08-07 19:21:33 -0500794 bbappendlines.append(('do_install:append%s()' % appendoverride, '', instfunclines))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500795
Brad Bishop316dfdd2018-06-25 12:45:53 -0400796 if redirect_output:
797 bb.note('Writing append file %s (dry-run)' % appendpath)
798 outfile = os.path.join(redirect_output, os.path.basename(appendpath))
799 # Only take a copy if the file isn't already there (this function may be called
800 # multiple times per operation when we're handling overrides)
801 if os.path.exists(appendpath) and not os.path.exists(outfile):
802 shutil.copy2(appendpath, outfile)
803 else:
804 bb.note('Writing append file %s' % appendpath)
805 outfile = appendpath
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500806
Brad Bishop316dfdd2018-06-25 12:45:53 -0400807 if os.path.exists(outfile):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500808 # Work around lack of nonlocal in python 2
809 extvars = {'destsubdir': destsubdir}
810
811 def appendfile_varfunc(varname, origvalue, op, newlines):
Patrick Williams213cb262021-08-07 19:21:33 -0500812 if varname == 'FILESEXTRAPATHS:prepend':
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500813 if origvalue.startswith('${THISDIR}/'):
Patrick Williams213cb262021-08-07 19:21:33 -0500814 popline('FILESEXTRAPATHS:prepend')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500815 extvars['destsubdir'] = rd.expand(origvalue.split('${THISDIR}/', 1)[1].rstrip(':'))
816 elif varname == 'PACKAGE_ARCH':
817 if machine:
818 popline('PACKAGE_ARCH')
819 return (machine, None, 4, False)
Patrick Williams213cb262021-08-07 19:21:33 -0500820 elif varname.startswith('do_install:append'):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500821 func = popline(varname)
822 if func:
823 instfunclines = [line.strip() for line in origvalue.strip('\n').splitlines()]
824 for line in func[2]:
825 if not line in instfunclines:
826 instfunclines.append(line)
827 return (instfunclines, None, 4, False)
828 else:
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500829 splitval = split_var_value(origvalue, assignment=False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500830 changed = False
831 removevar = varname
Patrick Williams213cb262021-08-07 19:21:33 -0500832 if varname in ['SRC_URI', 'SRC_URI:append%s' % appendoverride]:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500833 removevar = 'SRC_URI'
834 line = popline(varname)
835 if line:
836 if line[2] not in splitval:
837 splitval.append(line[2])
838 changed = True
839 else:
840 line = popline(varname)
841 if line:
842 splitval = [line[2]]
843 changed = True
844
845 if removevar in removevalues:
846 remove = removevalues[removevar]
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600847 if isinstance(remove, str):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500848 if remove in splitval:
849 splitval.remove(remove)
850 changed = True
851 else:
852 for removeitem in remove:
853 if removeitem in splitval:
854 splitval.remove(removeitem)
855 changed = True
856
857 if changed:
858 newvalue = splitval
859 if len(newvalue) == 1:
860 # Ensure it's written out as one line
Patrick Williams213cb262021-08-07 19:21:33 -0500861 if ':append' in varname:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500862 newvalue = ' ' + newvalue[0]
863 else:
864 newvalue = newvalue[0]
Patrick Williams213cb262021-08-07 19:21:33 -0500865 if not newvalue and (op in ['+=', '.='] or ':append' in varname):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500866 # There's no point appending nothing
867 newvalue = None
868 if varname.endswith('()'):
869 indent = 4
870 else:
871 indent = -1
872 return (newvalue, None, indent, True)
873 return (origvalue, None, 4, False)
874
875 varnames = [item[0] for item in bbappendlines]
876 if removevalues:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600877 varnames.extend(list(removevalues.keys()))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500878
Brad Bishop316dfdd2018-06-25 12:45:53 -0400879 with open(outfile, 'r') as f:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500880 (updated, newlines) = bb.utils.edit_metadata(f, varnames, appendfile_varfunc)
881
882 destsubdir = extvars['destsubdir']
883 else:
884 updated = False
885 newlines = []
886
887 if bbappendlines:
888 for line in bbappendlines:
889 if line[0].endswith('()'):
890 newlines.append('%s {\n %s\n}\n' % (line[0], '\n '.join(line[2])))
891 else:
892 newlines.append('%s %s "%s"\n\n' % line)
893 updated = True
894
895 if updated:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400896 with open(outfile, 'w') as f:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500897 f.writelines(newlines)
898
899 if copyfiles:
900 if machine:
901 destsubdir = os.path.join(destsubdir, machine)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400902 if redirect_output:
903 outdir = redirect_output
904 else:
905 outdir = appenddir
Patrick Williamsda295312023-12-05 16:48:56 -0600906 for newfile, param in copyfiles.items():
907 srcfile = param['path']
908 patchdir = param.get('patchdir', ".")
909
910 if patchdir != ".":
911 newfile = os.path.join(os.path.split(newfile)[0], patchdir, os.path.split(newfile)[1])
Brad Bishop316dfdd2018-06-25 12:45:53 -0400912 filedest = os.path.join(outdir, destsubdir, os.path.basename(srcfile))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500913 if os.path.abspath(newfile) != os.path.abspath(filedest):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500914 if newfile.startswith(tempfile.gettempdir()):
915 newfiledisp = os.path.basename(newfile)
916 else:
917 newfiledisp = newfile
Brad Bishop316dfdd2018-06-25 12:45:53 -0400918 if redirect_output:
919 bb.note('Copying %s to %s (dry-run)' % (newfiledisp, os.path.join(appenddir, destsubdir, os.path.basename(srcfile))))
920 else:
921 bb.note('Copying %s to %s' % (newfiledisp, filedest))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500922 bb.utils.mkdirhier(os.path.dirname(filedest))
923 shutil.copyfile(newfile, filedest)
924
925 return (appendpath, os.path.join(appenddir, destsubdir))
926
927
928def find_layerdir(fn):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600929 """ Figure out the path to the base of the layer containing a file (e.g. a recipe)"""
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500930 pth = os.path.abspath(fn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500931 layerdir = ''
932 while pth:
933 if os.path.exists(os.path.join(pth, 'conf', 'layer.conf')):
934 layerdir = pth
935 break
936 pth = os.path.dirname(pth)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600937 if pth == '/':
938 return None
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500939 return layerdir
940
941
942def replace_dir_vars(path, d):
943 """Replace common directory paths with appropriate variable references (e.g. /etc becomes ${sysconfdir})"""
944 dirvars = {}
945 # Sort by length so we get the variables we're interested in first
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600946 for var in sorted(list(d.keys()), key=len):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500947 if var.endswith('dir') and var.lower() == var:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500948 value = d.getVar(var)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500949 if value.startswith('/') and not '\n' in value and value not in dirvars:
950 dirvars[value] = var
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600951 for dirpath in sorted(list(dirvars.keys()), reverse=True):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500952 path = path.replace(dirpath, '${%s}' % dirvars[dirpath])
953 return path
954
Andrew Geissler5082cc72023-09-11 08:41:39 -0400955def get_recipe_pv_with_pfx_sfx(pv, uri_type):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500956 """
Andrew Geissler5082cc72023-09-11 08:41:39 -0400957 Get PV separating prefix and suffix components.
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500958
959 Returns tuple with pv, prefix and suffix.
960 """
961 pfx = ''
962 sfx = ''
963
964 if uri_type == 'git':
Andrew Geissler5082cc72023-09-11 08:41:39 -0400965 git_regex = re.compile(r"(?P<pfx>v?)(?P<ver>.*?)(?P<sfx>\+[^\+]*(git)?r?(AUTOINC\+)?)(?P<rev>.*)")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500966 m = git_regex.match(pv)
967
968 if m:
969 pv = m.group('ver')
970 pfx = m.group('pfx')
971 sfx = m.group('sfx')
972 else:
Brad Bishop19323692019-04-05 15:28:33 -0400973 regex = re.compile(r"(?P<pfx>(v|r)?)(?P<ver>.*)")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500974 m = regex.match(pv)
975 if m:
976 pv = m.group('ver')
977 pfx = m.group('pfx')
978
979 return (pv, pfx, sfx)
980
981def get_recipe_upstream_version(rd):
982 """
983 Get upstream version of recipe using bb.fetch2 methods with support for
984 http, https, ftp and git.
985
986 bb.fetch2 exceptions can be raised,
987 FetchError when don't have network access or upstream site don't response.
988 NoMethodError when uri latest_versionstring method isn't implemented.
989
Brad Bishop316dfdd2018-06-25 12:45:53 -0400990 Returns a dictonary with version, repository revision, current_version, type and datetime.
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500991 Type can be A for Automatic, M for Manual and U for Unknown.
992 """
993 from bb.fetch2 import decodeurl
994 from datetime import datetime
995
996 ru = {}
Brad Bishop316dfdd2018-06-25 12:45:53 -0400997 ru['current_version'] = rd.getVar('PV')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500998 ru['version'] = ''
999 ru['type'] = 'U'
1000 ru['datetime'] = ''
Brad Bishop316dfdd2018-06-25 12:45:53 -04001001 ru['revision'] = ''
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001002
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001003 # XXX: If don't have SRC_URI means that don't have upstream sources so
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001004 # returns the current recipe version, so that upstream version check
1005 # declares a match.
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001006 src_uris = rd.getVar('SRC_URI')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001007 if not src_uris:
Brad Bishop316dfdd2018-06-25 12:45:53 -04001008 ru['version'] = ru['current_version']
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001009 ru['type'] = 'M'
1010 ru['datetime'] = datetime.now()
1011 return ru
1012
1013 # XXX: we suppose that the first entry points to the upstream sources
1014 src_uri = src_uris.split()[0]
1015 uri_type, _, _, _, _, _ = decodeurl(src_uri)
1016
Andrew Geissler5082cc72023-09-11 08:41:39 -04001017 (pv, pfx, sfx) = get_recipe_pv_with_pfx_sfx(rd.getVar('PV'), uri_type)
Brad Bishop316dfdd2018-06-25 12:45:53 -04001018 ru['current_version'] = pv
1019
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001020 manual_upstream_version = rd.getVar("RECIPE_UPSTREAM_VERSION")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001021 if manual_upstream_version:
1022 # manual tracking of upstream version.
1023 ru['version'] = manual_upstream_version
1024 ru['type'] = 'M'
1025
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001026 manual_upstream_date = rd.getVar("CHECK_DATE")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001027 if manual_upstream_date:
1028 date = datetime.strptime(manual_upstream_date, "%b %d, %Y")
1029 else:
1030 date = datetime.now()
1031 ru['datetime'] = date
1032
1033 elif uri_type == "file":
1034 # files are always up-to-date
1035 ru['version'] = pv
1036 ru['type'] = 'A'
1037 ru['datetime'] = datetime.now()
1038 else:
1039 ud = bb.fetch2.FetchData(src_uri, rd)
Brad Bishop316dfdd2018-06-25 12:45:53 -04001040 if rd.getVar("UPSTREAM_CHECK_COMMITS") == "1":
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001041 bb.fetch2.get_srcrev(rd)
Brad Bishop316dfdd2018-06-25 12:45:53 -04001042 revision = ud.method.latest_revision(ud, rd, 'default')
1043 upversion = pv
1044 if revision != rd.getVar("SRCREV"):
Patrick Williamsda295312023-12-05 16:48:56 -06001045 upversion = upversion + "-new-commits-available"
Brad Bishop316dfdd2018-06-25 12:45:53 -04001046 else:
1047 pupver = ud.method.latest_versionstring(ud, rd)
1048 (upversion, revision) = pupver
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001049
1050 if upversion:
1051 ru['version'] = upversion
1052 ru['type'] = 'A'
1053
Brad Bishop316dfdd2018-06-25 12:45:53 -04001054 if revision:
1055 ru['revision'] = revision
1056
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001057 ru['datetime'] = datetime.now()
1058
1059 return ru
Brad Bishop19323692019-04-05 15:28:33 -04001060
1061def _get_recipe_upgrade_status(data):
1062 uv = get_recipe_upstream_version(data)
1063
1064 pn = data.getVar('PN')
1065 cur_ver = uv['current_version']
1066
1067 upstream_version_unknown = data.getVar('UPSTREAM_VERSION_UNKNOWN')
1068 if not uv['version']:
1069 status = "UNKNOWN" if upstream_version_unknown else "UNKNOWN_BROKEN"
1070 else:
1071 cmp = vercmp_string(uv['current_version'], uv['version'])
1072 if cmp == -1:
1073 status = "UPDATE" if not upstream_version_unknown else "KNOWN_BROKEN"
1074 elif cmp == 0:
1075 status = "MATCH" if not upstream_version_unknown else "KNOWN_BROKEN"
1076 else:
1077 status = "UNKNOWN" if upstream_version_unknown else "UNKNOWN_BROKEN"
1078
1079 next_ver = uv['version'] if uv['version'] else "N/A"
1080 revision = uv['revision'] if uv['revision'] else "N/A"
1081 maintainer = data.getVar('RECIPE_MAINTAINER')
1082 no_upgrade_reason = data.getVar('RECIPE_NO_UPDATE_REASON')
1083
1084 return (pn, status, cur_ver, next_ver, maintainer, revision, no_upgrade_reason)
1085
1086def get_recipe_upgrade_status(recipes=None):
1087 pkgs_list = []
1088 data_copy_list = []
1089 copy_vars = ('SRC_URI',
1090 'PV',
Brad Bishop19323692019-04-05 15:28:33 -04001091 'DL_DIR',
1092 'PN',
1093 'CACHE',
1094 'PERSISTENT_DIR',
1095 'BB_URI_HEADREVS',
1096 'UPSTREAM_CHECK_COMMITS',
1097 'UPSTREAM_CHECK_GITTAGREGEX',
1098 'UPSTREAM_CHECK_REGEX',
1099 'UPSTREAM_CHECK_URI',
1100 'UPSTREAM_VERSION_UNKNOWN',
1101 'RECIPE_MAINTAINER',
1102 'RECIPE_NO_UPDATE_REASON',
1103 'RECIPE_UPSTREAM_VERSION',
1104 'RECIPE_UPSTREAM_DATE',
1105 'CHECK_DATE',
Andrew Geissler1e34c2d2020-05-29 16:02:59 -05001106 'FETCHCMD_bzr',
1107 'FETCHCMD_ccrc',
1108 'FETCHCMD_cvs',
1109 'FETCHCMD_git',
1110 'FETCHCMD_hg',
1111 'FETCHCMD_npm',
1112 'FETCHCMD_osc',
1113 'FETCHCMD_p4',
1114 'FETCHCMD_repo',
1115 'FETCHCMD_s3',
1116 'FETCHCMD_svn',
1117 'FETCHCMD_wget',
Brad Bishop19323692019-04-05 15:28:33 -04001118 )
1119
1120 with bb.tinfoil.Tinfoil() as tinfoil:
1121 tinfoil.prepare(config_only=False)
1122
1123 if not recipes:
1124 recipes = tinfoil.all_recipe_files(variants=False)
1125
1126 for fn in recipes:
1127 try:
1128 if fn.startswith("/"):
1129 data = tinfoil.parse_recipe_file(fn)
1130 else:
1131 data = tinfoil.parse_recipe(fn)
1132 except bb.providers.NoProvider:
1133 bb.note(" No provider for %s" % fn)
1134 continue
1135
1136 unreliable = data.getVar('UPSTREAM_CHECK_UNRELIABLE')
1137 if unreliable == "1":
1138 bb.note(" Skip package %s as upstream check unreliable" % pn)
1139 continue
1140
1141 data_copy = bb.data.init()
1142 for var in copy_vars:
1143 data_copy.setVar(var, data.getVar(var))
1144 for k in data:
1145 if k.startswith('SRCREV'):
1146 data_copy.setVar(k, data.getVar(k))
1147
1148 data_copy_list.append(data_copy)
1149
1150 from concurrent.futures import ProcessPoolExecutor
1151 with ProcessPoolExecutor(max_workers=utils.cpu_count()) as executor:
1152 pkgs_list = executor.map(_get_recipe_upgrade_status, data_copy_list)
1153
1154 return pkgs_list