blob: 407d16889464246dfbd9d26ce8debb0331e0d0fc [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# Utility functions for reading and modifying recipes
2#
3# Some code borrowed from the OE layer index
4#
Brad Bishopd7bf8c12018-02-25 22:55:05 -05005# Copyright (C) 2013-2017 Intel Corporation
Patrick Williamsc124f4f2015-09-15 14:41:29 -05006#
Brad Bishopc342db32019-05-15 21:57:59 -04007# SPDX-License-Identifier: GPL-2.0-only
8#
Patrick Williamsc124f4f2015-09-15 14:41:29 -05009
10import sys
11import os
12import os.path
13import tempfile
14import textwrap
15import difflib
Patrick Williamsc0f7c042017-02-23 20:41:17 -060016from . import utils
Patrick Williamsc124f4f2015-09-15 14:41:29 -050017import shutil
18import re
19import fnmatch
Patrick Williamsc0f7c042017-02-23 20:41:17 -060020import glob
Brad Bishop19323692019-04-05 15:28:33 -040021import bb.tinfoil
Patrick Williamsc124f4f2015-09-15 14:41:29 -050022
Brad Bishop19323692019-04-05 15:28:33 -040023from collections import OrderedDict, defaultdict
24from bb.utils import vercmp_string
Patrick Williamsc124f4f2015-09-15 14:41:29 -050025
26# Help us to find places to insert values
Andrew Geisslerb7d28612020-07-24 16:15:54 -050027recipe_progression = ['SUMMARY', 'DESCRIPTION', 'AUTHOR', 'HOMEPAGE', 'BUGTRACKER', 'SECTION', 'LICENSE', 'LICENSE_FLAGS', 'LIC_FILES_CHKSUM', 'PROVIDES', 'DEPENDS', 'PR', 'PV', 'SRCREV', 'SRCPV', 'SRC_URI', 'S', 'do_fetch()', 'do_unpack()', 'do_patch()', 'EXTRA_OECONF', 'EXTRA_OECMAKE', 'EXTRA_OESCONS', 'do_configure()', 'EXTRA_OEMAKE', 'do_compile()', 'do_install()', 'do_populate_sysroot()', 'INITSCRIPT', 'USERADD', 'GROUPADD', 'PACKAGES', 'FILES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RPROVIDES', 'RREPLACES', 'RCONFLICTS', 'ALLOW_EMPTY', 'populate_packages()', 'do_package()', 'do_deploy()', 'BBCLASSEXTEND']
Patrick Williamsc124f4f2015-09-15 14:41:29 -050028# Variables that sometimes are a bit long but shouldn't be wrapped
Brad Bishop19323692019-04-05 15:28:33 -040029nowrap_vars = ['SUMMARY', 'HOMEPAGE', 'BUGTRACKER', r'SRC_URI\[(.+\.)?md5sum\]', r'SRC_URI\[(.+\.)?sha256sum\]']
Patrick Williamsc124f4f2015-09-15 14:41:29 -050030list_vars = ['SRC_URI', 'LIC_FILES_CHKSUM']
31meta_vars = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION']
32
33
Brad Bishop19323692019-04-05 15:28:33 -040034def simplify_history(history, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050035 """
Brad Bishop19323692019-04-05 15:28:33 -040036 Eliminate any irrelevant events from a variable history
Patrick Williamsc124f4f2015-09-15 14:41:29 -050037 """
Brad Bishop19323692019-04-05 15:28:33 -040038 ret_history = []
39 has_set = False
40 # Go backwards through the history and remove any immediate operations
41 # before the most recent set
42 for event in reversed(history):
43 if 'flag' in event or not 'file' in event:
44 continue
45 if event['op'] == 'set':
46 if has_set:
47 continue
48 has_set = True
49 elif event['op'] in ('append', 'prepend', 'postdot', 'predot'):
50 # Reminder: "append" and "prepend" mean += and =+ respectively, NOT _append / _prepend
51 if has_set:
52 continue
53 ret_history.insert(0, event)
54 return ret_history
Patrick Williamsc124f4f2015-09-15 14:41:29 -050055
56
Patrick Williamsc124f4f2015-09-15 14:41:29 -050057def get_var_files(fn, varlist, d):
58 """Find the file in which each of a list of variables is set.
59 Note: requires variable history to be enabled when parsing.
60 """
61 varfiles = {}
62 for v in varlist:
Patrick Williamsc124f4f2015-09-15 14:41:29 -050063 files = []
Brad Bishop19323692019-04-05 15:28:33 -040064 if '[' in v:
65 varsplit = v.split('[')
66 varflag = varsplit[1].split(']')[0]
67 history = d.varhistory.variable(varsplit[0])
68 for event in history:
69 if 'file' in event and event.get('flag', '') == varflag:
70 files.append(event['file'])
71 else:
72 history = d.varhistory.variable(v)
73 for event in history:
74 if 'file' in event and not 'flag' in event:
75 files.append(event['file'])
Patrick Williamsc124f4f2015-09-15 14:41:29 -050076 if files:
77 actualfile = files[-1]
78 else:
79 actualfile = None
80 varfiles[v] = actualfile
81
82 return varfiles
83
84
Patrick Williamsf1e5d692016-03-30 15:21:19 -050085def split_var_value(value, assignment=True):
86 """
87 Split a space-separated variable's value into a list of items,
88 taking into account that some of the items might be made up of
89 expressions containing spaces that should not be split.
90 Parameters:
91 value:
92 The string value to split
93 assignment:
94 True to assume that the value represents an assignment
95 statement, False otherwise. If True, and an assignment
96 statement is passed in the first item in
97 the returned list will be the part of the assignment
98 statement up to and including the opening quote character,
99 and the last item will be the closing quote.
100 """
101 inexpr = 0
102 lastchar = None
103 out = []
104 buf = ''
105 for char in value:
106 if char == '{':
107 if lastchar == '$':
108 inexpr += 1
109 elif char == '}':
110 inexpr -= 1
111 elif assignment and char in '"\'' and inexpr == 0:
112 if buf:
113 out.append(buf)
114 out.append(char)
115 char = ''
116 buf = ''
117 elif char.isspace() and inexpr == 0:
118 char = ''
119 if buf:
120 out.append(buf)
121 buf = ''
122 buf += char
123 lastchar = char
124 if buf:
125 out.append(buf)
126
127 # Join together assignment statement and opening quote
128 outlist = out
129 if assignment:
130 assigfound = False
131 for idx, item in enumerate(out):
132 if '=' in item:
133 assigfound = True
134 if assigfound:
135 if '"' in item or "'" in item:
136 outlist = [' '.join(out[:idx+1])]
137 outlist.extend(out[idx+1:])
138 break
139 return outlist
140
141
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600142def patch_recipe_lines(fromlines, values, trailing_newline=True):
143 """Update or insert variable values into lines from a recipe.
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500144 Note that some manual inspection/intervention may be required
145 since this cannot handle all situations.
146 """
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500147
148 import bb.utils
149
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600150 if trailing_newline:
151 newline = '\n'
152 else:
153 newline = ''
154
Brad Bishop316dfdd2018-06-25 12:45:53 -0400155 nowrap_vars_res = []
156 for item in nowrap_vars:
157 nowrap_vars_res.append(re.compile('^%s$' % item))
158
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500159 recipe_progression_res = []
160 recipe_progression_restrs = []
161 for item in recipe_progression:
162 if item.endswith('()'):
163 key = item[:-2]
164 else:
165 key = item
Brad Bishop19323692019-04-05 15:28:33 -0400166 restr = r'%s(_[a-zA-Z0-9-_$(){}]+|\[[^\]]*\])?' % key
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500167 if item.endswith('()'):
168 recipe_progression_restrs.append(restr + '()')
169 else:
170 recipe_progression_restrs.append(restr)
171 recipe_progression_res.append(re.compile('^%s$' % restr))
172
173 def get_recipe_pos(variable):
174 for i, p in enumerate(recipe_progression_res):
175 if p.match(variable):
176 return i
177 return -1
178
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500179 remainingnames = {}
180 for k in values.keys():
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500181 remainingnames[k] = get_recipe_pos(k)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600182 remainingnames = OrderedDict(sorted(remainingnames.items(), key=lambda x: x[1]))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500183
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500184 modifying = False
185
186 def outputvalue(name, lines, rewindcomments=False):
187 if values[name] is None:
188 return
Brad Bishop19323692019-04-05 15:28:33 -0400189 if isinstance(values[name], tuple):
190 op, value = values[name]
191 if op == '+=' and value.strip() == '':
192 return
193 else:
194 value = values[name]
195 op = '='
196 rawtext = '%s %s "%s"%s' % (name, op, value, newline)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500197 addlines = []
Brad Bishop316dfdd2018-06-25 12:45:53 -0400198 nowrap = False
199 for nowrap_re in nowrap_vars_res:
200 if nowrap_re.match(name):
201 nowrap = True
202 break
203 if nowrap:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500204 addlines.append(rawtext)
205 elif name in list_vars:
Brad Bishop19323692019-04-05 15:28:33 -0400206 splitvalue = split_var_value(value, assignment=False)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500207 if len(splitvalue) > 1:
208 linesplit = ' \\\n' + (' ' * (len(name) + 4))
Brad Bishop19323692019-04-05 15:28:33 -0400209 addlines.append('%s %s "%s%s"%s' % (name, op, linesplit.join(splitvalue), linesplit, newline))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500210 else:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500211 addlines.append(rawtext)
212 else:
213 wrapped = textwrap.wrap(rawtext)
214 for wrapline in wrapped[:-1]:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600215 addlines.append('%s \\%s' % (wrapline, newline))
216 addlines.append('%s%s' % (wrapped[-1], newline))
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500217
218 # Split on newlines - this isn't strictly necessary if you are only
219 # going to write the output to disk, but if you want to compare it
220 # (as patch_recipe_file() will do if patch=True) then it's important.
221 addlines = [line for l in addlines for line in l.splitlines(True)]
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500222 if rewindcomments:
223 # Ensure we insert the lines before any leading comments
224 # (that we'd want to ensure remain leading the next value)
225 for i, ln in reversed(list(enumerate(lines))):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600226 if not ln.startswith('#'):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500227 lines[i+1:i+1] = addlines
228 break
229 else:
230 lines.extend(addlines)
231 else:
232 lines.extend(addlines)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500233
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500234 existingnames = []
235 def patch_recipe_varfunc(varname, origvalue, op, newlines):
236 if modifying:
237 # Insert anything that should come before this variable
238 pos = get_recipe_pos(varname)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600239 for k in list(remainingnames):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500240 if remainingnames[k] > -1 and pos >= remainingnames[k] and not k in existingnames:
241 outputvalue(k, newlines, rewindcomments=True)
242 del remainingnames[k]
243 # Now change this variable, if it needs to be changed
244 if varname in existingnames and op in ['+=', '=', '=+']:
245 if varname in remainingnames:
246 outputvalue(varname, newlines)
247 del remainingnames[varname]
248 return None, None, 0, True
249 else:
250 if varname in values:
251 existingnames.append(varname)
252 return origvalue, None, 0, True
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500253
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500254 # First run - establish which values we want to set are already in the file
255 varlist = [re.escape(item) for item in values.keys()]
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600256 bb.utils.edit_metadata(fromlines, varlist, patch_recipe_varfunc)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500257 # Second run - actually set everything
258 modifying = True
259 varlist.extend(recipe_progression_restrs)
260 changed, tolines = bb.utils.edit_metadata(fromlines, varlist, patch_recipe_varfunc, match_overrides=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500261
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500262 if remainingnames:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600263 if tolines and tolines[-1].strip() != '':
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500264 tolines.append('\n')
265 for k in remainingnames.keys():
266 outputvalue(k, tolines)
267
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600268 return changed, tolines
269
270
Brad Bishop316dfdd2018-06-25 12:45:53 -0400271def patch_recipe_file(fn, values, patch=False, relpath='', redirect_output=None):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600272 """Update or insert variable values into a recipe file (assuming you
273 have already identified the exact file you want to update.)
274 Note that some manual inspection/intervention may be required
275 since this cannot handle all situations.
276 """
277
278 with open(fn, 'r') as f:
279 fromlines = f.readlines()
280
281 _, tolines = patch_recipe_lines(fromlines, values)
282
Brad Bishop316dfdd2018-06-25 12:45:53 -0400283 if redirect_output:
284 with open(os.path.join(redirect_output, os.path.basename(fn)), 'w') as f:
285 f.writelines(tolines)
286 return None
287 elif patch:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500288 relfn = os.path.relpath(fn, relpath)
289 diff = difflib.unified_diff(fromlines, tolines, 'a/%s' % relfn, 'b/%s' % relfn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500290 return diff
291 else:
292 with open(fn, 'w') as f:
293 f.writelines(tolines)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500294 return None
295
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500296
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500297def localise_file_vars(fn, varfiles, varlist):
298 """Given a list of variables and variable history (fetched with get_var_files())
299 find where each variable should be set/changed. This handles for example where a
300 recipe includes an inc file where variables might be changed - in most cases
301 we want to update the inc file when changing the variable value rather than adding
302 it to the recipe itself.
303 """
304 fndir = os.path.dirname(fn) + os.sep
305
306 first_meta_file = None
307 for v in meta_vars:
308 f = varfiles.get(v, None)
309 if f:
310 actualdir = os.path.dirname(f) + os.sep
311 if actualdir.startswith(fndir):
312 first_meta_file = f
313 break
314
315 filevars = defaultdict(list)
316 for v in varlist:
317 f = varfiles[v]
318 # Only return files that are in the same directory as the recipe or in some directory below there
319 # (this excludes bbclass files and common inc files that wouldn't be appropriate to set the variable
320 # in if we were going to set a value specific to this recipe)
321 if f:
322 actualfile = f
323 else:
324 # Variable isn't in a file, if it's one of the "meta" vars, use the first file with a meta var in it
325 if first_meta_file:
326 actualfile = first_meta_file
327 else:
328 actualfile = fn
329
330 actualdir = os.path.dirname(actualfile) + os.sep
331 if not actualdir.startswith(fndir):
332 actualfile = fn
333 filevars[actualfile].append(v)
334
335 return filevars
336
Brad Bishop316dfdd2018-06-25 12:45:53 -0400337def patch_recipe(d, fn, varvalues, patch=False, relpath='', redirect_output=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500338 """Modify a list of variable values in the specified recipe. Handles inc files if
339 used by the recipe.
340 """
Brad Bishop19323692019-04-05 15:28:33 -0400341 overrides = d.getVar('OVERRIDES').split(':')
342 def override_applicable(hevent):
343 op = hevent['op']
344 if '[' in op:
345 opoverrides = op.split('[')[1].split(']')[0].split('_')
346 for opoverride in opoverrides:
347 if not opoverride in overrides:
348 return False
349 return True
350
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500351 varlist = varvalues.keys()
Brad Bishop19323692019-04-05 15:28:33 -0400352 fn = os.path.abspath(fn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500353 varfiles = get_var_files(fn, varlist, d)
354 locs = localise_file_vars(fn, varfiles, varlist)
355 patches = []
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600356 for f,v in locs.items():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500357 vals = {k: varvalues[k] for k in v}
Brad Bishop19323692019-04-05 15:28:33 -0400358 f = os.path.abspath(f)
359 if f == fn:
360 extravals = {}
361 for var, value in vals.items():
362 if var in list_vars:
363 history = simplify_history(d.varhistory.variable(var), d)
364 recipe_set = False
365 for event in history:
366 if os.path.abspath(event['file']) == fn:
367 if event['op'] == 'set':
368 recipe_set = True
369 if not recipe_set:
370 for event in history:
371 if event['op'].startswith('_remove'):
372 continue
373 if not override_applicable(event):
374 continue
375 newvalue = value.replace(event['detail'], '')
376 if newvalue == value and os.path.abspath(event['file']) == fn and event['op'].startswith('_'):
377 op = event['op'].replace('[', '_').replace(']', '')
378 extravals[var + op] = None
379 value = newvalue
380 vals[var] = ('+=', value)
381 vals.update(extravals)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400382 patchdata = patch_recipe_file(f, vals, patch, relpath, redirect_output)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500383 if patch:
384 patches.append(patchdata)
385
386 if patch:
387 return patches
388 else:
389 return None
390
391
392
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500393def copy_recipe_files(d, tgt_dir, whole_dir=False, download=True, all_variants=False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500394 """Copy (local) recipe files, including both files included via include/require,
395 and files referred to in the SRC_URI variable."""
396 import bb.fetch2
397 import oe.path
398
399 # FIXME need a warning if the unexpanded SRC_URI value contains variable references
400
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500401 uri_values = []
402 localpaths = []
403 def fetch_urls(rdata):
404 # Collect the local paths from SRC_URI
405 srcuri = rdata.getVar('SRC_URI') or ""
406 if srcuri not in uri_values:
407 fetch = bb.fetch2.Fetch(srcuri.split(), rdata)
408 if download:
409 fetch.download()
410 for pth in fetch.localpaths():
411 if pth not in localpaths:
Andrew Geissler9b4d8b02021-02-19 12:26:16 -0600412 localpaths.append(os.path.abspath(pth))
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500413 uri_values.append(srcuri)
414
415 fetch_urls(d)
416 if all_variants:
417 # Get files for other variants e.g. in the case of a SRC_URI_append
418 localdata = bb.data.createCopy(d)
419 variants = (localdata.getVar('BBCLASSEXTEND') or '').split()
420 if variants:
421 # Ensure we handle class-target if we're dealing with one of the variants
422 variants.append('target')
423 for variant in variants:
Andrew Geissler82c905d2020-04-13 13:39:40 -0500424 if variant.startswith("devupstream"):
425 localdata.setVar('SRCPV', 'git')
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500426 localdata.setVar('CLASSOVERRIDE', 'class-%s' % variant)
427 fetch_urls(localdata)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500428
429 # Copy local files to target directory and gather any remote files
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500430 bb_dir = os.path.abspath(os.path.dirname(d.getVar('FILE'))) + os.sep
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500431 remotes = []
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600432 copied = []
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500433 # Need to do this in two steps since we want to check against the absolute path
434 includes = [os.path.abspath(path) for path in d.getVar('BBINCLUDED').split() if os.path.exists(path)]
435 # We also check this below, but we don't want any items in this list being considered remotes
436 includes = [path for path in includes if path.startswith(bb_dir)]
437 for path in localpaths + includes:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500438 # Only import files that are under the meta directory
439 if path.startswith(bb_dir):
440 if not whole_dir:
441 relpath = os.path.relpath(path, bb_dir)
442 subdir = os.path.join(tgt_dir, os.path.dirname(relpath))
443 if not os.path.exists(subdir):
444 os.makedirs(subdir)
445 shutil.copy2(path, os.path.join(tgt_dir, relpath))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600446 copied.append(relpath)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500447 else:
448 remotes.append(path)
449 # Simply copy whole meta dir, if requested
450 if whole_dir:
451 shutil.copytree(bb_dir, tgt_dir)
452
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600453 return copied, remotes
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500454
455
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500456def get_recipe_local_files(d, patches=False, archives=False):
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500457 """Get a list of local files in SRC_URI within a recipe."""
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500458 import oe.patch
459 uris = (d.getVar('SRC_URI') or "").split()
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500460 fetch = bb.fetch2.Fetch(uris, d)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500461 # FIXME this list should be factored out somewhere else (such as the
462 # fetcher) though note that this only encompasses actual container formats
463 # i.e. that can contain multiple files as opposed to those that only
464 # contain a compressed stream (i.e. .tar.gz as opposed to just .gz)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400465 archive_exts = ['.tar', '.tgz', '.tar.gz', '.tar.Z', '.tbz', '.tbz2', '.tar.bz2', '.txz', '.tar.xz', '.tar.lz', '.zip', '.jar', '.rpm', '.srpm', '.deb', '.ipk', '.tar.7z', '.7z']
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500466 ret = {}
467 for uri in uris:
468 if fetch.ud[uri].type == 'file':
469 if (not patches and
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500470 oe.patch.patch_path(uri, fetch, '', expand=False)):
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500471 continue
472 # Skip files that are referenced by absolute path
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600473 fname = fetch.ud[uri].basepath
474 if os.path.isabs(fname):
475 continue
476 # Handle subdir=
477 subdir = fetch.ud[uri].parm.get('subdir', '')
478 if subdir:
479 if os.path.isabs(subdir):
480 continue
481 fname = os.path.join(subdir, fname)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500482 localpath = fetch.localpath(uri)
483 if not archives:
484 # Ignore archives that will be unpacked
485 if localpath.endswith(tuple(archive_exts)):
486 unpack = fetch.ud[uri].parm.get('unpack', True)
487 if unpack:
488 continue
Brad Bishop19323692019-04-05 15:28:33 -0400489 if os.path.isdir(localpath):
490 for root, dirs, files in os.walk(localpath):
491 for fname in files:
492 fileabspath = os.path.join(root,fname)
493 srcdir = os.path.dirname(localpath)
494 ret[os.path.relpath(fileabspath,srcdir)] = fileabspath
495 else:
496 ret[fname] = localpath
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500497 return ret
498
499
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500500def get_recipe_patches(d):
501 """Get a list of the patches included in SRC_URI within a recipe."""
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500502 import oe.patch
503 patches = oe.patch.src_patches(d, expand=False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500504 patchfiles = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500505 for patch in patches:
506 _, _, local, _, _, parm = bb.fetch.decodeurl(patch)
507 patchfiles.append(local)
508 return patchfiles
509
510
511def get_recipe_patched_files(d):
512 """
513 Get the list of patches for a recipe along with the files each patch modifies.
514 Params:
515 d: the datastore for the recipe
516 Returns:
517 a dict mapping patch file path to a list of tuples of changed files and
518 change mode ('A' for add, 'D' for delete or 'M' for modify)
519 """
520 import oe.patch
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500521 patches = oe.patch.src_patches(d, expand=False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500522 patchedfiles = {}
523 for patch in patches:
524 _, _, patchfile, _, _, parm = bb.fetch.decodeurl(patch)
525 striplevel = int(parm['striplevel'])
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500526 patchedfiles[patchfile] = oe.patch.PatchSet.getPatchedFiles(patchfile, striplevel, os.path.join(d.getVar('S'), parm.get('patchdir', '')))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500527 return patchedfiles
528
529
530def validate_pn(pn):
531 """Perform validation on a recipe name (PN) for a new recipe."""
532 reserved_names = ['forcevariable', 'append', 'prepend', 'remove']
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600533 if not re.match('^[0-9a-z-.+]+$', pn):
534 return 'Recipe name "%s" is invalid: only characters 0-9, a-z, -, + and . are allowed' % pn
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500535 elif pn in reserved_names:
536 return 'Recipe name "%s" is invalid: is a reserved keyword' % pn
537 elif pn.startswith('pn-'):
538 return 'Recipe name "%s" is invalid: names starting with "pn-" are reserved' % pn
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500539 elif pn.endswith(('.bb', '.bbappend', '.bbclass', '.inc', '.conf')):
540 return 'Recipe name "%s" is invalid: should be just a name, not a file name' % pn
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500541 return ''
542
543
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600544def get_bbfile_path(d, destdir, extrapathhint=None):
545 """
546 Determine the correct path for a recipe within a layer
547 Parameters:
548 d: Recipe-specific datastore
549 destdir: destination directory. Can be the path to the base of the layer or a
550 partial path somewhere within the layer.
551 extrapathhint: a path relative to the base of the layer to try
552 """
553 import bb.cookerdata
554
555 destdir = os.path.abspath(destdir)
556 destlayerdir = find_layerdir(destdir)
557
558 # Parse the specified layer's layer.conf file directly, in case the layer isn't in bblayers.conf
559 confdata = d.createCopy()
560 confdata.setVar('BBFILES', '')
561 confdata.setVar('LAYERDIR', destlayerdir)
562 destlayerconf = os.path.join(destlayerdir, "conf", "layer.conf")
563 confdata = bb.cookerdata.parse_config_file(destlayerconf, confdata)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500564 pn = d.getVar('PN')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600565
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500566 # Parse BBFILES_DYNAMIC and append to BBFILES
567 bbfiles_dynamic = (confdata.getVar('BBFILES_DYNAMIC') or "").split()
568 collections = (confdata.getVar('BBFILE_COLLECTIONS') or "").split()
569 invalid = []
570 for entry in bbfiles_dynamic:
571 parts = entry.split(":", 1)
572 if len(parts) != 2:
573 invalid.append(entry)
574 continue
575 l, f = parts
576 invert = l[0] == "!"
577 if invert:
578 l = l[1:]
579 if (l in collections and not invert) or (l not in collections and invert):
580 confdata.appendVar("BBFILES", " " + f)
581 if invalid:
582 return None
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500583 bbfilespecs = (confdata.getVar('BBFILES') or '').split()
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600584 if destdir == destlayerdir:
585 for bbfilespec in bbfilespecs:
586 if not bbfilespec.endswith('.bbappend'):
587 for match in glob.glob(bbfilespec):
588 splitext = os.path.splitext(os.path.basename(match))
589 if splitext[1] == '.bb':
590 mpn = splitext[0].split('_')[0]
591 if mpn == pn:
592 return os.path.dirname(match)
593
594 # Try to make up a path that matches BBFILES
595 # this is a little crude, but better than nothing
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500596 bpn = d.getVar('BPN')
597 recipefn = os.path.basename(d.getVar('FILE'))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600598 pathoptions = [destdir]
599 if extrapathhint:
600 pathoptions.append(os.path.join(destdir, extrapathhint))
601 if destdir == destlayerdir:
602 pathoptions.append(os.path.join(destdir, 'recipes-%s' % bpn, bpn))
603 pathoptions.append(os.path.join(destdir, 'recipes', bpn))
604 pathoptions.append(os.path.join(destdir, bpn))
605 elif not destdir.endswith(('/' + pn, '/' + bpn)):
606 pathoptions.append(os.path.join(destdir, bpn))
607 closepath = ''
608 for pathoption in pathoptions:
609 bbfilepath = os.path.join(pathoption, 'test.bb')
610 for bbfilespec in bbfilespecs:
611 if fnmatch.fnmatchcase(bbfilepath, bbfilespec):
612 return pathoption
613 return None
614
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500615def get_bbappend_path(d, destlayerdir, wildcardver=False):
616 """Determine how a bbappend for a recipe should be named and located within another layer"""
617
618 import bb.cookerdata
619
620 destlayerdir = os.path.abspath(destlayerdir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500621 recipefile = d.getVar('FILE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500622 recipefn = os.path.splitext(os.path.basename(recipefile))[0]
623 if wildcardver and '_' in recipefn:
624 recipefn = recipefn.split('_', 1)[0] + '_%'
625 appendfn = recipefn + '.bbappend'
626
627 # Parse the specified layer's layer.conf file directly, in case the layer isn't in bblayers.conf
628 confdata = d.createCopy()
629 confdata.setVar('BBFILES', '')
630 confdata.setVar('LAYERDIR', destlayerdir)
631 destlayerconf = os.path.join(destlayerdir, "conf", "layer.conf")
632 confdata = bb.cookerdata.parse_config_file(destlayerconf, confdata)
633
634 origlayerdir = find_layerdir(recipefile)
635 if not origlayerdir:
636 return (None, False)
637 # Now join this to the path where the bbappend is going and check if it is covered by BBFILES
638 appendpath = os.path.join(destlayerdir, os.path.relpath(os.path.dirname(recipefile), origlayerdir), appendfn)
639 closepath = ''
640 pathok = True
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500641 for bbfilespec in confdata.getVar('BBFILES').split():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500642 if fnmatch.fnmatchcase(appendpath, bbfilespec):
643 # Our append path works, we're done
644 break
645 elif bbfilespec.startswith(destlayerdir) and fnmatch.fnmatchcase('test.bbappend', os.path.basename(bbfilespec)):
646 # Try to find the longest matching path
647 if len(bbfilespec) > len(closepath):
648 closepath = bbfilespec
649 else:
650 # Unfortunately the bbappend layer and the original recipe's layer don't have the same structure
651 if closepath:
652 # bbappend layer's layer.conf at least has a spec that picks up .bbappend files
653 # Now we just need to substitute out any wildcards
654 appendsubdir = os.path.relpath(os.path.dirname(closepath), destlayerdir)
655 if 'recipes-*' in appendsubdir:
656 # Try to copy this part from the original recipe path
657 res = re.search('/recipes-[^/]+/', recipefile)
658 if res:
659 appendsubdir = appendsubdir.replace('/recipes-*/', res.group(0))
660 # This is crude, but we have to do something
661 appendsubdir = appendsubdir.replace('*', recipefn.split('_')[0])
662 appendsubdir = appendsubdir.replace('?', 'a')
663 appendpath = os.path.join(destlayerdir, appendsubdir, appendfn)
664 else:
665 pathok = False
666 return (appendpath, pathok)
667
668
Brad Bishop316dfdd2018-06-25 12:45:53 -0400669def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, machine=None, extralines=None, removevalues=None, redirect_output=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500670 """
671 Writes a bbappend file for a recipe
672 Parameters:
673 rd: data dictionary for the recipe
674 destlayerdir: base directory of the layer to place the bbappend in
675 (subdirectory path from there will be determined automatically)
676 srcfiles: dict of source files to add to SRC_URI, where the value
677 is the full path to the file to be added, and the value is the
678 original filename as it would appear in SRC_URI or None if it
679 isn't already present. You may pass None for this parameter if
680 you simply want to specify your own content via the extralines
681 parameter.
682 install: dict mapping entries in srcfiles to a tuple of two elements:
683 install path (*without* ${D} prefix) and permission value (as a
684 string, e.g. '0644').
685 wildcardver: True to use a % wildcard in the bbappend filename, or
686 False to make the bbappend specific to the recipe version.
687 machine:
688 If specified, make the changes in the bbappend specific to this
689 machine. This will also cause PACKAGE_ARCH = "${MACHINE_ARCH}"
690 to be added to the bbappend.
691 extralines:
692 Extra lines to add to the bbappend. This may be a dict of name
693 value pairs, or simply a list of the lines.
694 removevalues:
695 Variable values to remove - a dict of names/values.
Brad Bishop316dfdd2018-06-25 12:45:53 -0400696 redirect_output:
697 If specified, redirects writing the output file to the
698 specified directory (for dry-run purposes)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500699 """
700
701 if not removevalues:
702 removevalues = {}
703
704 # Determine how the bbappend should be named
705 appendpath, pathok = get_bbappend_path(rd, destlayerdir, wildcardver)
706 if not appendpath:
707 bb.error('Unable to determine layer directory containing %s' % recipefile)
708 return (None, None)
709 if not pathok:
710 bb.warn('Unable to determine correct subdirectory path for bbappend file - check that what %s adds to BBFILES also matches .bbappend files. Using %s for now, but until you fix this the bbappend will not be applied.' % (os.path.join(destlayerdir, 'conf', 'layer.conf'), os.path.dirname(appendpath)))
711
712 appenddir = os.path.dirname(appendpath)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400713 if not redirect_output:
714 bb.utils.mkdirhier(appenddir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500715
716 # FIXME check if the bbappend doesn't get overridden by a higher priority layer?
717
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500718 layerdirs = [os.path.abspath(layerdir) for layerdir in rd.getVar('BBLAYERS').split()]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500719 if not os.path.abspath(destlayerdir) in layerdirs:
720 bb.warn('Specified layer is not currently enabled in bblayers.conf, you will need to add it before this bbappend will be active')
721
722 bbappendlines = []
723 if extralines:
724 if isinstance(extralines, dict):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600725 for name, value in extralines.items():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500726 bbappendlines.append((name, '=', value))
727 else:
728 # Do our best to split it
729 for line in extralines:
730 if line[-1] == '\n':
731 line = line[:-1]
732 splitline = line.split(None, 2)
733 if len(splitline) == 3:
734 bbappendlines.append(tuple(splitline))
735 else:
736 raise Exception('Invalid extralines value passed')
737
738 def popline(varname):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600739 for i in range(0, len(bbappendlines)):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500740 if bbappendlines[i][0] == varname:
741 line = bbappendlines.pop(i)
742 return line
743 return None
744
745 def appendline(varname, op, value):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600746 for i in range(0, len(bbappendlines)):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500747 item = bbappendlines[i]
748 if item[0] == varname:
749 bbappendlines[i] = (item[0], item[1], item[2] + ' ' + value)
750 break
751 else:
752 bbappendlines.append((varname, op, value))
753
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500754 destsubdir = rd.getVar('PN')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500755 if srcfiles:
756 bbappendlines.append(('FILESEXTRAPATHS_prepend', ':=', '${THISDIR}/${PN}:'))
757
758 appendoverride = ''
759 if machine:
760 bbappendlines.append(('PACKAGE_ARCH', '=', '${MACHINE_ARCH}'))
761 appendoverride = '_%s' % machine
762 copyfiles = {}
763 if srcfiles:
764 instfunclines = []
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600765 for newfile, origsrcfile in srcfiles.items():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500766 srcfile = origsrcfile
767 srcurientry = None
768 if not srcfile:
769 srcfile = os.path.basename(newfile)
770 srcurientry = 'file://%s' % srcfile
771 # Double-check it's not there already
772 # FIXME do we care if the entry is added by another bbappend that might go away?
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500773 if not srcurientry in rd.getVar('SRC_URI').split():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500774 if machine:
775 appendline('SRC_URI_append%s' % appendoverride, '=', ' ' + srcurientry)
776 else:
777 appendline('SRC_URI', '+=', srcurientry)
778 copyfiles[newfile] = srcfile
779 if install:
780 institem = install.pop(newfile, None)
781 if institem:
782 (destpath, perms) = institem
783 instdestpath = replace_dir_vars(destpath, rd)
784 instdirline = 'install -d ${D}%s' % os.path.dirname(instdestpath)
785 if not instdirline in instfunclines:
786 instfunclines.append(instdirline)
787 instfunclines.append('install -m %s ${WORKDIR}/%s ${D}%s' % (perms, os.path.basename(srcfile), instdestpath))
788 if instfunclines:
789 bbappendlines.append(('do_install_append%s()' % appendoverride, '', instfunclines))
790
Brad Bishop316dfdd2018-06-25 12:45:53 -0400791 if redirect_output:
792 bb.note('Writing append file %s (dry-run)' % appendpath)
793 outfile = os.path.join(redirect_output, os.path.basename(appendpath))
794 # Only take a copy if the file isn't already there (this function may be called
795 # multiple times per operation when we're handling overrides)
796 if os.path.exists(appendpath) and not os.path.exists(outfile):
797 shutil.copy2(appendpath, outfile)
798 else:
799 bb.note('Writing append file %s' % appendpath)
800 outfile = appendpath
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500801
Brad Bishop316dfdd2018-06-25 12:45:53 -0400802 if os.path.exists(outfile):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500803 # Work around lack of nonlocal in python 2
804 extvars = {'destsubdir': destsubdir}
805
806 def appendfile_varfunc(varname, origvalue, op, newlines):
807 if varname == 'FILESEXTRAPATHS_prepend':
808 if origvalue.startswith('${THISDIR}/'):
809 popline('FILESEXTRAPATHS_prepend')
810 extvars['destsubdir'] = rd.expand(origvalue.split('${THISDIR}/', 1)[1].rstrip(':'))
811 elif varname == 'PACKAGE_ARCH':
812 if machine:
813 popline('PACKAGE_ARCH')
814 return (machine, None, 4, False)
815 elif varname.startswith('do_install_append'):
816 func = popline(varname)
817 if func:
818 instfunclines = [line.strip() for line in origvalue.strip('\n').splitlines()]
819 for line in func[2]:
820 if not line in instfunclines:
821 instfunclines.append(line)
822 return (instfunclines, None, 4, False)
823 else:
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500824 splitval = split_var_value(origvalue, assignment=False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500825 changed = False
826 removevar = varname
827 if varname in ['SRC_URI', 'SRC_URI_append%s' % appendoverride]:
828 removevar = 'SRC_URI'
829 line = popline(varname)
830 if line:
831 if line[2] not in splitval:
832 splitval.append(line[2])
833 changed = True
834 else:
835 line = popline(varname)
836 if line:
837 splitval = [line[2]]
838 changed = True
839
840 if removevar in removevalues:
841 remove = removevalues[removevar]
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600842 if isinstance(remove, str):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500843 if remove in splitval:
844 splitval.remove(remove)
845 changed = True
846 else:
847 for removeitem in remove:
848 if removeitem in splitval:
849 splitval.remove(removeitem)
850 changed = True
851
852 if changed:
853 newvalue = splitval
854 if len(newvalue) == 1:
855 # Ensure it's written out as one line
856 if '_append' in varname:
857 newvalue = ' ' + newvalue[0]
858 else:
859 newvalue = newvalue[0]
860 if not newvalue and (op in ['+=', '.='] or '_append' in varname):
861 # There's no point appending nothing
862 newvalue = None
863 if varname.endswith('()'):
864 indent = 4
865 else:
866 indent = -1
867 return (newvalue, None, indent, True)
868 return (origvalue, None, 4, False)
869
870 varnames = [item[0] for item in bbappendlines]
871 if removevalues:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600872 varnames.extend(list(removevalues.keys()))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500873
Brad Bishop316dfdd2018-06-25 12:45:53 -0400874 with open(outfile, 'r') as f:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500875 (updated, newlines) = bb.utils.edit_metadata(f, varnames, appendfile_varfunc)
876
877 destsubdir = extvars['destsubdir']
878 else:
879 updated = False
880 newlines = []
881
882 if bbappendlines:
883 for line in bbappendlines:
884 if line[0].endswith('()'):
885 newlines.append('%s {\n %s\n}\n' % (line[0], '\n '.join(line[2])))
886 else:
887 newlines.append('%s %s "%s"\n\n' % line)
888 updated = True
889
890 if updated:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400891 with open(outfile, 'w') as f:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500892 f.writelines(newlines)
893
894 if copyfiles:
895 if machine:
896 destsubdir = os.path.join(destsubdir, machine)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400897 if redirect_output:
898 outdir = redirect_output
899 else:
900 outdir = appenddir
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600901 for newfile, srcfile in copyfiles.items():
Brad Bishop316dfdd2018-06-25 12:45:53 -0400902 filedest = os.path.join(outdir, destsubdir, os.path.basename(srcfile))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500903 if os.path.abspath(newfile) != os.path.abspath(filedest):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500904 if newfile.startswith(tempfile.gettempdir()):
905 newfiledisp = os.path.basename(newfile)
906 else:
907 newfiledisp = newfile
Brad Bishop316dfdd2018-06-25 12:45:53 -0400908 if redirect_output:
909 bb.note('Copying %s to %s (dry-run)' % (newfiledisp, os.path.join(appenddir, destsubdir, os.path.basename(srcfile))))
910 else:
911 bb.note('Copying %s to %s' % (newfiledisp, filedest))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500912 bb.utils.mkdirhier(os.path.dirname(filedest))
913 shutil.copyfile(newfile, filedest)
914
915 return (appendpath, os.path.join(appenddir, destsubdir))
916
917
918def find_layerdir(fn):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600919 """ Figure out the path to the base of the layer containing a file (e.g. a recipe)"""
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500920 pth = os.path.abspath(fn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500921 layerdir = ''
922 while pth:
923 if os.path.exists(os.path.join(pth, 'conf', 'layer.conf')):
924 layerdir = pth
925 break
926 pth = os.path.dirname(pth)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600927 if pth == '/':
928 return None
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500929 return layerdir
930
931
932def replace_dir_vars(path, d):
933 """Replace common directory paths with appropriate variable references (e.g. /etc becomes ${sysconfdir})"""
934 dirvars = {}
935 # Sort by length so we get the variables we're interested in first
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600936 for var in sorted(list(d.keys()), key=len):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500937 if var.endswith('dir') and var.lower() == var:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500938 value = d.getVar(var)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500939 if value.startswith('/') and not '\n' in value and value not in dirvars:
940 dirvars[value] = var
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600941 for dirpath in sorted(list(dirvars.keys()), reverse=True):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500942 path = path.replace(dirpath, '${%s}' % dirvars[dirpath])
943 return path
944
945def get_recipe_pv_without_srcpv(pv, uri_type):
946 """
947 Get PV without SRCPV common in SCM's for now only
948 support git.
949
950 Returns tuple with pv, prefix and suffix.
951 """
952 pfx = ''
953 sfx = ''
954
955 if uri_type == 'git':
Brad Bishop15ae2502019-06-18 21:44:24 -0400956 git_regex = re.compile(r"(?P<pfx>v?)(?P<ver>.*?)(?P<sfx>\+[^\+]*(git)?r?(AUTOINC\+))(?P<rev>.*)")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500957 m = git_regex.match(pv)
958
959 if m:
960 pv = m.group('ver')
961 pfx = m.group('pfx')
962 sfx = m.group('sfx')
963 else:
Brad Bishop19323692019-04-05 15:28:33 -0400964 regex = re.compile(r"(?P<pfx>(v|r)?)(?P<ver>.*)")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500965 m = regex.match(pv)
966 if m:
967 pv = m.group('ver')
968 pfx = m.group('pfx')
969
970 return (pv, pfx, sfx)
971
972def get_recipe_upstream_version(rd):
973 """
974 Get upstream version of recipe using bb.fetch2 methods with support for
975 http, https, ftp and git.
976
977 bb.fetch2 exceptions can be raised,
978 FetchError when don't have network access or upstream site don't response.
979 NoMethodError when uri latest_versionstring method isn't implemented.
980
Brad Bishop316dfdd2018-06-25 12:45:53 -0400981 Returns a dictonary with version, repository revision, current_version, type and datetime.
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500982 Type can be A for Automatic, M for Manual and U for Unknown.
983 """
984 from bb.fetch2 import decodeurl
985 from datetime import datetime
986
987 ru = {}
Brad Bishop316dfdd2018-06-25 12:45:53 -0400988 ru['current_version'] = rd.getVar('PV')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500989 ru['version'] = ''
990 ru['type'] = 'U'
991 ru['datetime'] = ''
Brad Bishop316dfdd2018-06-25 12:45:53 -0400992 ru['revision'] = ''
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500993
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500994 # XXX: If don't have SRC_URI means that don't have upstream sources so
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500995 # returns the current recipe version, so that upstream version check
996 # declares a match.
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500997 src_uris = rd.getVar('SRC_URI')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500998 if not src_uris:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400999 ru['version'] = ru['current_version']
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001000 ru['type'] = 'M'
1001 ru['datetime'] = datetime.now()
1002 return ru
1003
1004 # XXX: we suppose that the first entry points to the upstream sources
1005 src_uri = src_uris.split()[0]
1006 uri_type, _, _, _, _, _ = decodeurl(src_uri)
1007
Brad Bishop316dfdd2018-06-25 12:45:53 -04001008 (pv, pfx, sfx) = get_recipe_pv_without_srcpv(rd.getVar('PV'), uri_type)
1009 ru['current_version'] = pv
1010
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001011 manual_upstream_version = rd.getVar("RECIPE_UPSTREAM_VERSION")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001012 if manual_upstream_version:
1013 # manual tracking of upstream version.
1014 ru['version'] = manual_upstream_version
1015 ru['type'] = 'M'
1016
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001017 manual_upstream_date = rd.getVar("CHECK_DATE")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001018 if manual_upstream_date:
1019 date = datetime.strptime(manual_upstream_date, "%b %d, %Y")
1020 else:
1021 date = datetime.now()
1022 ru['datetime'] = date
1023
1024 elif uri_type == "file":
1025 # files are always up-to-date
1026 ru['version'] = pv
1027 ru['type'] = 'A'
1028 ru['datetime'] = datetime.now()
1029 else:
1030 ud = bb.fetch2.FetchData(src_uri, rd)
Brad Bishop316dfdd2018-06-25 12:45:53 -04001031 if rd.getVar("UPSTREAM_CHECK_COMMITS") == "1":
1032 revision = ud.method.latest_revision(ud, rd, 'default')
1033 upversion = pv
1034 if revision != rd.getVar("SRCREV"):
1035 upversion = upversion + "-new-commits-available"
1036 else:
1037 pupver = ud.method.latest_versionstring(ud, rd)
1038 (upversion, revision) = pupver
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001039
1040 if upversion:
1041 ru['version'] = upversion
1042 ru['type'] = 'A'
1043
Brad Bishop316dfdd2018-06-25 12:45:53 -04001044 if revision:
1045 ru['revision'] = revision
1046
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001047 ru['datetime'] = datetime.now()
1048
1049 return ru
Brad Bishop19323692019-04-05 15:28:33 -04001050
1051def _get_recipe_upgrade_status(data):
1052 uv = get_recipe_upstream_version(data)
1053
1054 pn = data.getVar('PN')
1055 cur_ver = uv['current_version']
1056
1057 upstream_version_unknown = data.getVar('UPSTREAM_VERSION_UNKNOWN')
1058 if not uv['version']:
1059 status = "UNKNOWN" if upstream_version_unknown else "UNKNOWN_BROKEN"
1060 else:
1061 cmp = vercmp_string(uv['current_version'], uv['version'])
1062 if cmp == -1:
1063 status = "UPDATE" if not upstream_version_unknown else "KNOWN_BROKEN"
1064 elif cmp == 0:
1065 status = "MATCH" if not upstream_version_unknown else "KNOWN_BROKEN"
1066 else:
1067 status = "UNKNOWN" if upstream_version_unknown else "UNKNOWN_BROKEN"
1068
1069 next_ver = uv['version'] if uv['version'] else "N/A"
1070 revision = uv['revision'] if uv['revision'] else "N/A"
1071 maintainer = data.getVar('RECIPE_MAINTAINER')
1072 no_upgrade_reason = data.getVar('RECIPE_NO_UPDATE_REASON')
1073
1074 return (pn, status, cur_ver, next_ver, maintainer, revision, no_upgrade_reason)
1075
1076def get_recipe_upgrade_status(recipes=None):
1077 pkgs_list = []
1078 data_copy_list = []
1079 copy_vars = ('SRC_URI',
1080 'PV',
Brad Bishop19323692019-04-05 15:28:33 -04001081 'DL_DIR',
1082 'PN',
1083 'CACHE',
1084 'PERSISTENT_DIR',
1085 'BB_URI_HEADREVS',
1086 'UPSTREAM_CHECK_COMMITS',
1087 'UPSTREAM_CHECK_GITTAGREGEX',
1088 'UPSTREAM_CHECK_REGEX',
1089 'UPSTREAM_CHECK_URI',
1090 'UPSTREAM_VERSION_UNKNOWN',
1091 'RECIPE_MAINTAINER',
1092 'RECIPE_NO_UPDATE_REASON',
1093 'RECIPE_UPSTREAM_VERSION',
1094 'RECIPE_UPSTREAM_DATE',
1095 'CHECK_DATE',
Andrew Geissler1e34c2d2020-05-29 16:02:59 -05001096 'FETCHCMD_bzr',
1097 'FETCHCMD_ccrc',
1098 'FETCHCMD_cvs',
1099 'FETCHCMD_git',
1100 'FETCHCMD_hg',
1101 'FETCHCMD_npm',
1102 'FETCHCMD_osc',
1103 'FETCHCMD_p4',
1104 'FETCHCMD_repo',
1105 'FETCHCMD_s3',
1106 'FETCHCMD_svn',
1107 'FETCHCMD_wget',
Brad Bishop19323692019-04-05 15:28:33 -04001108 )
1109
1110 with bb.tinfoil.Tinfoil() as tinfoil:
1111 tinfoil.prepare(config_only=False)
1112
1113 if not recipes:
1114 recipes = tinfoil.all_recipe_files(variants=False)
1115
1116 for fn in recipes:
1117 try:
1118 if fn.startswith("/"):
1119 data = tinfoil.parse_recipe_file(fn)
1120 else:
1121 data = tinfoil.parse_recipe(fn)
1122 except bb.providers.NoProvider:
1123 bb.note(" No provider for %s" % fn)
1124 continue
1125
1126 unreliable = data.getVar('UPSTREAM_CHECK_UNRELIABLE')
1127 if unreliable == "1":
1128 bb.note(" Skip package %s as upstream check unreliable" % pn)
1129 continue
1130
1131 data_copy = bb.data.init()
1132 for var in copy_vars:
1133 data_copy.setVar(var, data.getVar(var))
1134 for k in data:
1135 if k.startswith('SRCREV'):
1136 data_copy.setVar(k, data.getVar(k))
1137
1138 data_copy_list.append(data_copy)
1139
1140 from concurrent.futures import ProcessPoolExecutor
1141 with ProcessPoolExecutor(max_workers=utils.cpu_count()) as executor:
1142 pkgs_list = executor.map(_get_recipe_upgrade_status, data_copy_list)
1143
1144 return pkgs_list