blob: 36427eec9186953e71849dd6e01bbdc17c3cf7a7 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# Utility functions for reading and modifying recipes
2#
3# Some code borrowed from the OE layer index
4#
Brad Bishopd7bf8c12018-02-25 22:55:05 -05005# Copyright (C) 2013-2017 Intel Corporation
Patrick Williamsc124f4f2015-09-15 14:41:29 -05006#
Brad Bishopc342db32019-05-15 21:57:59 -04007# SPDX-License-Identifier: GPL-2.0-only
8#
Patrick Williamsc124f4f2015-09-15 14:41:29 -05009
10import sys
11import os
12import os.path
13import tempfile
14import textwrap
15import difflib
Patrick Williamsc0f7c042017-02-23 20:41:17 -060016from . import utils
Patrick Williamsc124f4f2015-09-15 14:41:29 -050017import shutil
18import re
19import fnmatch
Patrick Williamsc0f7c042017-02-23 20:41:17 -060020import glob
Brad Bishop19323692019-04-05 15:28:33 -040021import bb.tinfoil
Patrick Williamsc124f4f2015-09-15 14:41:29 -050022
Brad Bishop19323692019-04-05 15:28:33 -040023from collections import OrderedDict, defaultdict
24from bb.utils import vercmp_string
Patrick Williamsc124f4f2015-09-15 14:41:29 -050025
26# Help us to find places to insert values
Andrew Geisslerb7d28612020-07-24 16:15:54 -050027recipe_progression = ['SUMMARY', 'DESCRIPTION', 'AUTHOR', 'HOMEPAGE', 'BUGTRACKER', 'SECTION', 'LICENSE', 'LICENSE_FLAGS', 'LIC_FILES_CHKSUM', 'PROVIDES', 'DEPENDS', 'PR', 'PV', 'SRCREV', 'SRCPV', 'SRC_URI', 'S', 'do_fetch()', 'do_unpack()', 'do_patch()', 'EXTRA_OECONF', 'EXTRA_OECMAKE', 'EXTRA_OESCONS', 'do_configure()', 'EXTRA_OEMAKE', 'do_compile()', 'do_install()', 'do_populate_sysroot()', 'INITSCRIPT', 'USERADD', 'GROUPADD', 'PACKAGES', 'FILES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RPROVIDES', 'RREPLACES', 'RCONFLICTS', 'ALLOW_EMPTY', 'populate_packages()', 'do_package()', 'do_deploy()', 'BBCLASSEXTEND']
Patrick Williamsc124f4f2015-09-15 14:41:29 -050028# Variables that sometimes are a bit long but shouldn't be wrapped
Brad Bishop19323692019-04-05 15:28:33 -040029nowrap_vars = ['SUMMARY', 'HOMEPAGE', 'BUGTRACKER', r'SRC_URI\[(.+\.)?md5sum\]', r'SRC_URI\[(.+\.)?sha256sum\]']
Patrick Williamsc124f4f2015-09-15 14:41:29 -050030list_vars = ['SRC_URI', 'LIC_FILES_CHKSUM']
31meta_vars = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION']
32
33
Brad Bishop19323692019-04-05 15:28:33 -040034def simplify_history(history, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050035 """
Brad Bishop19323692019-04-05 15:28:33 -040036 Eliminate any irrelevant events from a variable history
Patrick Williamsc124f4f2015-09-15 14:41:29 -050037 """
Brad Bishop19323692019-04-05 15:28:33 -040038 ret_history = []
39 has_set = False
40 # Go backwards through the history and remove any immediate operations
41 # before the most recent set
42 for event in reversed(history):
43 if 'flag' in event or not 'file' in event:
44 continue
45 if event['op'] == 'set':
46 if has_set:
47 continue
48 has_set = True
49 elif event['op'] in ('append', 'prepend', 'postdot', 'predot'):
50 # Reminder: "append" and "prepend" mean += and =+ respectively, NOT _append / _prepend
51 if has_set:
52 continue
53 ret_history.insert(0, event)
54 return ret_history
Patrick Williamsc124f4f2015-09-15 14:41:29 -050055
56
Patrick Williamsc124f4f2015-09-15 14:41:29 -050057def get_var_files(fn, varlist, d):
58 """Find the file in which each of a list of variables is set.
59 Note: requires variable history to be enabled when parsing.
60 """
61 varfiles = {}
62 for v in varlist:
Patrick Williamsc124f4f2015-09-15 14:41:29 -050063 files = []
Brad Bishop19323692019-04-05 15:28:33 -040064 if '[' in v:
65 varsplit = v.split('[')
66 varflag = varsplit[1].split(']')[0]
67 history = d.varhistory.variable(varsplit[0])
68 for event in history:
69 if 'file' in event and event.get('flag', '') == varflag:
70 files.append(event['file'])
71 else:
72 history = d.varhistory.variable(v)
73 for event in history:
74 if 'file' in event and not 'flag' in event:
75 files.append(event['file'])
Patrick Williamsc124f4f2015-09-15 14:41:29 -050076 if files:
77 actualfile = files[-1]
78 else:
79 actualfile = None
80 varfiles[v] = actualfile
81
82 return varfiles
83
84
Patrick Williamsf1e5d692016-03-30 15:21:19 -050085def split_var_value(value, assignment=True):
86 """
87 Split a space-separated variable's value into a list of items,
88 taking into account that some of the items might be made up of
89 expressions containing spaces that should not be split.
90 Parameters:
91 value:
92 The string value to split
93 assignment:
94 True to assume that the value represents an assignment
95 statement, False otherwise. If True, and an assignment
96 statement is passed in the first item in
97 the returned list will be the part of the assignment
98 statement up to and including the opening quote character,
99 and the last item will be the closing quote.
100 """
101 inexpr = 0
102 lastchar = None
103 out = []
104 buf = ''
105 for char in value:
106 if char == '{':
107 if lastchar == '$':
108 inexpr += 1
109 elif char == '}':
110 inexpr -= 1
111 elif assignment and char in '"\'' and inexpr == 0:
112 if buf:
113 out.append(buf)
114 out.append(char)
115 char = ''
116 buf = ''
117 elif char.isspace() and inexpr == 0:
118 char = ''
119 if buf:
120 out.append(buf)
121 buf = ''
122 buf += char
123 lastchar = char
124 if buf:
125 out.append(buf)
126
127 # Join together assignment statement and opening quote
128 outlist = out
129 if assignment:
130 assigfound = False
131 for idx, item in enumerate(out):
132 if '=' in item:
133 assigfound = True
134 if assigfound:
135 if '"' in item or "'" in item:
136 outlist = [' '.join(out[:idx+1])]
137 outlist.extend(out[idx+1:])
138 break
139 return outlist
140
141
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600142def patch_recipe_lines(fromlines, values, trailing_newline=True):
143 """Update or insert variable values into lines from a recipe.
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500144 Note that some manual inspection/intervention may be required
145 since this cannot handle all situations.
146 """
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500147
148 import bb.utils
149
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600150 if trailing_newline:
151 newline = '\n'
152 else:
153 newline = ''
154
Brad Bishop316dfdd2018-06-25 12:45:53 -0400155 nowrap_vars_res = []
156 for item in nowrap_vars:
157 nowrap_vars_res.append(re.compile('^%s$' % item))
158
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500159 recipe_progression_res = []
160 recipe_progression_restrs = []
161 for item in recipe_progression:
162 if item.endswith('()'):
163 key = item[:-2]
164 else:
165 key = item
Brad Bishop19323692019-04-05 15:28:33 -0400166 restr = r'%s(_[a-zA-Z0-9-_$(){}]+|\[[^\]]*\])?' % key
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500167 if item.endswith('()'):
168 recipe_progression_restrs.append(restr + '()')
169 else:
170 recipe_progression_restrs.append(restr)
171 recipe_progression_res.append(re.compile('^%s$' % restr))
172
173 def get_recipe_pos(variable):
174 for i, p in enumerate(recipe_progression_res):
175 if p.match(variable):
176 return i
177 return -1
178
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500179 remainingnames = {}
180 for k in values.keys():
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500181 remainingnames[k] = get_recipe_pos(k)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600182 remainingnames = OrderedDict(sorted(remainingnames.items(), key=lambda x: x[1]))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500183
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500184 modifying = False
185
186 def outputvalue(name, lines, rewindcomments=False):
187 if values[name] is None:
188 return
Brad Bishop19323692019-04-05 15:28:33 -0400189 if isinstance(values[name], tuple):
190 op, value = values[name]
191 if op == '+=' and value.strip() == '':
192 return
193 else:
194 value = values[name]
195 op = '='
196 rawtext = '%s %s "%s"%s' % (name, op, value, newline)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500197 addlines = []
Brad Bishop316dfdd2018-06-25 12:45:53 -0400198 nowrap = False
199 for nowrap_re in nowrap_vars_res:
200 if nowrap_re.match(name):
201 nowrap = True
202 break
203 if nowrap:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500204 addlines.append(rawtext)
205 elif name in list_vars:
Brad Bishop19323692019-04-05 15:28:33 -0400206 splitvalue = split_var_value(value, assignment=False)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500207 if len(splitvalue) > 1:
208 linesplit = ' \\\n' + (' ' * (len(name) + 4))
Brad Bishop19323692019-04-05 15:28:33 -0400209 addlines.append('%s %s "%s%s"%s' % (name, op, linesplit.join(splitvalue), linesplit, newline))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500210 else:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500211 addlines.append(rawtext)
212 else:
213 wrapped = textwrap.wrap(rawtext)
214 for wrapline in wrapped[:-1]:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600215 addlines.append('%s \\%s' % (wrapline, newline))
216 addlines.append('%s%s' % (wrapped[-1], newline))
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500217
218 # Split on newlines - this isn't strictly necessary if you are only
219 # going to write the output to disk, but if you want to compare it
220 # (as patch_recipe_file() will do if patch=True) then it's important.
221 addlines = [line for l in addlines for line in l.splitlines(True)]
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500222 if rewindcomments:
223 # Ensure we insert the lines before any leading comments
224 # (that we'd want to ensure remain leading the next value)
225 for i, ln in reversed(list(enumerate(lines))):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600226 if not ln.startswith('#'):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500227 lines[i+1:i+1] = addlines
228 break
229 else:
230 lines.extend(addlines)
231 else:
232 lines.extend(addlines)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500233
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500234 existingnames = []
235 def patch_recipe_varfunc(varname, origvalue, op, newlines):
236 if modifying:
237 # Insert anything that should come before this variable
238 pos = get_recipe_pos(varname)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600239 for k in list(remainingnames):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500240 if remainingnames[k] > -1 and pos >= remainingnames[k] and not k in existingnames:
241 outputvalue(k, newlines, rewindcomments=True)
242 del remainingnames[k]
243 # Now change this variable, if it needs to be changed
244 if varname in existingnames and op in ['+=', '=', '=+']:
245 if varname in remainingnames:
246 outputvalue(varname, newlines)
247 del remainingnames[varname]
248 return None, None, 0, True
249 else:
250 if varname in values:
251 existingnames.append(varname)
252 return origvalue, None, 0, True
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500253
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500254 # First run - establish which values we want to set are already in the file
255 varlist = [re.escape(item) for item in values.keys()]
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600256 bb.utils.edit_metadata(fromlines, varlist, patch_recipe_varfunc)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500257 # Second run - actually set everything
258 modifying = True
259 varlist.extend(recipe_progression_restrs)
260 changed, tolines = bb.utils.edit_metadata(fromlines, varlist, patch_recipe_varfunc, match_overrides=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500261
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500262 if remainingnames:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600263 if tolines and tolines[-1].strip() != '':
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500264 tolines.append('\n')
265 for k in remainingnames.keys():
266 outputvalue(k, tolines)
267
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600268 return changed, tolines
269
270
Brad Bishop316dfdd2018-06-25 12:45:53 -0400271def patch_recipe_file(fn, values, patch=False, relpath='', redirect_output=None):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600272 """Update or insert variable values into a recipe file (assuming you
273 have already identified the exact file you want to update.)
274 Note that some manual inspection/intervention may be required
275 since this cannot handle all situations.
276 """
277
278 with open(fn, 'r') as f:
279 fromlines = f.readlines()
280
281 _, tolines = patch_recipe_lines(fromlines, values)
282
Brad Bishop316dfdd2018-06-25 12:45:53 -0400283 if redirect_output:
284 with open(os.path.join(redirect_output, os.path.basename(fn)), 'w') as f:
285 f.writelines(tolines)
286 return None
287 elif patch:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500288 relfn = os.path.relpath(fn, relpath)
289 diff = difflib.unified_diff(fromlines, tolines, 'a/%s' % relfn, 'b/%s' % relfn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500290 return diff
291 else:
292 with open(fn, 'w') as f:
293 f.writelines(tolines)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500294 return None
295
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500296
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500297def localise_file_vars(fn, varfiles, varlist):
298 """Given a list of variables and variable history (fetched with get_var_files())
299 find where each variable should be set/changed. This handles for example where a
300 recipe includes an inc file where variables might be changed - in most cases
301 we want to update the inc file when changing the variable value rather than adding
302 it to the recipe itself.
303 """
304 fndir = os.path.dirname(fn) + os.sep
305
306 first_meta_file = None
307 for v in meta_vars:
308 f = varfiles.get(v, None)
309 if f:
310 actualdir = os.path.dirname(f) + os.sep
311 if actualdir.startswith(fndir):
312 first_meta_file = f
313 break
314
315 filevars = defaultdict(list)
316 for v in varlist:
317 f = varfiles[v]
318 # Only return files that are in the same directory as the recipe or in some directory below there
319 # (this excludes bbclass files and common inc files that wouldn't be appropriate to set the variable
320 # in if we were going to set a value specific to this recipe)
321 if f:
322 actualfile = f
323 else:
324 # Variable isn't in a file, if it's one of the "meta" vars, use the first file with a meta var in it
325 if first_meta_file:
326 actualfile = first_meta_file
327 else:
328 actualfile = fn
329
330 actualdir = os.path.dirname(actualfile) + os.sep
331 if not actualdir.startswith(fndir):
332 actualfile = fn
333 filevars[actualfile].append(v)
334
335 return filevars
336
Brad Bishop316dfdd2018-06-25 12:45:53 -0400337def patch_recipe(d, fn, varvalues, patch=False, relpath='', redirect_output=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500338 """Modify a list of variable values in the specified recipe. Handles inc files if
339 used by the recipe.
340 """
Brad Bishop19323692019-04-05 15:28:33 -0400341 overrides = d.getVar('OVERRIDES').split(':')
342 def override_applicable(hevent):
343 op = hevent['op']
344 if '[' in op:
345 opoverrides = op.split('[')[1].split(']')[0].split('_')
346 for opoverride in opoverrides:
347 if not opoverride in overrides:
348 return False
349 return True
350
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500351 varlist = varvalues.keys()
Brad Bishop19323692019-04-05 15:28:33 -0400352 fn = os.path.abspath(fn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500353 varfiles = get_var_files(fn, varlist, d)
354 locs = localise_file_vars(fn, varfiles, varlist)
355 patches = []
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600356 for f,v in locs.items():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500357 vals = {k: varvalues[k] for k in v}
Brad Bishop19323692019-04-05 15:28:33 -0400358 f = os.path.abspath(f)
359 if f == fn:
360 extravals = {}
361 for var, value in vals.items():
362 if var in list_vars:
363 history = simplify_history(d.varhistory.variable(var), d)
364 recipe_set = False
365 for event in history:
366 if os.path.abspath(event['file']) == fn:
367 if event['op'] == 'set':
368 recipe_set = True
369 if not recipe_set:
370 for event in history:
371 if event['op'].startswith('_remove'):
372 continue
373 if not override_applicable(event):
374 continue
375 newvalue = value.replace(event['detail'], '')
376 if newvalue == value and os.path.abspath(event['file']) == fn and event['op'].startswith('_'):
377 op = event['op'].replace('[', '_').replace(']', '')
378 extravals[var + op] = None
379 value = newvalue
380 vals[var] = ('+=', value)
381 vals.update(extravals)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400382 patchdata = patch_recipe_file(f, vals, patch, relpath, redirect_output)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500383 if patch:
384 patches.append(patchdata)
385
386 if patch:
387 return patches
388 else:
389 return None
390
391
392
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500393def copy_recipe_files(d, tgt_dir, whole_dir=False, download=True, all_variants=False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500394 """Copy (local) recipe files, including both files included via include/require,
395 and files referred to in the SRC_URI variable."""
396 import bb.fetch2
397 import oe.path
398
399 # FIXME need a warning if the unexpanded SRC_URI value contains variable references
400
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500401 uri_values = []
402 localpaths = []
403 def fetch_urls(rdata):
404 # Collect the local paths from SRC_URI
405 srcuri = rdata.getVar('SRC_URI') or ""
406 if srcuri not in uri_values:
407 fetch = bb.fetch2.Fetch(srcuri.split(), rdata)
408 if download:
409 fetch.download()
410 for pth in fetch.localpaths():
411 if pth not in localpaths:
412 localpaths.append(pth)
413 uri_values.append(srcuri)
414
415 fetch_urls(d)
416 if all_variants:
417 # Get files for other variants e.g. in the case of a SRC_URI_append
418 localdata = bb.data.createCopy(d)
419 variants = (localdata.getVar('BBCLASSEXTEND') or '').split()
420 if variants:
421 # Ensure we handle class-target if we're dealing with one of the variants
422 variants.append('target')
423 for variant in variants:
Andrew Geissler82c905d2020-04-13 13:39:40 -0500424 if variant.startswith("devupstream"):
425 localdata.setVar('SRCPV', 'git')
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500426 localdata.setVar('CLASSOVERRIDE', 'class-%s' % variant)
427 fetch_urls(localdata)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500428
429 # Copy local files to target directory and gather any remote files
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500430 bb_dir = os.path.abspath(os.path.dirname(d.getVar('FILE'))) + os.sep
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500431 remotes = []
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600432 copied = []
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500433 # Need to do this in two steps since we want to check against the absolute path
434 includes = [os.path.abspath(path) for path in d.getVar('BBINCLUDED').split() if os.path.exists(path)]
435 # We also check this below, but we don't want any items in this list being considered remotes
436 includes = [path for path in includes if path.startswith(bb_dir)]
437 for path in localpaths + includes:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500438 # Only import files that are under the meta directory
439 if path.startswith(bb_dir):
440 if not whole_dir:
441 relpath = os.path.relpath(path, bb_dir)
442 subdir = os.path.join(tgt_dir, os.path.dirname(relpath))
443 if not os.path.exists(subdir):
444 os.makedirs(subdir)
445 shutil.copy2(path, os.path.join(tgt_dir, relpath))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600446 copied.append(relpath)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500447 else:
448 remotes.append(path)
449 # Simply copy whole meta dir, if requested
450 if whole_dir:
451 shutil.copytree(bb_dir, tgt_dir)
452
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600453 return copied, remotes
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500454
455
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500456def get_recipe_local_files(d, patches=False, archives=False):
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500457 """Get a list of local files in SRC_URI within a recipe."""
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500458 import oe.patch
459 uris = (d.getVar('SRC_URI') or "").split()
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500460 fetch = bb.fetch2.Fetch(uris, d)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500461 # FIXME this list should be factored out somewhere else (such as the
462 # fetcher) though note that this only encompasses actual container formats
463 # i.e. that can contain multiple files as opposed to those that only
464 # contain a compressed stream (i.e. .tar.gz as opposed to just .gz)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400465 archive_exts = ['.tar', '.tgz', '.tar.gz', '.tar.Z', '.tbz', '.tbz2', '.tar.bz2', '.txz', '.tar.xz', '.tar.lz', '.zip', '.jar', '.rpm', '.srpm', '.deb', '.ipk', '.tar.7z', '.7z']
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500466 ret = {}
467 for uri in uris:
468 if fetch.ud[uri].type == 'file':
469 if (not patches and
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500470 oe.patch.patch_path(uri, fetch, '', expand=False)):
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500471 continue
472 # Skip files that are referenced by absolute path
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600473 fname = fetch.ud[uri].basepath
474 if os.path.isabs(fname):
475 continue
476 # Handle subdir=
477 subdir = fetch.ud[uri].parm.get('subdir', '')
478 if subdir:
479 if os.path.isabs(subdir):
480 continue
481 fname = os.path.join(subdir, fname)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500482 localpath = fetch.localpath(uri)
483 if not archives:
484 # Ignore archives that will be unpacked
485 if localpath.endswith(tuple(archive_exts)):
486 unpack = fetch.ud[uri].parm.get('unpack', True)
487 if unpack:
488 continue
Brad Bishop19323692019-04-05 15:28:33 -0400489 if os.path.isdir(localpath):
490 for root, dirs, files in os.walk(localpath):
491 for fname in files:
492 fileabspath = os.path.join(root,fname)
493 srcdir = os.path.dirname(localpath)
494 ret[os.path.relpath(fileabspath,srcdir)] = fileabspath
495 else:
496 ret[fname] = localpath
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500497 return ret
498
499
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500500def get_recipe_patches(d):
501 """Get a list of the patches included in SRC_URI within a recipe."""
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500502 import oe.patch
503 patches = oe.patch.src_patches(d, expand=False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500504 patchfiles = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500505 for patch in patches:
506 _, _, local, _, _, parm = bb.fetch.decodeurl(patch)
507 patchfiles.append(local)
508 return patchfiles
509
510
511def get_recipe_patched_files(d):
512 """
513 Get the list of patches for a recipe along with the files each patch modifies.
514 Params:
515 d: the datastore for the recipe
516 Returns:
517 a dict mapping patch file path to a list of tuples of changed files and
518 change mode ('A' for add, 'D' for delete or 'M' for modify)
519 """
520 import oe.patch
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500521 patches = oe.patch.src_patches(d, expand=False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500522 patchedfiles = {}
523 for patch in patches:
524 _, _, patchfile, _, _, parm = bb.fetch.decodeurl(patch)
525 striplevel = int(parm['striplevel'])
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500526 patchedfiles[patchfile] = oe.patch.PatchSet.getPatchedFiles(patchfile, striplevel, os.path.join(d.getVar('S'), parm.get('patchdir', '')))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500527 return patchedfiles
528
529
530def validate_pn(pn):
531 """Perform validation on a recipe name (PN) for a new recipe."""
532 reserved_names = ['forcevariable', 'append', 'prepend', 'remove']
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600533 if not re.match('^[0-9a-z-.+]+$', pn):
534 return 'Recipe name "%s" is invalid: only characters 0-9, a-z, -, + and . are allowed' % pn
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500535 elif pn in reserved_names:
536 return 'Recipe name "%s" is invalid: is a reserved keyword' % pn
537 elif pn.startswith('pn-'):
538 return 'Recipe name "%s" is invalid: names starting with "pn-" are reserved' % pn
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500539 elif pn.endswith(('.bb', '.bbappend', '.bbclass', '.inc', '.conf')):
540 return 'Recipe name "%s" is invalid: should be just a name, not a file name' % pn
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500541 return ''
542
543
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600544def get_bbfile_path(d, destdir, extrapathhint=None):
545 """
546 Determine the correct path for a recipe within a layer
547 Parameters:
548 d: Recipe-specific datastore
549 destdir: destination directory. Can be the path to the base of the layer or a
550 partial path somewhere within the layer.
551 extrapathhint: a path relative to the base of the layer to try
552 """
553 import bb.cookerdata
554
555 destdir = os.path.abspath(destdir)
556 destlayerdir = find_layerdir(destdir)
557
558 # Parse the specified layer's layer.conf file directly, in case the layer isn't in bblayers.conf
559 confdata = d.createCopy()
560 confdata.setVar('BBFILES', '')
561 confdata.setVar('LAYERDIR', destlayerdir)
562 destlayerconf = os.path.join(destlayerdir, "conf", "layer.conf")
563 confdata = bb.cookerdata.parse_config_file(destlayerconf, confdata)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500564 pn = d.getVar('PN')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600565
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500566 bbfilespecs = (confdata.getVar('BBFILES') or '').split()
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600567 if destdir == destlayerdir:
568 for bbfilespec in bbfilespecs:
569 if not bbfilespec.endswith('.bbappend'):
570 for match in glob.glob(bbfilespec):
571 splitext = os.path.splitext(os.path.basename(match))
572 if splitext[1] == '.bb':
573 mpn = splitext[0].split('_')[0]
574 if mpn == pn:
575 return os.path.dirname(match)
576
577 # Try to make up a path that matches BBFILES
578 # this is a little crude, but better than nothing
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500579 bpn = d.getVar('BPN')
580 recipefn = os.path.basename(d.getVar('FILE'))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600581 pathoptions = [destdir]
582 if extrapathhint:
583 pathoptions.append(os.path.join(destdir, extrapathhint))
584 if destdir == destlayerdir:
585 pathoptions.append(os.path.join(destdir, 'recipes-%s' % bpn, bpn))
586 pathoptions.append(os.path.join(destdir, 'recipes', bpn))
587 pathoptions.append(os.path.join(destdir, bpn))
588 elif not destdir.endswith(('/' + pn, '/' + bpn)):
589 pathoptions.append(os.path.join(destdir, bpn))
590 closepath = ''
591 for pathoption in pathoptions:
592 bbfilepath = os.path.join(pathoption, 'test.bb')
593 for bbfilespec in bbfilespecs:
594 if fnmatch.fnmatchcase(bbfilepath, bbfilespec):
595 return pathoption
596 return None
597
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500598def get_bbappend_path(d, destlayerdir, wildcardver=False):
599 """Determine how a bbappend for a recipe should be named and located within another layer"""
600
601 import bb.cookerdata
602
603 destlayerdir = os.path.abspath(destlayerdir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500604 recipefile = d.getVar('FILE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500605 recipefn = os.path.splitext(os.path.basename(recipefile))[0]
606 if wildcardver and '_' in recipefn:
607 recipefn = recipefn.split('_', 1)[0] + '_%'
608 appendfn = recipefn + '.bbappend'
609
610 # Parse the specified layer's layer.conf file directly, in case the layer isn't in bblayers.conf
611 confdata = d.createCopy()
612 confdata.setVar('BBFILES', '')
613 confdata.setVar('LAYERDIR', destlayerdir)
614 destlayerconf = os.path.join(destlayerdir, "conf", "layer.conf")
615 confdata = bb.cookerdata.parse_config_file(destlayerconf, confdata)
616
617 origlayerdir = find_layerdir(recipefile)
618 if not origlayerdir:
619 return (None, False)
620 # Now join this to the path where the bbappend is going and check if it is covered by BBFILES
621 appendpath = os.path.join(destlayerdir, os.path.relpath(os.path.dirname(recipefile), origlayerdir), appendfn)
622 closepath = ''
623 pathok = True
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500624 for bbfilespec in confdata.getVar('BBFILES').split():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500625 if fnmatch.fnmatchcase(appendpath, bbfilespec):
626 # Our append path works, we're done
627 break
628 elif bbfilespec.startswith(destlayerdir) and fnmatch.fnmatchcase('test.bbappend', os.path.basename(bbfilespec)):
629 # Try to find the longest matching path
630 if len(bbfilespec) > len(closepath):
631 closepath = bbfilespec
632 else:
633 # Unfortunately the bbappend layer and the original recipe's layer don't have the same structure
634 if closepath:
635 # bbappend layer's layer.conf at least has a spec that picks up .bbappend files
636 # Now we just need to substitute out any wildcards
637 appendsubdir = os.path.relpath(os.path.dirname(closepath), destlayerdir)
638 if 'recipes-*' in appendsubdir:
639 # Try to copy this part from the original recipe path
640 res = re.search('/recipes-[^/]+/', recipefile)
641 if res:
642 appendsubdir = appendsubdir.replace('/recipes-*/', res.group(0))
643 # This is crude, but we have to do something
644 appendsubdir = appendsubdir.replace('*', recipefn.split('_')[0])
645 appendsubdir = appendsubdir.replace('?', 'a')
646 appendpath = os.path.join(destlayerdir, appendsubdir, appendfn)
647 else:
648 pathok = False
649 return (appendpath, pathok)
650
651
Brad Bishop316dfdd2018-06-25 12:45:53 -0400652def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, machine=None, extralines=None, removevalues=None, redirect_output=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500653 """
654 Writes a bbappend file for a recipe
655 Parameters:
656 rd: data dictionary for the recipe
657 destlayerdir: base directory of the layer to place the bbappend in
658 (subdirectory path from there will be determined automatically)
659 srcfiles: dict of source files to add to SRC_URI, where the value
660 is the full path to the file to be added, and the value is the
661 original filename as it would appear in SRC_URI or None if it
662 isn't already present. You may pass None for this parameter if
663 you simply want to specify your own content via the extralines
664 parameter.
665 install: dict mapping entries in srcfiles to a tuple of two elements:
666 install path (*without* ${D} prefix) and permission value (as a
667 string, e.g. '0644').
668 wildcardver: True to use a % wildcard in the bbappend filename, or
669 False to make the bbappend specific to the recipe version.
670 machine:
671 If specified, make the changes in the bbappend specific to this
672 machine. This will also cause PACKAGE_ARCH = "${MACHINE_ARCH}"
673 to be added to the bbappend.
674 extralines:
675 Extra lines to add to the bbappend. This may be a dict of name
676 value pairs, or simply a list of the lines.
677 removevalues:
678 Variable values to remove - a dict of names/values.
Brad Bishop316dfdd2018-06-25 12:45:53 -0400679 redirect_output:
680 If specified, redirects writing the output file to the
681 specified directory (for dry-run purposes)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500682 """
683
684 if not removevalues:
685 removevalues = {}
686
687 # Determine how the bbappend should be named
688 appendpath, pathok = get_bbappend_path(rd, destlayerdir, wildcardver)
689 if not appendpath:
690 bb.error('Unable to determine layer directory containing %s' % recipefile)
691 return (None, None)
692 if not pathok:
693 bb.warn('Unable to determine correct subdirectory path for bbappend file - check that what %s adds to BBFILES also matches .bbappend files. Using %s for now, but until you fix this the bbappend will not be applied.' % (os.path.join(destlayerdir, 'conf', 'layer.conf'), os.path.dirname(appendpath)))
694
695 appenddir = os.path.dirname(appendpath)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400696 if not redirect_output:
697 bb.utils.mkdirhier(appenddir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500698
699 # FIXME check if the bbappend doesn't get overridden by a higher priority layer?
700
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500701 layerdirs = [os.path.abspath(layerdir) for layerdir in rd.getVar('BBLAYERS').split()]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500702 if not os.path.abspath(destlayerdir) in layerdirs:
703 bb.warn('Specified layer is not currently enabled in bblayers.conf, you will need to add it before this bbappend will be active')
704
705 bbappendlines = []
706 if extralines:
707 if isinstance(extralines, dict):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600708 for name, value in extralines.items():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500709 bbappendlines.append((name, '=', value))
710 else:
711 # Do our best to split it
712 for line in extralines:
713 if line[-1] == '\n':
714 line = line[:-1]
715 splitline = line.split(None, 2)
716 if len(splitline) == 3:
717 bbappendlines.append(tuple(splitline))
718 else:
719 raise Exception('Invalid extralines value passed')
720
721 def popline(varname):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600722 for i in range(0, len(bbappendlines)):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500723 if bbappendlines[i][0] == varname:
724 line = bbappendlines.pop(i)
725 return line
726 return None
727
728 def appendline(varname, op, value):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600729 for i in range(0, len(bbappendlines)):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500730 item = bbappendlines[i]
731 if item[0] == varname:
732 bbappendlines[i] = (item[0], item[1], item[2] + ' ' + value)
733 break
734 else:
735 bbappendlines.append((varname, op, value))
736
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500737 destsubdir = rd.getVar('PN')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500738 if srcfiles:
739 bbappendlines.append(('FILESEXTRAPATHS_prepend', ':=', '${THISDIR}/${PN}:'))
740
741 appendoverride = ''
742 if machine:
743 bbappendlines.append(('PACKAGE_ARCH', '=', '${MACHINE_ARCH}'))
744 appendoverride = '_%s' % machine
745 copyfiles = {}
746 if srcfiles:
747 instfunclines = []
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600748 for newfile, origsrcfile in srcfiles.items():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500749 srcfile = origsrcfile
750 srcurientry = None
751 if not srcfile:
752 srcfile = os.path.basename(newfile)
753 srcurientry = 'file://%s' % srcfile
754 # Double-check it's not there already
755 # FIXME do we care if the entry is added by another bbappend that might go away?
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500756 if not srcurientry in rd.getVar('SRC_URI').split():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500757 if machine:
758 appendline('SRC_URI_append%s' % appendoverride, '=', ' ' + srcurientry)
759 else:
760 appendline('SRC_URI', '+=', srcurientry)
761 copyfiles[newfile] = srcfile
762 if install:
763 institem = install.pop(newfile, None)
764 if institem:
765 (destpath, perms) = institem
766 instdestpath = replace_dir_vars(destpath, rd)
767 instdirline = 'install -d ${D}%s' % os.path.dirname(instdestpath)
768 if not instdirline in instfunclines:
769 instfunclines.append(instdirline)
770 instfunclines.append('install -m %s ${WORKDIR}/%s ${D}%s' % (perms, os.path.basename(srcfile), instdestpath))
771 if instfunclines:
772 bbappendlines.append(('do_install_append%s()' % appendoverride, '', instfunclines))
773
Brad Bishop316dfdd2018-06-25 12:45:53 -0400774 if redirect_output:
775 bb.note('Writing append file %s (dry-run)' % appendpath)
776 outfile = os.path.join(redirect_output, os.path.basename(appendpath))
777 # Only take a copy if the file isn't already there (this function may be called
778 # multiple times per operation when we're handling overrides)
779 if os.path.exists(appendpath) and not os.path.exists(outfile):
780 shutil.copy2(appendpath, outfile)
781 else:
782 bb.note('Writing append file %s' % appendpath)
783 outfile = appendpath
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500784
Brad Bishop316dfdd2018-06-25 12:45:53 -0400785 if os.path.exists(outfile):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500786 # Work around lack of nonlocal in python 2
787 extvars = {'destsubdir': destsubdir}
788
789 def appendfile_varfunc(varname, origvalue, op, newlines):
790 if varname == 'FILESEXTRAPATHS_prepend':
791 if origvalue.startswith('${THISDIR}/'):
792 popline('FILESEXTRAPATHS_prepend')
793 extvars['destsubdir'] = rd.expand(origvalue.split('${THISDIR}/', 1)[1].rstrip(':'))
794 elif varname == 'PACKAGE_ARCH':
795 if machine:
796 popline('PACKAGE_ARCH')
797 return (machine, None, 4, False)
798 elif varname.startswith('do_install_append'):
799 func = popline(varname)
800 if func:
801 instfunclines = [line.strip() for line in origvalue.strip('\n').splitlines()]
802 for line in func[2]:
803 if not line in instfunclines:
804 instfunclines.append(line)
805 return (instfunclines, None, 4, False)
806 else:
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500807 splitval = split_var_value(origvalue, assignment=False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500808 changed = False
809 removevar = varname
810 if varname in ['SRC_URI', 'SRC_URI_append%s' % appendoverride]:
811 removevar = 'SRC_URI'
812 line = popline(varname)
813 if line:
814 if line[2] not in splitval:
815 splitval.append(line[2])
816 changed = True
817 else:
818 line = popline(varname)
819 if line:
820 splitval = [line[2]]
821 changed = True
822
823 if removevar in removevalues:
824 remove = removevalues[removevar]
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600825 if isinstance(remove, str):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500826 if remove in splitval:
827 splitval.remove(remove)
828 changed = True
829 else:
830 for removeitem in remove:
831 if removeitem in splitval:
832 splitval.remove(removeitem)
833 changed = True
834
835 if changed:
836 newvalue = splitval
837 if len(newvalue) == 1:
838 # Ensure it's written out as one line
839 if '_append' in varname:
840 newvalue = ' ' + newvalue[0]
841 else:
842 newvalue = newvalue[0]
843 if not newvalue and (op in ['+=', '.='] or '_append' in varname):
844 # There's no point appending nothing
845 newvalue = None
846 if varname.endswith('()'):
847 indent = 4
848 else:
849 indent = -1
850 return (newvalue, None, indent, True)
851 return (origvalue, None, 4, False)
852
853 varnames = [item[0] for item in bbappendlines]
854 if removevalues:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600855 varnames.extend(list(removevalues.keys()))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500856
Brad Bishop316dfdd2018-06-25 12:45:53 -0400857 with open(outfile, 'r') as f:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500858 (updated, newlines) = bb.utils.edit_metadata(f, varnames, appendfile_varfunc)
859
860 destsubdir = extvars['destsubdir']
861 else:
862 updated = False
863 newlines = []
864
865 if bbappendlines:
866 for line in bbappendlines:
867 if line[0].endswith('()'):
868 newlines.append('%s {\n %s\n}\n' % (line[0], '\n '.join(line[2])))
869 else:
870 newlines.append('%s %s "%s"\n\n' % line)
871 updated = True
872
873 if updated:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400874 with open(outfile, 'w') as f:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500875 f.writelines(newlines)
876
877 if copyfiles:
878 if machine:
879 destsubdir = os.path.join(destsubdir, machine)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400880 if redirect_output:
881 outdir = redirect_output
882 else:
883 outdir = appenddir
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600884 for newfile, srcfile in copyfiles.items():
Brad Bishop316dfdd2018-06-25 12:45:53 -0400885 filedest = os.path.join(outdir, destsubdir, os.path.basename(srcfile))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500886 if os.path.abspath(newfile) != os.path.abspath(filedest):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500887 if newfile.startswith(tempfile.gettempdir()):
888 newfiledisp = os.path.basename(newfile)
889 else:
890 newfiledisp = newfile
Brad Bishop316dfdd2018-06-25 12:45:53 -0400891 if redirect_output:
892 bb.note('Copying %s to %s (dry-run)' % (newfiledisp, os.path.join(appenddir, destsubdir, os.path.basename(srcfile))))
893 else:
894 bb.note('Copying %s to %s' % (newfiledisp, filedest))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500895 bb.utils.mkdirhier(os.path.dirname(filedest))
896 shutil.copyfile(newfile, filedest)
897
898 return (appendpath, os.path.join(appenddir, destsubdir))
899
900
901def find_layerdir(fn):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600902 """ Figure out the path to the base of the layer containing a file (e.g. a recipe)"""
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500903 pth = os.path.abspath(fn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500904 layerdir = ''
905 while pth:
906 if os.path.exists(os.path.join(pth, 'conf', 'layer.conf')):
907 layerdir = pth
908 break
909 pth = os.path.dirname(pth)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600910 if pth == '/':
911 return None
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500912 return layerdir
913
914
915def replace_dir_vars(path, d):
916 """Replace common directory paths with appropriate variable references (e.g. /etc becomes ${sysconfdir})"""
917 dirvars = {}
918 # Sort by length so we get the variables we're interested in first
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600919 for var in sorted(list(d.keys()), key=len):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500920 if var.endswith('dir') and var.lower() == var:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500921 value = d.getVar(var)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500922 if value.startswith('/') and not '\n' in value and value not in dirvars:
923 dirvars[value] = var
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600924 for dirpath in sorted(list(dirvars.keys()), reverse=True):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500925 path = path.replace(dirpath, '${%s}' % dirvars[dirpath])
926 return path
927
928def get_recipe_pv_without_srcpv(pv, uri_type):
929 """
930 Get PV without SRCPV common in SCM's for now only
931 support git.
932
933 Returns tuple with pv, prefix and suffix.
934 """
935 pfx = ''
936 sfx = ''
937
938 if uri_type == 'git':
Brad Bishop15ae2502019-06-18 21:44:24 -0400939 git_regex = re.compile(r"(?P<pfx>v?)(?P<ver>.*?)(?P<sfx>\+[^\+]*(git)?r?(AUTOINC\+))(?P<rev>.*)")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500940 m = git_regex.match(pv)
941
942 if m:
943 pv = m.group('ver')
944 pfx = m.group('pfx')
945 sfx = m.group('sfx')
946 else:
Brad Bishop19323692019-04-05 15:28:33 -0400947 regex = re.compile(r"(?P<pfx>(v|r)?)(?P<ver>.*)")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500948 m = regex.match(pv)
949 if m:
950 pv = m.group('ver')
951 pfx = m.group('pfx')
952
953 return (pv, pfx, sfx)
954
955def get_recipe_upstream_version(rd):
956 """
957 Get upstream version of recipe using bb.fetch2 methods with support for
958 http, https, ftp and git.
959
960 bb.fetch2 exceptions can be raised,
961 FetchError when don't have network access or upstream site don't response.
962 NoMethodError when uri latest_versionstring method isn't implemented.
963
Brad Bishop316dfdd2018-06-25 12:45:53 -0400964 Returns a dictonary with version, repository revision, current_version, type and datetime.
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500965 Type can be A for Automatic, M for Manual and U for Unknown.
966 """
967 from bb.fetch2 import decodeurl
968 from datetime import datetime
969
970 ru = {}
Brad Bishop316dfdd2018-06-25 12:45:53 -0400971 ru['current_version'] = rd.getVar('PV')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500972 ru['version'] = ''
973 ru['type'] = 'U'
974 ru['datetime'] = ''
Brad Bishop316dfdd2018-06-25 12:45:53 -0400975 ru['revision'] = ''
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500976
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500977 # XXX: If don't have SRC_URI means that don't have upstream sources so
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500978 # returns the current recipe version, so that upstream version check
979 # declares a match.
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500980 src_uris = rd.getVar('SRC_URI')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500981 if not src_uris:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400982 ru['version'] = ru['current_version']
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500983 ru['type'] = 'M'
984 ru['datetime'] = datetime.now()
985 return ru
986
987 # XXX: we suppose that the first entry points to the upstream sources
988 src_uri = src_uris.split()[0]
989 uri_type, _, _, _, _, _ = decodeurl(src_uri)
990
Brad Bishop316dfdd2018-06-25 12:45:53 -0400991 (pv, pfx, sfx) = get_recipe_pv_without_srcpv(rd.getVar('PV'), uri_type)
992 ru['current_version'] = pv
993
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500994 manual_upstream_version = rd.getVar("RECIPE_UPSTREAM_VERSION")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500995 if manual_upstream_version:
996 # manual tracking of upstream version.
997 ru['version'] = manual_upstream_version
998 ru['type'] = 'M'
999
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001000 manual_upstream_date = rd.getVar("CHECK_DATE")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001001 if manual_upstream_date:
1002 date = datetime.strptime(manual_upstream_date, "%b %d, %Y")
1003 else:
1004 date = datetime.now()
1005 ru['datetime'] = date
1006
1007 elif uri_type == "file":
1008 # files are always up-to-date
1009 ru['version'] = pv
1010 ru['type'] = 'A'
1011 ru['datetime'] = datetime.now()
1012 else:
1013 ud = bb.fetch2.FetchData(src_uri, rd)
Brad Bishop316dfdd2018-06-25 12:45:53 -04001014 if rd.getVar("UPSTREAM_CHECK_COMMITS") == "1":
1015 revision = ud.method.latest_revision(ud, rd, 'default')
1016 upversion = pv
1017 if revision != rd.getVar("SRCREV"):
1018 upversion = upversion + "-new-commits-available"
1019 else:
1020 pupver = ud.method.latest_versionstring(ud, rd)
1021 (upversion, revision) = pupver
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001022
1023 if upversion:
1024 ru['version'] = upversion
1025 ru['type'] = 'A'
1026
Brad Bishop316dfdd2018-06-25 12:45:53 -04001027 if revision:
1028 ru['revision'] = revision
1029
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001030 ru['datetime'] = datetime.now()
1031
1032 return ru
Brad Bishop19323692019-04-05 15:28:33 -04001033
1034def _get_recipe_upgrade_status(data):
1035 uv = get_recipe_upstream_version(data)
1036
1037 pn = data.getVar('PN')
1038 cur_ver = uv['current_version']
1039
1040 upstream_version_unknown = data.getVar('UPSTREAM_VERSION_UNKNOWN')
1041 if not uv['version']:
1042 status = "UNKNOWN" if upstream_version_unknown else "UNKNOWN_BROKEN"
1043 else:
1044 cmp = vercmp_string(uv['current_version'], uv['version'])
1045 if cmp == -1:
1046 status = "UPDATE" if not upstream_version_unknown else "KNOWN_BROKEN"
1047 elif cmp == 0:
1048 status = "MATCH" if not upstream_version_unknown else "KNOWN_BROKEN"
1049 else:
1050 status = "UNKNOWN" if upstream_version_unknown else "UNKNOWN_BROKEN"
1051
1052 next_ver = uv['version'] if uv['version'] else "N/A"
1053 revision = uv['revision'] if uv['revision'] else "N/A"
1054 maintainer = data.getVar('RECIPE_MAINTAINER')
1055 no_upgrade_reason = data.getVar('RECIPE_NO_UPDATE_REASON')
1056
1057 return (pn, status, cur_ver, next_ver, maintainer, revision, no_upgrade_reason)
1058
1059def get_recipe_upgrade_status(recipes=None):
1060 pkgs_list = []
1061 data_copy_list = []
1062 copy_vars = ('SRC_URI',
1063 'PV',
Brad Bishop19323692019-04-05 15:28:33 -04001064 'DL_DIR',
1065 'PN',
1066 'CACHE',
1067 'PERSISTENT_DIR',
1068 'BB_URI_HEADREVS',
1069 'UPSTREAM_CHECK_COMMITS',
1070 'UPSTREAM_CHECK_GITTAGREGEX',
1071 'UPSTREAM_CHECK_REGEX',
1072 'UPSTREAM_CHECK_URI',
1073 'UPSTREAM_VERSION_UNKNOWN',
1074 'RECIPE_MAINTAINER',
1075 'RECIPE_NO_UPDATE_REASON',
1076 'RECIPE_UPSTREAM_VERSION',
1077 'RECIPE_UPSTREAM_DATE',
1078 'CHECK_DATE',
Andrew Geissler1e34c2d2020-05-29 16:02:59 -05001079 'FETCHCMD_bzr',
1080 'FETCHCMD_ccrc',
1081 'FETCHCMD_cvs',
1082 'FETCHCMD_git',
1083 'FETCHCMD_hg',
1084 'FETCHCMD_npm',
1085 'FETCHCMD_osc',
1086 'FETCHCMD_p4',
1087 'FETCHCMD_repo',
1088 'FETCHCMD_s3',
1089 'FETCHCMD_svn',
1090 'FETCHCMD_wget',
Brad Bishop19323692019-04-05 15:28:33 -04001091 )
1092
1093 with bb.tinfoil.Tinfoil() as tinfoil:
1094 tinfoil.prepare(config_only=False)
1095
1096 if not recipes:
1097 recipes = tinfoil.all_recipe_files(variants=False)
1098
1099 for fn in recipes:
1100 try:
1101 if fn.startswith("/"):
1102 data = tinfoil.parse_recipe_file(fn)
1103 else:
1104 data = tinfoil.parse_recipe(fn)
1105 except bb.providers.NoProvider:
1106 bb.note(" No provider for %s" % fn)
1107 continue
1108
1109 unreliable = data.getVar('UPSTREAM_CHECK_UNRELIABLE')
1110 if unreliable == "1":
1111 bb.note(" Skip package %s as upstream check unreliable" % pn)
1112 continue
1113
1114 data_copy = bb.data.init()
1115 for var in copy_vars:
1116 data_copy.setVar(var, data.getVar(var))
1117 for k in data:
1118 if k.startswith('SRCREV'):
1119 data_copy.setVar(k, data.getVar(k))
1120
1121 data_copy_list.append(data_copy)
1122
1123 from concurrent.futures import ProcessPoolExecutor
1124 with ProcessPoolExecutor(max_workers=utils.cpu_count()) as executor:
1125 pkgs_list = executor.map(_get_recipe_upgrade_status, data_copy_list)
1126
1127 return pkgs_list