blob: 8244cf07208afb76c9d0f53b622c3e2b3f8b32b1 [file] [log] [blame]
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001# Yocto Project layer check tool
2#
3# Copyright (C) 2017 Intel Corporation
Brad Bishopc342db32019-05-15 21:57:59 -04004#
5# SPDX-License-Identifier: MIT
6#
Brad Bishopd7bf8c12018-02-25 22:55:05 -05007
8import os
9import re
10import subprocess
11from enum import Enum
12
13import bb.tinfoil
14
15class LayerType(Enum):
16 BSP = 0
17 DISTRO = 1
18 SOFTWARE = 2
19 ERROR_NO_LAYER_CONF = 98
20 ERROR_BSP_DISTRO = 99
21
22def _get_configurations(path):
23 configs = []
24
25 for f in os.listdir(path):
26 file_path = os.path.join(path, f)
27 if os.path.isfile(file_path) and f.endswith('.conf'):
28 configs.append(f[:-5]) # strip .conf
29 return configs
30
31def _get_layer_collections(layer_path, lconf=None, data=None):
32 import bb.parse
33 import bb.data
34
35 if lconf is None:
36 lconf = os.path.join(layer_path, 'conf', 'layer.conf')
37
38 if data is None:
39 ldata = bb.data.init()
40 bb.parse.init_parser(ldata)
41 else:
42 ldata = data.createCopy()
43
44 ldata.setVar('LAYERDIR', layer_path)
45 try:
46 ldata = bb.parse.handle(lconf, ldata, include=True)
Brad Bishop00111322018-04-01 22:23:53 -040047 except:
48 raise RuntimeError("Parsing of layer.conf from layer: %s failed" % layer_path)
Brad Bishopd7bf8c12018-02-25 22:55:05 -050049 ldata.expandVarref('LAYERDIR')
50
51 collections = (ldata.getVar('BBFILE_COLLECTIONS') or '').split()
52 if not collections:
53 name = os.path.basename(layer_path)
54 collections = [name]
55
56 collections = {c: {} for c in collections}
57 for name in collections:
58 priority = ldata.getVar('BBFILE_PRIORITY_%s' % name)
59 pattern = ldata.getVar('BBFILE_PATTERN_%s' % name)
60 depends = ldata.getVar('LAYERDEPENDS_%s' % name)
Brad Bishop316dfdd2018-06-25 12:45:53 -040061 compat = ldata.getVar('LAYERSERIES_COMPAT_%s' % name)
Brad Bishopd7bf8c12018-02-25 22:55:05 -050062 collections[name]['priority'] = priority
63 collections[name]['pattern'] = pattern
64 collections[name]['depends'] = depends
Brad Bishop316dfdd2018-06-25 12:45:53 -040065 collections[name]['compat'] = compat
Brad Bishopd7bf8c12018-02-25 22:55:05 -050066
67 return collections
68
69def _detect_layer(layer_path):
70 """
71 Scans layer directory to detect what type of layer
72 is BSP, Distro or Software.
73
74 Returns a dictionary with layer name, type and path.
75 """
76
77 layer = {}
78 layer_name = os.path.basename(layer_path)
79
80 layer['name'] = layer_name
81 layer['path'] = layer_path
82 layer['conf'] = {}
83
84 if not os.path.isfile(os.path.join(layer_path, 'conf', 'layer.conf')):
85 layer['type'] = LayerType.ERROR_NO_LAYER_CONF
86 return layer
87
88 machine_conf = os.path.join(layer_path, 'conf', 'machine')
89 distro_conf = os.path.join(layer_path, 'conf', 'distro')
90
91 is_bsp = False
92 is_distro = False
93
94 if os.path.isdir(machine_conf):
95 machines = _get_configurations(machine_conf)
96 if machines:
97 is_bsp = True
98
99 if os.path.isdir(distro_conf):
100 distros = _get_configurations(distro_conf)
101 if distros:
102 is_distro = True
103
104 if is_bsp and is_distro:
105 layer['type'] = LayerType.ERROR_BSP_DISTRO
106 elif is_bsp:
107 layer['type'] = LayerType.BSP
108 layer['conf']['machines'] = machines
109 elif is_distro:
110 layer['type'] = LayerType.DISTRO
111 layer['conf']['distros'] = distros
112 else:
113 layer['type'] = LayerType.SOFTWARE
114
115 layer['collections'] = _get_layer_collections(layer['path'])
116
117 return layer
118
119def detect_layers(layer_directories, no_auto):
120 layers = []
121
122 for directory in layer_directories:
123 directory = os.path.realpath(directory)
124 if directory[-1] == '/':
125 directory = directory[0:-1]
126
127 if no_auto:
128 conf_dir = os.path.join(directory, 'conf')
129 if os.path.isdir(conf_dir):
130 layer = _detect_layer(directory)
131 if layer:
132 layers.append(layer)
133 else:
134 for root, dirs, files in os.walk(directory):
135 dir_name = os.path.basename(root)
136 conf_dir = os.path.join(root, 'conf')
137 if os.path.isdir(conf_dir):
138 layer = _detect_layer(root)
139 if layer:
140 layers.append(layer)
141
142 return layers
143
144def _find_layer_depends(depend, layers):
145 for layer in layers:
146 for collection in layer['collections']:
147 if depend == collection:
148 return layer
149 return None
150
151def add_layer_dependencies(bblayersconf, layer, layers, logger):
152 def recurse_dependencies(depends, layer, layers, logger, ret = []):
153 logger.debug('Processing dependencies %s for layer %s.' % \
154 (depends, layer['name']))
155
156 for depend in depends.split():
157 # core (oe-core) is suppose to be provided
158 if depend == 'core':
159 continue
160
161 layer_depend = _find_layer_depends(depend, layers)
162 if not layer_depend:
163 logger.error('Layer %s depends on %s and isn\'t found.' % \
164 (layer['name'], depend))
165 ret = None
166 continue
167
168 # We keep processing, even if ret is None, this allows us to report
169 # multiple errors at once
170 if ret is not None and layer_depend not in ret:
171 ret.append(layer_depend)
Brad Bishop004d4992018-10-02 23:54:45 +0200172 else:
173 # we might have processed this dependency already, in which case
174 # we should not do it again (avoid recursive loop)
175 continue
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500176
177 # Recursively process...
178 if 'collections' not in layer_depend:
179 continue
180
181 for collection in layer_depend['collections']:
182 collect_deps = layer_depend['collections'][collection]['depends']
183 if not collect_deps:
184 continue
185 ret = recurse_dependencies(collect_deps, layer_depend, layers, logger, ret)
186
187 return ret
188
189 layer_depends = []
190 for collection in layer['collections']:
191 depends = layer['collections'][collection]['depends']
192 if not depends:
193 continue
194
195 layer_depends = recurse_dependencies(depends, layer, layers, logger, layer_depends)
196
197 # Note: [] (empty) is allowed, None is not!
198 if layer_depends is None:
199 return False
200 else:
Andrew Geissler99467da2019-02-25 18:54:23 -0600201 add_layers(bblayersconf, layer_depends, logger)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500202
Andrew Geissler99467da2019-02-25 18:54:23 -0600203 return True
204
205def add_layers(bblayersconf, layers, logger):
206 # Don't add a layer that is already present.
207 added = set()
208 output = check_command('Getting existing layers failed.', 'bitbake-layers show-layers').decode('utf-8')
209 for layer, path, pri in re.findall(r'^(\S+) +([^\n]*?) +(\d+)$', output, re.MULTILINE):
210 added.add(path)
211
212 with open(bblayersconf, 'a+') as f:
213 for layer in layers:
214 logger.info('Adding layer %s' % layer['name'])
215 name = layer['name']
216 path = layer['path']
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500217 if path in added:
Andrew Geissler99467da2019-02-25 18:54:23 -0600218 logger.info('%s is already in %s' % (name, bblayersconf))
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500219 else:
220 added.add(path)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500221 f.write("\nBBLAYERS += \"%s\"\n" % path)
222 return True
223
Andrew Geissler99467da2019-02-25 18:54:23 -0600224def check_command(error_msg, cmd, cwd=None):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500225 '''
226 Run a command under a shell, capture stdout and stderr in a single stream,
227 throw an error when command returns non-zero exit code. Returns the output.
228 '''
229
Andrew Geissler99467da2019-02-25 18:54:23 -0600230 p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=cwd)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500231 output, _ = p.communicate()
232 if p.returncode:
233 msg = "%s\nCommand: %s\nOutput:\n%s" % (error_msg, cmd, output.decode('utf-8'))
234 raise RuntimeError(msg)
235 return output
236
237def get_signatures(builddir, failsafe=False, machine=None):
238 import re
239
240 # some recipes needs to be excluded like meta-world-pkgdata
241 # because a layer can add recipes to a world build so signature
242 # will be change
243 exclude_recipes = ('meta-world-pkgdata',)
244
245 sigs = {}
246 tune2tasks = {}
247
248 cmd = ''
249 if machine:
250 cmd += 'MACHINE=%s ' % machine
251 cmd += 'bitbake '
252 if failsafe:
253 cmd += '-k '
254 cmd += '-S none world'
255 sigs_file = os.path.join(builddir, 'locked-sigs.inc')
256 if os.path.exists(sigs_file):
257 os.unlink(sigs_file)
258 try:
259 check_command('Generating signatures failed. This might be due to some parse error and/or general layer incompatibilities.',
Andrew Geissler99467da2019-02-25 18:54:23 -0600260 cmd, builddir)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500261 except RuntimeError as ex:
262 if failsafe and os.path.exists(sigs_file):
263 # Ignore the error here. Most likely some recipes active
264 # in a world build lack some dependencies. There is a
265 # separate test_machine_world_build which exposes the
266 # failure.
267 pass
268 else:
269 raise
270
271 sig_regex = re.compile("^(?P<task>.*:.*):(?P<hash>.*) .$")
272 tune_regex = re.compile("(^|\s)SIGGEN_LOCKEDSIGS_t-(?P<tune>\S*)\s*=\s*")
273 current_tune = None
274 with open(sigs_file, 'r') as f:
275 for line in f.readlines():
276 line = line.strip()
277 t = tune_regex.search(line)
278 if t:
279 current_tune = t.group('tune')
280 s = sig_regex.match(line)
281 if s:
282 exclude = False
283 for er in exclude_recipes:
284 (recipe, task) = s.group('task').split(':')
285 if er == recipe:
286 exclude = True
287 break
288 if exclude:
289 continue
290
291 sigs[s.group('task')] = s.group('hash')
292 tune2tasks.setdefault(current_tune, []).append(s.group('task'))
293
294 if not sigs:
295 raise RuntimeError('Can\'t load signatures from %s' % sigs_file)
296
297 return (sigs, tune2tasks)
298
299def get_depgraph(targets=['world'], failsafe=False):
300 '''
301 Returns the dependency graph for the given target(s).
302 The dependency graph is taken directly from DepTreeEvent.
303 '''
304 depgraph = None
305 with bb.tinfoil.Tinfoil() as tinfoil:
306 tinfoil.prepare(config_only=False)
307 tinfoil.set_event_mask(['bb.event.NoProvider', 'bb.event.DepTreeGenerated', 'bb.command.CommandCompleted'])
308 if not tinfoil.run_command('generateDepTreeEvent', targets, 'do_build'):
309 raise RuntimeError('starting generateDepTreeEvent failed')
310 while True:
311 event = tinfoil.wait_event(timeout=1000)
312 if event:
313 if isinstance(event, bb.command.CommandFailed):
314 raise RuntimeError('Generating dependency information failed: %s' % event.error)
315 elif isinstance(event, bb.command.CommandCompleted):
316 break
317 elif isinstance(event, bb.event.NoProvider):
318 if failsafe:
319 # The event is informational, we will get information about the
320 # remaining dependencies eventually and thus can ignore this
321 # here like we do in get_signatures(), if desired.
322 continue
323 if event._reasons:
324 raise RuntimeError('Nothing provides %s: %s' % (event._item, event._reasons))
325 else:
326 raise RuntimeError('Nothing provides %s.' % (event._item))
327 elif isinstance(event, bb.event.DepTreeGenerated):
328 depgraph = event._depgraph
329
330 if depgraph is None:
331 raise RuntimeError('Could not retrieve the depgraph.')
332 return depgraph
333
334def compare_signatures(old_sigs, curr_sigs):
335 '''
336 Compares the result of two get_signatures() calls. Returns None if no
337 problems found, otherwise a string that can be used as additional
338 explanation in self.fail().
339 '''
340 # task -> (old signature, new signature)
341 sig_diff = {}
342 for task in old_sigs:
343 if task in curr_sigs and \
344 old_sigs[task] != curr_sigs[task]:
345 sig_diff[task] = (old_sigs[task], curr_sigs[task])
346
347 if not sig_diff:
348 return None
349
350 # Beware, depgraph uses task=<pn>.<taskname> whereas get_signatures()
351 # uses <pn>:<taskname>. Need to convert sometimes. The output follows
352 # the convention from get_signatures() because that seems closer to
353 # normal bitbake output.
354 def sig2graph(task):
355 pn, taskname = task.rsplit(':', 1)
356 return pn + '.' + taskname
357 def graph2sig(task):
358 pn, taskname = task.rsplit('.', 1)
359 return pn + ':' + taskname
360 depgraph = get_depgraph(failsafe=True)
361 depends = depgraph['tdepends']
362
363 # If a task A has a changed signature, but none of its
364 # dependencies, then we need to report it because it is
365 # the one which introduces a change. Any task depending on
366 # A (directly or indirectly) will also have a changed
367 # signature, but we don't need to report it. It might have
368 # its own changes, which will become apparent once the
369 # issues that we do report are fixed and the test gets run
370 # again.
371 sig_diff_filtered = []
372 for task, (old_sig, new_sig) in sig_diff.items():
373 deps_tainted = False
374 for dep in depends.get(sig2graph(task), ()):
375 if graph2sig(dep) in sig_diff:
376 deps_tainted = True
377 break
378 if not deps_tainted:
379 sig_diff_filtered.append((task, old_sig, new_sig))
380
381 msg = []
382 msg.append('%d signatures changed, initial differences (first hash before, second after):' %
383 len(sig_diff))
384 for diff in sorted(sig_diff_filtered):
385 recipe, taskname = diff[0].rsplit(':', 1)
386 cmd = 'bitbake-diffsigs --task %s %s --signature %s %s' % \
387 (recipe, taskname, diff[1], diff[2])
388 msg.append(' %s: %s -> %s' % diff)
389 msg.append(' %s' % cmd)
390 try:
391 output = check_command('Determining signature difference failed.',
392 cmd).decode('utf-8')
393 except RuntimeError as error:
394 output = str(error)
395 if output:
396 msg.extend([' ' + line for line in output.splitlines()])
397 msg.append('')
398 return '\n'.join(msg)