blob: 670f0eea39222069bfa53ab194045018425bad03 [file] [log] [blame]
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001# Yocto Project layer check tool
2#
3# Copyright (C) 2017 Intel Corporation
4# Released under the MIT license (see COPYING.MIT)
5
6import os
7import re
8import subprocess
9from enum import Enum
10
11import bb.tinfoil
12
13class LayerType(Enum):
14 BSP = 0
15 DISTRO = 1
16 SOFTWARE = 2
17 ERROR_NO_LAYER_CONF = 98
18 ERROR_BSP_DISTRO = 99
19
20def _get_configurations(path):
21 configs = []
22
23 for f in os.listdir(path):
24 file_path = os.path.join(path, f)
25 if os.path.isfile(file_path) and f.endswith('.conf'):
26 configs.append(f[:-5]) # strip .conf
27 return configs
28
29def _get_layer_collections(layer_path, lconf=None, data=None):
30 import bb.parse
31 import bb.data
32
33 if lconf is None:
34 lconf = os.path.join(layer_path, 'conf', 'layer.conf')
35
36 if data is None:
37 ldata = bb.data.init()
38 bb.parse.init_parser(ldata)
39 else:
40 ldata = data.createCopy()
41
42 ldata.setVar('LAYERDIR', layer_path)
43 try:
44 ldata = bb.parse.handle(lconf, ldata, include=True)
Brad Bishop00111322018-04-01 22:23:53 -040045 except:
46 raise RuntimeError("Parsing of layer.conf from layer: %s failed" % layer_path)
Brad Bishopd7bf8c12018-02-25 22:55:05 -050047 ldata.expandVarref('LAYERDIR')
48
49 collections = (ldata.getVar('BBFILE_COLLECTIONS') or '').split()
50 if not collections:
51 name = os.path.basename(layer_path)
52 collections = [name]
53
54 collections = {c: {} for c in collections}
55 for name in collections:
56 priority = ldata.getVar('BBFILE_PRIORITY_%s' % name)
57 pattern = ldata.getVar('BBFILE_PATTERN_%s' % name)
58 depends = ldata.getVar('LAYERDEPENDS_%s' % name)
Brad Bishop316dfdd2018-06-25 12:45:53 -040059 compat = ldata.getVar('LAYERSERIES_COMPAT_%s' % name)
Brad Bishopd7bf8c12018-02-25 22:55:05 -050060 collections[name]['priority'] = priority
61 collections[name]['pattern'] = pattern
62 collections[name]['depends'] = depends
Brad Bishop316dfdd2018-06-25 12:45:53 -040063 collections[name]['compat'] = compat
Brad Bishopd7bf8c12018-02-25 22:55:05 -050064
65 return collections
66
67def _detect_layer(layer_path):
68 """
69 Scans layer directory to detect what type of layer
70 is BSP, Distro or Software.
71
72 Returns a dictionary with layer name, type and path.
73 """
74
75 layer = {}
76 layer_name = os.path.basename(layer_path)
77
78 layer['name'] = layer_name
79 layer['path'] = layer_path
80 layer['conf'] = {}
81
82 if not os.path.isfile(os.path.join(layer_path, 'conf', 'layer.conf')):
83 layer['type'] = LayerType.ERROR_NO_LAYER_CONF
84 return layer
85
86 machine_conf = os.path.join(layer_path, 'conf', 'machine')
87 distro_conf = os.path.join(layer_path, 'conf', 'distro')
88
89 is_bsp = False
90 is_distro = False
91
92 if os.path.isdir(machine_conf):
93 machines = _get_configurations(machine_conf)
94 if machines:
95 is_bsp = True
96
97 if os.path.isdir(distro_conf):
98 distros = _get_configurations(distro_conf)
99 if distros:
100 is_distro = True
101
102 if is_bsp and is_distro:
103 layer['type'] = LayerType.ERROR_BSP_DISTRO
104 elif is_bsp:
105 layer['type'] = LayerType.BSP
106 layer['conf']['machines'] = machines
107 elif is_distro:
108 layer['type'] = LayerType.DISTRO
109 layer['conf']['distros'] = distros
110 else:
111 layer['type'] = LayerType.SOFTWARE
112
113 layer['collections'] = _get_layer_collections(layer['path'])
114
115 return layer
116
117def detect_layers(layer_directories, no_auto):
118 layers = []
119
120 for directory in layer_directories:
121 directory = os.path.realpath(directory)
122 if directory[-1] == '/':
123 directory = directory[0:-1]
124
125 if no_auto:
126 conf_dir = os.path.join(directory, 'conf')
127 if os.path.isdir(conf_dir):
128 layer = _detect_layer(directory)
129 if layer:
130 layers.append(layer)
131 else:
132 for root, dirs, files in os.walk(directory):
133 dir_name = os.path.basename(root)
134 conf_dir = os.path.join(root, 'conf')
135 if os.path.isdir(conf_dir):
136 layer = _detect_layer(root)
137 if layer:
138 layers.append(layer)
139
140 return layers
141
142def _find_layer_depends(depend, layers):
143 for layer in layers:
144 for collection in layer['collections']:
145 if depend == collection:
146 return layer
147 return None
148
149def add_layer_dependencies(bblayersconf, layer, layers, logger):
150 def recurse_dependencies(depends, layer, layers, logger, ret = []):
151 logger.debug('Processing dependencies %s for layer %s.' % \
152 (depends, layer['name']))
153
154 for depend in depends.split():
155 # core (oe-core) is suppose to be provided
156 if depend == 'core':
157 continue
158
159 layer_depend = _find_layer_depends(depend, layers)
160 if not layer_depend:
161 logger.error('Layer %s depends on %s and isn\'t found.' % \
162 (layer['name'], depend))
163 ret = None
164 continue
165
166 # We keep processing, even if ret is None, this allows us to report
167 # multiple errors at once
168 if ret is not None and layer_depend not in ret:
169 ret.append(layer_depend)
Brad Bishop004d4992018-10-02 23:54:45 +0200170 else:
171 # we might have processed this dependency already, in which case
172 # we should not do it again (avoid recursive loop)
173 continue
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500174
175 # Recursively process...
176 if 'collections' not in layer_depend:
177 continue
178
179 for collection in layer_depend['collections']:
180 collect_deps = layer_depend['collections'][collection]['depends']
181 if not collect_deps:
182 continue
183 ret = recurse_dependencies(collect_deps, layer_depend, layers, logger, ret)
184
185 return ret
186
187 layer_depends = []
188 for collection in layer['collections']:
189 depends = layer['collections'][collection]['depends']
190 if not depends:
191 continue
192
193 layer_depends = recurse_dependencies(depends, layer, layers, logger, layer_depends)
194
195 # Note: [] (empty) is allowed, None is not!
196 if layer_depends is None:
197 return False
198 else:
Andrew Geissler99467da2019-02-25 18:54:23 -0600199 add_layers(bblayersconf, layer_depends, logger)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500200
Andrew Geissler99467da2019-02-25 18:54:23 -0600201 return True
202
203def add_layers(bblayersconf, layers, logger):
204 # Don't add a layer that is already present.
205 added = set()
206 output = check_command('Getting existing layers failed.', 'bitbake-layers show-layers').decode('utf-8')
207 for layer, path, pri in re.findall(r'^(\S+) +([^\n]*?) +(\d+)$', output, re.MULTILINE):
208 added.add(path)
209
210 with open(bblayersconf, 'a+') as f:
211 for layer in layers:
212 logger.info('Adding layer %s' % layer['name'])
213 name = layer['name']
214 path = layer['path']
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500215 if path in added:
Andrew Geissler99467da2019-02-25 18:54:23 -0600216 logger.info('%s is already in %s' % (name, bblayersconf))
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500217 else:
218 added.add(path)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500219 f.write("\nBBLAYERS += \"%s\"\n" % path)
220 return True
221
Andrew Geissler99467da2019-02-25 18:54:23 -0600222def check_command(error_msg, cmd, cwd=None):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500223 '''
224 Run a command under a shell, capture stdout and stderr in a single stream,
225 throw an error when command returns non-zero exit code. Returns the output.
226 '''
227
Andrew Geissler99467da2019-02-25 18:54:23 -0600228 p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=cwd)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500229 output, _ = p.communicate()
230 if p.returncode:
231 msg = "%s\nCommand: %s\nOutput:\n%s" % (error_msg, cmd, output.decode('utf-8'))
232 raise RuntimeError(msg)
233 return output
234
235def get_signatures(builddir, failsafe=False, machine=None):
236 import re
237
238 # some recipes needs to be excluded like meta-world-pkgdata
239 # because a layer can add recipes to a world build so signature
240 # will be change
241 exclude_recipes = ('meta-world-pkgdata',)
242
243 sigs = {}
244 tune2tasks = {}
245
246 cmd = ''
247 if machine:
248 cmd += 'MACHINE=%s ' % machine
249 cmd += 'bitbake '
250 if failsafe:
251 cmd += '-k '
252 cmd += '-S none world'
253 sigs_file = os.path.join(builddir, 'locked-sigs.inc')
254 if os.path.exists(sigs_file):
255 os.unlink(sigs_file)
256 try:
257 check_command('Generating signatures failed. This might be due to some parse error and/or general layer incompatibilities.',
Andrew Geissler99467da2019-02-25 18:54:23 -0600258 cmd, builddir)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500259 except RuntimeError as ex:
260 if failsafe and os.path.exists(sigs_file):
261 # Ignore the error here. Most likely some recipes active
262 # in a world build lack some dependencies. There is a
263 # separate test_machine_world_build which exposes the
264 # failure.
265 pass
266 else:
267 raise
268
269 sig_regex = re.compile("^(?P<task>.*:.*):(?P<hash>.*) .$")
270 tune_regex = re.compile("(^|\s)SIGGEN_LOCKEDSIGS_t-(?P<tune>\S*)\s*=\s*")
271 current_tune = None
272 with open(sigs_file, 'r') as f:
273 for line in f.readlines():
274 line = line.strip()
275 t = tune_regex.search(line)
276 if t:
277 current_tune = t.group('tune')
278 s = sig_regex.match(line)
279 if s:
280 exclude = False
281 for er in exclude_recipes:
282 (recipe, task) = s.group('task').split(':')
283 if er == recipe:
284 exclude = True
285 break
286 if exclude:
287 continue
288
289 sigs[s.group('task')] = s.group('hash')
290 tune2tasks.setdefault(current_tune, []).append(s.group('task'))
291
292 if not sigs:
293 raise RuntimeError('Can\'t load signatures from %s' % sigs_file)
294
295 return (sigs, tune2tasks)
296
297def get_depgraph(targets=['world'], failsafe=False):
298 '''
299 Returns the dependency graph for the given target(s).
300 The dependency graph is taken directly from DepTreeEvent.
301 '''
302 depgraph = None
303 with bb.tinfoil.Tinfoil() as tinfoil:
304 tinfoil.prepare(config_only=False)
305 tinfoil.set_event_mask(['bb.event.NoProvider', 'bb.event.DepTreeGenerated', 'bb.command.CommandCompleted'])
306 if not tinfoil.run_command('generateDepTreeEvent', targets, 'do_build'):
307 raise RuntimeError('starting generateDepTreeEvent failed')
308 while True:
309 event = tinfoil.wait_event(timeout=1000)
310 if event:
311 if isinstance(event, bb.command.CommandFailed):
312 raise RuntimeError('Generating dependency information failed: %s' % event.error)
313 elif isinstance(event, bb.command.CommandCompleted):
314 break
315 elif isinstance(event, bb.event.NoProvider):
316 if failsafe:
317 # The event is informational, we will get information about the
318 # remaining dependencies eventually and thus can ignore this
319 # here like we do in get_signatures(), if desired.
320 continue
321 if event._reasons:
322 raise RuntimeError('Nothing provides %s: %s' % (event._item, event._reasons))
323 else:
324 raise RuntimeError('Nothing provides %s.' % (event._item))
325 elif isinstance(event, bb.event.DepTreeGenerated):
326 depgraph = event._depgraph
327
328 if depgraph is None:
329 raise RuntimeError('Could not retrieve the depgraph.')
330 return depgraph
331
332def compare_signatures(old_sigs, curr_sigs):
333 '''
334 Compares the result of two get_signatures() calls. Returns None if no
335 problems found, otherwise a string that can be used as additional
336 explanation in self.fail().
337 '''
338 # task -> (old signature, new signature)
339 sig_diff = {}
340 for task in old_sigs:
341 if task in curr_sigs and \
342 old_sigs[task] != curr_sigs[task]:
343 sig_diff[task] = (old_sigs[task], curr_sigs[task])
344
345 if not sig_diff:
346 return None
347
348 # Beware, depgraph uses task=<pn>.<taskname> whereas get_signatures()
349 # uses <pn>:<taskname>. Need to convert sometimes. The output follows
350 # the convention from get_signatures() because that seems closer to
351 # normal bitbake output.
352 def sig2graph(task):
353 pn, taskname = task.rsplit(':', 1)
354 return pn + '.' + taskname
355 def graph2sig(task):
356 pn, taskname = task.rsplit('.', 1)
357 return pn + ':' + taskname
358 depgraph = get_depgraph(failsafe=True)
359 depends = depgraph['tdepends']
360
361 # If a task A has a changed signature, but none of its
362 # dependencies, then we need to report it because it is
363 # the one which introduces a change. Any task depending on
364 # A (directly or indirectly) will also have a changed
365 # signature, but we don't need to report it. It might have
366 # its own changes, which will become apparent once the
367 # issues that we do report are fixed and the test gets run
368 # again.
369 sig_diff_filtered = []
370 for task, (old_sig, new_sig) in sig_diff.items():
371 deps_tainted = False
372 for dep in depends.get(sig2graph(task), ()):
373 if graph2sig(dep) in sig_diff:
374 deps_tainted = True
375 break
376 if not deps_tainted:
377 sig_diff_filtered.append((task, old_sig, new_sig))
378
379 msg = []
380 msg.append('%d signatures changed, initial differences (first hash before, second after):' %
381 len(sig_diff))
382 for diff in sorted(sig_diff_filtered):
383 recipe, taskname = diff[0].rsplit(':', 1)
384 cmd = 'bitbake-diffsigs --task %s %s --signature %s %s' % \
385 (recipe, taskname, diff[1], diff[2])
386 msg.append(' %s: %s -> %s' % diff)
387 msg.append(' %s' % cmd)
388 try:
389 output = check_command('Determining signature difference failed.',
390 cmd).decode('utf-8')
391 except RuntimeError as error:
392 output = str(error)
393 if output:
394 msg.extend([' ' + line for line in output.splitlines()])
395 msg.append('')
396 return '\n'.join(msg)