blob: 77196b408f36abac8788d0012583a8e9c9de4c7f [file] [log] [blame]
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001# Copyright (C) 2016-2018 Wind River Systems, Inc.
2#
Brad Bishopc342db32019-05-15 21:57:59 -04003# SPDX-License-Identifier: GPL-2.0-only
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08004#
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08005
6import datetime
7
8import logging
9import imp
10
11from collections import OrderedDict
12from layerindexlib.plugin import LayerIndexPluginUrlError
13
14logger = logging.getLogger('BitBake.layerindexlib')
15
16# Exceptions
17
18class LayerIndexException(Exception):
19 '''LayerIndex Generic Exception'''
20 def __init__(self, message):
21 self.msg = message
22 Exception.__init__(self, message)
23
24 def __str__(self):
25 return self.msg
26
27class LayerIndexUrlError(LayerIndexException):
28 '''Exception raised when unable to access a URL for some reason'''
29 def __init__(self, url, message=""):
30 if message:
31 msg = "Unable to access layerindex url %s: %s" % (url, message)
32 else:
33 msg = "Unable to access layerindex url %s" % url
34 self.url = url
35 LayerIndexException.__init__(self, msg)
36
37class LayerIndexFetchError(LayerIndexException):
38 '''General layerindex fetcher exception when something fails'''
39 def __init__(self, url, message=""):
40 if message:
41 msg = "Unable to fetch layerindex url %s: %s" % (url, message)
42 else:
43 msg = "Unable to fetch layerindex url %s" % url
44 self.url = url
45 LayerIndexException.__init__(self, msg)
46
47
48# Interface to the overall layerindex system
49# the layer may contain one or more individual indexes
50class LayerIndex():
51 def __init__(self, d):
52 if not d:
53 raise LayerIndexException("Must be initialized with bb.data.")
54
55 self.data = d
56
57 # List of LayerIndexObj
58 self.indexes = []
59
60 self.plugins = []
61
62 import bb.utils
63 bb.utils.load_plugins(logger, self.plugins, os.path.dirname(__file__))
64 for plugin in self.plugins:
65 if hasattr(plugin, 'init'):
66 plugin.init(self)
67
68 def __add__(self, other):
69 newIndex = LayerIndex(self.data)
70
71 if self.__class__ != newIndex.__class__ or \
72 other.__class__ != newIndex.__class__:
73 raise TypeException("Can not add different types.")
74
75 for indexEnt in self.indexes:
76 newIndex.indexes.append(indexEnt)
77
78 for indexEnt in other.indexes:
79 newIndex.indexes.append(indexEnt)
80
81 return newIndex
82
83 def _parse_params(self, params):
84 '''Take a parameter list, return a dictionary of parameters.
85
86 Expected to be called from the data of urllib.parse.urlparse(url).params
87
88 If there are two conflicting parameters, last in wins...
89 '''
90
91 param_dict = {}
92 for param in params.split(';'):
93 if not param:
94 continue
95 item = param.split('=', 1)
96 logger.debug(1, item)
97 param_dict[item[0]] = item[1]
98
99 return param_dict
100
101 def _fetch_url(self, url, username=None, password=None, debuglevel=0):
102 '''Fetch data from a specific URL.
103
104 Fetch something from a specific URL. This is specifically designed to
105 fetch data from a layerindex-web instance, but may be useful for other
106 raw fetch actions.
107
108 It is not designed to be used to fetch recipe sources or similar. the
109 regular fetcher class should used for that.
110
111 It is the responsibility of the caller to check BB_NO_NETWORK and related
112 BB_ALLOWED_NETWORKS.
113 '''
114
115 if not url:
116 raise LayerIndexUrlError(url, "empty url")
117
118 import urllib
119 from urllib.request import urlopen, Request
120 from urllib.parse import urlparse
121
122 up = urlparse(url)
123
124 if username:
125 logger.debug(1, "Configuring authentication for %s..." % url)
126 password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
127 password_mgr.add_password(None, "%s://%s" % (up.scheme, up.netloc), username, password)
128 handler = urllib.request.HTTPBasicAuthHandler(password_mgr)
129 opener = urllib.request.build_opener(handler, urllib.request.HTTPSHandler(debuglevel=debuglevel))
130 else:
131 opener = urllib.request.build_opener(urllib.request.HTTPSHandler(debuglevel=debuglevel))
132
133 urllib.request.install_opener(opener)
134
135 logger.debug(1, "Fetching %s (%s)..." % (url, ["without authentication", "with authentication"][bool(username)]))
136
137 try:
138 res = urlopen(Request(url, headers={'User-Agent': 'Mozilla/5.0 (bitbake/lib/layerindex)'}, unverifiable=True))
139 except urllib.error.HTTPError as e:
140 logger.debug(1, "HTTP Error: %s: %s" % (e.code, e.reason))
141 logger.debug(1, " Requested: %s" % (url))
142 logger.debug(1, " Actual: %s" % (e.geturl()))
143
144 if e.code == 404:
145 logger.debug(1, "Request not found.")
146 raise LayerIndexFetchError(url, e)
147 else:
148 logger.debug(1, "Headers:\n%s" % (e.headers))
149 raise LayerIndexFetchError(url, e)
150 except OSError as e:
151 error = 0
152 reason = ""
153
154 # Process base OSError first...
155 if hasattr(e, 'errno'):
156 error = e.errno
157 reason = e.strerror
158
159 # Process gaierror (socket error) subclass if available.
160 if hasattr(e, 'reason') and hasattr(e.reason, 'errno') and hasattr(e.reason, 'strerror'):
161 error = e.reason.errno
162 reason = e.reason.strerror
163 if error == -2:
164 raise LayerIndexFetchError(url, "%s: %s" % (e, reason))
165
166 if error and error != 0:
167 raise LayerIndexFetchError(url, "Unexpected exception: [Error %s] %s" % (error, reason))
168 else:
169 raise LayerIndexFetchError(url, "Unable to fetch OSError exception: %s" % e)
170
171 finally:
172 logger.debug(1, "...fetching %s (%s), done." % (url, ["without authentication", "with authentication"][bool(username)]))
173
174 return res
175
176
177 def load_layerindex(self, indexURI, load=['layerDependencies', 'recipes', 'machines', 'distros'], reload=False):
178 '''Load the layerindex.
179
180 indexURI - An index to load. (Use multiple calls to load multiple indexes)
181
182 reload - If reload is True, then any previously loaded indexes will be forgotten.
183
184 load - List of elements to load. Default loads all items.
185 Note: plugs may ignore this.
186
187The format of the indexURI:
188
189 <url>;branch=<branch>;cache=<cache>;desc=<description>
190
191 Note: the 'branch' parameter if set can select multiple branches by using
192 comma, such as 'branch=master,morty,pyro'. However, many operations only look
193 at the -first- branch specified!
194
195 The cache value may be undefined, in this case a network failure will
196 result in an error, otherwise the system will look for a file of the cache
197 name and load that instead.
198
199 For example:
200
201 http://layers.openembedded.org/layerindex/api/;branch=master;desc=OpenEmbedded%20Layer%20Index
202 cooker://
203'''
204 if reload:
205 self.indexes = []
206
207 logger.debug(1, 'Loading: %s' % indexURI)
208
209 if not self.plugins:
210 raise LayerIndexException("No LayerIndex Plugins available")
211
212 for plugin in self.plugins:
213 # Check if the plugin was initialized
214 logger.debug(1, 'Trying %s' % plugin.__class__)
215 if not hasattr(plugin, 'type') or not plugin.type:
216 continue
217 try:
218 # TODO: Implement 'cache', for when the network is not available
219 indexEnt = plugin.load_index(indexURI, load)
220 break
221 except LayerIndexPluginUrlError as e:
222 logger.debug(1, "%s doesn't support %s" % (plugin.type, e.url))
223 except NotImplementedError:
224 pass
225 else:
226 logger.debug(1, "No plugins support %s" % indexURI)
227 raise LayerIndexException("No plugins support %s" % indexURI)
228
229 # Mark CONFIG data as something we've added...
230 indexEnt.config['local'] = []
231 indexEnt.config['local'].append('config')
232
233 # No longer permit changes..
234 indexEnt.lockData()
235
236 self.indexes.append(indexEnt)
237
238 def store_layerindex(self, indexURI, index=None):
239 '''Store one layerindex
240
241Typically this will be used to create a local cache file of a remote index.
242
243 file://<path>;branch=<branch>
244
245We can write out in either the restapi or django formats. The split option
246will write out the individual elements split by layer and related components.
247'''
248 if not index:
249 logger.warning('No index to write, nothing to do.')
250 return
251
252 if not self.plugins:
253 raise LayerIndexException("No LayerIndex Plugins available")
254
255 for plugin in self.plugins:
256 # Check if the plugin was initialized
257 logger.debug(1, 'Trying %s' % plugin.__class__)
258 if not hasattr(plugin, 'type') or not plugin.type:
259 continue
260 try:
261 plugin.store_index(indexURI, index)
262 break
263 except LayerIndexPluginUrlError as e:
264 logger.debug(1, "%s doesn't support %s" % (plugin.type, e.url))
265 except NotImplementedError:
266 logger.debug(1, "Store not implemented in %s" % plugin.type)
267 pass
268 else:
269 logger.debug(1, "No plugins support %s" % url)
270 raise LayerIndexException("No plugins support %s" % url)
271
272
273 def is_empty(self):
274 '''Return True or False if the index has any usable data.
275
276We check the indexes entries to see if they have a branch set, as well as
277layerBranches set. If not, they are effectively blank.'''
278
279 found = False
280 for index in self.indexes:
281 if index.__bool__():
282 found = True
283 break
284 return not found
285
286
287 def find_vcs_url(self, vcs_url, branch=None):
288 '''Return the first layerBranch with the given vcs_url
289
290 If a branch has not been specified, we will iterate over the branches in
291 the default configuration until the first vcs_url/branch match.'''
292
293 for index in self.indexes:
294 logger.debug(1, ' searching %s' % index.config['DESCRIPTION'])
295 layerBranch = index.find_vcs_url(vcs_url, [branch])
296 if layerBranch:
297 return layerBranch
298 return None
299
300 def find_collection(self, collection, version=None, branch=None):
301 '''Return the first layerBranch with the given collection name
302
303 If a branch has not been specified, we will iterate over the branches in
304 the default configuration until the first collection/branch match.'''
305
306 logger.debug(1, 'find_collection: %s (%s) %s' % (collection, version, branch))
307
308 if branch:
309 branches = [branch]
310 else:
311 branches = None
312
313 for index in self.indexes:
314 logger.debug(1, ' searching %s' % index.config['DESCRIPTION'])
315 layerBranch = index.find_collection(collection, version, branches)
316 if layerBranch:
317 return layerBranch
318 else:
319 logger.debug(1, 'Collection %s (%s) not found for branch (%s)' % (collection, version, branch))
320 return None
321
322 def find_layerbranch(self, name, branch=None):
323 '''Return the layerBranch item for a given name and branch
324
325 If a branch has not been specified, we will iterate over the branches in
326 the default configuration until the first name/branch match.'''
327
328 if branch:
329 branches = [branch]
330 else:
331 branches = None
332
333 for index in self.indexes:
334 layerBranch = index.find_layerbranch(name, branches)
335 if layerBranch:
336 return layerBranch
337 return None
338
339 def find_dependencies(self, names=None, layerbranches=None, ignores=None):
340 '''Return a tuple of all dependencies and valid items for the list of (layer) names
341
342 The dependency scanning happens depth-first. The returned
343 dependencies should be in the best order to define bblayers.
344
345 names - list of layer names (searching layerItems)
346 branches - when specified (with names) only this list of branches are evaluated
347
348 layerbranches - list of layerbranches to resolve dependencies
349
350 ignores - list of layer names to ignore
351
352 return: (dependencies, invalid)
353
354 dependencies[LayerItem.name] = [ LayerBranch, LayerDependency1, LayerDependency2, ... ]
355 invalid = [ LayerItem.name1, LayerItem.name2, ... ]
356 '''
357
358 invalid = []
359
360 # Convert name/branch to layerbranches
361 if layerbranches is None:
362 layerbranches = []
363
364 for name in names:
365 if ignores and name in ignores:
366 continue
367
368 for index in self.indexes:
369 layerbranch = index.find_layerbranch(name)
370 if not layerbranch:
371 # Not in this index, hopefully it's in another...
372 continue
373 layerbranches.append(layerbranch)
374 break
375 else:
376 invalid.append(name)
377
378
Brad Bishop08902b02019-08-20 09:16:51 -0400379 def _resolve_dependencies(layerbranches, ignores, dependencies, invalid, processed=None):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800380 for layerbranch in layerbranches:
381 if ignores and layerbranch.layer.name in ignores:
382 continue
383
384 # Get a list of dependencies and then recursively process them
385 for layerdependency in layerbranch.index.layerDependencies_layerBranchId[layerbranch.id]:
386 deplayerbranch = layerdependency.dependency_layerBranch
387
388 if ignores and deplayerbranch.layer.name in ignores:
389 continue
390
Brad Bishop08902b02019-08-20 09:16:51 -0400391 # Since this is depth first, we need to know what we're currently processing
392 # in order to avoid infinite recursion on a loop.
393 if processed and deplayerbranch.layer.name in processed:
394 # We have found a recursion...
395 logger.warning('Circular layer dependency found: %s -> %s' % (processed, deplayerbranch.layer.name))
396 continue
397
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800398 # This little block is why we can't re-use the LayerIndexObj version,
399 # we must be able to satisfy each dependencies across layer indexes and
400 # use the layer index order for priority. (r stands for replacement below)
401
402 # If this is the primary index, we can fast path and skip this
403 if deplayerbranch.index != self.indexes[0]:
404 # Is there an entry in a prior index for this collection/version?
405 rdeplayerbranch = self.find_collection(
406 collection=deplayerbranch.collection,
407 version=deplayerbranch.version
408 )
409 if rdeplayerbranch != deplayerbranch:
410 logger.debug(1, 'Replaced %s:%s:%s with %s:%s:%s' % \
411 (deplayerbranch.index.config['DESCRIPTION'],
412 deplayerbranch.branch.name,
413 deplayerbranch.layer.name,
414 rdeplayerbranch.index.config['DESCRIPTION'],
415 rdeplayerbranch.branch.name,
416 rdeplayerbranch.layer.name))
417 deplayerbranch = rdeplayerbranch
418
419 # New dependency, we need to resolve it now... depth-first
420 if deplayerbranch.layer.name not in dependencies:
Brad Bishop08902b02019-08-20 09:16:51 -0400421 # Avoid recursion on this branch.
422 # We copy so we don't end up polluting the depth-first branch with other
423 # branches. Duplication between individual branches IS expected and
424 # handled by 'dependencies' processing.
425 if not processed:
426 local_processed = []
427 else:
428 local_processed = processed.copy()
429 local_processed.append(deplayerbranch.layer.name)
430
431 (dependencies, invalid) = _resolve_dependencies([deplayerbranch], ignores, dependencies, invalid, local_processed)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800432
433 if deplayerbranch.layer.name not in dependencies:
434 dependencies[deplayerbranch.layer.name] = [deplayerbranch, layerdependency]
435 else:
436 if layerdependency not in dependencies[deplayerbranch.layer.name]:
437 dependencies[deplayerbranch.layer.name].append(layerdependency)
438
439 return (dependencies, invalid)
440
441 # OK, resolve this one...
442 dependencies = OrderedDict()
443 (dependencies, invalid) = _resolve_dependencies(layerbranches, ignores, dependencies, invalid)
444
445 for layerbranch in layerbranches:
446 if layerbranch.layer.name not in dependencies:
447 dependencies[layerbranch.layer.name] = [layerbranch]
448
449 return (dependencies, invalid)
450
451
452 def list_obj(self, object):
453 '''Print via the plain logger object information
454
455This function is used to implement debugging and provide the user info.
456'''
457 for lix in self.indexes:
458 if not hasattr(lix, object):
459 continue
460
461 logger.plain ('')
462 logger.plain ('Index: %s' % lix.config['DESCRIPTION'])
463
464 output = []
465
466 if object == 'branches':
467 logger.plain ('%s %s %s' % ('{:26}'.format('branch'), '{:34}'.format('description'), '{:22}'.format('bitbake branch')))
468 logger.plain ('{:-^80}'.format(""))
469 for branchid in lix.branches:
470 output.append('%s %s %s' % (
471 '{:26}'.format(lix.branches[branchid].name),
472 '{:34}'.format(lix.branches[branchid].short_description),
473 '{:22}'.format(lix.branches[branchid].bitbake_branch)
474 ))
475 for line in sorted(output):
476 logger.plain (line)
477
478 continue
479
480 if object == 'layerItems':
481 logger.plain ('%s %s' % ('{:26}'.format('layer'), '{:34}'.format('description')))
482 logger.plain ('{:-^80}'.format(""))
483 for layerid in lix.layerItems:
484 output.append('%s %s' % (
485 '{:26}'.format(lix.layerItems[layerid].name),
486 '{:34}'.format(lix.layerItems[layerid].summary)
487 ))
488 for line in sorted(output):
489 logger.plain (line)
490
491 continue
492
493 if object == 'layerBranches':
494 logger.plain ('%s %s %s' % ('{:26}'.format('layer'), '{:34}'.format('description'), '{:19}'.format('collection:version')))
495 logger.plain ('{:-^80}'.format(""))
496 for layerbranchid in lix.layerBranches:
497 output.append('%s %s %s' % (
498 '{:26}'.format(lix.layerBranches[layerbranchid].layer.name),
499 '{:34}'.format(lix.layerBranches[layerbranchid].layer.summary),
500 '{:19}'.format("%s:%s" %
501 (lix.layerBranches[layerbranchid].collection,
502 lix.layerBranches[layerbranchid].version)
503 )
504 ))
505 for line in sorted(output):
506 logger.plain (line)
507
508 continue
509
510 if object == 'layerDependencies':
511 logger.plain ('%s %s %s %s' % ('{:19}'.format('branch'), '{:26}'.format('layer'), '{:11}'.format('dependency'), '{:26}'.format('layer')))
512 logger.plain ('{:-^80}'.format(""))
513 for layerDependency in lix.layerDependencies:
514 if not lix.layerDependencies[layerDependency].dependency_layerBranch:
515 continue
516
517 output.append('%s %s %s %s' % (
518 '{:19}'.format(lix.layerDependencies[layerDependency].layerbranch.branch.name),
519 '{:26}'.format(lix.layerDependencies[layerDependency].layerbranch.layer.name),
520 '{:11}'.format('requires' if lix.layerDependencies[layerDependency].required else 'recommends'),
521 '{:26}'.format(lix.layerDependencies[layerDependency].dependency_layerBranch.layer.name)
522 ))
523 for line in sorted(output):
524 logger.plain (line)
525
526 continue
527
528 if object == 'recipes':
529 logger.plain ('%s %s %s' % ('{:20}'.format('recipe'), '{:10}'.format('version'), 'layer'))
530 logger.plain ('{:-^80}'.format(""))
531 output = []
532 for recipe in lix.recipes:
533 output.append('%s %s %s' % (
534 '{:30}'.format(lix.recipes[recipe].pn),
535 '{:30}'.format(lix.recipes[recipe].pv),
536 lix.recipes[recipe].layer.name
537 ))
538 for line in sorted(output):
539 logger.plain (line)
540
541 continue
542
543 if object == 'machines':
544 logger.plain ('%s %s %s' % ('{:24}'.format('machine'), '{:34}'.format('description'), '{:19}'.format('layer')))
545 logger.plain ('{:-^80}'.format(""))
546 for machine in lix.machines:
547 output.append('%s %s %s' % (
548 '{:24}'.format(lix.machines[machine].name),
549 '{:34}'.format(lix.machines[machine].description)[:34],
550 '{:19}'.format(lix.machines[machine].layerbranch.layer.name)
551 ))
552 for line in sorted(output):
553 logger.plain (line)
554
555 continue
556
557 if object == 'distros':
558 logger.plain ('%s %s %s' % ('{:24}'.format('distro'), '{:34}'.format('description'), '{:19}'.format('layer')))
559 logger.plain ('{:-^80}'.format(""))
560 for distro in lix.distros:
561 output.append('%s %s %s' % (
562 '{:24}'.format(lix.distros[distro].name),
563 '{:34}'.format(lix.distros[distro].description)[:34],
564 '{:19}'.format(lix.distros[distro].layerbranch.layer.name)
565 ))
566 for line in sorted(output):
567 logger.plain (line)
568
569 continue
570
571 logger.plain ('')
572
573
574# This class holds a single layer index instance
575# The LayerIndexObj is made up of dictionary of elements, such as:
576# index['config'] - configuration data for this index
577# index['branches'] - dictionary of Branch objects, by id number
578# index['layerItems'] - dictionary of layerItem objects, by id number
579# ...etc... (See: http://layers.openembedded.org/layerindex/api/)
580#
581# The class needs to manage the 'index' entries and allow easily adding
582# of new items, as well as simply loading of the items.
583class LayerIndexObj():
584 def __init__(self):
585 super().__setattr__('_index', {})
586 super().__setattr__('_lock', False)
587
588 def __bool__(self):
589 '''False if the index is effectively empty
590
591 We check the index to see if it has a branch set, as well as
592 layerbranches set. If not, it is effectively blank.'''
593
594 if not bool(self._index):
595 return False
596
597 try:
598 if self.branches and self.layerBranches:
599 return True
600 except AttributeError:
601 pass
602
603 return False
604
605 def __getattr__(self, name):
606 if name.startswith('_'):
607 return super().__getattribute__(name)
608
609 if name not in self._index:
610 raise AttributeError('%s not in index datastore' % name)
611
612 return self._index[name]
613
614 def __setattr__(self, name, value):
615 if self.isLocked():
616 raise TypeError("Can not set attribute '%s': index is locked" % name)
617
618 if name.startswith('_'):
619 super().__setattr__(name, value)
620 return
621
622 self._index[name] = value
623
624 def __delattr__(self, name):
625 if self.isLocked():
626 raise TypeError("Can not delete attribute '%s': index is locked" % name)
627
628 if name.startswith('_'):
629 super().__delattr__(name)
630
631 self._index.pop(name)
632
633 def lockData(self):
634 '''Lock data object (make it readonly)'''
635 super().__setattr__("_lock", True)
636
637 def unlockData(self):
638 '''unlock data object (make it readonly)'''
639 super().__setattr__("_lock", False)
640
641 # When the data is unlocked, we have to clear the caches, as
642 # modification is allowed!
643 del(self._layerBranches_layerId_branchId)
644 del(self._layerDependencies_layerBranchId)
645 del(self._layerBranches_vcsUrl)
646
647 def isLocked(self):
648 '''Is this object locked (readonly)?'''
649 return self._lock
650
651 def add_element(self, indexname, objs):
652 '''Add a layer index object to index.<indexname>'''
653 if indexname not in self._index:
654 self._index[indexname] = {}
655
656 for obj in objs:
657 if obj.id in self._index[indexname]:
658 if self._index[indexname][obj.id] == obj:
659 continue
660 raise LayerIndexError('Conflict adding object %s(%s) to index' % (indexname, obj.id))
661 self._index[indexname][obj.id] = obj
662
663 def add_raw_element(self, indexname, objtype, rawobjs):
664 '''Convert a raw layer index data item to a layer index item object and add to the index'''
665 objs = []
666 for entry in rawobjs:
667 objs.append(objtype(self, entry))
668 self.add_element(indexname, objs)
669
670 # Quick lookup table for searching layerId and branchID combos
671 @property
672 def layerBranches_layerId_branchId(self):
673 def createCache(self):
674 cache = {}
675 for layerbranchid in self.layerBranches:
676 layerbranch = self.layerBranches[layerbranchid]
677 cache["%s:%s" % (layerbranch.layer_id, layerbranch.branch_id)] = layerbranch
678 return cache
679
680 if self.isLocked():
681 cache = getattr(self, '_layerBranches_layerId_branchId', None)
682 else:
683 cache = None
684
685 if not cache:
686 cache = createCache(self)
687
688 if self.isLocked():
689 super().__setattr__('_layerBranches_layerId_branchId', cache)
690
691 return cache
692
693 # Quick lookup table for finding all dependencies of a layerBranch
694 @property
695 def layerDependencies_layerBranchId(self):
696 def createCache(self):
697 cache = {}
698 # This ensures empty lists for all branchids
699 for layerbranchid in self.layerBranches:
700 cache[layerbranchid] = []
701
702 for layerdependencyid in self.layerDependencies:
703 layerdependency = self.layerDependencies[layerdependencyid]
704 cache[layerdependency.layerbranch_id].append(layerdependency)
705 return cache
706
707 if self.isLocked():
708 cache = getattr(self, '_layerDependencies_layerBranchId', None)
709 else:
710 cache = None
711
712 if not cache:
713 cache = createCache(self)
714
715 if self.isLocked():
716 super().__setattr__('_layerDependencies_layerBranchId', cache)
717
718 return cache
719
720 # Quick lookup table for finding all instances of a vcs_url
721 @property
722 def layerBranches_vcsUrl(self):
723 def createCache(self):
724 cache = {}
725 for layerbranchid in self.layerBranches:
726 layerbranch = self.layerBranches[layerbranchid]
727 if layerbranch.layer.vcs_url not in cache:
728 cache[layerbranch.layer.vcs_url] = [layerbranch]
729 else:
730 cache[layerbranch.layer.vcs_url].append(layerbranch)
731 return cache
732
733 if self.isLocked():
734 cache = getattr(self, '_layerBranches_vcsUrl', None)
735 else:
736 cache = None
737
738 if not cache:
739 cache = createCache(self)
740
741 if self.isLocked():
742 super().__setattr__('_layerBranches_vcsUrl', cache)
743
744 return cache
745
746
747 def find_vcs_url(self, vcs_url, branches=None):
748 ''''Return the first layerBranch with the given vcs_url
749
750 If a list of branches has not been specified, we will iterate on
751 all branches until the first vcs_url is found.'''
752
753 if not self.__bool__():
754 return None
755
756 for layerbranch in self.layerBranches_vcsUrl:
757 if branches and layerbranch.branch.name not in branches:
758 continue
759
760 return layerbranch
761
762 return None
763
764
765 def find_collection(self, collection, version=None, branches=None):
766 '''Return the first layerBranch with the given collection name
767
768 If a list of branches has not been specified, we will iterate on
769 all branches until the first collection is found.'''
770
771 if not self.__bool__():
772 return None
773
774 for layerbranchid in self.layerBranches:
775 layerbranch = self.layerBranches[layerbranchid]
776 if branches and layerbranch.branch.name not in branches:
777 continue
778
779 if layerbranch.collection == collection and \
780 (version is None or version == layerbranch.version):
781 return layerbranch
782
783 return None
784
785
786 def find_layerbranch(self, name, branches=None):
787 '''Return the first layerbranch whose layer name matches
788
789 If a list of branches has not been specified, we will iterate on
790 all branches until the first layer with that name is found.'''
791
792 if not self.__bool__():
793 return None
794
795 for layerbranchid in self.layerBranches:
796 layerbranch = self.layerBranches[layerbranchid]
797 if branches and layerbranch.branch.name not in branches:
798 continue
799
800 if layerbranch.layer.name == name:
801 return layerbranch
802
803 return None
804
805 def find_dependencies(self, names=None, branches=None, layerBranches=None, ignores=None):
806 '''Return a tuple of all dependencies and valid items for the list of (layer) names
807
808 The dependency scanning happens depth-first. The returned
809 dependencies should be in the best order to define bblayers.
810
811 names - list of layer names (searching layerItems)
812 branches - when specified (with names) only this list of branches are evaluated
813
814 layerBranches - list of layerBranches to resolve dependencies
815
816 ignores - list of layer names to ignore
817
818 return: (dependencies, invalid)
819
820 dependencies[LayerItem.name] = [ LayerBranch, LayerDependency1, LayerDependency2, ... ]
821 invalid = [ LayerItem.name1, LayerItem.name2, ... ]'''
822
823 invalid = []
824
825 # Convert name/branch to layerBranches
826 if layerbranches is None:
827 layerbranches = []
828
829 for name in names:
830 if ignores and name in ignores:
831 continue
832
833 layerbranch = self.find_layerbranch(name, branches)
834 if not layerbranch:
835 invalid.append(name)
836 else:
837 layerbranches.append(layerbranch)
838
839 for layerbranch in layerbranches:
840 if layerbranch.index != self:
841 raise LayerIndexException("Can not resolve dependencies across indexes with this class function!")
842
843 def _resolve_dependencies(layerbranches, ignores, dependencies, invalid):
844 for layerbranch in layerbranches:
845 if ignores and layerBranch.layer.name in ignores:
846 continue
847
848 for layerdependency in layerbranch.index.layerDependencies_layerBranchId[layerBranch.id]:
849 deplayerbranch = layerDependency.dependency_layerBranch
850
851 if ignores and deplayerbranch.layer.name in ignores:
852 continue
853
854 # New dependency, we need to resolve it now... depth-first
855 if deplayerbranch.layer.name not in dependencies:
856 (dependencies, invalid) = _resolve_dependencies([deplayerbranch], ignores, dependencies, invalid)
857
858 if deplayerbranch.layer.name not in dependencies:
859 dependencies[deplayerbranch.layer.name] = [deplayerbranch, layerdependency]
860 else:
861 if layerdependency not in dependencies[deplayerbranch.layer.name]:
862 dependencies[deplayerbranch.layer.name].append(layerdependency)
863
864 return (dependencies, invalid)
865
866 # OK, resolve this one...
867 dependencies = OrderedDict()
868 (dependencies, invalid) = _resolve_dependencies(layerbranches, ignores, dependencies, invalid)
869
870 # Is this item already in the list, if not add it
871 for layerbranch in layerbranches:
872 if layerbranch.layer.name not in dependencies:
873 dependencies[layerbranch.layer.name] = [layerbranch]
874
875 return (dependencies, invalid)
876
877
878# Define a basic LayerIndexItemObj. This object forms the basis for all other
879# objects. The raw Layer Index data is stored in the _data element, but we
880# do not want users to access data directly. So wrap this and protect it
881# from direct manipulation.
882#
883# It is up to the insantiators of the objects to fill them out, and once done
884# lock the objects to prevent further accidently manipulation.
885#
886# Using the getattr, setattr and properties we can access and manipulate
887# the data within the data element.
888class LayerIndexItemObj():
889 def __init__(self, index, data=None, lock=False):
890 if data is None:
891 data = {}
892
893 if type(data) != type(dict()):
894 raise TypeError('data (%s) is not a dict' % type(data))
895
896 super().__setattr__('_lock', lock)
897 super().__setattr__('index', index)
898 super().__setattr__('_data', data)
899
900 def __eq__(self, other):
901 if self.__class__ != other.__class__:
902 return False
903 res=(self._data == other._data)
904 return res
905
906 def __bool__(self):
907 return bool(self._data)
908
909 def __getattr__(self, name):
910 # These are internal to THIS class, and not part of data
911 if name == "index" or name.startswith('_'):
912 return super().__getattribute__(name)
913
914 if name not in self._data:
915 raise AttributeError('%s not in datastore' % name)
916
917 return self._data[name]
918
919 def _setattr(self, name, value, prop=True):
920 '''__setattr__ like function, but with control over property object behavior'''
921 if self.isLocked():
922 raise TypeError("Can not set attribute '%s': Object data is locked" % name)
923
924 if name.startswith('_'):
925 super().__setattr__(name, value)
926 return
927
928 # Since __setattr__ runs before properties, we need to check if
929 # there is a setter property and then execute it
930 # ... or return self._data[name]
931 propertyobj = getattr(self.__class__, name, None)
932 if prop and isinstance(propertyobj, property):
933 if propertyobj.fset:
934 propertyobj.fset(self, value)
935 else:
936 raise AttributeError('Attribute %s is readonly, and may not be set' % name)
937 else:
938 self._data[name] = value
939
940 def __setattr__(self, name, value):
941 self._setattr(name, value, prop=True)
942
943 def _delattr(self, name, prop=True):
944 # Since __delattr__ runs before properties, we need to check if
945 # there is a deleter property and then execute it
946 # ... or we pop it ourselves..
947 propertyobj = getattr(self.__class__, name, None)
948 if prop and isinstance(propertyobj, property):
949 if propertyobj.fdel:
950 propertyobj.fdel(self)
951 else:
952 raise AttributeError('Attribute %s is readonly, and may not be deleted' % name)
953 else:
954 self._data.pop(name)
955
956 def __delattr__(self, name):
957 self._delattr(name, prop=True)
958
959 def lockData(self):
960 '''Lock data object (make it readonly)'''
961 super().__setattr__("_lock", True)
962
963 def unlockData(self):
964 '''unlock data object (make it readonly)'''
965 super().__setattr__("_lock", False)
966
967 def isLocked(self):
968 '''Is this object locked (readonly)?'''
969 return self._lock
970
971# Branch object
972class Branch(LayerIndexItemObj):
973 def define_data(self, id, name, bitbake_branch,
974 short_description=None, sort_priority=1,
975 updates_enabled=True, updated=None,
976 update_environment=None):
977 self.id = id
978 self.name = name
979 self.bitbake_branch = bitbake_branch
980 self.short_description = short_description or name
981 self.sort_priority = sort_priority
982 self.updates_enabled = updates_enabled
983 self.updated = updated or datetime.datetime.today().isoformat()
984 self.update_environment = update_environment
985
986 @property
987 def name(self):
988 return self.__getattr__('name')
989
990 @name.setter
991 def name(self, value):
992 self._data['name'] = value
993
994 if self.bitbake_branch == value:
995 self.bitbake_branch = ""
996
997 @name.deleter
998 def name(self):
999 self._delattr('name', prop=False)
1000
1001 @property
1002 def bitbake_branch(self):
1003 try:
1004 return self.__getattr__('bitbake_branch')
1005 except AttributeError:
1006 return self.name
1007
1008 @bitbake_branch.setter
1009 def bitbake_branch(self, value):
1010 if self.name == value:
1011 self._data['bitbake_branch'] = ""
1012 else:
1013 self._data['bitbake_branch'] = value
1014
1015 @bitbake_branch.deleter
1016 def bitbake_branch(self):
1017 self._delattr('bitbake_branch', prop=False)
1018
1019
1020class LayerItem(LayerIndexItemObj):
1021 def define_data(self, id, name, status='P',
1022 layer_type='A', summary=None,
1023 description=None,
1024 vcs_url=None, vcs_web_url=None,
1025 vcs_web_tree_base_url=None,
1026 vcs_web_file_base_url=None,
1027 usage_url=None,
1028 mailing_list_url=None,
1029 index_preference=1,
1030 classic=False,
1031 updated=None):
1032 self.id = id
1033 self.name = name
1034 self.status = status
1035 self.layer_type = layer_type
1036 self.summary = summary or name
1037 self.description = description or summary or name
1038 self.vcs_url = vcs_url
1039 self.vcs_web_url = vcs_web_url
1040 self.vcs_web_tree_base_url = vcs_web_tree_base_url
1041 self.vcs_web_file_base_url = vcs_web_file_base_url
1042 self.index_preference = index_preference
1043 self.classic = classic
1044 self.updated = updated or datetime.datetime.today().isoformat()
1045
1046
1047class LayerBranch(LayerIndexItemObj):
1048 def define_data(self, id, collection, version, layer, branch,
1049 vcs_subdir="", vcs_last_fetch=None,
1050 vcs_last_rev=None, vcs_last_commit=None,
1051 actual_branch="",
1052 updated=None):
1053 self.id = id
1054 self.collection = collection
1055 self.version = version
1056 if isinstance(layer, LayerItem):
1057 self.layer = layer
1058 else:
1059 self.layer_id = layer
1060
1061 if isinstance(branch, Branch):
1062 self.branch = branch
1063 else:
1064 self.branch_id = branch
1065
1066 self.vcs_subdir = vcs_subdir
1067 self.vcs_last_fetch = vcs_last_fetch
1068 self.vcs_last_rev = vcs_last_rev
1069 self.vcs_last_commit = vcs_last_commit
1070 self.actual_branch = actual_branch
1071 self.updated = updated or datetime.datetime.today().isoformat()
1072
1073 # This is a little odd, the _data attribute is 'layer', but it's really
1074 # referring to the layer id.. so lets adjust this to make it useful
1075 @property
1076 def layer_id(self):
1077 return self.__getattr__('layer')
1078
1079 @layer_id.setter
1080 def layer_id(self, value):
1081 self._setattr('layer', value, prop=False)
1082
1083 @layer_id.deleter
1084 def layer_id(self):
1085 self._delattr('layer', prop=False)
1086
1087 @property
1088 def layer(self):
1089 try:
1090 return self.index.layerItems[self.layer_id]
1091 except KeyError:
1092 raise AttributeError('Unable to find layerItems in index to map layer_id %s' % self.layer_id)
1093 except IndexError:
1094 raise AttributeError('Unable to find layer_id %s in index layerItems' % self.layer_id)
1095
1096 @layer.setter
1097 def layer(self, value):
1098 if not isinstance(value, LayerItem):
1099 raise TypeError('value is not a LayerItem')
1100 if self.index != value.index:
1101 raise AttributeError('Object and value do not share the same index and thus key set.')
1102 self.layer_id = value.id
1103
1104 @layer.deleter
1105 def layer(self):
1106 del self.layer_id
1107
1108 @property
1109 def branch_id(self):
1110 return self.__getattr__('branch')
1111
1112 @branch_id.setter
1113 def branch_id(self, value):
1114 self._setattr('branch', value, prop=False)
1115
1116 @branch_id.deleter
1117 def branch_id(self):
1118 self._delattr('branch', prop=False)
1119
1120 @property
1121 def branch(self):
1122 try:
1123 logger.debug(1, "Get branch object from branches[%s]" % (self.branch_id))
1124 return self.index.branches[self.branch_id]
1125 except KeyError:
1126 raise AttributeError('Unable to find branches in index to map branch_id %s' % self.branch_id)
1127 except IndexError:
1128 raise AttributeError('Unable to find branch_id %s in index branches' % self.branch_id)
1129
1130 @branch.setter
1131 def branch(self, value):
1132 if not isinstance(value, LayerItem):
1133 raise TypeError('value is not a LayerItem')
1134 if self.index != value.index:
1135 raise AttributeError('Object and value do not share the same index and thus key set.')
1136 self.branch_id = value.id
1137
1138 @branch.deleter
1139 def branch(self):
1140 del self.branch_id
1141
1142 @property
1143 def actual_branch(self):
1144 if self.__getattr__('actual_branch'):
1145 return self.__getattr__('actual_branch')
1146 else:
1147 return self.branch.name
1148
1149 @actual_branch.setter
1150 def actual_branch(self, value):
1151 logger.debug(1, "Set actual_branch to %s .. name is %s" % (value, self.branch.name))
1152 if value != self.branch.name:
1153 self._setattr('actual_branch', value, prop=False)
1154 else:
1155 self._setattr('actual_branch', '', prop=False)
1156
1157 @actual_branch.deleter
1158 def actual_branch(self):
1159 self._delattr('actual_branch', prop=False)
1160
1161# Extend LayerIndexItemObj with common LayerBranch manipulations
1162# All of the remaining LayerIndex objects refer to layerbranch, and it is
1163# up to the user to follow that back through the LayerBranch object into
1164# the layer object to get various attributes. So add an intermediate set
1165# of attributes that can easily get us the layerbranch as well as layer.
1166
1167class LayerIndexItemObj_LayerBranch(LayerIndexItemObj):
1168 @property
1169 def layerbranch_id(self):
1170 return self.__getattr__('layerbranch')
1171
1172 @layerbranch_id.setter
1173 def layerbranch_id(self, value):
1174 self._setattr('layerbranch', value, prop=False)
1175
1176 @layerbranch_id.deleter
1177 def layerbranch_id(self):
1178 self._delattr('layerbranch', prop=False)
1179
1180 @property
1181 def layerbranch(self):
1182 try:
1183 return self.index.layerBranches[self.layerbranch_id]
1184 except KeyError:
1185 raise AttributeError('Unable to find layerBranches in index to map layerbranch_id %s' % self.layerbranch_id)
1186 except IndexError:
1187 raise AttributeError('Unable to find layerbranch_id %s in index branches' % self.layerbranch_id)
1188
1189 @layerbranch.setter
1190 def layerbranch(self, value):
1191 if not isinstance(value, LayerBranch):
1192 raise TypeError('value (%s) is not a layerBranch' % type(value))
1193 if self.index != value.index:
1194 raise AttributeError('Object and value do not share the same index and thus key set.')
1195 self.layerbranch_id = value.id
1196
1197 @layerbranch.deleter
1198 def layerbranch(self):
1199 del self.layerbranch_id
1200
1201 @property
1202 def layer_id(self):
1203 return self.layerbranch.layer_id
1204
1205 # Doesn't make sense to set or delete layer_id
1206
1207 @property
1208 def layer(self):
1209 return self.layerbranch.layer
1210
1211 # Doesn't make sense to set or delete layer
1212
1213
1214class LayerDependency(LayerIndexItemObj_LayerBranch):
1215 def define_data(self, id, layerbranch, dependency, required=True):
1216 self.id = id
1217 if isinstance(layerbranch, LayerBranch):
1218 self.layerbranch = layerbranch
1219 else:
1220 self.layerbranch_id = layerbranch
1221 if isinstance(dependency, LayerDependency):
1222 self.dependency = dependency
1223 else:
1224 self.dependency_id = dependency
1225 self.required = required
1226
1227 @property
1228 def dependency_id(self):
1229 return self.__getattr__('dependency')
1230
1231 @dependency_id.setter
1232 def dependency_id(self, value):
1233 self._setattr('dependency', value, prop=False)
1234
1235 @dependency_id.deleter
1236 def dependency_id(self):
1237 self._delattr('dependency', prop=False)
1238
1239 @property
1240 def dependency(self):
1241 try:
1242 return self.index.layerItems[self.dependency_id]
1243 except KeyError:
1244 raise AttributeError('Unable to find layerItems in index to map layerbranch_id %s' % self.dependency_id)
1245 except IndexError:
1246 raise AttributeError('Unable to find dependency_id %s in index layerItems' % self.dependency_id)
1247
1248 @dependency.setter
1249 def dependency(self, value):
1250 if not isinstance(value, LayerDependency):
1251 raise TypeError('value (%s) is not a dependency' % type(value))
1252 if self.index != value.index:
1253 raise AttributeError('Object and value do not share the same index and thus key set.')
1254 self.dependency_id = value.id
1255
1256 @dependency.deleter
1257 def dependency(self):
1258 self._delattr('dependency', prop=False)
1259
1260 @property
1261 def dependency_layerBranch(self):
1262 layerid = self.dependency_id
1263 branchid = self.layerbranch.branch_id
1264
1265 try:
1266 return self.index.layerBranches_layerId_branchId["%s:%s" % (layerid, branchid)]
1267 except IndexError:
1268 # layerBranches_layerId_branchId -- but not layerId:branchId
1269 raise AttributeError('Unable to find layerId:branchId %s:%s in index layerBranches_layerId_branchId' % (layerid, branchid))
1270 except KeyError:
1271 raise AttributeError('Unable to find layerId:branchId %s:%s in layerItems and layerBranches' % (layerid, branchid))
1272
1273 # dependency_layerBranch doesn't make sense to set or del
1274
1275
1276class Recipe(LayerIndexItemObj_LayerBranch):
1277 def define_data(self, id,
1278 filename, filepath, pn, pv, layerbranch,
1279 summary="", description="", section="", license="",
1280 homepage="", bugtracker="", provides="", bbclassextend="",
1281 inherits="", blacklisted="", updated=None):
1282 self.id = id
1283 self.filename = filename
1284 self.filepath = filepath
1285 self.pn = pn
1286 self.pv = pv
1287 self.summary = summary
1288 self.description = description
1289 self.section = section
1290 self.license = license
1291 self.homepage = homepage
1292 self.bugtracker = bugtracker
1293 self.provides = provides
1294 self.bbclassextend = bbclassextend
1295 self.inherits = inherits
1296 self.updated = updated or datetime.datetime.today().isoformat()
1297 self.blacklisted = blacklisted
1298 if isinstance(layerbranch, LayerBranch):
1299 self.layerbranch = layerbranch
1300 else:
1301 self.layerbranch_id = layerbranch
1302
1303 @property
1304 def fullpath(self):
1305 return os.path.join(self.filepath, self.filename)
1306
1307 # Set would need to understand how to split it
1308 # del would we del both parts?
1309
1310 @property
1311 def inherits(self):
1312 if 'inherits' not in self._data:
1313 # Older indexes may not have this, so emulate it
1314 if '-image-' in self.pn:
1315 return 'image'
1316 return self.__getattr__('inherits')
1317
1318 @inherits.setter
1319 def inherits(self, value):
1320 return self._setattr('inherits', value, prop=False)
1321
1322 @inherits.deleter
1323 def inherits(self):
1324 return self._delattr('inherits', prop=False)
1325
1326
1327class Machine(LayerIndexItemObj_LayerBranch):
1328 def define_data(self, id,
1329 name, description, layerbranch,
1330 updated=None):
1331 self.id = id
1332 self.name = name
1333 self.description = description
1334 if isinstance(layerbranch, LayerBranch):
1335 self.layerbranch = layerbranch
1336 else:
1337 self.layerbranch_id = layerbranch
1338 self.updated = updated or datetime.datetime.today().isoformat()
1339
1340class Distro(LayerIndexItemObj_LayerBranch):
1341 def define_data(self, id,
1342 name, description, layerbranch,
1343 updated=None):
1344 self.id = id
1345 self.name = name
1346 self.description = description
1347 if isinstance(layerbranch, LayerBranch):
1348 self.layerbranch = layerbranch
1349 else:
1350 self.layerbranch_id = layerbranch
1351 self.updated = updated or datetime.datetime.today().isoformat()
1352
1353# When performing certain actions, we may need to sort the data.
1354# This will allow us to keep it consistent from run to run.
1355def sort_entry(item):
1356 newitem = item
1357 try:
1358 if type(newitem) == type(dict()):
1359 newitem = OrderedDict(sorted(newitem.items(), key=lambda t: t[0]))
1360 for index in newitem:
1361 newitem[index] = sort_entry(newitem[index])
1362 elif type(newitem) == type(list()):
1363 newitem.sort(key=lambda obj: obj['id'])
1364 for index, _ in enumerate(newitem):
1365 newitem[index] = sort_entry(newitem[index])
1366 except:
1367 logger.error('Sort failed for item %s' % type(item))
1368 pass
1369
1370 return newitem