blob: 2de6e5faa0b03998fd3c2f255fa62261d5cc6100 [file] [log] [blame]
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001# Copyright (C) 2016-2018 Wind River Systems, Inc.
2#
Brad Bishopc342db32019-05-15 21:57:59 -04003# SPDX-License-Identifier: GPL-2.0-only
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08004#
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08005
6import logging
Andrew Geisslerc9f78652020-09-18 14:11:35 -05007import os
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08008
Andrew Geissler82c905d2020-04-13 13:39:40 -05009from collections import defaultdict
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080010
11from urllib.parse import unquote, urlparse
12
13import layerindexlib
14
15import layerindexlib.plugin
16
17logger = logging.getLogger('BitBake.layerindexlib.cooker')
18
19import bb.utils
20
21def plugin_init(plugins):
22 return CookerPlugin()
23
24class CookerPlugin(layerindexlib.plugin.IndexPlugin):
25 def __init__(self):
26 self.type = "cooker"
27
28 self.server_connection = None
29 self.ui_module = None
30 self.server = None
31
32 def _run_command(self, command, path, default=None):
33 try:
34 result, _ = bb.process.run(command, cwd=path)
35 result = result.strip()
36 except bb.process.ExecutionError:
37 result = default
38 return result
39
40 def _handle_git_remote(self, remote):
41 if "://" not in remote:
42 if ':' in remote:
43 # This is assumed to be ssh
44 remote = "ssh://" + remote
45 else:
46 # This is assumed to be a file path
47 remote = "file://" + remote
48 return remote
49
50 def _get_bitbake_info(self):
51 """Return a tuple of bitbake information"""
52
53 # Our path SHOULD be .../bitbake/lib/layerindex/cooker.py
54 bb_path = os.path.dirname(__file__) # .../bitbake/lib/layerindex/cooker.py
55 bb_path = os.path.dirname(bb_path) # .../bitbake/lib/layerindex
56 bb_path = os.path.dirname(bb_path) # .../bitbake/lib
57 bb_path = os.path.dirname(bb_path) # .../bitbake
58 bb_path = self._run_command('git rev-parse --show-toplevel', os.path.dirname(__file__), default=bb_path)
59 bb_branch = self._run_command('git rev-parse --abbrev-ref HEAD', bb_path, default="<unknown>")
60 bb_rev = self._run_command('git rev-parse HEAD', bb_path, default="<unknown>")
61 for remotes in self._run_command('git remote -v', bb_path, default="").split("\n"):
62 remote = remotes.split("\t")[1].split(" ")[0]
63 if "(fetch)" == remotes.split("\t")[1].split(" ")[1]:
64 bb_remote = self._handle_git_remote(remote)
65 break
66 else:
67 bb_remote = self._handle_git_remote(bb_path)
68
69 return (bb_remote, bb_branch, bb_rev, bb_path)
70
71 def _load_bblayers(self, branches=None):
72 """Load the BBLAYERS and related collection information"""
73
74 d = self.layerindex.data
75
76 if not branches:
Andrew Geisslerc9f78652020-09-18 14:11:35 -050077 raise layerindexlib.LayerIndexFetchError("No branches specified for _load_bblayers!")
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080078
79 index = layerindexlib.LayerIndexObj()
80
81 branchId = 0
82 index.branches = {}
83
84 layerItemId = 0
85 index.layerItems = {}
86
87 layerBranchId = 0
88 index.layerBranches = {}
89
90 bblayers = d.getVar('BBLAYERS').split()
91
92 if not bblayers:
93 # It's blank! Nothing to process...
94 return index
95
96 collections = d.getVar('BBFILE_COLLECTIONS')
Andrew Geissler82c905d2020-04-13 13:39:40 -050097 layerconfs = d.varhistory.get_variable_items_files('BBFILE_COLLECTIONS')
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080098 bbfile_collections = {layer: os.path.dirname(os.path.dirname(path)) for layer, path in layerconfs.items()}
99
100 (_, bb_branch, _, _) = self._get_bitbake_info()
101
102 for branch in branches:
103 branchId += 1
104 index.branches[branchId] = layerindexlib.Branch(index, None)
105 index.branches[branchId].define_data(branchId, branch, bb_branch)
106
107 for entry in collections.split():
108 layerpath = entry
109 if entry in bbfile_collections:
110 layerpath = bbfile_collections[entry]
111
112 layername = d.getVar('BBLAYERS_LAYERINDEX_NAME_%s' % entry) or os.path.basename(layerpath)
113 layerversion = d.getVar('LAYERVERSION_%s' % entry) or ""
114 layerurl = self._handle_git_remote(layerpath)
115
116 layersubdir = ""
117 layerrev = "<unknown>"
118 layerbranch = "<unknown>"
119
120 if os.path.isdir(layerpath):
121 layerbasepath = self._run_command('git rev-parse --show-toplevel', layerpath, default=layerpath)
122 if os.path.abspath(layerpath) != os.path.abspath(layerbasepath):
123 layersubdir = os.path.abspath(layerpath)[len(layerbasepath) + 1:]
124
125 layerbranch = self._run_command('git rev-parse --abbrev-ref HEAD', layerpath, default="<unknown>")
126 layerrev = self._run_command('git rev-parse HEAD', layerpath, default="<unknown>")
127
128 for remotes in self._run_command('git remote -v', layerpath, default="").split("\n"):
129 if not remotes:
130 layerurl = self._handle_git_remote(layerpath)
131 else:
132 remote = remotes.split("\t")[1].split(" ")[0]
133 if "(fetch)" == remotes.split("\t")[1].split(" ")[1]:
134 layerurl = self._handle_git_remote(remote)
135 break
136
137 layerItemId += 1
138 index.layerItems[layerItemId] = layerindexlib.LayerItem(index, None)
139 index.layerItems[layerItemId].define_data(layerItemId, layername, description=layerpath, vcs_url=layerurl)
140
141 for branchId in index.branches:
142 layerBranchId += 1
143 index.layerBranches[layerBranchId] = layerindexlib.LayerBranch(index, None)
144 index.layerBranches[layerBranchId].define_data(layerBranchId, entry, layerversion, layerItemId, branchId,
145 vcs_subdir=layersubdir, vcs_last_rev=layerrev, actual_branch=layerbranch)
146
147 return index
148
149
150 def load_index(self, url, load):
151 """
152 Fetches layer information from a build configuration.
153
154 The return value is a dictionary containing API,
155 layer, branch, dependency, recipe, machine, distro, information.
156
157 url type should be 'cooker'.
158 url path is ignored
159 """
160
161 up = urlparse(url)
162
163 if up.scheme != 'cooker':
164 raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url)
165
166 d = self.layerindex.data
167
168 params = self.layerindex._parse_params(up.params)
169
170 # Only reason to pass a branch is to emulate them...
171 if 'branch' in params:
172 branches = params['branch'].split(',')
173 else:
174 branches = ['HEAD']
175
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600176 logger.debug("Loading cooker data branches %s" % branches)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800177
178 index = self._load_bblayers(branches=branches)
179
180 index.config = {}
181 index.config['TYPE'] = self.type
182 index.config['URL'] = url
183
184 if 'desc' in params:
185 index.config['DESCRIPTION'] = unquote(params['desc'])
186 else:
187 index.config['DESCRIPTION'] = 'local'
188
189 if 'cache' in params:
190 index.config['CACHE'] = params['cache']
191
192 index.config['BRANCH'] = branches
193
194 # ("layerDependencies", layerindexlib.LayerDependency)
195 layerDependencyId = 0
196 if "layerDependencies" in load:
197 index.layerDependencies = {}
198 for layerBranchId in index.layerBranches:
199 branchName = index.layerBranches[layerBranchId].branch.name
200 collection = index.layerBranches[layerBranchId].collection
201
202 def add_dependency(layerDependencyId, index, deps, required):
203 try:
204 depDict = bb.utils.explode_dep_versions2(deps)
205 except bb.utils.VersionStringException as vse:
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500206 bb.fatal('Error parsing LAYERDEPENDS_%s: %s' % (collection, str(vse)))
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800207
208 for dep, oplist in list(depDict.items()):
209 # We need to search ourselves, so use the _ version...
210 depLayerBranch = index.find_collection(dep, branches=[branchName])
211 if not depLayerBranch:
212 # Missing dependency?!
213 logger.error('Missing dependency %s (%s)' % (dep, branchName))
214 continue
215
216 # We assume that the oplist matches...
217 layerDependencyId += 1
218 layerDependency = layerindexlib.LayerDependency(index, None)
219 layerDependency.define_data(id=layerDependencyId,
220 required=required, layerbranch=layerBranchId,
221 dependency=depLayerBranch.layer_id)
222
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600223 logger.debug('%s requires %s' % (layerDependency.layer.name, layerDependency.dependency.name))
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800224 index.add_element("layerDependencies", [layerDependency])
225
226 return layerDependencyId
227
228 deps = d.getVar("LAYERDEPENDS_%s" % collection)
229 if deps:
230 layerDependencyId = add_dependency(layerDependencyId, index, deps, True)
231
232 deps = d.getVar("LAYERRECOMMENDS_%s" % collection)
233 if deps:
234 layerDependencyId = add_dependency(layerDependencyId, index, deps, False)
235
236 # Need to load recipes here (requires cooker access)
237 recipeId = 0
238 ## TODO: NOT IMPLEMENTED
239 # The code following this is an example of what needs to be
240 # implemented. However, it does not work as-is.
241 if False and 'recipes' in load:
242 index.recipes = {}
243
244 ret = self.ui_module.main(self.server_connection.connection, self.server_connection.events, config_params)
245
246 all_versions = self._run_command('allProviders')
247
248 all_versions_list = defaultdict(list, all_versions)
249 for pn in all_versions_list:
250 for ((pe, pv, pr), fpath) in all_versions_list[pn]:
251 realfn = bb.cache.virtualfn2realfn(fpath)
252
253 filepath = os.path.dirname(realfn[0])
254 filename = os.path.basename(realfn[0])
255
256 # This is all HORRIBLY slow, and likely unnecessary
257 #dscon = self._run_command('parseRecipeFile', fpath, False, [])
258 #connector = myDataStoreConnector(self, dscon.dsindex)
259 #recipe_data = bb.data.init()
260 #recipe_data.setVar('_remote_data', connector)
261
262 #summary = recipe_data.getVar('SUMMARY')
263 #description = recipe_data.getVar('DESCRIPTION')
264 #section = recipe_data.getVar('SECTION')
265 #license = recipe_data.getVar('LICENSE')
266 #homepage = recipe_data.getVar('HOMEPAGE')
267 #bugtracker = recipe_data.getVar('BUGTRACKER')
268 #provides = recipe_data.getVar('PROVIDES')
269
270 layer = bb.utils.get_file_layer(realfn[0], self.config_data)
271
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500272 depBranchId = collection[layer]
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800273
274 recipeId += 1
275 recipe = layerindexlib.Recipe(index, None)
276 recipe.define_data(id=recipeId,
277 filename=filename, filepath=filepath,
278 pn=pn, pv=pv,
279 summary=pn, description=pn, section='?',
280 license='?', homepage='?', bugtracker='?',
281 provides='?', bbclassextend='?', inherits='?',
282 blacklisted='?', layerbranch=depBranchId)
283
284 index = addElement("recipes", [recipe], index)
285
286 # ("machines", layerindexlib.Machine)
287 machineId = 0
288 if 'machines' in load:
289 index.machines = {}
290
291 for layerBranchId in index.layerBranches:
292 # load_bblayers uses the description to cache the actual path...
293 machine_path = index.layerBranches[layerBranchId].layer.description
294 machine_path = os.path.join(machine_path, 'conf/machine')
295 if os.path.isdir(machine_path):
296 for (dirpath, _, filenames) in os.walk(machine_path):
297 # Ignore subdirs...
298 if not dirpath.endswith('conf/machine'):
299 continue
300 for fname in filenames:
301 if fname.endswith('.conf'):
302 machineId += 1
303 machine = layerindexlib.Machine(index, None)
304 machine.define_data(id=machineId, name=fname[:-5],
305 description=fname[:-5],
306 layerbranch=index.layerBranches[layerBranchId])
307
308 index.add_element("machines", [machine])
309
310 # ("distros", layerindexlib.Distro)
311 distroId = 0
312 if 'distros' in load:
313 index.distros = {}
314
315 for layerBranchId in index.layerBranches:
316 # load_bblayers uses the description to cache the actual path...
317 distro_path = index.layerBranches[layerBranchId].layer.description
318 distro_path = os.path.join(distro_path, 'conf/distro')
319 if os.path.isdir(distro_path):
320 for (dirpath, _, filenames) in os.walk(distro_path):
321 # Ignore subdirs...
322 if not dirpath.endswith('conf/distro'):
323 continue
324 for fname in filenames:
325 if fname.endswith('.conf'):
326 distroId += 1
327 distro = layerindexlib.Distro(index, None)
328 distro.define_data(id=distroId, name=fname[:-5],
329 description=fname[:-5],
330 layerbranch=index.layerBranches[layerBranchId])
331
332 index.add_element("distros", [distro])
333
334 return index