blob: 7023f42f20ae58e3de0948394c9b749c0b29e01e [file] [log] [blame]
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001# Copyright (C) 2016-2018 Wind River Systems, Inc.
2#
Brad Bishopc342db32019-05-15 21:57:59 -04003# SPDX-License-Identifier: GPL-2.0-only
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08004#
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08005
6import logging
7import json
Andrew Geisslerc9f78652020-09-18 14:11:35 -05008import os
9
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080010from urllib.parse import unquote
11from urllib.parse import urlparse
12
Andrew Geisslerc9f78652020-09-18 14:11:35 -050013import bb
14
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080015import layerindexlib
16import layerindexlib.plugin
17
18logger = logging.getLogger('BitBake.layerindexlib.restapi')
19
20def plugin_init(plugins):
21 return RestApiPlugin()
22
23class RestApiPlugin(layerindexlib.plugin.IndexPlugin):
24 def __init__(self):
25 self.type = "restapi"
26
27 def load_index(self, url, load):
28 """
29 Fetches layer information from a local or remote layer index.
30
31 The return value is a LayerIndexObj.
32
33 url is the url to the rest api of the layer index, such as:
34 http://layers.openembedded.org/layerindex/api/
35
36 Or a local file...
37 """
38
39 up = urlparse(url)
40
41 if up.scheme == 'file':
42 return self.load_index_file(up, url, load)
43
44 if up.scheme == 'http' or up.scheme == 'https':
45 return self.load_index_web(up, url, load)
46
47 raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url)
48
49
50 def load_index_file(self, up, url, load):
51 """
52 Fetches layer information from a local file or directory.
53
54 The return value is a LayerIndexObj.
55
56 ud is the parsed url to the local file or directory.
57 """
58 if not os.path.exists(up.path):
59 raise FileNotFoundError(up.path)
60
61 index = layerindexlib.LayerIndexObj()
62
63 index.config = {}
64 index.config['TYPE'] = self.type
65 index.config['URL'] = url
66
67 params = self.layerindex._parse_params(up.params)
68
69 if 'desc' in params:
70 index.config['DESCRIPTION'] = unquote(params['desc'])
71 else:
72 index.config['DESCRIPTION'] = up.path
73
74 if 'cache' in params:
75 index.config['CACHE'] = params['cache']
76
77 if 'branch' in params:
78 branches = params['branch'].split(',')
79 index.config['BRANCH'] = branches
80 else:
81 branches = ['*']
82
83
84 def load_cache(path, index, branches=[]):
85 logger.debug(1, 'Loading json file %s' % path)
86 with open(path, 'rt', encoding='utf-8') as f:
87 pindex = json.load(f)
88
89 # Filter the branches on loaded files...
90 newpBranch = []
91 for branch in branches:
92 if branch != '*':
93 if 'branches' in pindex:
94 for br in pindex['branches']:
95 if br['name'] == branch:
96 newpBranch.append(br)
97 else:
98 if 'branches' in pindex:
99 for br in pindex['branches']:
100 newpBranch.append(br)
101
102 if newpBranch:
103 index.add_raw_element('branches', layerindexlib.Branch, newpBranch)
104 else:
105 logger.debug(1, 'No matching branches (%s) in index file(s)' % branches)
106 # No matching branches.. return nothing...
107 return
108
109 for (lName, lType) in [("layerItems", layerindexlib.LayerItem),
110 ("layerBranches", layerindexlib.LayerBranch),
111 ("layerDependencies", layerindexlib.LayerDependency),
112 ("recipes", layerindexlib.Recipe),
113 ("machines", layerindexlib.Machine),
114 ("distros", layerindexlib.Distro)]:
115 if lName in pindex:
116 index.add_raw_element(lName, lType, pindex[lName])
117
118
119 if not os.path.isdir(up.path):
120 load_cache(up.path, index, branches)
121 return index
122
123 logger.debug(1, 'Loading from dir %s...' % (up.path))
124 for (dirpath, _, filenames) in os.walk(up.path):
125 for filename in filenames:
126 if not filename.endswith('.json'):
127 continue
128 fpath = os.path.join(dirpath, filename)
129 load_cache(fpath, index, branches)
130
131 return index
132
133
134 def load_index_web(self, up, url, load):
135 """
136 Fetches layer information from a remote layer index.
137
138 The return value is a LayerIndexObj.
139
140 ud is the parsed url to the rest api of the layer index, such as:
141 http://layers.openembedded.org/layerindex/api/
142 """
143
144 def _get_json_response(apiurl=None, username=None, password=None, retry=True):
145 assert apiurl is not None
146
147 logger.debug(1, "fetching %s" % apiurl)
148
149 up = urlparse(apiurl)
150
151 username=up.username
152 password=up.password
153
154 # Strip username/password and params
155 if up.port:
156 up_stripped = up._replace(params="", netloc="%s:%s" % (up.hostname, up.port))
157 else:
158 up_stripped = up._replace(params="", netloc=up.hostname)
159
160 res = self.layerindex._fetch_url(up_stripped.geturl(), username=username, password=password)
161
162 try:
163 parsed = json.loads(res.read().decode('utf-8'))
164 except ConnectionResetError:
165 if retry:
166 logger.debug(1, "%s: Connection reset by peer. Retrying..." % url)
167 parsed = _get_json_response(apiurl=up_stripped.geturl(), username=username, password=password, retry=False)
168 logger.debug(1, "%s: retry successful.")
169 else:
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500170 raise layerindexlib.LayerIndexFetchError('%s: Connection reset by peer. Is there a firewall blocking your connection?' % apiurl)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800171
172 return parsed
173
174 index = layerindexlib.LayerIndexObj()
175
176 index.config = {}
177 index.config['TYPE'] = self.type
178 index.config['URL'] = url
179
180 params = self.layerindex._parse_params(up.params)
181
182 if 'desc' in params:
183 index.config['DESCRIPTION'] = unquote(params['desc'])
184 else:
185 index.config['DESCRIPTION'] = up.hostname
186
187 if 'cache' in params:
188 index.config['CACHE'] = params['cache']
189
190 if 'branch' in params:
191 branches = params['branch'].split(',')
192 index.config['BRANCH'] = branches
193 else:
194 branches = ['*']
195
196 try:
197 index.apilinks = _get_json_response(apiurl=url, username=up.username, password=up.password)
198 except Exception as e:
199 raise layerindexlib.LayerIndexFetchError(url, e)
200
201 # Local raw index set...
202 pindex = {}
203
204 # Load all the requested branches at the same time time,
205 # a special branch of '*' means load all branches
206 filter = ""
207 if "*" not in branches:
208 filter = "?filter=name:%s" % "OR".join(branches)
209
210 logger.debug(1, "Loading %s from %s" % (branches, index.apilinks['branches']))
211
212 # The link won't include username/password, so pull it from the original url
213 pindex['branches'] = _get_json_response(index.apilinks['branches'] + filter,
214 username=up.username, password=up.password)
215 if not pindex['branches']:
216 logger.debug(1, "No valid branches (%s) found at url %s." % (branch, url))
217 return index
218 index.add_raw_element("branches", layerindexlib.Branch, pindex['branches'])
219
220 # Load all of the layerItems (these can not be easily filtered)
221 logger.debug(1, "Loading %s from %s" % ('layerItems', index.apilinks['layerItems']))
222
223
224 # The link won't include username/password, so pull it from the original url
225 pindex['layerItems'] = _get_json_response(index.apilinks['layerItems'],
226 username=up.username, password=up.password)
227 if not pindex['layerItems']:
228 logger.debug(1, "No layers were found at url %s." % (url))
229 return index
230 index.add_raw_element("layerItems", layerindexlib.LayerItem, pindex['layerItems'])
231
232
233 # From this point on load the contents for each branch. Otherwise we
234 # could run into a timeout.
235 for branch in index.branches:
236 filter = "?filter=branch__name:%s" % index.branches[branch].name
237
238 logger.debug(1, "Loading %s from %s" % ('layerBranches', index.apilinks['layerBranches']))
239
240 # The link won't include username/password, so pull it from the original url
241 pindex['layerBranches'] = _get_json_response(index.apilinks['layerBranches'] + filter,
242 username=up.username, password=up.password)
243 if not pindex['layerBranches']:
244 logger.debug(1, "No valid layer branches (%s) found at url %s." % (branches or "*", url))
245 return index
246 index.add_raw_element("layerBranches", layerindexlib.LayerBranch, pindex['layerBranches'])
247
248
249 # Load the rest, they all have a similar format
250 # Note: the layer index has a few more items, we can add them if necessary
251 # in the future.
252 filter = "?filter=layerbranch__branch__name:%s" % index.branches[branch].name
253 for (lName, lType) in [("layerDependencies", layerindexlib.LayerDependency),
254 ("recipes", layerindexlib.Recipe),
255 ("machines", layerindexlib.Machine),
256 ("distros", layerindexlib.Distro)]:
257 if lName not in load:
258 continue
259 logger.debug(1, "Loading %s from %s" % (lName, index.apilinks[lName]))
260
261 # The link won't include username/password, so pull it from the original url
262 pindex[lName] = _get_json_response(index.apilinks[lName] + filter,
263 username=up.username, password=up.password)
264 index.add_raw_element(lName, lType, pindex[lName])
265
266 return index
267
268 def store_index(self, url, index):
269 """
270 Store layer information into a local file/dir.
271
272 The return value is a dictionary containing API,
273 layer, branch, dependency, recipe, machine, distro, information.
274
275 ud is a parsed url to a directory or file. If the path is a
276 directory, we will split the files into one file per layer.
277 If the path is to a file (exists or not) the entire DB will be
278 dumped into that one file.
279 """
280
281 up = urlparse(url)
282
283 if up.scheme != 'file':
284 raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url)
285
286 logger.debug(1, "Storing to %s..." % up.path)
287
288 try:
289 layerbranches = index.layerBranches
290 except KeyError:
291 logger.error('No layerBranches to write.')
292 return
293
294
295 def filter_item(layerbranchid, objects):
296 filtered = []
297 for obj in getattr(index, objects, None):
298 try:
299 if getattr(index, objects)[obj].layerbranch_id == layerbranchid:
300 filtered.append(getattr(index, objects)[obj]._data)
301 except AttributeError:
302 logger.debug(1, 'No obj.layerbranch_id: %s' % objects)
303 # No simple filter method, just include it...
304 try:
305 filtered.append(getattr(index, objects)[obj]._data)
306 except AttributeError:
307 logger.debug(1, 'No obj._data: %s %s' % (objects, type(obj)))
308 filtered.append(obj)
309 return filtered
310
311
312 # Write out to a single file.
313 # Filter out unnecessary items, then sort as we write for determinism
314 if not os.path.isdir(up.path):
315 pindex = {}
316
317 pindex['branches'] = []
318 pindex['layerItems'] = []
319 pindex['layerBranches'] = []
320
321 for layerbranchid in layerbranches:
322 if layerbranches[layerbranchid].branch._data not in pindex['branches']:
323 pindex['branches'].append(layerbranches[layerbranchid].branch._data)
324
325 if layerbranches[layerbranchid].layer._data not in pindex['layerItems']:
326 pindex['layerItems'].append(layerbranches[layerbranchid].layer._data)
327
328 if layerbranches[layerbranchid]._data not in pindex['layerBranches']:
329 pindex['layerBranches'].append(layerbranches[layerbranchid]._data)
330
331 for entry in index._index:
332 # Skip local items, apilinks and items already processed
333 if entry in index.config['local'] or \
334 entry == 'apilinks' or \
335 entry == 'branches' or \
336 entry == 'layerBranches' or \
337 entry == 'layerItems':
338 continue
339 if entry not in pindex:
340 pindex[entry] = []
341 pindex[entry].extend(filter_item(layerbranchid, entry))
342
343 bb.debug(1, 'Writing index to %s' % up.path)
344 with open(up.path, 'wt') as f:
345 json.dump(layerindexlib.sort_entry(pindex), f, indent=4)
346 return
347
348
349 # Write out to a directory one file per layerBranch
350 # Prepare all layer related items, to create a minimal file.
351 # We have to sort the entries as we write so they are deterministic
352 for layerbranchid in layerbranches:
353 pindex = {}
354
355 for entry in index._index:
356 # Skip local items, apilinks and items already processed
357 if entry in index.config['local'] or \
358 entry == 'apilinks' or \
359 entry == 'branches' or \
360 entry == 'layerBranches' or \
361 entry == 'layerItems':
362 continue
363 pindex[entry] = filter_item(layerbranchid, entry)
364
365 # Add the layer we're processing as the first one...
366 pindex['branches'] = [layerbranches[layerbranchid].branch._data]
367 pindex['layerItems'] = [layerbranches[layerbranchid].layer._data]
368 pindex['layerBranches'] = [layerbranches[layerbranchid]._data]
369
370 # We also need to include the layerbranch for any dependencies...
371 for layerdep in pindex['layerDependencies']:
372 layerdependency = layerindexlib.LayerDependency(index, layerdep)
373
374 layeritem = layerdependency.dependency
375 layerbranch = layerdependency.dependency_layerBranch
376
377 # We need to avoid duplicates...
378 if layeritem._data not in pindex['layerItems']:
379 pindex['layerItems'].append(layeritem._data)
380
381 if layerbranch._data not in pindex['layerBranches']:
382 pindex['layerBranches'].append(layerbranch._data)
383
384 # apply mirroring adjustments here....
385
386 fname = index.config['DESCRIPTION'] + '__' + pindex['branches'][0]['name'] + '__' + pindex['layerItems'][0]['name']
387 fname = fname.translate(str.maketrans('/ ', '__'))
388 fpath = os.path.join(up.path, fname)
389
390 bb.debug(1, 'Writing index to %s' % fpath + '.json')
391 with open(fpath + '.json', 'wt') as f:
392 json.dump(layerindexlib.sort_entry(pindex), f, indent=4)