blob: 21fd1441434c7db18b1ba13605794d92b07daf7f [file] [log] [blame]
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001# Copyright (C) 2016-2018 Wind River Systems, Inc.
2#
Brad Bishopc342db32019-05-15 21:57:59 -04003# SPDX-License-Identifier: GPL-2.0-only
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08004#
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08005
6import logging
7import json
8from urllib.parse import unquote
9from urllib.parse import urlparse
10
11import layerindexlib
12import layerindexlib.plugin
13
14logger = logging.getLogger('BitBake.layerindexlib.restapi')
15
16def plugin_init(plugins):
17 return RestApiPlugin()
18
19class RestApiPlugin(layerindexlib.plugin.IndexPlugin):
20 def __init__(self):
21 self.type = "restapi"
22
23 def load_index(self, url, load):
24 """
25 Fetches layer information from a local or remote layer index.
26
27 The return value is a LayerIndexObj.
28
29 url is the url to the rest api of the layer index, such as:
30 http://layers.openembedded.org/layerindex/api/
31
32 Or a local file...
33 """
34
35 up = urlparse(url)
36
37 if up.scheme == 'file':
38 return self.load_index_file(up, url, load)
39
40 if up.scheme == 'http' or up.scheme == 'https':
41 return self.load_index_web(up, url, load)
42
43 raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url)
44
45
46 def load_index_file(self, up, url, load):
47 """
48 Fetches layer information from a local file or directory.
49
50 The return value is a LayerIndexObj.
51
52 ud is the parsed url to the local file or directory.
53 """
54 if not os.path.exists(up.path):
55 raise FileNotFoundError(up.path)
56
57 index = layerindexlib.LayerIndexObj()
58
59 index.config = {}
60 index.config['TYPE'] = self.type
61 index.config['URL'] = url
62
63 params = self.layerindex._parse_params(up.params)
64
65 if 'desc' in params:
66 index.config['DESCRIPTION'] = unquote(params['desc'])
67 else:
68 index.config['DESCRIPTION'] = up.path
69
70 if 'cache' in params:
71 index.config['CACHE'] = params['cache']
72
73 if 'branch' in params:
74 branches = params['branch'].split(',')
75 index.config['BRANCH'] = branches
76 else:
77 branches = ['*']
78
79
80 def load_cache(path, index, branches=[]):
81 logger.debug(1, 'Loading json file %s' % path)
82 with open(path, 'rt', encoding='utf-8') as f:
83 pindex = json.load(f)
84
85 # Filter the branches on loaded files...
86 newpBranch = []
87 for branch in branches:
88 if branch != '*':
89 if 'branches' in pindex:
90 for br in pindex['branches']:
91 if br['name'] == branch:
92 newpBranch.append(br)
93 else:
94 if 'branches' in pindex:
95 for br in pindex['branches']:
96 newpBranch.append(br)
97
98 if newpBranch:
99 index.add_raw_element('branches', layerindexlib.Branch, newpBranch)
100 else:
101 logger.debug(1, 'No matching branches (%s) in index file(s)' % branches)
102 # No matching branches.. return nothing...
103 return
104
105 for (lName, lType) in [("layerItems", layerindexlib.LayerItem),
106 ("layerBranches", layerindexlib.LayerBranch),
107 ("layerDependencies", layerindexlib.LayerDependency),
108 ("recipes", layerindexlib.Recipe),
109 ("machines", layerindexlib.Machine),
110 ("distros", layerindexlib.Distro)]:
111 if lName in pindex:
112 index.add_raw_element(lName, lType, pindex[lName])
113
114
115 if not os.path.isdir(up.path):
116 load_cache(up.path, index, branches)
117 return index
118
119 logger.debug(1, 'Loading from dir %s...' % (up.path))
120 for (dirpath, _, filenames) in os.walk(up.path):
121 for filename in filenames:
122 if not filename.endswith('.json'):
123 continue
124 fpath = os.path.join(dirpath, filename)
125 load_cache(fpath, index, branches)
126
127 return index
128
129
130 def load_index_web(self, up, url, load):
131 """
132 Fetches layer information from a remote layer index.
133
134 The return value is a LayerIndexObj.
135
136 ud is the parsed url to the rest api of the layer index, such as:
137 http://layers.openembedded.org/layerindex/api/
138 """
139
140 def _get_json_response(apiurl=None, username=None, password=None, retry=True):
141 assert apiurl is not None
142
143 logger.debug(1, "fetching %s" % apiurl)
144
145 up = urlparse(apiurl)
146
147 username=up.username
148 password=up.password
149
150 # Strip username/password and params
151 if up.port:
152 up_stripped = up._replace(params="", netloc="%s:%s" % (up.hostname, up.port))
153 else:
154 up_stripped = up._replace(params="", netloc=up.hostname)
155
156 res = self.layerindex._fetch_url(up_stripped.geturl(), username=username, password=password)
157
158 try:
159 parsed = json.loads(res.read().decode('utf-8'))
160 except ConnectionResetError:
161 if retry:
162 logger.debug(1, "%s: Connection reset by peer. Retrying..." % url)
163 parsed = _get_json_response(apiurl=up_stripped.geturl(), username=username, password=password, retry=False)
164 logger.debug(1, "%s: retry successful.")
165 else:
166 raise LayerIndexFetchError('%s: Connection reset by peer. Is there a firewall blocking your connection?' % apiurl)
167
168 return parsed
169
170 index = layerindexlib.LayerIndexObj()
171
172 index.config = {}
173 index.config['TYPE'] = self.type
174 index.config['URL'] = url
175
176 params = self.layerindex._parse_params(up.params)
177
178 if 'desc' in params:
179 index.config['DESCRIPTION'] = unquote(params['desc'])
180 else:
181 index.config['DESCRIPTION'] = up.hostname
182
183 if 'cache' in params:
184 index.config['CACHE'] = params['cache']
185
186 if 'branch' in params:
187 branches = params['branch'].split(',')
188 index.config['BRANCH'] = branches
189 else:
190 branches = ['*']
191
192 try:
193 index.apilinks = _get_json_response(apiurl=url, username=up.username, password=up.password)
194 except Exception as e:
195 raise layerindexlib.LayerIndexFetchError(url, e)
196
197 # Local raw index set...
198 pindex = {}
199
200 # Load all the requested branches at the same time time,
201 # a special branch of '*' means load all branches
202 filter = ""
203 if "*" not in branches:
204 filter = "?filter=name:%s" % "OR".join(branches)
205
206 logger.debug(1, "Loading %s from %s" % (branches, index.apilinks['branches']))
207
208 # The link won't include username/password, so pull it from the original url
209 pindex['branches'] = _get_json_response(index.apilinks['branches'] + filter,
210 username=up.username, password=up.password)
211 if not pindex['branches']:
212 logger.debug(1, "No valid branches (%s) found at url %s." % (branch, url))
213 return index
214 index.add_raw_element("branches", layerindexlib.Branch, pindex['branches'])
215
216 # Load all of the layerItems (these can not be easily filtered)
217 logger.debug(1, "Loading %s from %s" % ('layerItems', index.apilinks['layerItems']))
218
219
220 # The link won't include username/password, so pull it from the original url
221 pindex['layerItems'] = _get_json_response(index.apilinks['layerItems'],
222 username=up.username, password=up.password)
223 if not pindex['layerItems']:
224 logger.debug(1, "No layers were found at url %s." % (url))
225 return index
226 index.add_raw_element("layerItems", layerindexlib.LayerItem, pindex['layerItems'])
227
228
229 # From this point on load the contents for each branch. Otherwise we
230 # could run into a timeout.
231 for branch in index.branches:
232 filter = "?filter=branch__name:%s" % index.branches[branch].name
233
234 logger.debug(1, "Loading %s from %s" % ('layerBranches', index.apilinks['layerBranches']))
235
236 # The link won't include username/password, so pull it from the original url
237 pindex['layerBranches'] = _get_json_response(index.apilinks['layerBranches'] + filter,
238 username=up.username, password=up.password)
239 if not pindex['layerBranches']:
240 logger.debug(1, "No valid layer branches (%s) found at url %s." % (branches or "*", url))
241 return index
242 index.add_raw_element("layerBranches", layerindexlib.LayerBranch, pindex['layerBranches'])
243
244
245 # Load the rest, they all have a similar format
246 # Note: the layer index has a few more items, we can add them if necessary
247 # in the future.
248 filter = "?filter=layerbranch__branch__name:%s" % index.branches[branch].name
249 for (lName, lType) in [("layerDependencies", layerindexlib.LayerDependency),
250 ("recipes", layerindexlib.Recipe),
251 ("machines", layerindexlib.Machine),
252 ("distros", layerindexlib.Distro)]:
253 if lName not in load:
254 continue
255 logger.debug(1, "Loading %s from %s" % (lName, index.apilinks[lName]))
256
257 # The link won't include username/password, so pull it from the original url
258 pindex[lName] = _get_json_response(index.apilinks[lName] + filter,
259 username=up.username, password=up.password)
260 index.add_raw_element(lName, lType, pindex[lName])
261
262 return index
263
264 def store_index(self, url, index):
265 """
266 Store layer information into a local file/dir.
267
268 The return value is a dictionary containing API,
269 layer, branch, dependency, recipe, machine, distro, information.
270
271 ud is a parsed url to a directory or file. If the path is a
272 directory, we will split the files into one file per layer.
273 If the path is to a file (exists or not) the entire DB will be
274 dumped into that one file.
275 """
276
277 up = urlparse(url)
278
279 if up.scheme != 'file':
280 raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url)
281
282 logger.debug(1, "Storing to %s..." % up.path)
283
284 try:
285 layerbranches = index.layerBranches
286 except KeyError:
287 logger.error('No layerBranches to write.')
288 return
289
290
291 def filter_item(layerbranchid, objects):
292 filtered = []
293 for obj in getattr(index, objects, None):
294 try:
295 if getattr(index, objects)[obj].layerbranch_id == layerbranchid:
296 filtered.append(getattr(index, objects)[obj]._data)
297 except AttributeError:
298 logger.debug(1, 'No obj.layerbranch_id: %s' % objects)
299 # No simple filter method, just include it...
300 try:
301 filtered.append(getattr(index, objects)[obj]._data)
302 except AttributeError:
303 logger.debug(1, 'No obj._data: %s %s' % (objects, type(obj)))
304 filtered.append(obj)
305 return filtered
306
307
308 # Write out to a single file.
309 # Filter out unnecessary items, then sort as we write for determinism
310 if not os.path.isdir(up.path):
311 pindex = {}
312
313 pindex['branches'] = []
314 pindex['layerItems'] = []
315 pindex['layerBranches'] = []
316
317 for layerbranchid in layerbranches:
318 if layerbranches[layerbranchid].branch._data not in pindex['branches']:
319 pindex['branches'].append(layerbranches[layerbranchid].branch._data)
320
321 if layerbranches[layerbranchid].layer._data not in pindex['layerItems']:
322 pindex['layerItems'].append(layerbranches[layerbranchid].layer._data)
323
324 if layerbranches[layerbranchid]._data not in pindex['layerBranches']:
325 pindex['layerBranches'].append(layerbranches[layerbranchid]._data)
326
327 for entry in index._index:
328 # Skip local items, apilinks and items already processed
329 if entry in index.config['local'] or \
330 entry == 'apilinks' or \
331 entry == 'branches' or \
332 entry == 'layerBranches' or \
333 entry == 'layerItems':
334 continue
335 if entry not in pindex:
336 pindex[entry] = []
337 pindex[entry].extend(filter_item(layerbranchid, entry))
338
339 bb.debug(1, 'Writing index to %s' % up.path)
340 with open(up.path, 'wt') as f:
341 json.dump(layerindexlib.sort_entry(pindex), f, indent=4)
342 return
343
344
345 # Write out to a directory one file per layerBranch
346 # Prepare all layer related items, to create a minimal file.
347 # We have to sort the entries as we write so they are deterministic
348 for layerbranchid in layerbranches:
349 pindex = {}
350
351 for entry in index._index:
352 # Skip local items, apilinks and items already processed
353 if entry in index.config['local'] or \
354 entry == 'apilinks' or \
355 entry == 'branches' or \
356 entry == 'layerBranches' or \
357 entry == 'layerItems':
358 continue
359 pindex[entry] = filter_item(layerbranchid, entry)
360
361 # Add the layer we're processing as the first one...
362 pindex['branches'] = [layerbranches[layerbranchid].branch._data]
363 pindex['layerItems'] = [layerbranches[layerbranchid].layer._data]
364 pindex['layerBranches'] = [layerbranches[layerbranchid]._data]
365
366 # We also need to include the layerbranch for any dependencies...
367 for layerdep in pindex['layerDependencies']:
368 layerdependency = layerindexlib.LayerDependency(index, layerdep)
369
370 layeritem = layerdependency.dependency
371 layerbranch = layerdependency.dependency_layerBranch
372
373 # We need to avoid duplicates...
374 if layeritem._data not in pindex['layerItems']:
375 pindex['layerItems'].append(layeritem._data)
376
377 if layerbranch._data not in pindex['layerBranches']:
378 pindex['layerBranches'].append(layerbranch._data)
379
380 # apply mirroring adjustments here....
381
382 fname = index.config['DESCRIPTION'] + '__' + pindex['branches'][0]['name'] + '__' + pindex['layerItems'][0]['name']
383 fname = fname.translate(str.maketrans('/ ', '__'))
384 fpath = os.path.join(up.path, fname)
385
386 bb.debug(1, 'Writing index to %s' % fpath + '.json')
387 with open(fpath + '.json', 'wt') as f:
388 json.dump(layerindexlib.sort_entry(pindex), f, indent=4)