| # Copyright (C) 2016-2018 Wind River Systems, Inc. |
| # |
| # This program is free software; you can redistribute it and/or modify |
| # it under the terms of the GNU General Public License version 2 as |
| # published by the Free Software Foundation. |
| # |
| # This program is distributed in the hope that it will be useful, |
| # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. |
| # See the GNU General Public License for more details. |
| # |
| # You should have received a copy of the GNU General Public License |
| # along with this program; if not, write to the Free Software |
| # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA |
| |
| import logging |
| import json |
| from urllib.parse import unquote |
| from urllib.parse import urlparse |
| |
| import layerindexlib |
| import layerindexlib.plugin |
| |
| logger = logging.getLogger('BitBake.layerindexlib.restapi') |
| |
| def plugin_init(plugins): |
| return RestApiPlugin() |
| |
| class RestApiPlugin(layerindexlib.plugin.IndexPlugin): |
| def __init__(self): |
| self.type = "restapi" |
| |
| def load_index(self, url, load): |
| """ |
| Fetches layer information from a local or remote layer index. |
| |
| The return value is a LayerIndexObj. |
| |
| url is the url to the rest api of the layer index, such as: |
| http://layers.openembedded.org/layerindex/api/ |
| |
| Or a local file... |
| """ |
| |
| up = urlparse(url) |
| |
| if up.scheme == 'file': |
| return self.load_index_file(up, url, load) |
| |
| if up.scheme == 'http' or up.scheme == 'https': |
| return self.load_index_web(up, url, load) |
| |
| raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url) |
| |
| |
| def load_index_file(self, up, url, load): |
| """ |
| Fetches layer information from a local file or directory. |
| |
| The return value is a LayerIndexObj. |
| |
| ud is the parsed url to the local file or directory. |
| """ |
| if not os.path.exists(up.path): |
| raise FileNotFoundError(up.path) |
| |
| index = layerindexlib.LayerIndexObj() |
| |
| index.config = {} |
| index.config['TYPE'] = self.type |
| index.config['URL'] = url |
| |
| params = self.layerindex._parse_params(up.params) |
| |
| if 'desc' in params: |
| index.config['DESCRIPTION'] = unquote(params['desc']) |
| else: |
| index.config['DESCRIPTION'] = up.path |
| |
| if 'cache' in params: |
| index.config['CACHE'] = params['cache'] |
| |
| if 'branch' in params: |
| branches = params['branch'].split(',') |
| index.config['BRANCH'] = branches |
| else: |
| branches = ['*'] |
| |
| |
| def load_cache(path, index, branches=[]): |
| logger.debug(1, 'Loading json file %s' % path) |
| with open(path, 'rt', encoding='utf-8') as f: |
| pindex = json.load(f) |
| |
| # Filter the branches on loaded files... |
| newpBranch = [] |
| for branch in branches: |
| if branch != '*': |
| if 'branches' in pindex: |
| for br in pindex['branches']: |
| if br['name'] == branch: |
| newpBranch.append(br) |
| else: |
| if 'branches' in pindex: |
| for br in pindex['branches']: |
| newpBranch.append(br) |
| |
| if newpBranch: |
| index.add_raw_element('branches', layerindexlib.Branch, newpBranch) |
| else: |
| logger.debug(1, 'No matching branches (%s) in index file(s)' % branches) |
| # No matching branches.. return nothing... |
| return |
| |
| for (lName, lType) in [("layerItems", layerindexlib.LayerItem), |
| ("layerBranches", layerindexlib.LayerBranch), |
| ("layerDependencies", layerindexlib.LayerDependency), |
| ("recipes", layerindexlib.Recipe), |
| ("machines", layerindexlib.Machine), |
| ("distros", layerindexlib.Distro)]: |
| if lName in pindex: |
| index.add_raw_element(lName, lType, pindex[lName]) |
| |
| |
| if not os.path.isdir(up.path): |
| load_cache(up.path, index, branches) |
| return index |
| |
| logger.debug(1, 'Loading from dir %s...' % (up.path)) |
| for (dirpath, _, filenames) in os.walk(up.path): |
| for filename in filenames: |
| if not filename.endswith('.json'): |
| continue |
| fpath = os.path.join(dirpath, filename) |
| load_cache(fpath, index, branches) |
| |
| return index |
| |
| |
| def load_index_web(self, up, url, load): |
| """ |
| Fetches layer information from a remote layer index. |
| |
| The return value is a LayerIndexObj. |
| |
| ud is the parsed url to the rest api of the layer index, such as: |
| http://layers.openembedded.org/layerindex/api/ |
| """ |
| |
| def _get_json_response(apiurl=None, username=None, password=None, retry=True): |
| assert apiurl is not None |
| |
| logger.debug(1, "fetching %s" % apiurl) |
| |
| up = urlparse(apiurl) |
| |
| username=up.username |
| password=up.password |
| |
| # Strip username/password and params |
| if up.port: |
| up_stripped = up._replace(params="", netloc="%s:%s" % (up.hostname, up.port)) |
| else: |
| up_stripped = up._replace(params="", netloc=up.hostname) |
| |
| res = self.layerindex._fetch_url(up_stripped.geturl(), username=username, password=password) |
| |
| try: |
| parsed = json.loads(res.read().decode('utf-8')) |
| except ConnectionResetError: |
| if retry: |
| logger.debug(1, "%s: Connection reset by peer. Retrying..." % url) |
| parsed = _get_json_response(apiurl=up_stripped.geturl(), username=username, password=password, retry=False) |
| logger.debug(1, "%s: retry successful.") |
| else: |
| raise LayerIndexFetchError('%s: Connection reset by peer. Is there a firewall blocking your connection?' % apiurl) |
| |
| return parsed |
| |
| index = layerindexlib.LayerIndexObj() |
| |
| index.config = {} |
| index.config['TYPE'] = self.type |
| index.config['URL'] = url |
| |
| params = self.layerindex._parse_params(up.params) |
| |
| if 'desc' in params: |
| index.config['DESCRIPTION'] = unquote(params['desc']) |
| else: |
| index.config['DESCRIPTION'] = up.hostname |
| |
| if 'cache' in params: |
| index.config['CACHE'] = params['cache'] |
| |
| if 'branch' in params: |
| branches = params['branch'].split(',') |
| index.config['BRANCH'] = branches |
| else: |
| branches = ['*'] |
| |
| try: |
| index.apilinks = _get_json_response(apiurl=url, username=up.username, password=up.password) |
| except Exception as e: |
| raise layerindexlib.LayerIndexFetchError(url, e) |
| |
| # Local raw index set... |
| pindex = {} |
| |
| # Load all the requested branches at the same time time, |
| # a special branch of '*' means load all branches |
| filter = "" |
| if "*" not in branches: |
| filter = "?filter=name:%s" % "OR".join(branches) |
| |
| logger.debug(1, "Loading %s from %s" % (branches, index.apilinks['branches'])) |
| |
| # The link won't include username/password, so pull it from the original url |
| pindex['branches'] = _get_json_response(index.apilinks['branches'] + filter, |
| username=up.username, password=up.password) |
| if not pindex['branches']: |
| logger.debug(1, "No valid branches (%s) found at url %s." % (branch, url)) |
| return index |
| index.add_raw_element("branches", layerindexlib.Branch, pindex['branches']) |
| |
| # Load all of the layerItems (these can not be easily filtered) |
| logger.debug(1, "Loading %s from %s" % ('layerItems', index.apilinks['layerItems'])) |
| |
| |
| # The link won't include username/password, so pull it from the original url |
| pindex['layerItems'] = _get_json_response(index.apilinks['layerItems'], |
| username=up.username, password=up.password) |
| if not pindex['layerItems']: |
| logger.debug(1, "No layers were found at url %s." % (url)) |
| return index |
| index.add_raw_element("layerItems", layerindexlib.LayerItem, pindex['layerItems']) |
| |
| |
| # From this point on load the contents for each branch. Otherwise we |
| # could run into a timeout. |
| for branch in index.branches: |
| filter = "?filter=branch__name:%s" % index.branches[branch].name |
| |
| logger.debug(1, "Loading %s from %s" % ('layerBranches', index.apilinks['layerBranches'])) |
| |
| # The link won't include username/password, so pull it from the original url |
| pindex['layerBranches'] = _get_json_response(index.apilinks['layerBranches'] + filter, |
| username=up.username, password=up.password) |
| if not pindex['layerBranches']: |
| logger.debug(1, "No valid layer branches (%s) found at url %s." % (branches or "*", url)) |
| return index |
| index.add_raw_element("layerBranches", layerindexlib.LayerBranch, pindex['layerBranches']) |
| |
| |
| # Load the rest, they all have a similar format |
| # Note: the layer index has a few more items, we can add them if necessary |
| # in the future. |
| filter = "?filter=layerbranch__branch__name:%s" % index.branches[branch].name |
| for (lName, lType) in [("layerDependencies", layerindexlib.LayerDependency), |
| ("recipes", layerindexlib.Recipe), |
| ("machines", layerindexlib.Machine), |
| ("distros", layerindexlib.Distro)]: |
| if lName not in load: |
| continue |
| logger.debug(1, "Loading %s from %s" % (lName, index.apilinks[lName])) |
| |
| # The link won't include username/password, so pull it from the original url |
| pindex[lName] = _get_json_response(index.apilinks[lName] + filter, |
| username=up.username, password=up.password) |
| index.add_raw_element(lName, lType, pindex[lName]) |
| |
| return index |
| |
| def store_index(self, url, index): |
| """ |
| Store layer information into a local file/dir. |
| |
| The return value is a dictionary containing API, |
| layer, branch, dependency, recipe, machine, distro, information. |
| |
| ud is a parsed url to a directory or file. If the path is a |
| directory, we will split the files into one file per layer. |
| If the path is to a file (exists or not) the entire DB will be |
| dumped into that one file. |
| """ |
| |
| up = urlparse(url) |
| |
| if up.scheme != 'file': |
| raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url) |
| |
| logger.debug(1, "Storing to %s..." % up.path) |
| |
| try: |
| layerbranches = index.layerBranches |
| except KeyError: |
| logger.error('No layerBranches to write.') |
| return |
| |
| |
| def filter_item(layerbranchid, objects): |
| filtered = [] |
| for obj in getattr(index, objects, None): |
| try: |
| if getattr(index, objects)[obj].layerbranch_id == layerbranchid: |
| filtered.append(getattr(index, objects)[obj]._data) |
| except AttributeError: |
| logger.debug(1, 'No obj.layerbranch_id: %s' % objects) |
| # No simple filter method, just include it... |
| try: |
| filtered.append(getattr(index, objects)[obj]._data) |
| except AttributeError: |
| logger.debug(1, 'No obj._data: %s %s' % (objects, type(obj))) |
| filtered.append(obj) |
| return filtered |
| |
| |
| # Write out to a single file. |
| # Filter out unnecessary items, then sort as we write for determinism |
| if not os.path.isdir(up.path): |
| pindex = {} |
| |
| pindex['branches'] = [] |
| pindex['layerItems'] = [] |
| pindex['layerBranches'] = [] |
| |
| for layerbranchid in layerbranches: |
| if layerbranches[layerbranchid].branch._data not in pindex['branches']: |
| pindex['branches'].append(layerbranches[layerbranchid].branch._data) |
| |
| if layerbranches[layerbranchid].layer._data not in pindex['layerItems']: |
| pindex['layerItems'].append(layerbranches[layerbranchid].layer._data) |
| |
| if layerbranches[layerbranchid]._data not in pindex['layerBranches']: |
| pindex['layerBranches'].append(layerbranches[layerbranchid]._data) |
| |
| for entry in index._index: |
| # Skip local items, apilinks and items already processed |
| if entry in index.config['local'] or \ |
| entry == 'apilinks' or \ |
| entry == 'branches' or \ |
| entry == 'layerBranches' or \ |
| entry == 'layerItems': |
| continue |
| if entry not in pindex: |
| pindex[entry] = [] |
| pindex[entry].extend(filter_item(layerbranchid, entry)) |
| |
| bb.debug(1, 'Writing index to %s' % up.path) |
| with open(up.path, 'wt') as f: |
| json.dump(layerindexlib.sort_entry(pindex), f, indent=4) |
| return |
| |
| |
| # Write out to a directory one file per layerBranch |
| # Prepare all layer related items, to create a minimal file. |
| # We have to sort the entries as we write so they are deterministic |
| for layerbranchid in layerbranches: |
| pindex = {} |
| |
| for entry in index._index: |
| # Skip local items, apilinks and items already processed |
| if entry in index.config['local'] or \ |
| entry == 'apilinks' or \ |
| entry == 'branches' or \ |
| entry == 'layerBranches' or \ |
| entry == 'layerItems': |
| continue |
| pindex[entry] = filter_item(layerbranchid, entry) |
| |
| # Add the layer we're processing as the first one... |
| pindex['branches'] = [layerbranches[layerbranchid].branch._data] |
| pindex['layerItems'] = [layerbranches[layerbranchid].layer._data] |
| pindex['layerBranches'] = [layerbranches[layerbranchid]._data] |
| |
| # We also need to include the layerbranch for any dependencies... |
| for layerdep in pindex['layerDependencies']: |
| layerdependency = layerindexlib.LayerDependency(index, layerdep) |
| |
| layeritem = layerdependency.dependency |
| layerbranch = layerdependency.dependency_layerBranch |
| |
| # We need to avoid duplicates... |
| if layeritem._data not in pindex['layerItems']: |
| pindex['layerItems'].append(layeritem._data) |
| |
| if layerbranch._data not in pindex['layerBranches']: |
| pindex['layerBranches'].append(layerbranch._data) |
| |
| # apply mirroring adjustments here.... |
| |
| fname = index.config['DESCRIPTION'] + '__' + pindex['branches'][0]['name'] + '__' + pindex['layerItems'][0]['name'] |
| fname = fname.translate(str.maketrans('/ ', '__')) |
| fpath = os.path.join(up.path, fname) |
| |
| bb.debug(1, 'Writing index to %s' % fpath + '.json') |
| with open(fpath + '.json', 'wt') as f: |
| json.dump(layerindexlib.sort_entry(pindex), f, indent=4) |