blob: 9af385db598f38882a6e46c79c0fbcff9fb1fecf [file] [log] [blame]
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001import argparse
2import http.client
3import json
4import logging
5import os
6import subprocess
7import urllib.parse
8
9from bblayers.action import ActionPlugin
10
11logger = logging.getLogger('bitbake-layers')
12
13
14def plugin_init(plugins):
15 return LayerIndexPlugin()
16
17
18class LayerIndexPlugin(ActionPlugin):
19 """Subcommands for interacting with the layer index.
20
21 This class inherits ActionPlugin to get do_add_layer.
22 """
23
24 def get_json_data(self, apiurl):
25 proxy_settings = os.environ.get("http_proxy", None)
26 conn = None
27 _parsedurl = urllib.parse.urlparse(apiurl)
28 path = _parsedurl.path
29 query = _parsedurl.query
30
31 def parse_url(url):
32 parsedurl = urllib.parse.urlparse(url)
33 if parsedurl.netloc[0] == '[':
34 host, port = parsedurl.netloc[1:].split(']', 1)
35 if ':' in port:
36 port = port.rsplit(':', 1)[1]
37 else:
38 port = None
39 else:
40 if parsedurl.netloc.count(':') == 1:
41 (host, port) = parsedurl.netloc.split(":")
42 else:
43 host = parsedurl.netloc
44 port = None
45 return (host, 80 if port is None else int(port))
46
47 if proxy_settings is None:
48 host, port = parse_url(apiurl)
49 conn = http.client.HTTPConnection(host, port)
50 conn.request("GET", path + "?" + query)
51 else:
52 host, port = parse_url(proxy_settings)
53 conn = http.client.HTTPConnection(host, port)
54 conn.request("GET", apiurl)
55
56 r = conn.getresponse()
57 if r.status != 200:
58 raise Exception("Failed to read " + path + ": %d %s" % (r.status, r.reason))
Brad Bishop6e60e8b2018-02-01 10:27:11 -050059 return json.loads(r.read().decode())
Patrick Williamsc0f7c042017-02-23 20:41:17 -060060
61 def get_layer_deps(self, layername, layeritems, layerbranches, layerdependencies, branchnum, selfname=False):
62 def layeritems_info_id(items_name, layeritems):
63 litems_id = None
64 for li in layeritems:
65 if li['name'] == items_name:
66 litems_id = li['id']
67 break
68 return litems_id
69
70 def layerbranches_info(items_id, layerbranches):
71 lbranch = {}
72 for lb in layerbranches:
73 if lb['layer'] == items_id and lb['branch'] == branchnum:
74 lbranch['id'] = lb['id']
75 lbranch['vcs_subdir'] = lb['vcs_subdir']
76 break
77 return lbranch
78
79 def layerdependencies_info(lb_id, layerdependencies):
80 ld_deps = []
81 for ld in layerdependencies:
82 if ld['layerbranch'] == lb_id and not ld['dependency'] in ld_deps:
83 ld_deps.append(ld['dependency'])
84 if not ld_deps:
85 logger.error("The dependency of layerDependencies is not found.")
86 return ld_deps
87
88 def layeritems_info_name_subdir(items_id, layeritems):
89 litems = {}
90 for li in layeritems:
91 if li['id'] == items_id:
92 litems['vcs_url'] = li['vcs_url']
93 litems['name'] = li['name']
94 break
95 return litems
96
97 if selfname:
98 selfid = layeritems_info_id(layername, layeritems)
99 lbinfo = layerbranches_info(selfid, layerbranches)
100 if lbinfo:
101 selfsubdir = lbinfo['vcs_subdir']
102 else:
103 logger.error("%s is not found in the specified branch" % layername)
104 return
105 selfurl = layeritems_info_name_subdir(selfid, layeritems)['vcs_url']
106 if selfurl:
107 return selfurl, selfsubdir
108 else:
109 logger.error("Cannot get layer %s git repo and subdir" % layername)
110 return
111 ldict = {}
112 itemsid = layeritems_info_id(layername, layeritems)
113 if not itemsid:
114 return layername, None
115 lbid = layerbranches_info(itemsid, layerbranches)
116 if lbid:
117 lbid = layerbranches_info(itemsid, layerbranches)['id']
118 else:
119 logger.error("%s is not found in the specified branch" % layername)
120 return None, None
121 for dependency in layerdependencies_info(lbid, layerdependencies):
122 lname = layeritems_info_name_subdir(dependency, layeritems)['name']
123 lurl = layeritems_info_name_subdir(dependency, layeritems)['vcs_url']
124 lsubdir = layerbranches_info(dependency, layerbranches)['vcs_subdir']
125 ldict[lname] = lurl, lsubdir
126 return None, ldict
127
128 def get_fetch_layer(self, fetchdir, url, subdir, fetch_layer):
129 layername = self.get_layer_name(url)
130 if os.path.splitext(layername)[1] == '.git':
131 layername = os.path.splitext(layername)[0]
132 repodir = os.path.join(fetchdir, layername)
133 layerdir = os.path.join(repodir, subdir)
134 if not os.path.exists(repodir):
135 if fetch_layer:
136 result = subprocess.call('git clone %s %s' % (url, repodir), shell = True)
137 if result:
138 logger.error("Failed to download %s" % url)
139 return None, None
140 else:
141 return layername, layerdir
142 else:
143 logger.plain("Repository %s needs to be fetched" % url)
144 return layername, layerdir
145 elif os.path.exists(layerdir):
146 return layername, layerdir
147 else:
148 logger.error("%s is not in %s" % (url, subdir))
149 return None, None
150
151 def do_layerindex_fetch(self, args):
152 """Fetches a layer from a layer index along with its dependent layers, and adds them to conf/bblayers.conf.
153"""
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500154 apiurl = self.tinfoil.config_data.getVar('BBLAYERS_LAYERINDEX_URL')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600155 if not apiurl:
156 logger.error("Cannot get BBLAYERS_LAYERINDEX_URL")
157 return 1
158 else:
159 if apiurl[-1] != '/':
160 apiurl += '/'
161 apiurl += "api/"
162 apilinks = self.get_json_data(apiurl)
163 branches = self.get_json_data(apilinks['branches'])
164
165 branchnum = 0
166 for branch in branches:
167 if branch['name'] == args.branch:
168 branchnum = branch['id']
169 break
170 if branchnum == 0:
171 validbranches = ', '.join([branch['name'] for branch in branches])
172 logger.error('Invalid layer branch name "%s". Valid branches: %s' % (args.branch, validbranches))
173 return 1
174
175 ignore_layers = []
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500176 for collection in self.tinfoil.config_data.getVar('BBFILE_COLLECTIONS').split():
177 lname = self.tinfoil.config_data.getVar('BBLAYERS_LAYERINDEX_NAME_%s' % collection)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600178 if lname:
179 ignore_layers.append(lname)
180
181 if args.ignore:
182 ignore_layers.extend(args.ignore.split(','))
183
184 layeritems = self.get_json_data(apilinks['layerItems'])
185 layerbranches = self.get_json_data(apilinks['layerBranches'])
186 layerdependencies = self.get_json_data(apilinks['layerDependencies'])
187 invaluenames = []
188 repourls = {}
189 printlayers = []
190
191 def query_dependencies(layers, layeritems, layerbranches, layerdependencies, branchnum):
192 depslayer = []
193 for layername in layers:
194 invaluename, layerdict = self.get_layer_deps(layername, layeritems, layerbranches, layerdependencies, branchnum)
195 if layerdict:
196 repourls[layername] = self.get_layer_deps(layername, layeritems, layerbranches, layerdependencies, branchnum, selfname=True)
197 for layer in layerdict:
198 if not layer in ignore_layers:
199 depslayer.append(layer)
200 printlayers.append((layername, layer, layerdict[layer][0], layerdict[layer][1]))
201 if not layer in ignore_layers and not layer in repourls:
202 repourls[layer] = (layerdict[layer][0], layerdict[layer][1])
203 if invaluename and not invaluename in invaluenames:
204 invaluenames.append(invaluename)
205 return depslayer
206
207 depslayers = query_dependencies(args.layername, layeritems, layerbranches, layerdependencies, branchnum)
208 while depslayers:
209 depslayer = query_dependencies(depslayers, layeritems, layerbranches, layerdependencies, branchnum)
210 depslayers = depslayer
211 if invaluenames:
212 for invaluename in invaluenames:
213 logger.error('Layer "%s" not found in layer index' % invaluename)
214 return 1
215 logger.plain("%s %s %s %s" % ("Layer".ljust(19), "Required by".ljust(19), "Git repository".ljust(54), "Subdirectory"))
216 logger.plain('=' * 115)
217 for layername in args.layername:
218 layerurl = repourls[layername]
219 logger.plain("%s %s %s %s" % (layername.ljust(20), '-'.ljust(20), layerurl[0].ljust(55), layerurl[1]))
220 printedlayers = []
221 for layer, dependency, gitrepo, subdirectory in printlayers:
222 if dependency in printedlayers:
223 continue
224 logger.plain("%s %s %s %s" % (dependency.ljust(20), layer.ljust(20), gitrepo.ljust(55), subdirectory))
225 printedlayers.append(dependency)
226
227 if repourls:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500228 fetchdir = self.tinfoil.config_data.getVar('BBLAYERS_FETCH_DIR')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600229 if not fetchdir:
230 logger.error("Cannot get BBLAYERS_FETCH_DIR")
231 return 1
232 if not os.path.exists(fetchdir):
233 os.makedirs(fetchdir)
234 addlayers = []
235 for repourl, subdir in repourls.values():
236 name, layerdir = self.get_fetch_layer(fetchdir, repourl, subdir, not args.show_only)
237 if not name:
238 # Error already shown
239 return 1
240 addlayers.append((subdir, name, layerdir))
241 if not args.show_only:
242 for subdir, name, layerdir in set(addlayers):
243 if os.path.exists(layerdir):
244 if subdir:
245 logger.plain("Adding layer \"%s\" to conf/bblayers.conf" % subdir)
246 else:
247 logger.plain("Adding layer \"%s\" to conf/bblayers.conf" % name)
248 localargs = argparse.Namespace()
249 localargs.layerdir = layerdir
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500250 localargs.force = args.force
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600251 self.do_add_layer(localargs)
252 else:
253 break
254
255 def do_layerindex_show_depends(self, args):
256 """Find layer dependencies from layer index.
257"""
258 args.show_only = True
259 args.ignore = []
260 self.do_layerindex_fetch(args)
261
262 def register_commands(self, sp):
263 parser_layerindex_fetch = self.add_command(sp, 'layerindex-fetch', self.do_layerindex_fetch)
264 parser_layerindex_fetch.add_argument('-n', '--show-only', help='show dependencies and do nothing else', action='store_true')
265 parser_layerindex_fetch.add_argument('-b', '--branch', help='branch name to fetch (default %(default)s)', default='master')
266 parser_layerindex_fetch.add_argument('-i', '--ignore', help='assume the specified layers do not need to be fetched/added (separate multiple layers with commas, no spaces)', metavar='LAYER')
267 parser_layerindex_fetch.add_argument('layername', nargs='+', help='layer to fetch')
268
269 parser_layerindex_show_depends = self.add_command(sp, 'layerindex-show-depends', self.do_layerindex_show_depends)
270 parser_layerindex_show_depends.add_argument('-b', '--branch', help='branch name to fetch (default %(default)s)', default='master')
271 parser_layerindex_show_depends.add_argument('layername', nargs='+', help='layer to query')