blob: f92cd2e42312cd423e7c8ae6ac81df45a4be5184 [file] [log] [blame]
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001from contextlib import contextmanager
2@contextmanager
3def create_socket(url, d):
4 import urllib
5 socket = urllib.urlopen(url, proxies=get_proxies(d))
6 try:
7 yield socket
8 finally:
9 socket.close()
10
11def get_proxies(d):
12 import os
13 proxykeys = ['http', 'https', 'ftp', 'ftps', 'no', 'all']
14 proxyvalues = map(lambda key: d.getVar(key+'_proxy', True), proxykeys)
15 return dict(zip(proxykeys, proxyvalues))
16
17def get_links_from_url(url, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050018 "Return all the href links found on the web location"
19
Patrick Williamsf1e5d692016-03-30 15:21:19 -050020 import sgmllib
Patrick Williamsc124f4f2015-09-15 14:41:29 -050021
22 class LinksParser(sgmllib.SGMLParser):
23 def parse(self, s):
24 "Parse the given string 's'."
25 self.feed(s)
26 self.close()
27
28 def __init__(self, verbose=0):
29 "Initialise an object passing 'verbose' to the superclass."
30 sgmllib.SGMLParser.__init__(self, verbose)
31 self.hyperlinks = []
32
33 def start_a(self, attributes):
34 "Process a hyperlink and its 'attributes'."
35 for name, value in attributes:
36 if name == "href":
37 self.hyperlinks.append(value.strip('/'))
38
39 def get_hyperlinks(self):
40 "Return the list of hyperlinks."
41 return self.hyperlinks
42
Patrick Williamsf1e5d692016-03-30 15:21:19 -050043 with create_socket(url,d) as sock:
44 webpage = sock.read()
Patrick Williamsc124f4f2015-09-15 14:41:29 -050045
46 linksparser = LinksParser()
47 linksparser.parse(webpage)
48 return linksparser.get_hyperlinks()
49
Patrick Williamsf1e5d692016-03-30 15:21:19 -050050def find_latest_numeric_release(url, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050051 "Find the latest listed numeric release on the given url"
52 max=0
53 maxstr=""
Patrick Williamsf1e5d692016-03-30 15:21:19 -050054 for link in get_links_from_url(url, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050055 try:
56 release = float(link)
57 except:
58 release = 0
59 if release > max:
60 max = release
61 maxstr = link
62 return maxstr
63
64def is_src_rpm(name):
65 "Check if the link is pointing to a src.rpm file"
66 if name[-8:] == ".src.rpm":
67 return True
68 else:
69 return False
70
71def package_name_from_srpm(srpm):
72 "Strip out the package name from the src.rpm filename"
73 strings = srpm.split('-')
74 package_name = strings[0]
75 for i in range(1, len (strings) - 1):
76 str = strings[i]
77 if not str[0].isdigit():
78 package_name += '-' + str
79 return package_name
80
81def clean_package_list(package_list):
82 "Removes multiple entries of packages and sorts the list"
83 set = {}
84 map(set.__setitem__, package_list, [])
85 return set.keys()
86
87
Patrick Williamsf1e5d692016-03-30 15:21:19 -050088def get_latest_released_meego_source_package_list(d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050089 "Returns list of all the name os packages in the latest meego distro"
90
91 package_names = []
92 try:
93 f = open("/tmp/Meego-1.1", "r")
94 for line in f:
95 package_names.append(line[:-1] + ":" + "main") # Also strip the '\n' at the end
96 except IOError: pass
97 package_list=clean_package_list(package_names)
98 return "1.0", package_list
99
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500100def get_source_package_list_from_url(url, section, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500101 "Return a sectioned list of package names from a URL list"
102
103 bb.note("Reading %s: %s" % (url, section))
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500104 links = get_links_from_url(url, d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500105 srpms = filter(is_src_rpm, links)
106 names_list = map(package_name_from_srpm, srpms)
107
108 new_pkgs = []
109 for pkgs in names_list:
110 new_pkgs.append(pkgs + ":" + section)
111
112 return new_pkgs
113
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500114def get_latest_released_fedora_source_package_list(d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500115 "Returns list of all the name os packages in the latest fedora distro"
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500116 latest = find_latest_numeric_release("http://archive.fedoraproject.org/pub/fedora/linux/releases/", d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500117
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500118 package_names = get_source_package_list_from_url("http://archive.fedoraproject.org/pub/fedora/linux/releases/%s/Fedora/source/SRPMS/" % latest, "main", d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500119
120# package_names += get_source_package_list_from_url("http://download.fedora.redhat.com/pub/fedora/linux/releases/%s/Everything/source/SPRMS/" % latest, "everything")
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500121 package_names += get_source_package_list_from_url("http://archive.fedoraproject.org/pub/fedora/linux/updates/%s/SRPMS/" % latest, "updates", d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500122
123 package_list=clean_package_list(package_names)
124
125 return latest, package_list
126
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500127def get_latest_released_opensuse_source_package_list(d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500128 "Returns list of all the name os packages in the latest opensuse distro"
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500129 latest = find_latest_numeric_release("http://download.opensuse.org/source/distribution/",d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500130
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500131 package_names = get_source_package_list_from_url("http://download.opensuse.org/source/distribution/%s/repo/oss/suse/src/" % latest, "main", d)
132 package_names += get_source_package_list_from_url("http://download.opensuse.org/update/%s/rpm/src/" % latest, "updates", d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500133
134 package_list=clean_package_list(package_names)
135 return latest, package_list
136
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500137def get_latest_released_mandriva_source_package_list(d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500138 "Returns list of all the name os packages in the latest mandriva distro"
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500139 latest = find_latest_numeric_release("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/", d)
140 package_names = get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/release/" % latest, "main", d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500141# package_names += get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/contrib/release/" % latest, "contrib")
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500142 package_names += get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/updates/" % latest, "updates", d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500143
144 package_list=clean_package_list(package_names)
145 return latest, package_list
146
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500147def find_latest_debian_release(url, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500148 "Find the latest listed debian release on the given url"
149
150 releases = []
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500151 for link in get_links_from_url(url, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500152 if link[:6] == "Debian":
153 if ';' not in link:
154 releases.append(link)
155 releases.sort()
156 try:
157 return releases.pop()[6:]
158 except:
159 return "_NotFound_"
160
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500161def get_debian_style_source_package_list(url, section, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500162 "Return the list of package-names stored in the debian style Sources.gz file"
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500163 with create_socket(url,d) as sock:
164 webpage = sock.read()
165 import tempfile
166 tmpfile = tempfile.NamedTemporaryFile(mode='wb', prefix='oecore.', suffix='.tmp', delete=False)
167 tmpfilename=tmpfile.name
168 tmpfile.write(sock.read())
169 tmpfile.close()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500170 import gzip
171 bb.note("Reading %s: %s" % (url, section))
172
173 f = gzip.open(tmpfilename)
174 package_names = []
175 for line in f:
176 if line[:9] == "Package: ":
177 package_names.append(line[9:-1] + ":" + section) # Also strip the '\n' at the end
178 os.unlink(tmpfilename)
179
180 return package_names
181
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500182def get_latest_released_debian_source_package_list(d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500183 "Returns list of all the name os packages in the latest debian distro"
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500184 latest = find_latest_debian_release("http://ftp.debian.org/debian/dists/", d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500185 url = "http://ftp.debian.org/debian/dists/stable/main/source/Sources.gz"
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500186 package_names = get_debian_style_source_package_list(url, "main", d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500187# url = "http://ftp.debian.org/debian/dists/stable/contrib/source/Sources.gz"
188# package_names += get_debian_style_source_package_list(url, "contrib")
189 url = "http://ftp.debian.org/debian/dists/stable-proposed-updates/main/source/Sources.gz"
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500190 package_names += get_debian_style_source_package_list(url, "updates", d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500191 package_list=clean_package_list(package_names)
192 return latest, package_list
193
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500194def find_latest_ubuntu_release(url, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500195 "Find the latest listed ubuntu release on the given url"
196 url += "?C=M;O=D" # Descending Sort by Last Modified
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500197 for link in get_links_from_url(url, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500198 if link[-8:] == "-updates":
199 return link[:-8]
200 return "_NotFound_"
201
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500202def get_latest_released_ubuntu_source_package_list(d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500203 "Returns list of all the name os packages in the latest ubuntu distro"
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500204 latest = find_latest_ubuntu_release("http://archive.ubuntu.com/ubuntu/dists/", d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500205 url = "http://archive.ubuntu.com/ubuntu/dists/%s/main/source/Sources.gz" % latest
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500206 package_names = get_debian_style_source_package_list(url, "main", d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500207# url = "http://archive.ubuntu.com/ubuntu/dists/%s/multiverse/source/Sources.gz" % latest
208# package_names += get_debian_style_source_package_list(url, "multiverse")
209# url = "http://archive.ubuntu.com/ubuntu/dists/%s/universe/source/Sources.gz" % latest
210# package_names += get_debian_style_source_package_list(url, "universe")
211 url = "http://archive.ubuntu.com/ubuntu/dists/%s-updates/main/source/Sources.gz" % latest
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500212 package_names += get_debian_style_source_package_list(url, "updates", d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500213 package_list=clean_package_list(package_names)
214 return latest, package_list
215
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500216def create_distro_packages_list(distro_check_dir, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500217 pkglst_dir = os.path.join(distro_check_dir, "package_lists")
218 if not os.path.isdir (pkglst_dir):
219 os.makedirs(pkglst_dir)
220 # first clear old stuff
221 for file in os.listdir(pkglst_dir):
222 os.unlink(os.path.join(pkglst_dir, file))
223
224 per_distro_functions = [
225 ["Debian", get_latest_released_debian_source_package_list],
226 ["Ubuntu", get_latest_released_ubuntu_source_package_list],
227 ["Fedora", get_latest_released_fedora_source_package_list],
228 ["OpenSuSE", get_latest_released_opensuse_source_package_list],
229 ["Mandriva", get_latest_released_mandriva_source_package_list],
230 ["Meego", get_latest_released_meego_source_package_list]
231 ]
232
233 from datetime import datetime
234 begin = datetime.now()
235 for distro in per_distro_functions:
236 name = distro[0]
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500237 release, package_list = distro[1](d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500238 bb.note("Distro: %s, Latest Release: %s, # src packages: %d" % (name, release, len(package_list)))
239 package_list_file = os.path.join(pkglst_dir, name + "-" + release)
240 f = open(package_list_file, "w+b")
241 for pkg in package_list:
242 f.write(pkg + "\n")
243 f.close()
244 end = datetime.now()
245 delta = end - begin
246 bb.note("package_list generatiosn took this much time: %d seconds" % delta.seconds)
247
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500248def update_distro_data(distro_check_dir, datetime, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500249 """
250 If distro packages list data is old then rebuild it.
251 The operations has to be protected by a lock so that
252 only one thread performes it at a time.
253 """
254 if not os.path.isdir (distro_check_dir):
255 try:
256 bb.note ("Making new directory: %s" % distro_check_dir)
257 os.makedirs (distro_check_dir)
258 except OSError:
259 raise Exception('Unable to create directory %s' % (distro_check_dir))
260
261
262 datetime_file = os.path.join(distro_check_dir, "build_datetime")
263 saved_datetime = "_invalid_"
264 import fcntl
265 try:
266 if not os.path.exists(datetime_file):
267 open(datetime_file, 'w+b').close() # touch the file so that the next open won't fail
268
269 f = open(datetime_file, "r+b")
270 fcntl.lockf(f, fcntl.LOCK_EX)
271 saved_datetime = f.read()
272 if saved_datetime[0:8] != datetime[0:8]:
273 bb.note("The build datetime did not match: saved:%s current:%s" % (saved_datetime, datetime))
274 bb.note("Regenerating distro package lists")
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500275 create_distro_packages_list(distro_check_dir, d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500276 f.seek(0)
277 f.write(datetime)
278
279 except OSError:
280 raise Exception('Unable to read/write this file: %s' % (datetime_file))
281 finally:
282 fcntl.lockf(f, fcntl.LOCK_UN)
283 f.close()
284
285def compare_in_distro_packages_list(distro_check_dir, d):
286 if not os.path.isdir(distro_check_dir):
287 raise Exception("compare_in_distro_packages_list: invalid distro_check_dir passed")
288
289 localdata = bb.data.createCopy(d)
290 pkglst_dir = os.path.join(distro_check_dir, "package_lists")
291 matching_distros = []
292 pn = d.getVar('PN', True)
293 recipe_name = d.getVar('PN', True)
294 bb.note("Checking: %s" % pn)
295
296 trim_dict = dict({"-native":"-native", "-cross":"-cross", "-initial":"-initial"})
297
298 if pn.find("-native") != -1:
299 pnstripped = pn.split("-native")
300 localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True))
301 bb.data.update_data(localdata)
302 recipe_name = pnstripped[0]
303
304 if pn.startswith("nativesdk-"):
305 pnstripped = pn.split("nativesdk-")
306 localdata.setVar('OVERRIDES', "pn-" + pnstripped[1] + ":" + d.getVar('OVERRIDES', True))
307 bb.data.update_data(localdata)
308 recipe_name = pnstripped[1]
309
310 if pn.find("-cross") != -1:
311 pnstripped = pn.split("-cross")
312 localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True))
313 bb.data.update_data(localdata)
314 recipe_name = pnstripped[0]
315
316 if pn.find("-initial") != -1:
317 pnstripped = pn.split("-initial")
318 localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True))
319 bb.data.update_data(localdata)
320 recipe_name = pnstripped[0]
321
322 bb.note("Recipe: %s" % recipe_name)
323 tmp = localdata.getVar('DISTRO_PN_ALIAS', True)
324
325 distro_exceptions = dict({"OE-Core":'OE-Core', "OpenedHand":'OpenedHand', "Intel":'Intel', "Upstream":'Upstream', "Windriver":'Windriver', "OSPDT":'OSPDT Approved', "Poky":'poky'})
326
327 if tmp:
328 list = tmp.split(' ')
329 for str in list:
330 if str and str.find("=") == -1 and distro_exceptions[str]:
331 matching_distros.append(str)
332
333 distro_pn_aliases = {}
334 if tmp:
335 list = tmp.split(' ')
336 for str in list:
337 if str.find("=") != -1:
338 (dist, pn_alias) = str.split('=')
339 distro_pn_aliases[dist.strip().lower()] = pn_alias.strip()
340
341 for file in os.listdir(pkglst_dir):
342 (distro, distro_release) = file.split("-")
343 f = open(os.path.join(pkglst_dir, file), "rb")
344 for line in f:
345 (pkg, section) = line.split(":")
346 if distro.lower() in distro_pn_aliases:
347 pn = distro_pn_aliases[distro.lower()]
348 else:
349 pn = recipe_name
350 if pn == pkg:
351 matching_distros.append(distro + "-" + section[:-1]) # strip the \n at the end
352 f.close()
353 break
354 f.close()
355
356
357 if tmp != None:
358 list = tmp.split(' ')
359 for item in list:
360 matching_distros.append(item)
361 bb.note("Matching: %s" % matching_distros)
362 return matching_distros
363
364def create_log_file(d, logname):
365 import subprocess
366 logpath = d.getVar('LOG_DIR', True)
367 bb.utils.mkdirhier(logpath)
368 logfn, logsuffix = os.path.splitext(logname)
369 logfile = os.path.join(logpath, "%s.%s%s" % (logfn, d.getVar('DATETIME', True), logsuffix))
370 if not os.path.exists(logfile):
371 slogfile = os.path.join(logpath, logname)
372 if os.path.exists(slogfile):
373 os.remove(slogfile)
374 subprocess.call("touch %s" % logfile, shell=True)
375 os.symlink(logfile, slogfile)
376 d.setVar('LOG_FILE', logfile)
377 return logfile
378
379
380def save_distro_check_result(result, datetime, result_file, d):
381 pn = d.getVar('PN', True)
382 logdir = d.getVar('LOG_DIR', True)
383 if not logdir:
384 bb.error("LOG_DIR variable is not defined, can't write the distro_check results")
385 return
386 if not os.path.isdir(logdir):
387 os.makedirs(logdir)
388 line = pn
389 for i in result:
390 line = line + "," + i
391 f = open(result_file, "a")
392 import fcntl
393 fcntl.lockf(f, fcntl.LOCK_EX)
394 f.seek(0, os.SEEK_END) # seek to the end of file
395 f.write(line + "\n")
396 fcntl.lockf(f, fcntl.LOCK_UN)
397 f.close()