blob: 8ed5b0ec80ee54173cdb3d0e47d7a452a8fea360 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001def get_links_from_url(url):
2 "Return all the href links found on the web location"
3
4 import urllib, sgmllib
5
6 class LinksParser(sgmllib.SGMLParser):
7 def parse(self, s):
8 "Parse the given string 's'."
9 self.feed(s)
10 self.close()
11
12 def __init__(self, verbose=0):
13 "Initialise an object passing 'verbose' to the superclass."
14 sgmllib.SGMLParser.__init__(self, verbose)
15 self.hyperlinks = []
16
17 def start_a(self, attributes):
18 "Process a hyperlink and its 'attributes'."
19 for name, value in attributes:
20 if name == "href":
21 self.hyperlinks.append(value.strip('/'))
22
23 def get_hyperlinks(self):
24 "Return the list of hyperlinks."
25 return self.hyperlinks
26
27 sock = urllib.urlopen(url)
28 webpage = sock.read()
29 sock.close()
30
31 linksparser = LinksParser()
32 linksparser.parse(webpage)
33 return linksparser.get_hyperlinks()
34
35def find_latest_numeric_release(url):
36 "Find the latest listed numeric release on the given url"
37 max=0
38 maxstr=""
39 for link in get_links_from_url(url):
40 try:
41 release = float(link)
42 except:
43 release = 0
44 if release > max:
45 max = release
46 maxstr = link
47 return maxstr
48
49def is_src_rpm(name):
50 "Check if the link is pointing to a src.rpm file"
51 if name[-8:] == ".src.rpm":
52 return True
53 else:
54 return False
55
56def package_name_from_srpm(srpm):
57 "Strip out the package name from the src.rpm filename"
58 strings = srpm.split('-')
59 package_name = strings[0]
60 for i in range(1, len (strings) - 1):
61 str = strings[i]
62 if not str[0].isdigit():
63 package_name += '-' + str
64 return package_name
65
66def clean_package_list(package_list):
67 "Removes multiple entries of packages and sorts the list"
68 set = {}
69 map(set.__setitem__, package_list, [])
70 return set.keys()
71
72
73def get_latest_released_meego_source_package_list():
74 "Returns list of all the name os packages in the latest meego distro"
75
76 package_names = []
77 try:
78 f = open("/tmp/Meego-1.1", "r")
79 for line in f:
80 package_names.append(line[:-1] + ":" + "main") # Also strip the '\n' at the end
81 except IOError: pass
82 package_list=clean_package_list(package_names)
83 return "1.0", package_list
84
85def get_source_package_list_from_url(url, section):
86 "Return a sectioned list of package names from a URL list"
87
88 bb.note("Reading %s: %s" % (url, section))
89 links = get_links_from_url(url)
90 srpms = filter(is_src_rpm, links)
91 names_list = map(package_name_from_srpm, srpms)
92
93 new_pkgs = []
94 for pkgs in names_list:
95 new_pkgs.append(pkgs + ":" + section)
96
97 return new_pkgs
98
99def get_latest_released_fedora_source_package_list():
100 "Returns list of all the name os packages in the latest fedora distro"
101 latest = find_latest_numeric_release("http://archive.fedoraproject.org/pub/fedora/linux/releases/")
102
103 package_names = get_source_package_list_from_url("http://archive.fedoraproject.org/pub/fedora/linux/releases/%s/Fedora/source/SRPMS/" % latest, "main")
104
105# package_names += get_source_package_list_from_url("http://download.fedora.redhat.com/pub/fedora/linux/releases/%s/Everything/source/SPRMS/" % latest, "everything")
106 package_names += get_source_package_list_from_url("http://archive.fedoraproject.org/pub/fedora/linux/updates/%s/SRPMS/" % latest, "updates")
107
108 package_list=clean_package_list(package_names)
109
110 return latest, package_list
111
112def get_latest_released_opensuse_source_package_list():
113 "Returns list of all the name os packages in the latest opensuse distro"
114 latest = find_latest_numeric_release("http://download.opensuse.org/source/distribution/")
115
116 package_names = get_source_package_list_from_url("http://download.opensuse.org/source/distribution/%s/repo/oss/suse/src/" % latest, "main")
117 package_names += get_source_package_list_from_url("http://download.opensuse.org/update/%s/rpm/src/" % latest, "updates")
118
119 package_list=clean_package_list(package_names)
120 return latest, package_list
121
122def get_latest_released_mandriva_source_package_list():
123 "Returns list of all the name os packages in the latest mandriva distro"
124 latest = find_latest_numeric_release("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/")
125 package_names = get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/release/" % latest, "main")
126# package_names += get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/contrib/release/" % latest, "contrib")
127 package_names += get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/updates/" % latest, "updates")
128
129 package_list=clean_package_list(package_names)
130 return latest, package_list
131
132def find_latest_debian_release(url):
133 "Find the latest listed debian release on the given url"
134
135 releases = []
136 for link in get_links_from_url(url):
137 if link[:6] == "Debian":
138 if ';' not in link:
139 releases.append(link)
140 releases.sort()
141 try:
142 return releases.pop()[6:]
143 except:
144 return "_NotFound_"
145
146def get_debian_style_source_package_list(url, section):
147 "Return the list of package-names stored in the debian style Sources.gz file"
148 import urllib
149 sock = urllib.urlopen(url)
150 import tempfile
151 tmpfile = tempfile.NamedTemporaryFile(mode='wb', prefix='oecore.', suffix='.tmp', delete=False)
152 tmpfilename=tmpfile.name
153 tmpfile.write(sock.read())
154 sock.close()
155 tmpfile.close()
156 import gzip
157 bb.note("Reading %s: %s" % (url, section))
158
159 f = gzip.open(tmpfilename)
160 package_names = []
161 for line in f:
162 if line[:9] == "Package: ":
163 package_names.append(line[9:-1] + ":" + section) # Also strip the '\n' at the end
164 os.unlink(tmpfilename)
165
166 return package_names
167
168def get_latest_released_debian_source_package_list():
169 "Returns list of all the name os packages in the latest debian distro"
170 latest = find_latest_debian_release("http://ftp.debian.org/debian/dists/")
171 url = "http://ftp.debian.org/debian/dists/stable/main/source/Sources.gz"
172 package_names = get_debian_style_source_package_list(url, "main")
173# url = "http://ftp.debian.org/debian/dists/stable/contrib/source/Sources.gz"
174# package_names += get_debian_style_source_package_list(url, "contrib")
175 url = "http://ftp.debian.org/debian/dists/stable-proposed-updates/main/source/Sources.gz"
176 package_names += get_debian_style_source_package_list(url, "updates")
177 package_list=clean_package_list(package_names)
178 return latest, package_list
179
180def find_latest_ubuntu_release(url):
181 "Find the latest listed ubuntu release on the given url"
182 url += "?C=M;O=D" # Descending Sort by Last Modified
183 for link in get_links_from_url(url):
184 if link[-8:] == "-updates":
185 return link[:-8]
186 return "_NotFound_"
187
188def get_latest_released_ubuntu_source_package_list():
189 "Returns list of all the name os packages in the latest ubuntu distro"
190 latest = find_latest_ubuntu_release("http://archive.ubuntu.com/ubuntu/dists/")
191 url = "http://archive.ubuntu.com/ubuntu/dists/%s/main/source/Sources.gz" % latest
192 package_names = get_debian_style_source_package_list(url, "main")
193# url = "http://archive.ubuntu.com/ubuntu/dists/%s/multiverse/source/Sources.gz" % latest
194# package_names += get_debian_style_source_package_list(url, "multiverse")
195# url = "http://archive.ubuntu.com/ubuntu/dists/%s/universe/source/Sources.gz" % latest
196# package_names += get_debian_style_source_package_list(url, "universe")
197 url = "http://archive.ubuntu.com/ubuntu/dists/%s-updates/main/source/Sources.gz" % latest
198 package_names += get_debian_style_source_package_list(url, "updates")
199 package_list=clean_package_list(package_names)
200 return latest, package_list
201
202def create_distro_packages_list(distro_check_dir):
203 pkglst_dir = os.path.join(distro_check_dir, "package_lists")
204 if not os.path.isdir (pkglst_dir):
205 os.makedirs(pkglst_dir)
206 # first clear old stuff
207 for file in os.listdir(pkglst_dir):
208 os.unlink(os.path.join(pkglst_dir, file))
209
210 per_distro_functions = [
211 ["Debian", get_latest_released_debian_source_package_list],
212 ["Ubuntu", get_latest_released_ubuntu_source_package_list],
213 ["Fedora", get_latest_released_fedora_source_package_list],
214 ["OpenSuSE", get_latest_released_opensuse_source_package_list],
215 ["Mandriva", get_latest_released_mandriva_source_package_list],
216 ["Meego", get_latest_released_meego_source_package_list]
217 ]
218
219 from datetime import datetime
220 begin = datetime.now()
221 for distro in per_distro_functions:
222 name = distro[0]
223 release, package_list = distro[1]()
224 bb.note("Distro: %s, Latest Release: %s, # src packages: %d" % (name, release, len(package_list)))
225 package_list_file = os.path.join(pkglst_dir, name + "-" + release)
226 f = open(package_list_file, "w+b")
227 for pkg in package_list:
228 f.write(pkg + "\n")
229 f.close()
230 end = datetime.now()
231 delta = end - begin
232 bb.note("package_list generatiosn took this much time: %d seconds" % delta.seconds)
233
234def update_distro_data(distro_check_dir, datetime):
235 """
236 If distro packages list data is old then rebuild it.
237 The operations has to be protected by a lock so that
238 only one thread performes it at a time.
239 """
240 if not os.path.isdir (distro_check_dir):
241 try:
242 bb.note ("Making new directory: %s" % distro_check_dir)
243 os.makedirs (distro_check_dir)
244 except OSError:
245 raise Exception('Unable to create directory %s' % (distro_check_dir))
246
247
248 datetime_file = os.path.join(distro_check_dir, "build_datetime")
249 saved_datetime = "_invalid_"
250 import fcntl
251 try:
252 if not os.path.exists(datetime_file):
253 open(datetime_file, 'w+b').close() # touch the file so that the next open won't fail
254
255 f = open(datetime_file, "r+b")
256 fcntl.lockf(f, fcntl.LOCK_EX)
257 saved_datetime = f.read()
258 if saved_datetime[0:8] != datetime[0:8]:
259 bb.note("The build datetime did not match: saved:%s current:%s" % (saved_datetime, datetime))
260 bb.note("Regenerating distro package lists")
261 create_distro_packages_list(distro_check_dir)
262 f.seek(0)
263 f.write(datetime)
264
265 except OSError:
266 raise Exception('Unable to read/write this file: %s' % (datetime_file))
267 finally:
268 fcntl.lockf(f, fcntl.LOCK_UN)
269 f.close()
270
271def compare_in_distro_packages_list(distro_check_dir, d):
272 if not os.path.isdir(distro_check_dir):
273 raise Exception("compare_in_distro_packages_list: invalid distro_check_dir passed")
274
275 localdata = bb.data.createCopy(d)
276 pkglst_dir = os.path.join(distro_check_dir, "package_lists")
277 matching_distros = []
278 pn = d.getVar('PN', True)
279 recipe_name = d.getVar('PN', True)
280 bb.note("Checking: %s" % pn)
281
282 trim_dict = dict({"-native":"-native", "-cross":"-cross", "-initial":"-initial"})
283
284 if pn.find("-native") != -1:
285 pnstripped = pn.split("-native")
286 localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True))
287 bb.data.update_data(localdata)
288 recipe_name = pnstripped[0]
289
290 if pn.startswith("nativesdk-"):
291 pnstripped = pn.split("nativesdk-")
292 localdata.setVar('OVERRIDES', "pn-" + pnstripped[1] + ":" + d.getVar('OVERRIDES', True))
293 bb.data.update_data(localdata)
294 recipe_name = pnstripped[1]
295
296 if pn.find("-cross") != -1:
297 pnstripped = pn.split("-cross")
298 localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True))
299 bb.data.update_data(localdata)
300 recipe_name = pnstripped[0]
301
302 if pn.find("-initial") != -1:
303 pnstripped = pn.split("-initial")
304 localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True))
305 bb.data.update_data(localdata)
306 recipe_name = pnstripped[0]
307
308 bb.note("Recipe: %s" % recipe_name)
309 tmp = localdata.getVar('DISTRO_PN_ALIAS', True)
310
311 distro_exceptions = dict({"OE-Core":'OE-Core', "OpenedHand":'OpenedHand', "Intel":'Intel', "Upstream":'Upstream', "Windriver":'Windriver', "OSPDT":'OSPDT Approved', "Poky":'poky'})
312
313 if tmp:
314 list = tmp.split(' ')
315 for str in list:
316 if str and str.find("=") == -1 and distro_exceptions[str]:
317 matching_distros.append(str)
318
319 distro_pn_aliases = {}
320 if tmp:
321 list = tmp.split(' ')
322 for str in list:
323 if str.find("=") != -1:
324 (dist, pn_alias) = str.split('=')
325 distro_pn_aliases[dist.strip().lower()] = pn_alias.strip()
326
327 for file in os.listdir(pkglst_dir):
328 (distro, distro_release) = file.split("-")
329 f = open(os.path.join(pkglst_dir, file), "rb")
330 for line in f:
331 (pkg, section) = line.split(":")
332 if distro.lower() in distro_pn_aliases:
333 pn = distro_pn_aliases[distro.lower()]
334 else:
335 pn = recipe_name
336 if pn == pkg:
337 matching_distros.append(distro + "-" + section[:-1]) # strip the \n at the end
338 f.close()
339 break
340 f.close()
341
342
343 if tmp != None:
344 list = tmp.split(' ')
345 for item in list:
346 matching_distros.append(item)
347 bb.note("Matching: %s" % matching_distros)
348 return matching_distros
349
350def create_log_file(d, logname):
351 import subprocess
352 logpath = d.getVar('LOG_DIR', True)
353 bb.utils.mkdirhier(logpath)
354 logfn, logsuffix = os.path.splitext(logname)
355 logfile = os.path.join(logpath, "%s.%s%s" % (logfn, d.getVar('DATETIME', True), logsuffix))
356 if not os.path.exists(logfile):
357 slogfile = os.path.join(logpath, logname)
358 if os.path.exists(slogfile):
359 os.remove(slogfile)
360 subprocess.call("touch %s" % logfile, shell=True)
361 os.symlink(logfile, slogfile)
362 d.setVar('LOG_FILE', logfile)
363 return logfile
364
365
366def save_distro_check_result(result, datetime, result_file, d):
367 pn = d.getVar('PN', True)
368 logdir = d.getVar('LOG_DIR', True)
369 if not logdir:
370 bb.error("LOG_DIR variable is not defined, can't write the distro_check results")
371 return
372 if not os.path.isdir(logdir):
373 os.makedirs(logdir)
374 line = pn
375 for i in result:
376 line = line + "," + i
377 f = open(result_file, "a")
378 import fcntl
379 fcntl.lockf(f, fcntl.LOCK_EX)
380 f.seek(0, os.SEEK_END) # seek to the end of file
381 f.write(line + "\n")
382 fcntl.lockf(f, fcntl.LOCK_UN)
383 f.close()