blob: 88e46c354d21358cabae53558ba14ca0990854f1 [file] [log] [blame]
Brad Bishopc342db32019-05-15 21:57:59 -04001#
2# SPDX-License-Identifier: GPL-2.0-only
3#
4
Patrick Williamsf1e5d692016-03-30 15:21:19 -05005def create_socket(url, d):
6 import urllib
Brad Bishop6e60e8b2018-02-01 10:27:11 -05007 from bb.utils import export_proxies
Patrick Williamsf1e5d692016-03-30 15:21:19 -05008
Brad Bishop6e60e8b2018-02-01 10:27:11 -05009 export_proxies(d)
10 return urllib.request.urlopen(url)
Patrick Williamsf1e5d692016-03-30 15:21:19 -050011
12def get_links_from_url(url, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050013 "Return all the href links found on the web location"
14
Patrick Williamsc0f7c042017-02-23 20:41:17 -060015 from bs4 import BeautifulSoup, SoupStrainer
Patrick Williamsc124f4f2015-09-15 14:41:29 -050016
Brad Bishop6e60e8b2018-02-01 10:27:11 -050017 soup = BeautifulSoup(create_socket(url,d), "html.parser", parse_only=SoupStrainer("a"))
Patrick Williamsc0f7c042017-02-23 20:41:17 -060018 hyperlinks = []
Patrick Williamsc0f7c042017-02-23 20:41:17 -060019 for line in soup.find_all('a', href=True):
20 hyperlinks.append(line['href'].strip('/'))
21 return hyperlinks
Patrick Williamsc124f4f2015-09-15 14:41:29 -050022
Patrick Williamsf1e5d692016-03-30 15:21:19 -050023def find_latest_numeric_release(url, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050024 "Find the latest listed numeric release on the given url"
25 max=0
26 maxstr=""
Patrick Williamsf1e5d692016-03-30 15:21:19 -050027 for link in get_links_from_url(url, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050028 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -050029 # TODO use LooseVersion
Patrick Williamsc124f4f2015-09-15 14:41:29 -050030 release = float(link)
31 except:
32 release = 0
33 if release > max:
34 max = release
35 maxstr = link
36 return maxstr
37
38def is_src_rpm(name):
39 "Check if the link is pointing to a src.rpm file"
Brad Bishop6e60e8b2018-02-01 10:27:11 -050040 return name.endswith(".src.rpm")
Patrick Williamsc124f4f2015-09-15 14:41:29 -050041
42def package_name_from_srpm(srpm):
43 "Strip out the package name from the src.rpm filename"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050044
Brad Bishop6e60e8b2018-02-01 10:27:11 -050045 # ca-certificates-2016.2.7-1.0.fc24.src.rpm
46 # ^name ^ver ^release^removed
47 (name, version, release) = srpm.replace(".src.rpm", "").rsplit("-", 2)
48 return name
Patrick Williamsc124f4f2015-09-15 14:41:29 -050049
Patrick Williamsf1e5d692016-03-30 15:21:19 -050050def get_source_package_list_from_url(url, section, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050051 "Return a sectioned list of package names from a URL list"
52
53 bb.note("Reading %s: %s" % (url, section))
Patrick Williamsf1e5d692016-03-30 15:21:19 -050054 links = get_links_from_url(url, d)
Brad Bishop6e60e8b2018-02-01 10:27:11 -050055 srpms = filter(is_src_rpm, links)
56 names_list = map(package_name_from_srpm, srpms)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050057
Brad Bishop6e60e8b2018-02-01 10:27:11 -050058 new_pkgs = set()
Patrick Williamsc124f4f2015-09-15 14:41:29 -050059 for pkgs in names_list:
Brad Bishop6e60e8b2018-02-01 10:27:11 -050060 new_pkgs.add(pkgs + ":" + section)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050061 return new_pkgs
62
Brad Bishop6e60e8b2018-02-01 10:27:11 -050063def get_source_package_list_from_url_by_letter(url, section, d):
64 import string
65 from urllib.error import HTTPError
66 packages = set()
67 for letter in (string.ascii_lowercase + string.digits):
68 # Not all subfolders may exist, so silently handle 404
69 try:
70 packages |= get_source_package_list_from_url(url + "/" + letter, section, d)
71 except HTTPError as e:
72 if e.code != 404: raise
73 return packages
74
Patrick Williamsf1e5d692016-03-30 15:21:19 -050075def get_latest_released_fedora_source_package_list(d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050076 "Returns list of all the name os packages in the latest fedora distro"
Patrick Williamsf1e5d692016-03-30 15:21:19 -050077 latest = find_latest_numeric_release("http://archive.fedoraproject.org/pub/fedora/linux/releases/", d)
Brad Bishop6e60e8b2018-02-01 10:27:11 -050078 package_names = get_source_package_list_from_url_by_letter("http://archive.fedoraproject.org/pub/fedora/linux/releases/%s/Everything/source/tree/Packages/" % latest, "main", d)
79 package_names |= get_source_package_list_from_url_by_letter("http://archive.fedoraproject.org/pub/fedora/linux/updates/%s/SRPMS/" % latest, "updates", d)
80 return latest, package_names
Patrick Williamsc124f4f2015-09-15 14:41:29 -050081
Patrick Williamsf1e5d692016-03-30 15:21:19 -050082def get_latest_released_opensuse_source_package_list(d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050083 "Returns list of all the name os packages in the latest opensuse distro"
Brad Bishopd7bf8c12018-02-25 22:55:05 -050084 latest = find_latest_numeric_release("http://download.opensuse.org/source/distribution/leap", d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050085
Brad Bishopd7bf8c12018-02-25 22:55:05 -050086 package_names = get_source_package_list_from_url("http://download.opensuse.org/source/distribution/leap/%s/repo/oss/suse/src/" % latest, "main", d)
87 package_names |= get_source_package_list_from_url("http://download.opensuse.org/update/leap/%s/oss/src/" % latest, "updates", d)
Brad Bishop6e60e8b2018-02-01 10:27:11 -050088 return latest, package_names
Patrick Williamsc124f4f2015-09-15 14:41:29 -050089
Brad Bishop6e60e8b2018-02-01 10:27:11 -050090def get_latest_released_clear_source_package_list(d):
91 latest = find_latest_numeric_release("https://download.clearlinux.org/releases/", d)
92 package_names = get_source_package_list_from_url("https://download.clearlinux.org/releases/%s/clear/source/SRPMS/" % latest, "main", d)
93 return latest, package_names
Patrick Williamsc124f4f2015-09-15 14:41:29 -050094
Patrick Williamsf1e5d692016-03-30 15:21:19 -050095def find_latest_debian_release(url, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050096 "Find the latest listed debian release on the given url"
97
Brad Bishop6e60e8b2018-02-01 10:27:11 -050098 releases = [link.replace("Debian", "")
99 for link in get_links_from_url(url, d)
100 if link.startswith("Debian")]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500101 releases.sort()
102 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500103 return releases[-1]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500104 except:
105 return "_NotFound_"
106
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500107def get_debian_style_source_package_list(url, section, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500108 "Return the list of package-names stored in the debian style Sources.gz file"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500109 import gzip
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600110
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500111 package_names = set()
112 for line in gzip.open(create_socket(url, d), mode="rt"):
113 if line.startswith("Package:"):
114 pkg = line.split(":", 1)[1].strip()
115 package_names.add(pkg + ":" + section)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500116 return package_names
117
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500118def get_latest_released_debian_source_package_list(d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500119 "Returns list of all the name of packages in the latest debian distro"
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500120 latest = find_latest_debian_release("http://ftp.debian.org/debian/dists/", d)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500121 url = "http://ftp.debian.org/debian/dists/stable/main/source/Sources.gz"
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500122 package_names = get_debian_style_source_package_list(url, "main", d)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500123 url = "http://ftp.debian.org/debian/dists/stable-proposed-updates/main/source/Sources.gz"
124 package_names |= get_debian_style_source_package_list(url, "updates", d)
125 return latest, package_names
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500126
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500127def find_latest_ubuntu_release(url, d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500128 """
129 Find the latest listed Ubuntu release on the given ubuntu/dists/ URL.
130
131 To avoid matching development releases look for distributions that have
132 updates, so the resulting distro could be any supported release.
133 """
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500134 url += "?C=M;O=D" # Descending Sort by Last Modified
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500135 for link in get_links_from_url(url, d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500136 if "-updates" in link:
137 distro = link.replace("-updates", "")
138 return distro
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500139 return "_NotFound_"
140
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500141def get_latest_released_ubuntu_source_package_list(d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500142 "Returns list of all the name os packages in the latest ubuntu distro"
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500143 latest = find_latest_ubuntu_release("http://archive.ubuntu.com/ubuntu/dists/", d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500144 url = "http://archive.ubuntu.com/ubuntu/dists/%s/main/source/Sources.gz" % latest
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500145 package_names = get_debian_style_source_package_list(url, "main", d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500146 url = "http://archive.ubuntu.com/ubuntu/dists/%s-updates/main/source/Sources.gz" % latest
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500147 package_names |= get_debian_style_source_package_list(url, "updates", d)
148 return latest, package_names
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500149
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500150def create_distro_packages_list(distro_check_dir, d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500151 import shutil
152
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500153 pkglst_dir = os.path.join(distro_check_dir, "package_lists")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500154 bb.utils.remove(pkglst_dir, True)
155 bb.utils.mkdirhier(pkglst_dir)
156
157 per_distro_functions = (
158 ("Debian", get_latest_released_debian_source_package_list),
159 ("Ubuntu", get_latest_released_ubuntu_source_package_list),
160 ("Fedora", get_latest_released_fedora_source_package_list),
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500161 ("openSUSE", get_latest_released_opensuse_source_package_list),
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500162 ("Clear", get_latest_released_clear_source_package_list),
163 )
164
165 for name, fetcher_func in per_distro_functions:
166 try:
167 release, package_list = fetcher_func(d)
168 except Exception as e:
169 bb.warn("Cannot fetch packages for %s: %s" % (name, e))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500170 bb.note("Distro: %s, Latest Release: %s, # src packages: %d" % (name, release, len(package_list)))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500171 if len(package_list) == 0:
172 bb.error("Didn't fetch any packages for %s %s" % (name, release))
173
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500174 package_list_file = os.path.join(pkglst_dir, name + "-" + release)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500175 with open(package_list_file, 'w') as f:
176 for pkg in sorted(package_list):
177 f.write(pkg + "\n")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500178
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500179def update_distro_data(distro_check_dir, datetime, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500180 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500181 If distro packages list data is old then rebuild it.
182 The operations has to be protected by a lock so that
183 only one thread performes it at a time.
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500184 """
185 if not os.path.isdir (distro_check_dir):
186 try:
187 bb.note ("Making new directory: %s" % distro_check_dir)
188 os.makedirs (distro_check_dir)
189 except OSError:
190 raise Exception('Unable to create directory %s' % (distro_check_dir))
191
192
193 datetime_file = os.path.join(distro_check_dir, "build_datetime")
194 saved_datetime = "_invalid_"
195 import fcntl
196 try:
197 if not os.path.exists(datetime_file):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600198 open(datetime_file, 'w+').close() # touch the file so that the next open won't fail
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500199
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600200 f = open(datetime_file, "r+")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500201 fcntl.lockf(f, fcntl.LOCK_EX)
202 saved_datetime = f.read()
203 if saved_datetime[0:8] != datetime[0:8]:
204 bb.note("The build datetime did not match: saved:%s current:%s" % (saved_datetime, datetime))
205 bb.note("Regenerating distro package lists")
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500206 create_distro_packages_list(distro_check_dir, d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500207 f.seek(0)
208 f.write(datetime)
209
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500210 except OSError as e:
211 raise Exception('Unable to open timestamp: %s' % e)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500212 finally:
213 fcntl.lockf(f, fcntl.LOCK_UN)
214 f.close()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500215
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500216def compare_in_distro_packages_list(distro_check_dir, d):
217 if not os.path.isdir(distro_check_dir):
218 raise Exception("compare_in_distro_packages_list: invalid distro_check_dir passed")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500219
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500220 localdata = bb.data.createCopy(d)
221 pkglst_dir = os.path.join(distro_check_dir, "package_lists")
222 matching_distros = []
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500223 pn = recipe_name = d.getVar('PN')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500224 bb.note("Checking: %s" % pn)
225
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500226 if pn.find("-native") != -1:
227 pnstripped = pn.split("-native")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500228 localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500229 recipe_name = pnstripped[0]
230
231 if pn.startswith("nativesdk-"):
232 pnstripped = pn.split("nativesdk-")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500233 localdata.setVar('OVERRIDES', "pn-" + pnstripped[1] + ":" + d.getVar('OVERRIDES'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500234 recipe_name = pnstripped[1]
235
236 if pn.find("-cross") != -1:
237 pnstripped = pn.split("-cross")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500238 localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500239 recipe_name = pnstripped[0]
240
241 if pn.find("-initial") != -1:
242 pnstripped = pn.split("-initial")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500243 localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500244 recipe_name = pnstripped[0]
245
246 bb.note("Recipe: %s" % recipe_name)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500247
248 distro_exceptions = dict({"OE-Core":'OE-Core', "OpenedHand":'OpenedHand', "Intel":'Intel', "Upstream":'Upstream', "Windriver":'Windriver', "OSPDT":'OSPDT Approved', "Poky":'poky'})
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500249 tmp = localdata.getVar('DISTRO_PN_ALIAS') or ""
250 for str in tmp.split():
251 if str and str.find("=") == -1 and distro_exceptions[str]:
252 matching_distros.append(str)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500253
254 distro_pn_aliases = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500255 for str in tmp.split():
256 if "=" in str:
257 (dist, pn_alias) = str.split('=')
258 distro_pn_aliases[dist.strip().lower()] = pn_alias.strip()
259
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500260 for file in os.listdir(pkglst_dir):
261 (distro, distro_release) = file.split("-")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500262 f = open(os.path.join(pkglst_dir, file), "r")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500263 for line in f:
264 (pkg, section) = line.split(":")
265 if distro.lower() in distro_pn_aliases:
266 pn = distro_pn_aliases[distro.lower()]
267 else:
268 pn = recipe_name
269 if pn == pkg:
270 matching_distros.append(distro + "-" + section[:-1]) # strip the \n at the end
271 f.close()
272 break
273 f.close()
274
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500275 for item in tmp.split():
276 matching_distros.append(item)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500277 bb.note("Matching: %s" % matching_distros)
278 return matching_distros
279
280def create_log_file(d, logname):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500281 logpath = d.getVar('LOG_DIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500282 bb.utils.mkdirhier(logpath)
283 logfn, logsuffix = os.path.splitext(logname)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500284 logfile = os.path.join(logpath, "%s.%s%s" % (logfn, d.getVar('DATETIME'), logsuffix))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500285 if not os.path.exists(logfile):
286 slogfile = os.path.join(logpath, logname)
287 if os.path.exists(slogfile):
288 os.remove(slogfile)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500289 open(logfile, 'w+').close()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500290 os.symlink(logfile, slogfile)
291 d.setVar('LOG_FILE', logfile)
292 return logfile
293
294
295def save_distro_check_result(result, datetime, result_file, d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500296 pn = d.getVar('PN')
297 logdir = d.getVar('LOG_DIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500298 if not logdir:
299 bb.error("LOG_DIR variable is not defined, can't write the distro_check results")
300 return
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500301 bb.utils.mkdirhier(logdir)
302
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500303 line = pn
304 for i in result:
305 line = line + "," + i
306 f = open(result_file, "a")
307 import fcntl
308 fcntl.lockf(f, fcntl.LOCK_EX)
309 f.seek(0, os.SEEK_END) # seek to the end of file
310 f.write(line + "\n")
311 fcntl.lockf(f, fcntl.LOCK_UN)
312 f.close()