Patrick Williams | f1e5d69 | 2016-03-30 15:21:19 -0500 | [diff] [blame] | 1 | def create_socket(url, d): |
| 2 | import urllib |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 3 | from bb.utils import export_proxies |
Patrick Williams | f1e5d69 | 2016-03-30 15:21:19 -0500 | [diff] [blame] | 4 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 5 | export_proxies(d) |
| 6 | return urllib.request.urlopen(url) |
Patrick Williams | f1e5d69 | 2016-03-30 15:21:19 -0500 | [diff] [blame] | 7 | |
| 8 | def get_links_from_url(url, d): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 9 | "Return all the href links found on the web location" |
| 10 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 11 | from bs4 import BeautifulSoup, SoupStrainer |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 12 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 13 | soup = BeautifulSoup(create_socket(url,d), "html.parser", parse_only=SoupStrainer("a")) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 14 | hyperlinks = [] |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 15 | for line in soup.find_all('a', href=True): |
| 16 | hyperlinks.append(line['href'].strip('/')) |
| 17 | return hyperlinks |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 18 | |
Patrick Williams | f1e5d69 | 2016-03-30 15:21:19 -0500 | [diff] [blame] | 19 | def find_latest_numeric_release(url, d): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 20 | "Find the latest listed numeric release on the given url" |
| 21 | max=0 |
| 22 | maxstr="" |
Patrick Williams | f1e5d69 | 2016-03-30 15:21:19 -0500 | [diff] [blame] | 23 | for link in get_links_from_url(url, d): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 24 | try: |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 25 | # TODO use LooseVersion |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 26 | release = float(link) |
| 27 | except: |
| 28 | release = 0 |
| 29 | if release > max: |
| 30 | max = release |
| 31 | maxstr = link |
| 32 | return maxstr |
| 33 | |
| 34 | def is_src_rpm(name): |
| 35 | "Check if the link is pointing to a src.rpm file" |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 36 | return name.endswith(".src.rpm") |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 37 | |
| 38 | def package_name_from_srpm(srpm): |
| 39 | "Strip out the package name from the src.rpm filename" |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 40 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 41 | # ca-certificates-2016.2.7-1.0.fc24.src.rpm |
| 42 | # ^name ^ver ^release^removed |
| 43 | (name, version, release) = srpm.replace(".src.rpm", "").rsplit("-", 2) |
| 44 | return name |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 45 | |
Patrick Williams | f1e5d69 | 2016-03-30 15:21:19 -0500 | [diff] [blame] | 46 | def get_source_package_list_from_url(url, section, d): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 47 | "Return a sectioned list of package names from a URL list" |
| 48 | |
| 49 | bb.note("Reading %s: %s" % (url, section)) |
Patrick Williams | f1e5d69 | 2016-03-30 15:21:19 -0500 | [diff] [blame] | 50 | links = get_links_from_url(url, d) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 51 | srpms = filter(is_src_rpm, links) |
| 52 | names_list = map(package_name_from_srpm, srpms) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 53 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 54 | new_pkgs = set() |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 55 | for pkgs in names_list: |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 56 | new_pkgs.add(pkgs + ":" + section) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 57 | return new_pkgs |
| 58 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 59 | def get_source_package_list_from_url_by_letter(url, section, d): |
| 60 | import string |
| 61 | from urllib.error import HTTPError |
| 62 | packages = set() |
| 63 | for letter in (string.ascii_lowercase + string.digits): |
| 64 | # Not all subfolders may exist, so silently handle 404 |
| 65 | try: |
| 66 | packages |= get_source_package_list_from_url(url + "/" + letter, section, d) |
| 67 | except HTTPError as e: |
| 68 | if e.code != 404: raise |
| 69 | return packages |
| 70 | |
Patrick Williams | f1e5d69 | 2016-03-30 15:21:19 -0500 | [diff] [blame] | 71 | def get_latest_released_fedora_source_package_list(d): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 72 | "Returns list of all the name os packages in the latest fedora distro" |
Patrick Williams | f1e5d69 | 2016-03-30 15:21:19 -0500 | [diff] [blame] | 73 | latest = find_latest_numeric_release("http://archive.fedoraproject.org/pub/fedora/linux/releases/", d) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 74 | package_names = get_source_package_list_from_url_by_letter("http://archive.fedoraproject.org/pub/fedora/linux/releases/%s/Everything/source/tree/Packages/" % latest, "main", d) |
| 75 | package_names |= get_source_package_list_from_url_by_letter("http://archive.fedoraproject.org/pub/fedora/linux/updates/%s/SRPMS/" % latest, "updates", d) |
| 76 | return latest, package_names |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 77 | |
Patrick Williams | f1e5d69 | 2016-03-30 15:21:19 -0500 | [diff] [blame] | 78 | def get_latest_released_opensuse_source_package_list(d): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 79 | "Returns list of all the name os packages in the latest opensuse distro" |
Patrick Williams | f1e5d69 | 2016-03-30 15:21:19 -0500 | [diff] [blame] | 80 | latest = find_latest_numeric_release("http://download.opensuse.org/source/distribution/",d) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 81 | |
Patrick Williams | f1e5d69 | 2016-03-30 15:21:19 -0500 | [diff] [blame] | 82 | package_names = get_source_package_list_from_url("http://download.opensuse.org/source/distribution/%s/repo/oss/suse/src/" % latest, "main", d) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 83 | package_names |= get_source_package_list_from_url("http://download.opensuse.org/update/%s/src/" % latest, "updates", d) |
| 84 | return latest, package_names |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 85 | |
Patrick Williams | f1e5d69 | 2016-03-30 15:21:19 -0500 | [diff] [blame] | 86 | def get_latest_released_mandriva_source_package_list(d): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 87 | "Returns list of all the name os packages in the latest mandriva distro" |
Patrick Williams | f1e5d69 | 2016-03-30 15:21:19 -0500 | [diff] [blame] | 88 | latest = find_latest_numeric_release("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/", d) |
| 89 | package_names = get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/release/" % latest, "main", d) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 90 | package_names |= get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/updates/" % latest, "updates", d) |
| 91 | return latest, package_names |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 92 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 93 | def get_latest_released_clear_source_package_list(d): |
| 94 | latest = find_latest_numeric_release("https://download.clearlinux.org/releases/", d) |
| 95 | package_names = get_source_package_list_from_url("https://download.clearlinux.org/releases/%s/clear/source/SRPMS/" % latest, "main", d) |
| 96 | return latest, package_names |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 97 | |
Patrick Williams | f1e5d69 | 2016-03-30 15:21:19 -0500 | [diff] [blame] | 98 | def find_latest_debian_release(url, d): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 99 | "Find the latest listed debian release on the given url" |
| 100 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 101 | releases = [link.replace("Debian", "") |
| 102 | for link in get_links_from_url(url, d) |
| 103 | if link.startswith("Debian")] |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 104 | releases.sort() |
| 105 | try: |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 106 | return releases[-1] |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 107 | except: |
| 108 | return "_NotFound_" |
| 109 | |
Patrick Williams | f1e5d69 | 2016-03-30 15:21:19 -0500 | [diff] [blame] | 110 | def get_debian_style_source_package_list(url, section, d): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 111 | "Return the list of package-names stored in the debian style Sources.gz file" |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 112 | import gzip |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 113 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 114 | package_names = set() |
| 115 | for line in gzip.open(create_socket(url, d), mode="rt"): |
| 116 | if line.startswith("Package:"): |
| 117 | pkg = line.split(":", 1)[1].strip() |
| 118 | package_names.add(pkg + ":" + section) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 119 | return package_names |
| 120 | |
Patrick Williams | f1e5d69 | 2016-03-30 15:21:19 -0500 | [diff] [blame] | 121 | def get_latest_released_debian_source_package_list(d): |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 122 | "Returns list of all the name of packages in the latest debian distro" |
Patrick Williams | f1e5d69 | 2016-03-30 15:21:19 -0500 | [diff] [blame] | 123 | latest = find_latest_debian_release("http://ftp.debian.org/debian/dists/", d) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 124 | url = "http://ftp.debian.org/debian/dists/stable/main/source/Sources.gz" |
Patrick Williams | f1e5d69 | 2016-03-30 15:21:19 -0500 | [diff] [blame] | 125 | package_names = get_debian_style_source_package_list(url, "main", d) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 126 | url = "http://ftp.debian.org/debian/dists/stable-proposed-updates/main/source/Sources.gz" |
| 127 | package_names |= get_debian_style_source_package_list(url, "updates", d) |
| 128 | return latest, package_names |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 129 | |
Patrick Williams | f1e5d69 | 2016-03-30 15:21:19 -0500 | [diff] [blame] | 130 | def find_latest_ubuntu_release(url, d): |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 131 | """ |
| 132 | Find the latest listed Ubuntu release on the given ubuntu/dists/ URL. |
| 133 | |
| 134 | To avoid matching development releases look for distributions that have |
| 135 | updates, so the resulting distro could be any supported release. |
| 136 | """ |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 137 | url += "?C=M;O=D" # Descending Sort by Last Modified |
Patrick Williams | f1e5d69 | 2016-03-30 15:21:19 -0500 | [diff] [blame] | 138 | for link in get_links_from_url(url, d): |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 139 | if "-updates" in link: |
| 140 | distro = link.replace("-updates", "") |
| 141 | return distro |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 142 | return "_NotFound_" |
| 143 | |
Patrick Williams | f1e5d69 | 2016-03-30 15:21:19 -0500 | [diff] [blame] | 144 | def get_latest_released_ubuntu_source_package_list(d): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 145 | "Returns list of all the name os packages in the latest ubuntu distro" |
Patrick Williams | f1e5d69 | 2016-03-30 15:21:19 -0500 | [diff] [blame] | 146 | latest = find_latest_ubuntu_release("http://archive.ubuntu.com/ubuntu/dists/", d) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 147 | url = "http://archive.ubuntu.com/ubuntu/dists/%s/main/source/Sources.gz" % latest |
Patrick Williams | f1e5d69 | 2016-03-30 15:21:19 -0500 | [diff] [blame] | 148 | package_names = get_debian_style_source_package_list(url, "main", d) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 149 | url = "http://archive.ubuntu.com/ubuntu/dists/%s-updates/main/source/Sources.gz" % latest |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 150 | package_names |= get_debian_style_source_package_list(url, "updates", d) |
| 151 | return latest, package_names |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 152 | |
Patrick Williams | f1e5d69 | 2016-03-30 15:21:19 -0500 | [diff] [blame] | 153 | def create_distro_packages_list(distro_check_dir, d): |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 154 | import shutil |
| 155 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 156 | pkglst_dir = os.path.join(distro_check_dir, "package_lists") |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 157 | bb.utils.remove(pkglst_dir, True) |
| 158 | bb.utils.mkdirhier(pkglst_dir) |
| 159 | |
| 160 | per_distro_functions = ( |
| 161 | ("Debian", get_latest_released_debian_source_package_list), |
| 162 | ("Ubuntu", get_latest_released_ubuntu_source_package_list), |
| 163 | ("Fedora", get_latest_released_fedora_source_package_list), |
| 164 | ("OpenSuSE", get_latest_released_opensuse_source_package_list), |
| 165 | ("Mandriva", get_latest_released_mandriva_source_package_list), |
| 166 | ("Clear", get_latest_released_clear_source_package_list), |
| 167 | ) |
| 168 | |
| 169 | for name, fetcher_func in per_distro_functions: |
| 170 | try: |
| 171 | release, package_list = fetcher_func(d) |
| 172 | except Exception as e: |
| 173 | bb.warn("Cannot fetch packages for %s: %s" % (name, e)) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 174 | bb.note("Distro: %s, Latest Release: %s, # src packages: %d" % (name, release, len(package_list))) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 175 | if len(package_list) == 0: |
| 176 | bb.error("Didn't fetch any packages for %s %s" % (name, release)) |
| 177 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 178 | package_list_file = os.path.join(pkglst_dir, name + "-" + release) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 179 | with open(package_list_file, 'w') as f: |
| 180 | for pkg in sorted(package_list): |
| 181 | f.write(pkg + "\n") |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 182 | |
Patrick Williams | f1e5d69 | 2016-03-30 15:21:19 -0500 | [diff] [blame] | 183 | def update_distro_data(distro_check_dir, datetime, d): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 184 | """ |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 185 | If distro packages list data is old then rebuild it. |
| 186 | The operations has to be protected by a lock so that |
| 187 | only one thread performes it at a time. |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 188 | """ |
| 189 | if not os.path.isdir (distro_check_dir): |
| 190 | try: |
| 191 | bb.note ("Making new directory: %s" % distro_check_dir) |
| 192 | os.makedirs (distro_check_dir) |
| 193 | except OSError: |
| 194 | raise Exception('Unable to create directory %s' % (distro_check_dir)) |
| 195 | |
| 196 | |
| 197 | datetime_file = os.path.join(distro_check_dir, "build_datetime") |
| 198 | saved_datetime = "_invalid_" |
| 199 | import fcntl |
| 200 | try: |
| 201 | if not os.path.exists(datetime_file): |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 202 | open(datetime_file, 'w+').close() # touch the file so that the next open won't fail |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 203 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 204 | f = open(datetime_file, "r+") |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 205 | fcntl.lockf(f, fcntl.LOCK_EX) |
| 206 | saved_datetime = f.read() |
| 207 | if saved_datetime[0:8] != datetime[0:8]: |
| 208 | bb.note("The build datetime did not match: saved:%s current:%s" % (saved_datetime, datetime)) |
| 209 | bb.note("Regenerating distro package lists") |
Patrick Williams | f1e5d69 | 2016-03-30 15:21:19 -0500 | [diff] [blame] | 210 | create_distro_packages_list(distro_check_dir, d) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 211 | f.seek(0) |
| 212 | f.write(datetime) |
| 213 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 214 | except OSError as e: |
| 215 | raise Exception('Unable to open timestamp: %s' % e) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 216 | finally: |
| 217 | fcntl.lockf(f, fcntl.LOCK_UN) |
| 218 | f.close() |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 219 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 220 | def compare_in_distro_packages_list(distro_check_dir, d): |
| 221 | if not os.path.isdir(distro_check_dir): |
| 222 | raise Exception("compare_in_distro_packages_list: invalid distro_check_dir passed") |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 223 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 224 | localdata = bb.data.createCopy(d) |
| 225 | pkglst_dir = os.path.join(distro_check_dir, "package_lists") |
| 226 | matching_distros = [] |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 227 | pn = recipe_name = d.getVar('PN') |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 228 | bb.note("Checking: %s" % pn) |
| 229 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 230 | if pn.find("-native") != -1: |
| 231 | pnstripped = pn.split("-native") |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 232 | localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 233 | recipe_name = pnstripped[0] |
| 234 | |
| 235 | if pn.startswith("nativesdk-"): |
| 236 | pnstripped = pn.split("nativesdk-") |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 237 | localdata.setVar('OVERRIDES', "pn-" + pnstripped[1] + ":" + d.getVar('OVERRIDES')) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 238 | recipe_name = pnstripped[1] |
| 239 | |
| 240 | if pn.find("-cross") != -1: |
| 241 | pnstripped = pn.split("-cross") |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 242 | localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 243 | recipe_name = pnstripped[0] |
| 244 | |
| 245 | if pn.find("-initial") != -1: |
| 246 | pnstripped = pn.split("-initial") |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 247 | localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 248 | recipe_name = pnstripped[0] |
| 249 | |
| 250 | bb.note("Recipe: %s" % recipe_name) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 251 | |
| 252 | distro_exceptions = dict({"OE-Core":'OE-Core', "OpenedHand":'OpenedHand', "Intel":'Intel', "Upstream":'Upstream', "Windriver":'Windriver', "OSPDT":'OSPDT Approved', "Poky":'poky'}) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 253 | tmp = localdata.getVar('DISTRO_PN_ALIAS') or "" |
| 254 | for str in tmp.split(): |
| 255 | if str and str.find("=") == -1 and distro_exceptions[str]: |
| 256 | matching_distros.append(str) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 257 | |
| 258 | distro_pn_aliases = {} |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 259 | for str in tmp.split(): |
| 260 | if "=" in str: |
| 261 | (dist, pn_alias) = str.split('=') |
| 262 | distro_pn_aliases[dist.strip().lower()] = pn_alias.strip() |
| 263 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 264 | for file in os.listdir(pkglst_dir): |
| 265 | (distro, distro_release) = file.split("-") |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 266 | f = open(os.path.join(pkglst_dir, file), "r") |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 267 | for line in f: |
| 268 | (pkg, section) = line.split(":") |
| 269 | if distro.lower() in distro_pn_aliases: |
| 270 | pn = distro_pn_aliases[distro.lower()] |
| 271 | else: |
| 272 | pn = recipe_name |
| 273 | if pn == pkg: |
| 274 | matching_distros.append(distro + "-" + section[:-1]) # strip the \n at the end |
| 275 | f.close() |
| 276 | break |
| 277 | f.close() |
| 278 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 279 | for item in tmp.split(): |
| 280 | matching_distros.append(item) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 281 | bb.note("Matching: %s" % matching_distros) |
| 282 | return matching_distros |
| 283 | |
| 284 | def create_log_file(d, logname): |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 285 | logpath = d.getVar('LOG_DIR') |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 286 | bb.utils.mkdirhier(logpath) |
| 287 | logfn, logsuffix = os.path.splitext(logname) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 288 | logfile = os.path.join(logpath, "%s.%s%s" % (logfn, d.getVar('DATETIME'), logsuffix)) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 289 | if not os.path.exists(logfile): |
| 290 | slogfile = os.path.join(logpath, logname) |
| 291 | if os.path.exists(slogfile): |
| 292 | os.remove(slogfile) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 293 | open(logfile, 'w+').close() |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 294 | os.symlink(logfile, slogfile) |
| 295 | d.setVar('LOG_FILE', logfile) |
| 296 | return logfile |
| 297 | |
| 298 | |
| 299 | def save_distro_check_result(result, datetime, result_file, d): |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 300 | pn = d.getVar('PN') |
| 301 | logdir = d.getVar('LOG_DIR') |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 302 | if not logdir: |
| 303 | bb.error("LOG_DIR variable is not defined, can't write the distro_check results") |
| 304 | return |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 305 | bb.utils.mkdirhier(logdir) |
| 306 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 307 | line = pn |
| 308 | for i in result: |
| 309 | line = line + "," + i |
| 310 | f = open(result_file, "a") |
| 311 | import fcntl |
| 312 | fcntl.lockf(f, fcntl.LOCK_EX) |
| 313 | f.seek(0, os.SEEK_END) # seek to the end of file |
| 314 | f.write(line + "\n") |
| 315 | fcntl.lockf(f, fcntl.LOCK_UN) |
| 316 | f.close() |