Andrew Geissler | 635e0e4 | 2020-08-21 15:58:33 -0500 | [diff] [blame] | 1 | # |
| 2 | # SPDX-License-Identifier: GPL-2.0-only |
| 3 | # |
| 4 | |
| 5 | import re |
| 6 | import filecmp |
| 7 | import shutil |
| 8 | from oe.rootfs import Rootfs |
| 9 | from oe.manifest import Manifest |
| 10 | from oe.utils import execute_pre_post_process |
Andrew Geissler | 6ce62a2 | 2020-11-30 19:58:47 -0600 | [diff] [blame] | 11 | from oe.package_manager.ipk.manifest import PkgManifest |
Andrew Geissler | 635e0e4 | 2020-08-21 15:58:33 -0500 | [diff] [blame] | 12 | from oe.package_manager.ipk import OpkgPM |
| 13 | |
| 14 | class DpkgOpkgRootfs(Rootfs): |
| 15 | def __init__(self, d, progress_reporter=None, logcatcher=None): |
| 16 | super(DpkgOpkgRootfs, self).__init__(d, progress_reporter, logcatcher) |
| 17 | |
| 18 | def _get_pkgs_postinsts(self, status_file): |
| 19 | def _get_pkg_depends_list(pkg_depends): |
| 20 | pkg_depends_list = [] |
| 21 | # filter version requirements like libc (>= 1.1) |
| 22 | for dep in pkg_depends.split(', '): |
| 23 | m_dep = re.match(r"^(.*) \(.*\)$", dep) |
| 24 | if m_dep: |
| 25 | dep = m_dep.group(1) |
| 26 | pkg_depends_list.append(dep) |
| 27 | |
| 28 | return pkg_depends_list |
| 29 | |
| 30 | pkgs = {} |
| 31 | pkg_name = "" |
| 32 | pkg_status_match = False |
| 33 | pkg_depends = "" |
| 34 | |
| 35 | with open(status_file) as status: |
| 36 | data = status.read() |
| 37 | status.close() |
| 38 | for line in data.split('\n'): |
| 39 | m_pkg = re.match(r"^Package: (.*)", line) |
| 40 | m_status = re.match(r"^Status:.*unpacked", line) |
| 41 | m_depends = re.match(r"^Depends: (.*)", line) |
| 42 | |
| 43 | #Only one of m_pkg, m_status or m_depends is not None at time |
| 44 | #If m_pkg is not None, we started a new package |
| 45 | if m_pkg is not None: |
| 46 | #Get Package name |
| 47 | pkg_name = m_pkg.group(1) |
| 48 | #Make sure we reset other variables |
| 49 | pkg_status_match = False |
| 50 | pkg_depends = "" |
| 51 | elif m_status is not None: |
| 52 | #New status matched |
| 53 | pkg_status_match = True |
| 54 | elif m_depends is not None: |
| 55 | #New depends macthed |
| 56 | pkg_depends = m_depends.group(1) |
| 57 | else: |
| 58 | pass |
| 59 | |
| 60 | #Now check if we can process package depends and postinst |
| 61 | if "" != pkg_name and pkg_status_match: |
| 62 | pkgs[pkg_name] = _get_pkg_depends_list(pkg_depends) |
| 63 | else: |
| 64 | #Not enough information |
| 65 | pass |
| 66 | |
| 67 | # remove package dependencies not in postinsts |
| 68 | pkg_names = list(pkgs.keys()) |
| 69 | for pkg_name in pkg_names: |
| 70 | deps = pkgs[pkg_name][:] |
| 71 | |
| 72 | for d in deps: |
| 73 | if d not in pkg_names: |
| 74 | pkgs[pkg_name].remove(d) |
| 75 | |
| 76 | return pkgs |
| 77 | |
| 78 | def _get_delayed_postinsts_common(self, status_file): |
| 79 | def _dep_resolve(graph, node, resolved, seen): |
| 80 | seen.append(node) |
| 81 | |
| 82 | for edge in graph[node]: |
| 83 | if edge not in resolved: |
| 84 | if edge in seen: |
| 85 | raise RuntimeError("Packages %s and %s have " \ |
| 86 | "a circular dependency in postinsts scripts." \ |
| 87 | % (node, edge)) |
| 88 | _dep_resolve(graph, edge, resolved, seen) |
| 89 | |
| 90 | resolved.append(node) |
| 91 | |
| 92 | pkg_list = [] |
| 93 | |
| 94 | pkgs = None |
| 95 | if not self.d.getVar('PACKAGE_INSTALL').strip(): |
| 96 | bb.note("Building empty image") |
| 97 | else: |
| 98 | pkgs = self._get_pkgs_postinsts(status_file) |
| 99 | if pkgs: |
| 100 | root = "__packagegroup_postinst__" |
| 101 | pkgs[root] = list(pkgs.keys()) |
| 102 | _dep_resolve(pkgs, root, pkg_list, []) |
| 103 | pkg_list.remove(root) |
| 104 | |
| 105 | if len(pkg_list) == 0: |
| 106 | return None |
| 107 | |
| 108 | return pkg_list |
| 109 | |
| 110 | def _save_postinsts_common(self, dst_postinst_dir, src_postinst_dir): |
| 111 | if bb.utils.contains("IMAGE_FEATURES", "package-management", |
| 112 | True, False, self.d): |
| 113 | return |
| 114 | num = 0 |
| 115 | for p in self._get_delayed_postinsts(): |
| 116 | bb.utils.mkdirhier(dst_postinst_dir) |
| 117 | |
| 118 | if os.path.exists(os.path.join(src_postinst_dir, p + ".postinst")): |
| 119 | shutil.copy(os.path.join(src_postinst_dir, p + ".postinst"), |
| 120 | os.path.join(dst_postinst_dir, "%03d-%s" % (num, p))) |
| 121 | |
| 122 | num += 1 |
| 123 | |
Andrew Geissler | 6ce62a2 | 2020-11-30 19:58:47 -0600 | [diff] [blame] | 124 | class PkgRootfs(DpkgOpkgRootfs): |
Andrew Geissler | 635e0e4 | 2020-08-21 15:58:33 -0500 | [diff] [blame] | 125 | def __init__(self, d, manifest_dir, progress_reporter=None, logcatcher=None): |
Andrew Geissler | 6ce62a2 | 2020-11-30 19:58:47 -0600 | [diff] [blame] | 126 | super(PkgRootfs, self).__init__(d, progress_reporter, logcatcher) |
Andrew Geissler | 635e0e4 | 2020-08-21 15:58:33 -0500 | [diff] [blame] | 127 | self.log_check_regex = '(exit 1|Collected errors)' |
| 128 | |
Andrew Geissler | 6ce62a2 | 2020-11-30 19:58:47 -0600 | [diff] [blame] | 129 | self.manifest = PkgManifest(d, manifest_dir) |
Andrew Geissler | 635e0e4 | 2020-08-21 15:58:33 -0500 | [diff] [blame] | 130 | self.opkg_conf = self.d.getVar("IPKGCONF_TARGET") |
| 131 | self.pkg_archs = self.d.getVar("ALL_MULTILIB_PACKAGE_ARCHS") |
| 132 | |
| 133 | self.inc_opkg_image_gen = self.d.getVar('INC_IPK_IMAGE_GEN') or "" |
| 134 | if self._remove_old_rootfs(): |
| 135 | bb.utils.remove(self.image_rootfs, True) |
| 136 | self.pm = OpkgPM(d, |
| 137 | self.image_rootfs, |
| 138 | self.opkg_conf, |
| 139 | self.pkg_archs) |
| 140 | else: |
| 141 | self.pm = OpkgPM(d, |
| 142 | self.image_rootfs, |
| 143 | self.opkg_conf, |
| 144 | self.pkg_archs) |
| 145 | self.pm.recover_packaging_data() |
| 146 | |
| 147 | bb.utils.remove(self.d.getVar('MULTILIB_TEMP_ROOTFS'), True) |
| 148 | |
| 149 | def _prelink_file(self, root_dir, filename): |
| 150 | bb.note('prelink %s in %s' % (filename, root_dir)) |
| 151 | prelink_cfg = oe.path.join(root_dir, |
| 152 | self.d.expand('${sysconfdir}/prelink.conf')) |
| 153 | if not os.path.exists(prelink_cfg): |
| 154 | shutil.copy(self.d.expand('${STAGING_DIR_NATIVE}${sysconfdir_native}/prelink.conf'), |
| 155 | prelink_cfg) |
| 156 | |
| 157 | cmd_prelink = self.d.expand('${STAGING_DIR_NATIVE}${sbindir_native}/prelink') |
| 158 | self._exec_shell_cmd([cmd_prelink, |
| 159 | '--root', |
| 160 | root_dir, |
| 161 | '-amR', |
| 162 | '-N', |
| 163 | '-c', |
| 164 | self.d.expand('${sysconfdir}/prelink.conf')]) |
| 165 | |
| 166 | ''' |
| 167 | Compare two files with the same key twice to see if they are equal. |
| 168 | If they are not equal, it means they are duplicated and come from |
| 169 | different packages. |
| 170 | 1st: Comapre them directly; |
| 171 | 2nd: While incremental image creation is enabled, one of the |
| 172 | files could be probaly prelinked in the previous image |
| 173 | creation and the file has been changed, so we need to |
| 174 | prelink the other one and compare them. |
| 175 | ''' |
| 176 | def _file_equal(self, key, f1, f2): |
| 177 | |
| 178 | # Both of them are not prelinked |
| 179 | if filecmp.cmp(f1, f2): |
| 180 | return True |
| 181 | |
| 182 | if bb.data.inherits_class('image-prelink', self.d): |
| 183 | if self.image_rootfs not in f1: |
| 184 | self._prelink_file(f1.replace(key, ''), f1) |
| 185 | |
| 186 | if self.image_rootfs not in f2: |
| 187 | self._prelink_file(f2.replace(key, ''), f2) |
| 188 | |
| 189 | # Both of them are prelinked |
| 190 | if filecmp.cmp(f1, f2): |
| 191 | return True |
| 192 | |
| 193 | # Not equal |
| 194 | return False |
| 195 | |
| 196 | """ |
| 197 | This function was reused from the old implementation. |
| 198 | See commit: "image.bbclass: Added variables for multilib support." by |
| 199 | Lianhao Lu. |
| 200 | """ |
| 201 | def _multilib_sanity_test(self, dirs): |
| 202 | |
| 203 | allow_replace = self.d.getVar("MULTILIBRE_ALLOW_REP") |
| 204 | if allow_replace is None: |
| 205 | allow_replace = "" |
| 206 | |
| 207 | allow_rep = re.compile(re.sub(r"\|$", r"", allow_replace)) |
| 208 | error_prompt = "Multilib check error:" |
| 209 | |
| 210 | files = {} |
| 211 | for dir in dirs: |
| 212 | for root, subfolders, subfiles in os.walk(dir): |
| 213 | for file in subfiles: |
| 214 | item = os.path.join(root, file) |
| 215 | key = str(os.path.join("/", os.path.relpath(item, dir))) |
| 216 | |
| 217 | valid = True |
| 218 | if key in files: |
| 219 | #check whether the file is allow to replace |
| 220 | if allow_rep.match(key): |
| 221 | valid = True |
| 222 | else: |
| 223 | if os.path.exists(files[key]) and \ |
| 224 | os.path.exists(item) and \ |
| 225 | not self._file_equal(key, files[key], item): |
| 226 | valid = False |
| 227 | bb.fatal("%s duplicate files %s %s is not the same\n" % |
| 228 | (error_prompt, item, files[key])) |
| 229 | |
| 230 | #pass the check, add to list |
| 231 | if valid: |
| 232 | files[key] = item |
| 233 | |
| 234 | def _multilib_test_install(self, pkgs): |
| 235 | ml_temp = self.d.getVar("MULTILIB_TEMP_ROOTFS") |
| 236 | bb.utils.mkdirhier(ml_temp) |
| 237 | |
| 238 | dirs = [self.image_rootfs] |
| 239 | |
| 240 | for variant in self.d.getVar("MULTILIB_VARIANTS").split(): |
| 241 | ml_target_rootfs = os.path.join(ml_temp, variant) |
| 242 | |
| 243 | bb.utils.remove(ml_target_rootfs, True) |
| 244 | |
| 245 | ml_opkg_conf = os.path.join(ml_temp, |
| 246 | variant + "-" + os.path.basename(self.opkg_conf)) |
| 247 | |
| 248 | ml_pm = OpkgPM(self.d, ml_target_rootfs, ml_opkg_conf, self.pkg_archs, prepare_index=False) |
| 249 | |
| 250 | ml_pm.update() |
| 251 | ml_pm.install(pkgs) |
| 252 | |
| 253 | dirs.append(ml_target_rootfs) |
| 254 | |
| 255 | self._multilib_sanity_test(dirs) |
| 256 | |
| 257 | ''' |
| 258 | While ipk incremental image generation is enabled, it will remove the |
| 259 | unneeded pkgs by comparing the old full manifest in previous existing |
| 260 | image and the new full manifest in the current image. |
| 261 | ''' |
| 262 | def _remove_extra_packages(self, pkgs_initial_install): |
| 263 | if self.inc_opkg_image_gen == "1": |
| 264 | # Parse full manifest in previous existing image creation session |
| 265 | old_full_manifest = self.manifest.parse_full_manifest() |
| 266 | |
| 267 | # Create full manifest for the current image session, the old one |
| 268 | # will be replaced by the new one. |
| 269 | self.manifest.create_full(self.pm) |
| 270 | |
| 271 | # Parse full manifest in current image creation session |
| 272 | new_full_manifest = self.manifest.parse_full_manifest() |
| 273 | |
| 274 | pkg_to_remove = list() |
| 275 | for pkg in old_full_manifest: |
| 276 | if pkg not in new_full_manifest: |
| 277 | pkg_to_remove.append(pkg) |
| 278 | |
| 279 | if pkg_to_remove != []: |
| 280 | bb.note('decremental removed: %s' % ' '.join(pkg_to_remove)) |
| 281 | self.pm.remove(pkg_to_remove) |
| 282 | |
| 283 | ''' |
| 284 | Compare with previous existing image creation, if some conditions |
| 285 | triggered, the previous old image should be removed. |
| 286 | The conditions include any of 'PACKAGE_EXCLUDE, NO_RECOMMENDATIONS |
| 287 | and BAD_RECOMMENDATIONS' has been changed. |
| 288 | ''' |
| 289 | def _remove_old_rootfs(self): |
| 290 | if self.inc_opkg_image_gen != "1": |
| 291 | return True |
| 292 | |
| 293 | vars_list_file = self.d.expand('${T}/vars_list') |
| 294 | |
| 295 | old_vars_list = "" |
| 296 | if os.path.exists(vars_list_file): |
| 297 | old_vars_list = open(vars_list_file, 'r+').read() |
| 298 | |
| 299 | new_vars_list = '%s:%s:%s\n' % \ |
| 300 | ((self.d.getVar('BAD_RECOMMENDATIONS') or '').strip(), |
| 301 | (self.d.getVar('NO_RECOMMENDATIONS') or '').strip(), |
| 302 | (self.d.getVar('PACKAGE_EXCLUDE') or '').strip()) |
| 303 | open(vars_list_file, 'w+').write(new_vars_list) |
| 304 | |
| 305 | if old_vars_list != new_vars_list: |
| 306 | return True |
| 307 | |
| 308 | return False |
| 309 | |
| 310 | def _create(self): |
| 311 | pkgs_to_install = self.manifest.parse_initial_manifest() |
| 312 | opkg_pre_process_cmds = self.d.getVar('OPKG_PREPROCESS_COMMANDS') |
| 313 | opkg_post_process_cmds = self.d.getVar('OPKG_POSTPROCESS_COMMANDS') |
| 314 | |
| 315 | # update PM index files |
| 316 | self.pm.write_index() |
| 317 | |
| 318 | execute_pre_post_process(self.d, opkg_pre_process_cmds) |
| 319 | |
| 320 | if self.progress_reporter: |
| 321 | self.progress_reporter.next_stage() |
| 322 | # Steps are a bit different in order, skip next |
| 323 | self.progress_reporter.next_stage() |
| 324 | |
| 325 | self.pm.update() |
| 326 | |
| 327 | if self.progress_reporter: |
| 328 | self.progress_reporter.next_stage() |
| 329 | |
| 330 | if self.inc_opkg_image_gen == "1": |
| 331 | self._remove_extra_packages(pkgs_to_install) |
| 332 | |
| 333 | if self.progress_reporter: |
| 334 | self.progress_reporter.next_stage() |
| 335 | |
| 336 | for pkg_type in self.install_order: |
| 337 | if pkg_type in pkgs_to_install: |
| 338 | # For multilib, we perform a sanity test before final install |
| 339 | # If sanity test fails, it will automatically do a bb.fatal() |
| 340 | # and the installation will stop |
| 341 | if pkg_type == Manifest.PKG_TYPE_MULTILIB: |
| 342 | self._multilib_test_install(pkgs_to_install[pkg_type]) |
| 343 | |
| 344 | self.pm.install(pkgs_to_install[pkg_type], |
| 345 | [False, True][pkg_type == Manifest.PKG_TYPE_ATTEMPT_ONLY]) |
| 346 | |
| 347 | if self.progress_reporter: |
| 348 | self.progress_reporter.next_stage() |
| 349 | |
| 350 | self.pm.install_complementary() |
| 351 | |
| 352 | if self.progress_reporter: |
| 353 | self.progress_reporter.next_stage() |
| 354 | |
| 355 | opkg_lib_dir = self.d.getVar('OPKGLIBDIR') |
| 356 | opkg_dir = os.path.join(opkg_lib_dir, 'opkg') |
| 357 | self._setup_dbg_rootfs([opkg_dir]) |
| 358 | |
| 359 | execute_pre_post_process(self.d, opkg_post_process_cmds) |
| 360 | |
| 361 | if self.inc_opkg_image_gen == "1": |
| 362 | self.pm.backup_packaging_data() |
| 363 | |
| 364 | if self.progress_reporter: |
| 365 | self.progress_reporter.next_stage() |
| 366 | |
| 367 | @staticmethod |
| 368 | def _depends_list(): |
| 369 | return ['IPKGCONF_SDK', 'IPK_FEED_URIS', 'DEPLOY_DIR_IPK', 'IPKGCONF_TARGET', 'INC_IPK_IMAGE_GEN', 'OPKG_ARGS', 'OPKGLIBDIR', 'OPKG_PREPROCESS_COMMANDS', 'OPKG_POSTPROCESS_COMMANDS', 'OPKGLIBDIR'] |
| 370 | |
| 371 | def _get_delayed_postinsts(self): |
| 372 | status_file = os.path.join(self.image_rootfs, |
| 373 | self.d.getVar('OPKGLIBDIR').strip('/'), |
| 374 | "opkg", "status") |
| 375 | return self._get_delayed_postinsts_common(status_file) |
| 376 | |
| 377 | def _save_postinsts(self): |
| 378 | dst_postinst_dir = self.d.expand("${IMAGE_ROOTFS}${sysconfdir}/ipk-postinsts") |
| 379 | src_postinst_dir = self.d.expand("${IMAGE_ROOTFS}${OPKGLIBDIR}/opkg/info") |
| 380 | return self._save_postinsts_common(dst_postinst_dir, src_postinst_dir) |
| 381 | |
| 382 | def _log_check(self): |
| 383 | self._log_check_warn() |
| 384 | self._log_check_error() |
| 385 | |
| 386 | def _cleanup(self): |
| 387 | self.pm.remove_lists() |