Andrew Geissler | 635e0e4 | 2020-08-21 15:58:33 -0500 | [diff] [blame] | 1 | # |
Patrick Williams | 92b42cb | 2022-09-03 06:53:57 -0500 | [diff] [blame^] | 2 | # Copyright OpenEmbedded Contributors |
| 3 | # |
Andrew Geissler | 635e0e4 | 2020-08-21 15:58:33 -0500 | [diff] [blame] | 4 | # SPDX-License-Identifier: GPL-2.0-only |
| 5 | # |
| 6 | |
| 7 | import re |
| 8 | import filecmp |
| 9 | import shutil |
| 10 | from oe.rootfs import Rootfs |
| 11 | from oe.manifest import Manifest |
| 12 | from oe.utils import execute_pre_post_process |
Andrew Geissler | 6ce62a2 | 2020-11-30 19:58:47 -0600 | [diff] [blame] | 13 | from oe.package_manager.ipk.manifest import PkgManifest |
Andrew Geissler | 635e0e4 | 2020-08-21 15:58:33 -0500 | [diff] [blame] | 14 | from oe.package_manager.ipk import OpkgPM |
| 15 | |
| 16 | class DpkgOpkgRootfs(Rootfs): |
| 17 | def __init__(self, d, progress_reporter=None, logcatcher=None): |
| 18 | super(DpkgOpkgRootfs, self).__init__(d, progress_reporter, logcatcher) |
| 19 | |
| 20 | def _get_pkgs_postinsts(self, status_file): |
| 21 | def _get_pkg_depends_list(pkg_depends): |
| 22 | pkg_depends_list = [] |
| 23 | # filter version requirements like libc (>= 1.1) |
| 24 | for dep in pkg_depends.split(', '): |
| 25 | m_dep = re.match(r"^(.*) \(.*\)$", dep) |
| 26 | if m_dep: |
| 27 | dep = m_dep.group(1) |
| 28 | pkg_depends_list.append(dep) |
| 29 | |
| 30 | return pkg_depends_list |
| 31 | |
| 32 | pkgs = {} |
| 33 | pkg_name = "" |
| 34 | pkg_status_match = False |
| 35 | pkg_depends = "" |
| 36 | |
| 37 | with open(status_file) as status: |
| 38 | data = status.read() |
| 39 | status.close() |
| 40 | for line in data.split('\n'): |
| 41 | m_pkg = re.match(r"^Package: (.*)", line) |
| 42 | m_status = re.match(r"^Status:.*unpacked", line) |
| 43 | m_depends = re.match(r"^Depends: (.*)", line) |
| 44 | |
| 45 | #Only one of m_pkg, m_status or m_depends is not None at time |
| 46 | #If m_pkg is not None, we started a new package |
| 47 | if m_pkg is not None: |
| 48 | #Get Package name |
| 49 | pkg_name = m_pkg.group(1) |
| 50 | #Make sure we reset other variables |
| 51 | pkg_status_match = False |
| 52 | pkg_depends = "" |
| 53 | elif m_status is not None: |
| 54 | #New status matched |
| 55 | pkg_status_match = True |
| 56 | elif m_depends is not None: |
| 57 | #New depends macthed |
| 58 | pkg_depends = m_depends.group(1) |
| 59 | else: |
| 60 | pass |
| 61 | |
| 62 | #Now check if we can process package depends and postinst |
| 63 | if "" != pkg_name and pkg_status_match: |
| 64 | pkgs[pkg_name] = _get_pkg_depends_list(pkg_depends) |
| 65 | else: |
| 66 | #Not enough information |
| 67 | pass |
| 68 | |
| 69 | # remove package dependencies not in postinsts |
| 70 | pkg_names = list(pkgs.keys()) |
| 71 | for pkg_name in pkg_names: |
| 72 | deps = pkgs[pkg_name][:] |
| 73 | |
| 74 | for d in deps: |
| 75 | if d not in pkg_names: |
| 76 | pkgs[pkg_name].remove(d) |
| 77 | |
| 78 | return pkgs |
| 79 | |
| 80 | def _get_delayed_postinsts_common(self, status_file): |
| 81 | def _dep_resolve(graph, node, resolved, seen): |
| 82 | seen.append(node) |
| 83 | |
| 84 | for edge in graph[node]: |
| 85 | if edge not in resolved: |
| 86 | if edge in seen: |
| 87 | raise RuntimeError("Packages %s and %s have " \ |
| 88 | "a circular dependency in postinsts scripts." \ |
| 89 | % (node, edge)) |
| 90 | _dep_resolve(graph, edge, resolved, seen) |
| 91 | |
| 92 | resolved.append(node) |
| 93 | |
| 94 | pkg_list = [] |
| 95 | |
| 96 | pkgs = None |
| 97 | if not self.d.getVar('PACKAGE_INSTALL').strip(): |
| 98 | bb.note("Building empty image") |
| 99 | else: |
| 100 | pkgs = self._get_pkgs_postinsts(status_file) |
| 101 | if pkgs: |
| 102 | root = "__packagegroup_postinst__" |
| 103 | pkgs[root] = list(pkgs.keys()) |
| 104 | _dep_resolve(pkgs, root, pkg_list, []) |
| 105 | pkg_list.remove(root) |
| 106 | |
| 107 | if len(pkg_list) == 0: |
| 108 | return None |
| 109 | |
| 110 | return pkg_list |
| 111 | |
| 112 | def _save_postinsts_common(self, dst_postinst_dir, src_postinst_dir): |
| 113 | if bb.utils.contains("IMAGE_FEATURES", "package-management", |
| 114 | True, False, self.d): |
| 115 | return |
| 116 | num = 0 |
| 117 | for p in self._get_delayed_postinsts(): |
| 118 | bb.utils.mkdirhier(dst_postinst_dir) |
| 119 | |
| 120 | if os.path.exists(os.path.join(src_postinst_dir, p + ".postinst")): |
| 121 | shutil.copy(os.path.join(src_postinst_dir, p + ".postinst"), |
| 122 | os.path.join(dst_postinst_dir, "%03d-%s" % (num, p))) |
| 123 | |
| 124 | num += 1 |
| 125 | |
Andrew Geissler | 6ce62a2 | 2020-11-30 19:58:47 -0600 | [diff] [blame] | 126 | class PkgRootfs(DpkgOpkgRootfs): |
Andrew Geissler | 635e0e4 | 2020-08-21 15:58:33 -0500 | [diff] [blame] | 127 | def __init__(self, d, manifest_dir, progress_reporter=None, logcatcher=None): |
Andrew Geissler | 6ce62a2 | 2020-11-30 19:58:47 -0600 | [diff] [blame] | 128 | super(PkgRootfs, self).__init__(d, progress_reporter, logcatcher) |
Andrew Geissler | 635e0e4 | 2020-08-21 15:58:33 -0500 | [diff] [blame] | 129 | self.log_check_regex = '(exit 1|Collected errors)' |
| 130 | |
Andrew Geissler | 6ce62a2 | 2020-11-30 19:58:47 -0600 | [diff] [blame] | 131 | self.manifest = PkgManifest(d, manifest_dir) |
Andrew Geissler | 635e0e4 | 2020-08-21 15:58:33 -0500 | [diff] [blame] | 132 | self.opkg_conf = self.d.getVar("IPKGCONF_TARGET") |
| 133 | self.pkg_archs = self.d.getVar("ALL_MULTILIB_PACKAGE_ARCHS") |
| 134 | |
| 135 | self.inc_opkg_image_gen = self.d.getVar('INC_IPK_IMAGE_GEN') or "" |
| 136 | if self._remove_old_rootfs(): |
| 137 | bb.utils.remove(self.image_rootfs, True) |
| 138 | self.pm = OpkgPM(d, |
| 139 | self.image_rootfs, |
| 140 | self.opkg_conf, |
| 141 | self.pkg_archs) |
| 142 | else: |
| 143 | self.pm = OpkgPM(d, |
| 144 | self.image_rootfs, |
| 145 | self.opkg_conf, |
| 146 | self.pkg_archs) |
| 147 | self.pm.recover_packaging_data() |
| 148 | |
| 149 | bb.utils.remove(self.d.getVar('MULTILIB_TEMP_ROOTFS'), True) |
Andrew Geissler | 635e0e4 | 2020-08-21 15:58:33 -0500 | [diff] [blame] | 150 | ''' |
| 151 | Compare two files with the same key twice to see if they are equal. |
| 152 | If they are not equal, it means they are duplicated and come from |
| 153 | different packages. |
Andrew Geissler | 635e0e4 | 2020-08-21 15:58:33 -0500 | [diff] [blame] | 154 | ''' |
| 155 | def _file_equal(self, key, f1, f2): |
Andrew Geissler | 635e0e4 | 2020-08-21 15:58:33 -0500 | [diff] [blame] | 156 | if filecmp.cmp(f1, f2): |
| 157 | return True |
Andrew Geissler | 635e0e4 | 2020-08-21 15:58:33 -0500 | [diff] [blame] | 158 | # Not equal |
| 159 | return False |
| 160 | |
| 161 | """ |
| 162 | This function was reused from the old implementation. |
| 163 | See commit: "image.bbclass: Added variables for multilib support." by |
| 164 | Lianhao Lu. |
| 165 | """ |
| 166 | def _multilib_sanity_test(self, dirs): |
| 167 | |
| 168 | allow_replace = self.d.getVar("MULTILIBRE_ALLOW_REP") |
| 169 | if allow_replace is None: |
| 170 | allow_replace = "" |
| 171 | |
| 172 | allow_rep = re.compile(re.sub(r"\|$", r"", allow_replace)) |
| 173 | error_prompt = "Multilib check error:" |
| 174 | |
| 175 | files = {} |
| 176 | for dir in dirs: |
| 177 | for root, subfolders, subfiles in os.walk(dir): |
| 178 | for file in subfiles: |
| 179 | item = os.path.join(root, file) |
| 180 | key = str(os.path.join("/", os.path.relpath(item, dir))) |
| 181 | |
| 182 | valid = True |
| 183 | if key in files: |
| 184 | #check whether the file is allow to replace |
| 185 | if allow_rep.match(key): |
| 186 | valid = True |
| 187 | else: |
| 188 | if os.path.exists(files[key]) and \ |
| 189 | os.path.exists(item) and \ |
| 190 | not self._file_equal(key, files[key], item): |
| 191 | valid = False |
| 192 | bb.fatal("%s duplicate files %s %s is not the same\n" % |
| 193 | (error_prompt, item, files[key])) |
| 194 | |
| 195 | #pass the check, add to list |
| 196 | if valid: |
| 197 | files[key] = item |
| 198 | |
| 199 | def _multilib_test_install(self, pkgs): |
| 200 | ml_temp = self.d.getVar("MULTILIB_TEMP_ROOTFS") |
| 201 | bb.utils.mkdirhier(ml_temp) |
| 202 | |
| 203 | dirs = [self.image_rootfs] |
| 204 | |
| 205 | for variant in self.d.getVar("MULTILIB_VARIANTS").split(): |
| 206 | ml_target_rootfs = os.path.join(ml_temp, variant) |
| 207 | |
| 208 | bb.utils.remove(ml_target_rootfs, True) |
| 209 | |
| 210 | ml_opkg_conf = os.path.join(ml_temp, |
| 211 | variant + "-" + os.path.basename(self.opkg_conf)) |
| 212 | |
| 213 | ml_pm = OpkgPM(self.d, ml_target_rootfs, ml_opkg_conf, self.pkg_archs, prepare_index=False) |
| 214 | |
| 215 | ml_pm.update() |
| 216 | ml_pm.install(pkgs) |
| 217 | |
| 218 | dirs.append(ml_target_rootfs) |
| 219 | |
| 220 | self._multilib_sanity_test(dirs) |
| 221 | |
| 222 | ''' |
| 223 | While ipk incremental image generation is enabled, it will remove the |
| 224 | unneeded pkgs by comparing the old full manifest in previous existing |
| 225 | image and the new full manifest in the current image. |
| 226 | ''' |
| 227 | def _remove_extra_packages(self, pkgs_initial_install): |
| 228 | if self.inc_opkg_image_gen == "1": |
| 229 | # Parse full manifest in previous existing image creation session |
| 230 | old_full_manifest = self.manifest.parse_full_manifest() |
| 231 | |
| 232 | # Create full manifest for the current image session, the old one |
| 233 | # will be replaced by the new one. |
| 234 | self.manifest.create_full(self.pm) |
| 235 | |
| 236 | # Parse full manifest in current image creation session |
| 237 | new_full_manifest = self.manifest.parse_full_manifest() |
| 238 | |
| 239 | pkg_to_remove = list() |
| 240 | for pkg in old_full_manifest: |
| 241 | if pkg not in new_full_manifest: |
| 242 | pkg_to_remove.append(pkg) |
| 243 | |
| 244 | if pkg_to_remove != []: |
| 245 | bb.note('decremental removed: %s' % ' '.join(pkg_to_remove)) |
| 246 | self.pm.remove(pkg_to_remove) |
| 247 | |
| 248 | ''' |
| 249 | Compare with previous existing image creation, if some conditions |
| 250 | triggered, the previous old image should be removed. |
| 251 | The conditions include any of 'PACKAGE_EXCLUDE, NO_RECOMMENDATIONS |
| 252 | and BAD_RECOMMENDATIONS' has been changed. |
| 253 | ''' |
| 254 | def _remove_old_rootfs(self): |
| 255 | if self.inc_opkg_image_gen != "1": |
| 256 | return True |
| 257 | |
| 258 | vars_list_file = self.d.expand('${T}/vars_list') |
| 259 | |
| 260 | old_vars_list = "" |
| 261 | if os.path.exists(vars_list_file): |
| 262 | old_vars_list = open(vars_list_file, 'r+').read() |
| 263 | |
| 264 | new_vars_list = '%s:%s:%s\n' % \ |
| 265 | ((self.d.getVar('BAD_RECOMMENDATIONS') or '').strip(), |
| 266 | (self.d.getVar('NO_RECOMMENDATIONS') or '').strip(), |
| 267 | (self.d.getVar('PACKAGE_EXCLUDE') or '').strip()) |
| 268 | open(vars_list_file, 'w+').write(new_vars_list) |
| 269 | |
| 270 | if old_vars_list != new_vars_list: |
| 271 | return True |
| 272 | |
| 273 | return False |
| 274 | |
| 275 | def _create(self): |
| 276 | pkgs_to_install = self.manifest.parse_initial_manifest() |
| 277 | opkg_pre_process_cmds = self.d.getVar('OPKG_PREPROCESS_COMMANDS') |
| 278 | opkg_post_process_cmds = self.d.getVar('OPKG_POSTPROCESS_COMMANDS') |
| 279 | |
| 280 | # update PM index files |
| 281 | self.pm.write_index() |
| 282 | |
| 283 | execute_pre_post_process(self.d, opkg_pre_process_cmds) |
| 284 | |
| 285 | if self.progress_reporter: |
| 286 | self.progress_reporter.next_stage() |
| 287 | # Steps are a bit different in order, skip next |
| 288 | self.progress_reporter.next_stage() |
| 289 | |
| 290 | self.pm.update() |
| 291 | |
| 292 | if self.progress_reporter: |
| 293 | self.progress_reporter.next_stage() |
| 294 | |
| 295 | if self.inc_opkg_image_gen == "1": |
| 296 | self._remove_extra_packages(pkgs_to_install) |
| 297 | |
| 298 | if self.progress_reporter: |
| 299 | self.progress_reporter.next_stage() |
| 300 | |
| 301 | for pkg_type in self.install_order: |
| 302 | if pkg_type in pkgs_to_install: |
| 303 | # For multilib, we perform a sanity test before final install |
| 304 | # If sanity test fails, it will automatically do a bb.fatal() |
| 305 | # and the installation will stop |
| 306 | if pkg_type == Manifest.PKG_TYPE_MULTILIB: |
| 307 | self._multilib_test_install(pkgs_to_install[pkg_type]) |
| 308 | |
| 309 | self.pm.install(pkgs_to_install[pkg_type], |
| 310 | [False, True][pkg_type == Manifest.PKG_TYPE_ATTEMPT_ONLY]) |
| 311 | |
| 312 | if self.progress_reporter: |
| 313 | self.progress_reporter.next_stage() |
| 314 | |
| 315 | self.pm.install_complementary() |
| 316 | |
| 317 | if self.progress_reporter: |
| 318 | self.progress_reporter.next_stage() |
| 319 | |
| 320 | opkg_lib_dir = self.d.getVar('OPKGLIBDIR') |
| 321 | opkg_dir = os.path.join(opkg_lib_dir, 'opkg') |
| 322 | self._setup_dbg_rootfs([opkg_dir]) |
| 323 | |
| 324 | execute_pre_post_process(self.d, opkg_post_process_cmds) |
| 325 | |
| 326 | if self.inc_opkg_image_gen == "1": |
| 327 | self.pm.backup_packaging_data() |
| 328 | |
| 329 | if self.progress_reporter: |
| 330 | self.progress_reporter.next_stage() |
| 331 | |
| 332 | @staticmethod |
| 333 | def _depends_list(): |
| 334 | return ['IPKGCONF_SDK', 'IPK_FEED_URIS', 'DEPLOY_DIR_IPK', 'IPKGCONF_TARGET', 'INC_IPK_IMAGE_GEN', 'OPKG_ARGS', 'OPKGLIBDIR', 'OPKG_PREPROCESS_COMMANDS', 'OPKG_POSTPROCESS_COMMANDS', 'OPKGLIBDIR'] |
| 335 | |
| 336 | def _get_delayed_postinsts(self): |
| 337 | status_file = os.path.join(self.image_rootfs, |
| 338 | self.d.getVar('OPKGLIBDIR').strip('/'), |
| 339 | "opkg", "status") |
| 340 | return self._get_delayed_postinsts_common(status_file) |
| 341 | |
| 342 | def _save_postinsts(self): |
| 343 | dst_postinst_dir = self.d.expand("${IMAGE_ROOTFS}${sysconfdir}/ipk-postinsts") |
| 344 | src_postinst_dir = self.d.expand("${IMAGE_ROOTFS}${OPKGLIBDIR}/opkg/info") |
| 345 | return self._save_postinsts_common(dst_postinst_dir, src_postinst_dir) |
| 346 | |
| 347 | def _log_check(self): |
| 348 | self._log_check_warn() |
| 349 | self._log_check_error() |
| 350 | |
| 351 | def _cleanup(self): |
| 352 | self.pm.remove_lists() |