blob: a95e1b73910e0fdabcfc016bf915fe763179c35c [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001from abc import ABCMeta, abstractmethod
2from oe.utils import execute_pre_post_process
3from oe.package_manager import *
4from oe.manifest import *
5import oe.path
6import filecmp
7import shutil
8import os
9import subprocess
10import re
11
12
13class Rootfs(object):
14 """
15 This is an abstract class. Do not instantiate this directly.
16 """
17 __metaclass__ = ABCMeta
18
19 def __init__(self, d):
20 self.d = d
21 self.pm = None
22 self.image_rootfs = self.d.getVar('IMAGE_ROOTFS', True)
23 self.deploy_dir_image = self.d.getVar('DEPLOY_DIR_IMAGE', True)
24
25 self.install_order = Manifest.INSTALL_ORDER
26
27 @abstractmethod
28 def _create(self):
29 pass
30
31 @abstractmethod
32 def _get_delayed_postinsts(self):
33 pass
34
35 @abstractmethod
36 def _save_postinsts(self):
37 pass
38
39 @abstractmethod
40 def _log_check(self):
41 pass
42
43 def _log_check_warn(self):
44 r = re.compile('^(warn|Warn|NOTE: warn|NOTE: Warn|WARNING:)')
45 log_path = self.d.expand("${T}/log.do_rootfs")
46 with open(log_path, 'r') as log:
47 for line in log:
48 if 'log_check' in line or 'NOTE:' in line:
49 continue
50
51 m = r.search(line)
52 if m:
53 bb.warn('[log_check] %s: found a warning message in the logfile (keyword \'%s\'):\n[log_check] %s'
54 % (self.d.getVar('PN', True), m.group(), line))
55
56 def _log_check_error(self):
57 r = re.compile(self.log_check_regex)
58 log_path = self.d.expand("${T}/log.do_rootfs")
59 with open(log_path, 'r') as log:
60 found_error = 0
61 message = "\n"
62 for line in log:
63 if 'log_check' in line:
64 continue
65
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050066 if hasattr(self, 'log_check_expected_errors_regexes'):
67 m = None
68 for ee in self.log_check_expected_errors_regexes:
69 m = re.search(ee, line)
70 if m:
71 break
72 if m:
73 continue
74
Patrick Williamsc124f4f2015-09-15 14:41:29 -050075 m = r.search(line)
76 if m:
77 found_error = 1
Patrick Williamsf1e5d692016-03-30 15:21:19 -050078 bb.warn('[log_check] In line: [%s]' % line)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050079 bb.warn('[log_check] %s: found an error message in the logfile (keyword \'%s\'):\n[log_check] %s'
80 % (self.d.getVar('PN', True), m.group(), line))
81
82 if found_error >= 1 and found_error <= 5:
83 message += line + '\n'
84 found_error += 1
85
86 if found_error == 6:
87 bb.fatal(message)
88
89 def _insert_feed_uris(self):
90 if bb.utils.contains("IMAGE_FEATURES", "package-management",
91 True, False, self.d):
92 self.pm.insert_feeds_uris()
93
94 @abstractmethod
95 def _handle_intercept_failure(self, failed_script):
96 pass
97
98 """
99 The _cleanup() method should be used to clean-up stuff that we don't really
100 want to end up on target. For example, in the case of RPM, the DB locks.
101 The method is called, once, at the end of create() method.
102 """
103 @abstractmethod
104 def _cleanup(self):
105 pass
106
107 def _setup_dbg_rootfs(self, dirs):
108 gen_debugfs = self.d.getVar('IMAGE_GEN_DEBUGFS', True) or '0'
109 if gen_debugfs != '1':
110 return
111
112 bb.note(" Renaming the original rootfs...")
113 try:
114 shutil.rmtree(self.image_rootfs + '-orig')
115 except:
116 pass
117 os.rename(self.image_rootfs, self.image_rootfs + '-orig')
118
119 bb.note(" Creating debug rootfs...")
120 bb.utils.mkdirhier(self.image_rootfs)
121
122 bb.note(" Copying back package database...")
123 for dir in dirs:
124 bb.utils.mkdirhier(self.image_rootfs + os.path.dirname(dir))
125 shutil.copytree(self.image_rootfs + '-orig' + dir, self.image_rootfs + dir)
126
127 cpath = oe.cachedpath.CachedPath()
128 # Copy files located in /usr/lib/debug or /usr/src/debug
129 for dir in ["/usr/lib/debug", "/usr/src/debug"]:
130 src = self.image_rootfs + '-orig' + dir
131 if cpath.exists(src):
132 dst = self.image_rootfs + dir
133 bb.utils.mkdirhier(os.path.dirname(dst))
134 shutil.copytree(src, dst)
135
136 # Copy files with suffix '.debug' or located in '.debug' dir.
137 for root, dirs, files in cpath.walk(self.image_rootfs + '-orig'):
138 relative_dir = root[len(self.image_rootfs + '-orig'):]
139 for f in files:
140 if f.endswith('.debug') or '/.debug' in relative_dir:
141 bb.utils.mkdirhier(self.image_rootfs + relative_dir)
142 shutil.copy(os.path.join(root, f),
143 self.image_rootfs + relative_dir)
144
145 bb.note(" Install complementary '*-dbg' packages...")
146 self.pm.install_complementary('*-dbg')
147
148 bb.note(" Rename debug rootfs...")
149 try:
150 shutil.rmtree(self.image_rootfs + '-dbg')
151 except:
152 pass
153 os.rename(self.image_rootfs, self.image_rootfs + '-dbg')
154
155 bb.note(" Restoreing original rootfs...")
156 os.rename(self.image_rootfs + '-orig', self.image_rootfs)
157
158 def _exec_shell_cmd(self, cmd):
159 fakerootcmd = self.d.getVar('FAKEROOT', True)
160 if fakerootcmd is not None:
161 exec_cmd = [fakerootcmd, cmd]
162 else:
163 exec_cmd = cmd
164
165 try:
166 subprocess.check_output(exec_cmd, stderr=subprocess.STDOUT)
167 except subprocess.CalledProcessError as e:
168 return("Command '%s' returned %d:\n%s" % (e.cmd, e.returncode, e.output))
169
170 return None
171
172 def create(self):
173 bb.note("###### Generate rootfs #######")
174 pre_process_cmds = self.d.getVar("ROOTFS_PREPROCESS_COMMAND", True)
175 post_process_cmds = self.d.getVar("ROOTFS_POSTPROCESS_COMMAND", True)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500176 rootfs_post_install_cmds = self.d.getVar('ROOTFS_POSTINSTALL_COMMAND', True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500177
178 postinst_intercepts_dir = self.d.getVar("POSTINST_INTERCEPTS_DIR", True)
179 if not postinst_intercepts_dir:
180 postinst_intercepts_dir = self.d.expand("${COREBASE}/scripts/postinst-intercepts")
181 intercepts_dir = os.path.join(self.d.getVar('WORKDIR', True),
182 "intercept_scripts")
183
184 bb.utils.remove(intercepts_dir, True)
185
186 bb.utils.mkdirhier(self.image_rootfs)
187
188 bb.utils.mkdirhier(self.deploy_dir_image)
189
190 shutil.copytree(postinst_intercepts_dir, intercepts_dir)
191
192 shutil.copy(self.d.expand("${COREBASE}/meta/files/deploydir_readme.txt"),
193 self.deploy_dir_image +
194 "/README_-_DO_NOT_DELETE_FILES_IN_THIS_DIRECTORY.txt")
195
196 execute_pre_post_process(self.d, pre_process_cmds)
197
198 # call the package manager dependent create method
199 self._create()
200
201 sysconfdir = self.image_rootfs + self.d.getVar('sysconfdir', True)
202 bb.utils.mkdirhier(sysconfdir)
203 with open(sysconfdir + "/version", "w+") as ver:
204 ver.write(self.d.getVar('BUILDNAME', True) + "\n")
205
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500206 execute_pre_post_process(self.d, rootfs_post_install_cmds)
207
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500208 self._run_intercepts()
209
210 execute_pre_post_process(self.d, post_process_cmds)
211
212 if bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs",
213 True, False, self.d):
214 delayed_postinsts = self._get_delayed_postinsts()
215 if delayed_postinsts is not None:
216 bb.fatal("The following packages could not be configured "
217 "offline and rootfs is read-only: %s" %
218 delayed_postinsts)
219
220 if self.d.getVar('USE_DEVFS', True) != "1":
221 self._create_devfs()
222
223 self._uninstall_unneeded()
224
225 self._insert_feed_uris()
226
227 self._run_ldconfig()
228
229 if self.d.getVar('USE_DEPMOD', True) != "0":
230 self._generate_kernel_module_deps()
231
232 self._cleanup()
233 self._log_check()
234
235 def _uninstall_unneeded(self):
236 # Remove unneeded init script symlinks
237 delayed_postinsts = self._get_delayed_postinsts()
238 if delayed_postinsts is None:
239 if os.path.exists(self.d.expand("${IMAGE_ROOTFS}${sysconfdir}/init.d/run-postinsts")):
240 self._exec_shell_cmd(["update-rc.d", "-f", "-r",
241 self.d.getVar('IMAGE_ROOTFS', True),
242 "run-postinsts", "remove"])
243
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500244 image_rorfs = bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs",
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500245 True, False, self.d)
246 if image_rorfs:
247 # Remove components that we don't need if it's a read-only rootfs
248 unneeded_pkgs = self.d.getVar("ROOTFS_RO_UNNEEDED", True).split()
249 pkgs_installed = image_list_installed_packages(self.d)
250 pkgs_to_remove = [pkg for pkg in pkgs_installed if pkg in unneeded_pkgs]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500251
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500252 if len(pkgs_to_remove) > 0:
253 self.pm.remove(pkgs_to_remove, False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500254
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500255 if delayed_postinsts:
256 self._save_postinsts()
257 if image_rorfs:
258 bb.warn("There are post install scripts "
259 "in a read-only rootfs")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500260
261 post_uninstall_cmds = self.d.getVar("ROOTFS_POSTUNINSTALL_COMMAND", True)
262 execute_pre_post_process(self.d, post_uninstall_cmds)
263
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500264 runtime_pkgmanage = bb.utils.contains("IMAGE_FEATURES", "package-management",
265 True, False, self.d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500266 if not runtime_pkgmanage:
267 # Remove the package manager data files
268 self.pm.remove_packaging_data()
269
270 def _run_intercepts(self):
271 intercepts_dir = os.path.join(self.d.getVar('WORKDIR', True),
272 "intercept_scripts")
273
274 bb.note("Running intercept scripts:")
275 os.environ['D'] = self.image_rootfs
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500276 os.environ['STAGING_DIR_NATIVE'] = self.d.getVar('STAGING_DIR_NATIVE', True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500277 for script in os.listdir(intercepts_dir):
278 script_full = os.path.join(intercepts_dir, script)
279
280 if script == "postinst_intercept" or not os.access(script_full, os.X_OK):
281 continue
282
283 bb.note("> Executing %s intercept ..." % script)
284
285 try:
286 subprocess.check_call(script_full)
287 except subprocess.CalledProcessError as e:
288 bb.warn("The postinstall intercept hook '%s' failed (exit code: %d)! See log for details!" %
289 (script, e.returncode))
290
291 with open(script_full) as intercept:
292 registered_pkgs = None
293 for line in intercept.read().split("\n"):
294 m = re.match("^##PKGS:(.*)", line)
295 if m is not None:
296 registered_pkgs = m.group(1).strip()
297 break
298
299 if registered_pkgs is not None:
300 bb.warn("The postinstalls for the following packages "
301 "will be postponed for first boot: %s" %
302 registered_pkgs)
303
304 # call the backend dependent handler
305 self._handle_intercept_failure(registered_pkgs)
306
307 def _run_ldconfig(self):
308 if self.d.getVar('LDCONFIGDEPEND', True):
309 bb.note("Executing: ldconfig -r" + self.image_rootfs + "-c new -v")
310 self._exec_shell_cmd(['ldconfig', '-r', self.image_rootfs, '-c',
311 'new', '-v'])
312
313 def _check_for_kernel_modules(self, modules_dir):
314 for root, dirs, files in os.walk(modules_dir, topdown=True):
315 for name in files:
316 found_ko = name.endswith(".ko")
317 if found_ko:
318 return found_ko
319 return False
320
321 def _generate_kernel_module_deps(self):
322 modules_dir = os.path.join(self.image_rootfs, 'lib', 'modules')
323 # if we don't have any modules don't bother to do the depmod
324 if not self._check_for_kernel_modules(modules_dir):
325 bb.note("No Kernel Modules found, not running depmod")
326 return
327
328 kernel_abi_ver_file = oe.path.join(self.d.getVar('PKGDATA_DIR', True), "kernel-depmod",
329 'kernel-abiversion')
330 if not os.path.exists(kernel_abi_ver_file):
331 bb.fatal("No kernel-abiversion file found (%s), cannot run depmod, aborting" % kernel_abi_ver_file)
332
333 kernel_ver = open(kernel_abi_ver_file).read().strip(' \n')
334 versioned_modules_dir = os.path.join(self.image_rootfs, modules_dir, kernel_ver)
335
336 bb.utils.mkdirhier(versioned_modules_dir)
337
338 self._exec_shell_cmd(['depmodwrapper', '-a', '-b', self.image_rootfs, kernel_ver])
339
340 """
341 Create devfs:
342 * IMAGE_DEVICE_TABLE is the old name to an absolute path to a device table file
343 * IMAGE_DEVICE_TABLES is a new name for a file, or list of files, seached
344 for in the BBPATH
345 If neither are specified then the default name of files/device_table-minimal.txt
346 is searched for in the BBPATH (same as the old version.)
347 """
348 def _create_devfs(self):
349 devtable_list = []
350 devtable = self.d.getVar('IMAGE_DEVICE_TABLE', True)
351 if devtable is not None:
352 devtable_list.append(devtable)
353 else:
354 devtables = self.d.getVar('IMAGE_DEVICE_TABLES', True)
355 if devtables is None:
356 devtables = 'files/device_table-minimal.txt'
357 for devtable in devtables.split():
358 devtable_list.append("%s" % bb.utils.which(self.d.getVar('BBPATH', True), devtable))
359
360 for devtable in devtable_list:
361 self._exec_shell_cmd(["makedevs", "-r",
362 self.image_rootfs, "-D", devtable])
363
364
365class RpmRootfs(Rootfs):
366 def __init__(self, d, manifest_dir):
367 super(RpmRootfs, self).__init__(d)
368 self.log_check_regex = '(unpacking of archive failed|Cannot find package'\
369 '|exit 1|ERROR: |Error: |Error |ERROR '\
370 '|Failed |Failed: |Failed$|Failed\(\d+\):)'
371 self.manifest = RpmManifest(d, manifest_dir)
372
373 self.pm = RpmPM(d,
374 d.getVar('IMAGE_ROOTFS', True),
375 self.d.getVar('TARGET_VENDOR', True)
376 )
377
378 self.inc_rpm_image_gen = self.d.getVar('INC_RPM_IMAGE_GEN', True)
379 if self.inc_rpm_image_gen != "1":
380 bb.utils.remove(self.image_rootfs, True)
381 else:
382 self.pm.recovery_packaging_data()
383 bb.utils.remove(self.d.getVar('MULTILIB_TEMP_ROOTFS', True), True)
384
385 self.pm.create_configs()
386
387 '''
388 While rpm incremental image generation is enabled, it will remove the
389 unneeded pkgs by comparing the new install solution manifest and the
390 old installed manifest.
391 '''
392 def _create_incremental(self, pkgs_initial_install):
393 if self.inc_rpm_image_gen == "1":
394
395 pkgs_to_install = list()
396 for pkg_type in pkgs_initial_install:
397 pkgs_to_install += pkgs_initial_install[pkg_type]
398
399 installed_manifest = self.pm.load_old_install_solution()
400 solution_manifest = self.pm.dump_install_solution(pkgs_to_install)
401
402 pkg_to_remove = list()
403 for pkg in installed_manifest:
404 if pkg not in solution_manifest:
405 pkg_to_remove.append(pkg)
406
407 self.pm.update()
408
409 bb.note('incremental update -- upgrade packages in place ')
410 self.pm.upgrade()
411 if pkg_to_remove != []:
412 bb.note('incremental removed: %s' % ' '.join(pkg_to_remove))
413 self.pm.remove(pkg_to_remove)
414
415 def _create(self):
416 pkgs_to_install = self.manifest.parse_initial_manifest()
417 rpm_pre_process_cmds = self.d.getVar('RPM_PREPROCESS_COMMANDS', True)
418 rpm_post_process_cmds = self.d.getVar('RPM_POSTPROCESS_COMMANDS', True)
419
420 # update PM index files
421 self.pm.write_index()
422
423 execute_pre_post_process(self.d, rpm_pre_process_cmds)
424
425 self.pm.dump_all_available_pkgs()
426
427 if self.inc_rpm_image_gen == "1":
428 self._create_incremental(pkgs_to_install)
429
430 self.pm.update()
431
432 pkgs = []
433 pkgs_attempt = []
434 for pkg_type in pkgs_to_install:
435 if pkg_type == Manifest.PKG_TYPE_ATTEMPT_ONLY:
436 pkgs_attempt += pkgs_to_install[pkg_type]
437 else:
438 pkgs += pkgs_to_install[pkg_type]
439
440 self.pm.install(pkgs)
441
442 self.pm.install(pkgs_attempt, True)
443
444 self.pm.install_complementary()
445
446 self._setup_dbg_rootfs(['/etc/rpm', '/var/lib/rpm', '/var/lib/smart'])
447
448 execute_pre_post_process(self.d, rpm_post_process_cmds)
449
450 self._log_check()
451
452 if self.inc_rpm_image_gen == "1":
453 self.pm.backup_packaging_data()
454
455 self.pm.rpm_setup_smart_target_config()
456
457 @staticmethod
458 def _depends_list():
459 return ['DEPLOY_DIR_RPM', 'INC_RPM_IMAGE_GEN', 'RPM_PREPROCESS_COMMANDS',
460 'RPM_POSTPROCESS_COMMANDS', 'RPM_PREFER_ELF_ARCH']
461
462 def _get_delayed_postinsts(self):
463 postinst_dir = self.d.expand("${IMAGE_ROOTFS}${sysconfdir}/rpm-postinsts")
464 if os.path.isdir(postinst_dir):
465 files = os.listdir(postinst_dir)
466 for f in files:
467 bb.note('Delayed package scriptlet: %s' % f)
468 return files
469
470 return None
471
472 def _save_postinsts(self):
473 # this is just a stub. For RPM, the failed postinstalls are
474 # already saved in /etc/rpm-postinsts
475 pass
476
477 def _log_check_error(self):
478 r = re.compile('(unpacking of archive failed|Cannot find package|exit 1|ERR|Fail)')
479 log_path = self.d.expand("${T}/log.do_rootfs")
480 with open(log_path, 'r') as log:
481 found_error = 0
482 message = "\n"
483 for line in log.read().split('\n'):
484 if 'log_check' in line:
485 continue
486 # sh -x may emit code which isn't actually executed
487 if line.startswith('+'):
488 continue
489
490 m = r.search(line)
491 if m:
492 found_error = 1
493 bb.warn('log_check: There were error messages in the logfile')
494 bb.warn('log_check: Matched keyword: [%s]\n\n' % m.group())
495
496 if found_error >= 1 and found_error <= 5:
497 message += line + '\n'
498 found_error += 1
499
500 if found_error == 6:
501 bb.fatal(message)
502
503 def _log_check(self):
504 self._log_check_warn()
505 self._log_check_error()
506
507 def _handle_intercept_failure(self, registered_pkgs):
508 rpm_postinsts_dir = self.image_rootfs + self.d.expand('${sysconfdir}/rpm-postinsts/')
509 bb.utils.mkdirhier(rpm_postinsts_dir)
510
511 # Save the package postinstalls in /etc/rpm-postinsts
512 for pkg in registered_pkgs.split():
513 self.pm.save_rpmpostinst(pkg)
514
515 def _cleanup(self):
516 # during the execution of postprocess commands, rpm is called several
517 # times to get the files installed, dependencies, etc. This creates the
518 # __db.00* (Berkeley DB files that hold locks, rpm specific environment
519 # settings, etc.), that should not get into the final rootfs
520 self.pm.unlock_rpm_db()
521 if os.path.isdir(self.pm.install_dir_path + "/tmp") and not os.listdir(self.pm.install_dir_path + "/tmp"):
522 bb.utils.remove(self.pm.install_dir_path + "/tmp", True)
523 if os.path.isdir(self.pm.install_dir_path) and not os.listdir(self.pm.install_dir_path):
524 bb.utils.remove(self.pm.install_dir_path, True)
525
526class DpkgOpkgRootfs(Rootfs):
527 def __init__(self, d):
528 super(DpkgOpkgRootfs, self).__init__(d)
529
530 def _get_pkgs_postinsts(self, status_file):
531 def _get_pkg_depends_list(pkg_depends):
532 pkg_depends_list = []
533 # filter version requirements like libc (>= 1.1)
534 for dep in pkg_depends.split(', '):
535 m_dep = re.match("^(.*) \(.*\)$", dep)
536 if m_dep:
537 dep = m_dep.group(1)
538 pkg_depends_list.append(dep)
539
540 return pkg_depends_list
541
542 pkgs = {}
543 pkg_name = ""
544 pkg_status_match = False
545 pkg_depends = ""
546
547 with open(status_file) as status:
548 data = status.read()
549 status.close()
550 for line in data.split('\n'):
551 m_pkg = re.match("^Package: (.*)", line)
552 m_status = re.match("^Status:.*unpacked", line)
553 m_depends = re.match("^Depends: (.*)", line)
554
555 if m_pkg is not None:
556 if pkg_name and pkg_status_match:
557 pkgs[pkg_name] = _get_pkg_depends_list(pkg_depends)
558
559 pkg_name = m_pkg.group(1)
560 pkg_status_match = False
561 pkg_depends = ""
562 elif m_status is not None:
563 pkg_status_match = True
564 elif m_depends is not None:
565 pkg_depends = m_depends.group(1)
566
567 # remove package dependencies not in postinsts
568 pkg_names = pkgs.keys()
569 for pkg_name in pkg_names:
570 deps = pkgs[pkg_name][:]
571
572 for d in deps:
573 if d not in pkg_names:
574 pkgs[pkg_name].remove(d)
575
576 return pkgs
577
578 def _get_delayed_postinsts_common(self, status_file):
579 def _dep_resolve(graph, node, resolved, seen):
580 seen.append(node)
581
582 for edge in graph[node]:
583 if edge not in resolved:
584 if edge in seen:
585 raise RuntimeError("Packages %s and %s have " \
586 "a circular dependency in postinsts scripts." \
587 % (node, edge))
588 _dep_resolve(graph, edge, resolved, seen)
589
590 resolved.append(node)
591
592 pkg_list = []
593
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500594 pkgs = None
595 if not self.d.getVar('PACKAGE_INSTALL', True).strip():
596 bb.note("Building empty image")
597 else:
598 pkgs = self._get_pkgs_postinsts(status_file)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500599 if pkgs:
600 root = "__packagegroup_postinst__"
601 pkgs[root] = pkgs.keys()
602 _dep_resolve(pkgs, root, pkg_list, [])
603 pkg_list.remove(root)
604
605 if len(pkg_list) == 0:
606 return None
607
608 return pkg_list
609
610 def _save_postinsts_common(self, dst_postinst_dir, src_postinst_dir):
611 num = 0
612 for p in self._get_delayed_postinsts():
613 bb.utils.mkdirhier(dst_postinst_dir)
614
615 if os.path.exists(os.path.join(src_postinst_dir, p + ".postinst")):
616 shutil.copy(os.path.join(src_postinst_dir, p + ".postinst"),
617 os.path.join(dst_postinst_dir, "%03d-%s" % (num, p)))
618
619 num += 1
620
621class DpkgRootfs(DpkgOpkgRootfs):
622 def __init__(self, d, manifest_dir):
623 super(DpkgRootfs, self).__init__(d)
624 self.log_check_regex = '^E:'
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500625 self.log_check_expected_errors_regexes = \
626 [
627 "^E: Unmet dependencies."
628 ]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500629
630 bb.utils.remove(self.image_rootfs, True)
631 bb.utils.remove(self.d.getVar('MULTILIB_TEMP_ROOTFS', True), True)
632 self.manifest = DpkgManifest(d, manifest_dir)
633 self.pm = DpkgPM(d, d.getVar('IMAGE_ROOTFS', True),
634 d.getVar('PACKAGE_ARCHS', True),
635 d.getVar('DPKG_ARCH', True))
636
637
638 def _create(self):
639 pkgs_to_install = self.manifest.parse_initial_manifest()
640 deb_pre_process_cmds = self.d.getVar('DEB_PREPROCESS_COMMANDS', True)
641 deb_post_process_cmds = self.d.getVar('DEB_POSTPROCESS_COMMANDS', True)
642
643 alt_dir = self.d.expand("${IMAGE_ROOTFS}/var/lib/dpkg/alternatives")
644 bb.utils.mkdirhier(alt_dir)
645
646 # update PM index files
647 self.pm.write_index()
648
649 execute_pre_post_process(self.d, deb_pre_process_cmds)
650
651 self.pm.update()
652
653 for pkg_type in self.install_order:
654 if pkg_type in pkgs_to_install:
655 self.pm.install(pkgs_to_install[pkg_type],
656 [False, True][pkg_type == Manifest.PKG_TYPE_ATTEMPT_ONLY])
657
658 self.pm.install_complementary()
659
660 self._setup_dbg_rootfs(['/var/lib/dpkg'])
661
662 self.pm.fix_broken_dependencies()
663
664 self.pm.mark_packages("installed")
665
666 self.pm.run_pre_post_installs()
667
668 execute_pre_post_process(self.d, deb_post_process_cmds)
669
670 @staticmethod
671 def _depends_list():
672 return ['DEPLOY_DIR_DEB', 'DEB_SDK_ARCH', 'APTCONF_TARGET', 'APT_ARGS', 'DPKG_ARCH', 'DEB_PREPROCESS_COMMANDS', 'DEB_POSTPROCESS_COMMANDS']
673
674 def _get_delayed_postinsts(self):
675 status_file = self.image_rootfs + "/var/lib/dpkg/status"
676 return self._get_delayed_postinsts_common(status_file)
677
678 def _save_postinsts(self):
679 dst_postinst_dir = self.d.expand("${IMAGE_ROOTFS}${sysconfdir}/deb-postinsts")
680 src_postinst_dir = self.d.expand("${IMAGE_ROOTFS}/var/lib/dpkg/info")
681 return self._save_postinsts_common(dst_postinst_dir, src_postinst_dir)
682
683 def _handle_intercept_failure(self, registered_pkgs):
684 self.pm.mark_packages("unpacked", registered_pkgs.split())
685
686 def _log_check(self):
687 self._log_check_warn()
688 self._log_check_error()
689
690 def _cleanup(self):
691 pass
692
693
694class OpkgRootfs(DpkgOpkgRootfs):
695 def __init__(self, d, manifest_dir):
696 super(OpkgRootfs, self).__init__(d)
697 self.log_check_regex = '(exit 1|Collected errors)'
698
699 self.manifest = OpkgManifest(d, manifest_dir)
700 self.opkg_conf = self.d.getVar("IPKGCONF_TARGET", True)
701 self.pkg_archs = self.d.getVar("ALL_MULTILIB_PACKAGE_ARCHS", True)
702
703 self.inc_opkg_image_gen = self.d.getVar('INC_IPK_IMAGE_GEN', True) or ""
704 if self._remove_old_rootfs():
705 bb.utils.remove(self.image_rootfs, True)
706 self.pm = OpkgPM(d,
707 self.image_rootfs,
708 self.opkg_conf,
709 self.pkg_archs)
710 else:
711 self.pm = OpkgPM(d,
712 self.image_rootfs,
713 self.opkg_conf,
714 self.pkg_archs)
715 self.pm.recover_packaging_data()
716
717 bb.utils.remove(self.d.getVar('MULTILIB_TEMP_ROOTFS', True), True)
718
719 def _prelink_file(self, root_dir, filename):
720 bb.note('prelink %s in %s' % (filename, root_dir))
721 prelink_cfg = oe.path.join(root_dir,
722 self.d.expand('${sysconfdir}/prelink.conf'))
723 if not os.path.exists(prelink_cfg):
724 shutil.copy(self.d.expand('${STAGING_DIR_NATIVE}${sysconfdir_native}/prelink.conf'),
725 prelink_cfg)
726
727 cmd_prelink = self.d.expand('${STAGING_DIR_NATIVE}${sbindir_native}/prelink')
728 self._exec_shell_cmd([cmd_prelink,
729 '--root',
730 root_dir,
731 '-amR',
732 '-N',
733 '-c',
734 self.d.expand('${sysconfdir}/prelink.conf')])
735
736 '''
737 Compare two files with the same key twice to see if they are equal.
738 If they are not equal, it means they are duplicated and come from
739 different packages.
740 1st: Comapre them directly;
741 2nd: While incremental image creation is enabled, one of the
742 files could be probaly prelinked in the previous image
743 creation and the file has been changed, so we need to
744 prelink the other one and compare them.
745 '''
746 def _file_equal(self, key, f1, f2):
747
748 # Both of them are not prelinked
749 if filecmp.cmp(f1, f2):
750 return True
751
752 if self.image_rootfs not in f1:
753 self._prelink_file(f1.replace(key, ''), f1)
754
755 if self.image_rootfs not in f2:
756 self._prelink_file(f2.replace(key, ''), f2)
757
758 # Both of them are prelinked
759 if filecmp.cmp(f1, f2):
760 return True
761
762 # Not equal
763 return False
764
765 """
766 This function was reused from the old implementation.
767 See commit: "image.bbclass: Added variables for multilib support." by
768 Lianhao Lu.
769 """
770 def _multilib_sanity_test(self, dirs):
771
772 allow_replace = self.d.getVar("MULTILIBRE_ALLOW_REP", True)
773 if allow_replace is None:
774 allow_replace = ""
775
776 allow_rep = re.compile(re.sub("\|$", "", allow_replace))
777 error_prompt = "Multilib check error:"
778
779 files = {}
780 for dir in dirs:
781 for root, subfolders, subfiles in os.walk(dir):
782 for file in subfiles:
783 item = os.path.join(root, file)
784 key = str(os.path.join("/", os.path.relpath(item, dir)))
785
786 valid = True
787 if key in files:
788 #check whether the file is allow to replace
789 if allow_rep.match(key):
790 valid = True
791 else:
792 if os.path.exists(files[key]) and \
793 os.path.exists(item) and \
794 not self._file_equal(key, files[key], item):
795 valid = False
796 bb.fatal("%s duplicate files %s %s is not the same\n" %
797 (error_prompt, item, files[key]))
798
799 #pass the check, add to list
800 if valid:
801 files[key] = item
802
803 def _multilib_test_install(self, pkgs):
804 ml_temp = self.d.getVar("MULTILIB_TEMP_ROOTFS", True)
805 bb.utils.mkdirhier(ml_temp)
806
807 dirs = [self.image_rootfs]
808
809 for variant in self.d.getVar("MULTILIB_VARIANTS", True).split():
810 ml_target_rootfs = os.path.join(ml_temp, variant)
811
812 bb.utils.remove(ml_target_rootfs, True)
813
814 ml_opkg_conf = os.path.join(ml_temp,
815 variant + "-" + os.path.basename(self.opkg_conf))
816
817 ml_pm = OpkgPM(self.d, ml_target_rootfs, ml_opkg_conf, self.pkg_archs)
818
819 ml_pm.update()
820 ml_pm.install(pkgs)
821
822 dirs.append(ml_target_rootfs)
823
824 self._multilib_sanity_test(dirs)
825
826 '''
827 While ipk incremental image generation is enabled, it will remove the
828 unneeded pkgs by comparing the old full manifest in previous existing
829 image and the new full manifest in the current image.
830 '''
831 def _remove_extra_packages(self, pkgs_initial_install):
832 if self.inc_opkg_image_gen == "1":
833 # Parse full manifest in previous existing image creation session
834 old_full_manifest = self.manifest.parse_full_manifest()
835
836 # Create full manifest for the current image session, the old one
837 # will be replaced by the new one.
838 self.manifest.create_full(self.pm)
839
840 # Parse full manifest in current image creation session
841 new_full_manifest = self.manifest.parse_full_manifest()
842
843 pkg_to_remove = list()
844 for pkg in old_full_manifest:
845 if pkg not in new_full_manifest:
846 pkg_to_remove.append(pkg)
847
848 if pkg_to_remove != []:
849 bb.note('decremental removed: %s' % ' '.join(pkg_to_remove))
850 self.pm.remove(pkg_to_remove)
851
852 '''
853 Compare with previous existing image creation, if some conditions
854 triggered, the previous old image should be removed.
855 The conditions include any of 'PACKAGE_EXCLUDE, NO_RECOMMENDATIONS
856 and BAD_RECOMMENDATIONS' has been changed.
857 '''
858 def _remove_old_rootfs(self):
859 if self.inc_opkg_image_gen != "1":
860 return True
861
862 vars_list_file = self.d.expand('${T}/vars_list')
863
864 old_vars_list = ""
865 if os.path.exists(vars_list_file):
866 old_vars_list = open(vars_list_file, 'r+').read()
867
868 new_vars_list = '%s:%s:%s\n' % \
869 ((self.d.getVar('BAD_RECOMMENDATIONS', True) or '').strip(),
870 (self.d.getVar('NO_RECOMMENDATIONS', True) or '').strip(),
871 (self.d.getVar('PACKAGE_EXCLUDE', True) or '').strip())
872 open(vars_list_file, 'w+').write(new_vars_list)
873
874 if old_vars_list != new_vars_list:
875 return True
876
877 return False
878
879 def _create(self):
880 pkgs_to_install = self.manifest.parse_initial_manifest()
881 opkg_pre_process_cmds = self.d.getVar('OPKG_PREPROCESS_COMMANDS', True)
882 opkg_post_process_cmds = self.d.getVar('OPKG_POSTPROCESS_COMMANDS', True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500883
884 # update PM index files, unless users provide their own feeds
885 if (self.d.getVar('BUILD_IMAGES_FROM_FEEDS', True) or "") != "1":
886 self.pm.write_index()
887
888 execute_pre_post_process(self.d, opkg_pre_process_cmds)
889
890 self.pm.update()
891
892 self.pm.handle_bad_recommendations()
893
894 if self.inc_opkg_image_gen == "1":
895 self._remove_extra_packages(pkgs_to_install)
896
897 for pkg_type in self.install_order:
898 if pkg_type in pkgs_to_install:
899 # For multilib, we perform a sanity test before final install
900 # If sanity test fails, it will automatically do a bb.fatal()
901 # and the installation will stop
902 if pkg_type == Manifest.PKG_TYPE_MULTILIB:
903 self._multilib_test_install(pkgs_to_install[pkg_type])
904
905 self.pm.install(pkgs_to_install[pkg_type],
906 [False, True][pkg_type == Manifest.PKG_TYPE_ATTEMPT_ONLY])
907
908 self.pm.install_complementary()
909
910 self._setup_dbg_rootfs(['/var/lib/opkg'])
911
912 execute_pre_post_process(self.d, opkg_post_process_cmds)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500913
914 if self.inc_opkg_image_gen == "1":
915 self.pm.backup_packaging_data()
916
917 @staticmethod
918 def _depends_list():
919 return ['IPKGCONF_SDK', 'IPK_FEED_URIS', 'DEPLOY_DIR_IPK', 'IPKGCONF_TARGET', 'INC_IPK_IMAGE_GEN', 'OPKG_ARGS', 'OPKGLIBDIR', 'OPKG_PREPROCESS_COMMANDS', 'OPKG_POSTPROCESS_COMMANDS', 'OPKGLIBDIR']
920
921 def _get_delayed_postinsts(self):
922 status_file = os.path.join(self.image_rootfs,
923 self.d.getVar('OPKGLIBDIR', True).strip('/'),
924 "opkg", "status")
925 return self._get_delayed_postinsts_common(status_file)
926
927 def _save_postinsts(self):
928 dst_postinst_dir = self.d.expand("${IMAGE_ROOTFS}${sysconfdir}/ipk-postinsts")
929 src_postinst_dir = self.d.expand("${IMAGE_ROOTFS}${OPKGLIBDIR}/opkg/info")
930 return self._save_postinsts_common(dst_postinst_dir, src_postinst_dir)
931
932 def _handle_intercept_failure(self, registered_pkgs):
933 self.pm.mark_packages("unpacked", registered_pkgs.split())
934
935 def _log_check(self):
936 self._log_check_warn()
937 self._log_check_error()
938
939 def _cleanup(self):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500940 self.pm.remove_lists()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500941
942def get_class_for_type(imgtype):
943 return {"rpm": RpmRootfs,
944 "ipk": OpkgRootfs,
945 "deb": DpkgRootfs}[imgtype]
946
947def variable_depends(d, manifest_dir=None):
948 img_type = d.getVar('IMAGE_PKGTYPE', True)
949 cls = get_class_for_type(img_type)
950 return cls._depends_list()
951
952def create_rootfs(d, manifest_dir=None):
953 env_bkp = os.environ.copy()
954
955 img_type = d.getVar('IMAGE_PKGTYPE', True)
956 if img_type == "rpm":
957 RpmRootfs(d, manifest_dir).create()
958 elif img_type == "ipk":
959 OpkgRootfs(d, manifest_dir).create()
960 elif img_type == "deb":
961 DpkgRootfs(d, manifest_dir).create()
962
963 os.environ.clear()
964 os.environ.update(env_bkp)
965
966
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500967def image_list_installed_packages(d, rootfs_dir=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500968 if not rootfs_dir:
969 rootfs_dir = d.getVar('IMAGE_ROOTFS', True)
970
971 img_type = d.getVar('IMAGE_PKGTYPE', True)
972 if img_type == "rpm":
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500973 return RpmPkgsList(d, rootfs_dir).list_pkgs()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500974 elif img_type == "ipk":
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500975 return OpkgPkgsList(d, rootfs_dir, d.getVar("IPKGCONF_TARGET", True)).list_pkgs()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500976 elif img_type == "deb":
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500977 return DpkgPkgsList(d, rootfs_dir).list_pkgs()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500978
979if __name__ == "__main__":
980 """
981 We should be able to run this as a standalone script, from outside bitbake
982 environment.
983 """
984 """
985 TBD
986 """