blob: 5f8102304059b065c3ae4bb7aaa00919074175e8 [file] [log] [blame]
Brad Bishopc342db32019-05-15 21:57:59 -04001#
2# SPDX-License-Identifier: GPL-2.0-only
3#
Patrick Williamsc124f4f2015-09-15 14:41:29 -05004from abc import ABCMeta, abstractmethod
5from oe.utils import execute_pre_post_process
6from oe.package_manager import *
7from oe.manifest import *
8import oe.path
Patrick Williamsc124f4f2015-09-15 14:41:29 -05009import shutil
10import os
11import subprocess
12import re
Patrick Williamsc124f4f2015-09-15 14:41:29 -050013
Patrick Williamsc0f7c042017-02-23 20:41:17 -060014class Rootfs(object, metaclass=ABCMeta):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050015 """
16 This is an abstract class. Do not instantiate this directly.
17 """
Patrick Williamsc124f4f2015-09-15 14:41:29 -050018
Brad Bishop6e60e8b2018-02-01 10:27:11 -050019 def __init__(self, d, progress_reporter=None, logcatcher=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050020 self.d = d
21 self.pm = None
Brad Bishop6e60e8b2018-02-01 10:27:11 -050022 self.image_rootfs = self.d.getVar('IMAGE_ROOTFS')
23 self.deploydir = self.d.getVar('IMGDEPLOYDIR')
Patrick Williamsc0f7c042017-02-23 20:41:17 -060024 self.progress_reporter = progress_reporter
Brad Bishop6e60e8b2018-02-01 10:27:11 -050025 self.logcatcher = logcatcher
Patrick Williamsc124f4f2015-09-15 14:41:29 -050026
27 self.install_order = Manifest.INSTALL_ORDER
28
29 @abstractmethod
30 def _create(self):
31 pass
32
33 @abstractmethod
34 def _get_delayed_postinsts(self):
35 pass
36
37 @abstractmethod
38 def _save_postinsts(self):
39 pass
40
41 @abstractmethod
42 def _log_check(self):
43 pass
44
Patrick Williamsc0f7c042017-02-23 20:41:17 -060045 def _log_check_common(self, type, match):
46 # Ignore any lines containing log_check to avoid recursion, and ignore
47 # lines beginning with a + since sh -x may emit code which isn't
48 # actually executed, but may contain error messages
49 excludes = [ 'log_check', r'^\+' ]
50 if hasattr(self, 'log_check_expected_regexes'):
51 excludes.extend(self.log_check_expected_regexes)
Andrew Geissler4c19ea12020-10-27 13:52:24 -050052 # Insert custom log_check excludes
53 excludes += [x for x in (self.d.getVar("IMAGE_LOG_CHECK_EXCLUDES") or "").split(" ") if x]
Patrick Williamsc0f7c042017-02-23 20:41:17 -060054 excludes = [re.compile(x) for x in excludes]
55 r = re.compile(match)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050056 log_path = self.d.expand("${T}/log.do_rootfs")
Patrick Williamsc0f7c042017-02-23 20:41:17 -060057 messages = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -050058 with open(log_path, 'r') as log:
59 for line in log:
Brad Bishop6e60e8b2018-02-01 10:27:11 -050060 if self.logcatcher and self.logcatcher.contains(line.rstrip()):
61 continue
Patrick Williamsc0f7c042017-02-23 20:41:17 -060062 for ee in excludes:
63 m = ee.search(line)
64 if m:
65 break
66 if m:
Patrick Williamsc124f4f2015-09-15 14:41:29 -050067 continue
68
69 m = r.search(line)
70 if m:
Patrick Williamsc0f7c042017-02-23 20:41:17 -060071 messages.append('[log_check] %s' % line)
72 if messages:
73 if len(messages) == 1:
74 msg = '1 %s message' % type
75 else:
76 msg = '%d %s messages' % (len(messages), type)
77 msg = '[log_check] %s: found %s in the logfile:\n%s' % \
Brad Bishop6e60e8b2018-02-01 10:27:11 -050078 (self.d.getVar('PN'), msg, ''.join(messages))
Patrick Williamsc0f7c042017-02-23 20:41:17 -060079 if type == 'error':
80 bb.fatal(msg)
81 else:
82 bb.warn(msg)
83
84 def _log_check_warn(self):
85 self._log_check_common('warning', '^(warn|Warn|WARNING:)')
Patrick Williamsc124f4f2015-09-15 14:41:29 -050086
87 def _log_check_error(self):
Patrick Williamsc0f7c042017-02-23 20:41:17 -060088 self._log_check_common('error', self.log_check_regex)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050089
90 def _insert_feed_uris(self):
91 if bb.utils.contains("IMAGE_FEATURES", "package-management",
92 True, False, self.d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -050093 self.pm.insert_feeds_uris(self.d.getVar('PACKAGE_FEED_URIS') or "",
94 self.d.getVar('PACKAGE_FEED_BASE_PATHS') or "",
95 self.d.getVar('PACKAGE_FEED_ARCHS'))
96
Patrick Williamsc124f4f2015-09-15 14:41:29 -050097
Patrick Williamsc124f4f2015-09-15 14:41:29 -050098 """
99 The _cleanup() method should be used to clean-up stuff that we don't really
100 want to end up on target. For example, in the case of RPM, the DB locks.
101 The method is called, once, at the end of create() method.
102 """
103 @abstractmethod
104 def _cleanup(self):
105 pass
106
107 def _setup_dbg_rootfs(self, dirs):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500108 gen_debugfs = self.d.getVar('IMAGE_GEN_DEBUGFS') or '0'
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500109 if gen_debugfs != '1':
110 return
111
112 bb.note(" Renaming the original rootfs...")
113 try:
114 shutil.rmtree(self.image_rootfs + '-orig')
115 except:
116 pass
117 os.rename(self.image_rootfs, self.image_rootfs + '-orig')
118
119 bb.note(" Creating debug rootfs...")
120 bb.utils.mkdirhier(self.image_rootfs)
121
122 bb.note(" Copying back package database...")
123 for dir in dirs:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600124 if not os.path.isdir(self.image_rootfs + '-orig' + dir):
125 continue
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500126 bb.utils.mkdirhier(self.image_rootfs + os.path.dirname(dir))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600127 shutil.copytree(self.image_rootfs + '-orig' + dir, self.image_rootfs + dir, symlinks=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500128
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500129 # Copy files located in /usr/lib/debug or /usr/src/debug
130 for dir in ["/usr/lib/debug", "/usr/src/debug"]:
131 src = self.image_rootfs + '-orig' + dir
Andrew Geissler82c905d2020-04-13 13:39:40 -0500132 if os.path.exists(src):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500133 dst = self.image_rootfs + dir
134 bb.utils.mkdirhier(os.path.dirname(dst))
135 shutil.copytree(src, dst)
136
137 # Copy files with suffix '.debug' or located in '.debug' dir.
Andrew Geissler82c905d2020-04-13 13:39:40 -0500138 for root, dirs, files in os.walk(self.image_rootfs + '-orig'):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500139 relative_dir = root[len(self.image_rootfs + '-orig'):]
140 for f in files:
141 if f.endswith('.debug') or '/.debug' in relative_dir:
142 bb.utils.mkdirhier(self.image_rootfs + relative_dir)
143 shutil.copy(os.path.join(root, f),
144 self.image_rootfs + relative_dir)
145
146 bb.note(" Install complementary '*-dbg' packages...")
147 self.pm.install_complementary('*-dbg')
148
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800149 if self.d.getVar('PACKAGE_DEBUG_SPLIT_STYLE') == 'debug-with-srcpkg':
150 bb.note(" Install complementary '*-src' packages...")
151 self.pm.install_complementary('*-src')
152
153 """
154 Install additional debug packages. Possibility to install additional packages,
155 which are not automatically installed as complementary package of
156 standard one, e.g. debug package of static libraries.
157 """
158 extra_debug_pkgs = self.d.getVar('IMAGE_INSTALL_DEBUGFS')
159 if extra_debug_pkgs:
160 bb.note(" Install extra debug packages...")
161 self.pm.install(extra_debug_pkgs.split(), True)
162
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500163 bb.note(" Rename debug rootfs...")
164 try:
165 shutil.rmtree(self.image_rootfs + '-dbg')
166 except:
167 pass
168 os.rename(self.image_rootfs, self.image_rootfs + '-dbg')
169
170 bb.note(" Restoreing original rootfs...")
171 os.rename(self.image_rootfs + '-orig', self.image_rootfs)
172
173 def _exec_shell_cmd(self, cmd):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500174 fakerootcmd = self.d.getVar('FAKEROOT')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500175 if fakerootcmd is not None:
176 exec_cmd = [fakerootcmd, cmd]
177 else:
178 exec_cmd = cmd
179
180 try:
181 subprocess.check_output(exec_cmd, stderr=subprocess.STDOUT)
182 except subprocess.CalledProcessError as e:
183 return("Command '%s' returned %d:\n%s" % (e.cmd, e.returncode, e.output))
184
185 return None
186
187 def create(self):
188 bb.note("###### Generate rootfs #######")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500189 pre_process_cmds = self.d.getVar("ROOTFS_PREPROCESS_COMMAND")
190 post_process_cmds = self.d.getVar("ROOTFS_POSTPROCESS_COMMAND")
191 rootfs_post_install_cmds = self.d.getVar('ROOTFS_POSTINSTALL_COMMAND')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500192
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500193 bb.utils.mkdirhier(self.image_rootfs)
194
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600195 bb.utils.mkdirhier(self.deploydir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500196
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500197 execute_pre_post_process(self.d, pre_process_cmds)
198
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600199 if self.progress_reporter:
200 self.progress_reporter.next_stage()
201
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500202 # call the package manager dependent create method
203 self._create()
204
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500205 sysconfdir = self.image_rootfs + self.d.getVar('sysconfdir')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500206 bb.utils.mkdirhier(sysconfdir)
207 with open(sysconfdir + "/version", "w+") as ver:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500208 ver.write(self.d.getVar('BUILDNAME') + "\n")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500209
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500210 execute_pre_post_process(self.d, rootfs_post_install_cmds)
211
Brad Bishop316dfdd2018-06-25 12:45:53 -0400212 self.pm.run_intercepts()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500213
214 execute_pre_post_process(self.d, post_process_cmds)
215
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600216 if self.progress_reporter:
217 self.progress_reporter.next_stage()
218
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500219 if bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs",
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600220 True, False, self.d) and \
221 not bb.utils.contains("IMAGE_FEATURES",
222 "read-only-rootfs-delayed-postinsts",
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500223 True, False, self.d):
224 delayed_postinsts = self._get_delayed_postinsts()
225 if delayed_postinsts is not None:
226 bb.fatal("The following packages could not be configured "
227 "offline and rootfs is read-only: %s" %
228 delayed_postinsts)
229
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500230 if self.d.getVar('USE_DEVFS') != "1":
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500231 self._create_devfs()
232
233 self._uninstall_unneeded()
234
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600235 if self.progress_reporter:
236 self.progress_reporter.next_stage()
237
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500238 self._insert_feed_uris()
239
240 self._run_ldconfig()
241
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500242 if self.d.getVar('USE_DEPMOD') != "0":
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500243 self._generate_kernel_module_deps()
244
245 self._cleanup()
246 self._log_check()
247
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600248 if self.progress_reporter:
249 self.progress_reporter.next_stage()
250
251
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500252 def _uninstall_unneeded(self):
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500253 # Remove the run-postinsts package if no delayed postinsts are found
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500254 delayed_postinsts = self._get_delayed_postinsts()
255 if delayed_postinsts is None:
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500256 if os.path.exists(self.d.expand("${IMAGE_ROOTFS}${sysconfdir}/init.d/run-postinsts")) or os.path.exists(self.d.expand("${IMAGE_ROOTFS}${systemd_unitdir}/system/run-postinsts.service")):
257 self.pm.remove(["run-postinsts"])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500258
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500259 image_rorfs = bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs",
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500260 True, False, self.d)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500261 image_rorfs_force = self.d.getVar('FORCE_RO_REMOVE')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600262
263 if image_rorfs or image_rorfs_force == "1":
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500264 # Remove components that we don't need if it's a read-only rootfs
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500265 unneeded_pkgs = self.d.getVar("ROOTFS_RO_UNNEEDED").split()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500266 pkgs_installed = image_list_installed_packages(self.d)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500267 # Make sure update-alternatives is removed last. This is
268 # because its database has to available while uninstalling
269 # other packages, allowing alternative symlinks of packages
270 # to be uninstalled or to be managed correctly otherwise.
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500271 provider = self.d.getVar("VIRTUAL-RUNTIME_update-alternatives")
272 pkgs_to_remove = sorted([pkg for pkg in pkgs_installed if pkg in unneeded_pkgs], key=lambda x: x == provider)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500273
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500274 # update-alternatives provider is removed in its own remove()
275 # call because all package managers do not guarantee the packages
276 # are removed in the order they given in the list (which is
277 # passed to the command line). The sorting done earlier is
278 # utilized to implement the 2-stage removal.
279 if len(pkgs_to_remove) > 1:
280 self.pm.remove(pkgs_to_remove[:-1], False)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500281 if len(pkgs_to_remove) > 0:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500282 self.pm.remove([pkgs_to_remove[-1]], False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500283
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500284 if delayed_postinsts:
285 self._save_postinsts()
286 if image_rorfs:
287 bb.warn("There are post install scripts "
288 "in a read-only rootfs")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500289
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500290 post_uninstall_cmds = self.d.getVar("ROOTFS_POSTUNINSTALL_COMMAND")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500291 execute_pre_post_process(self.d, post_uninstall_cmds)
292
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500293 runtime_pkgmanage = bb.utils.contains("IMAGE_FEATURES", "package-management",
294 True, False, self.d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500295 if not runtime_pkgmanage:
296 # Remove the package manager data files
297 self.pm.remove_packaging_data()
298
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500299 def _run_ldconfig(self):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500300 if self.d.getVar('LDCONFIGDEPEND'):
Andrew Geissler475cb722020-07-10 16:00:51 -0500301 bb.note("Executing: ldconfig -r " + self.image_rootfs + " -c new -v -X")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500302 self._exec_shell_cmd(['ldconfig', '-r', self.image_rootfs, '-c',
Andrew Geissler475cb722020-07-10 16:00:51 -0500303 'new', '-v', '-X'])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500304
305 def _check_for_kernel_modules(self, modules_dir):
306 for root, dirs, files in os.walk(modules_dir, topdown=True):
307 for name in files:
308 found_ko = name.endswith(".ko")
309 if found_ko:
310 return found_ko
311 return False
312
313 def _generate_kernel_module_deps(self):
314 modules_dir = os.path.join(self.image_rootfs, 'lib', 'modules')
315 # if we don't have any modules don't bother to do the depmod
316 if not self._check_for_kernel_modules(modules_dir):
317 bb.note("No Kernel Modules found, not running depmod")
318 return
319
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500320 kernel_abi_ver_file = oe.path.join(self.d.getVar('PKGDATA_DIR'), "kernel-depmod",
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500321 'kernel-abiversion')
322 if not os.path.exists(kernel_abi_ver_file):
323 bb.fatal("No kernel-abiversion file found (%s), cannot run depmod, aborting" % kernel_abi_ver_file)
324
325 kernel_ver = open(kernel_abi_ver_file).read().strip(' \n')
326 versioned_modules_dir = os.path.join(self.image_rootfs, modules_dir, kernel_ver)
327
328 bb.utils.mkdirhier(versioned_modules_dir)
329
330 self._exec_shell_cmd(['depmodwrapper', '-a', '-b', self.image_rootfs, kernel_ver])
331
332 """
333 Create devfs:
334 * IMAGE_DEVICE_TABLE is the old name to an absolute path to a device table file
335 * IMAGE_DEVICE_TABLES is a new name for a file, or list of files, seached
336 for in the BBPATH
337 If neither are specified then the default name of files/device_table-minimal.txt
338 is searched for in the BBPATH (same as the old version.)
339 """
340 def _create_devfs(self):
341 devtable_list = []
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500342 devtable = self.d.getVar('IMAGE_DEVICE_TABLE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500343 if devtable is not None:
344 devtable_list.append(devtable)
345 else:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500346 devtables = self.d.getVar('IMAGE_DEVICE_TABLES')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500347 if devtables is None:
348 devtables = 'files/device_table-minimal.txt'
349 for devtable in devtables.split():
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500350 devtable_list.append("%s" % bb.utils.which(self.d.getVar('BBPATH'), devtable))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500351
352 for devtable in devtable_list:
353 self._exec_shell_cmd(["makedevs", "-r",
354 self.image_rootfs, "-D", devtable])
355
356
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500357def get_class_for_type(imgtype):
Andrew Geissler6ce62a22020-11-30 19:58:47 -0600358 import importlib
359 mod = importlib.import_module('oe.package_manager.' + imgtype + '.rootfs')
360 return mod.PkgRootfs
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500361
362def variable_depends(d, manifest_dir=None):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500363 img_type = d.getVar('IMAGE_PKGTYPE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500364 cls = get_class_for_type(img_type)
365 return cls._depends_list()
366
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500367def create_rootfs(d, manifest_dir=None, progress_reporter=None, logcatcher=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500368 env_bkp = os.environ.copy()
369
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500370 img_type = d.getVar('IMAGE_PKGTYPE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500371
Andrew Geissler6ce62a22020-11-30 19:58:47 -0600372 cls = get_class_for_type(img_type)
373 cls(d, manifest_dir, progress_reporter, logcatcher).create()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500374 os.environ.clear()
375 os.environ.update(env_bkp)
376
377
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500378def image_list_installed_packages(d, rootfs_dir=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500379 if not rootfs_dir:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500380 rootfs_dir = d.getVar('IMAGE_ROOTFS')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500381
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500382 img_type = d.getVar('IMAGE_PKGTYPE')
Andrew Geissler6ce62a22020-11-30 19:58:47 -0600383
384 import importlib
385 cls = importlib.import_module('oe.package_manager.' + img_type)
386 return cls.PMPkgsList(d, rootfs_dir).list_pkgs()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500387
388if __name__ == "__main__":
389 """
390 We should be able to run this as a standalone script, from outside bitbake
391 environment.
392 """
393 """
394 TBD
395 """