blob: 4ff19cf09c4f319fe4fccefe48ac73f4b50c78d3 [file] [log] [blame]
Brad Bishopc342db32019-05-15 21:57:59 -04001#
2# SPDX-License-Identifier: GPL-2.0-only
3#
4
Patrick Williamsc124f4f2015-09-15 14:41:29 -05005from abc import ABCMeta, abstractmethod
6import os
7import glob
8import subprocess
9import shutil
Patrick Williamsc124f4f2015-09-15 14:41:29 -050010import re
Patrick Williamsc0f7c042017-02-23 20:41:17 -060011import collections
Patrick Williamsc124f4f2015-09-15 14:41:29 -050012import bb
13import tempfile
14import oe.utils
Patrick Williamsc0f7c042017-02-23 20:41:17 -060015import oe.path
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050016import string
17from oe.gpg_sign import get_signer
Brad Bishop316dfdd2018-06-25 12:45:53 -040018import hashlib
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080019import fnmatch
Patrick Williamsc124f4f2015-09-15 14:41:29 -050020
21# this can be used by all PM backends to create the index files in parallel
22def create_index(arg):
23 index_cmd = arg
24
Brad Bishopd7bf8c12018-02-25 22:55:05 -050025 bb.note("Executing '%s' ..." % index_cmd)
26 result = subprocess.check_output(index_cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8")
Patrick Williamsc124f4f2015-09-15 14:41:29 -050027 if result:
28 bb.note(result)
29
Patrick Williamsc0f7c042017-02-23 20:41:17 -060030def opkg_query(cmd_output):
Brad Bishop316dfdd2018-06-25 12:45:53 -040031 """
32 This method parse the output from the package managerand return
33 a dictionary with the information of the packages. This is used
34 when the packages are in deb or ipk format.
35 """
Brad Bishop19323692019-04-05 15:28:33 -040036 verregex = re.compile(r' \([=<>]* [^ )]*\)')
Patrick Williamsc0f7c042017-02-23 20:41:17 -060037 output = dict()
38 pkg = ""
39 arch = ""
40 ver = ""
41 filename = ""
42 dep = []
43 pkgarch = ""
44 for line in cmd_output.splitlines():
45 line = line.rstrip()
46 if ':' in line:
47 if line.startswith("Package: "):
48 pkg = line.split(": ")[1]
49 elif line.startswith("Architecture: "):
50 arch = line.split(": ")[1]
51 elif line.startswith("Version: "):
52 ver = line.split(": ")[1]
53 elif line.startswith("File: ") or line.startswith("Filename:"):
54 filename = line.split(": ")[1]
55 if "/" in filename:
56 filename = os.path.basename(filename)
57 elif line.startswith("Depends: "):
58 depends = verregex.sub('', line.split(": ")[1])
59 for depend in depends.split(", "):
60 dep.append(depend)
61 elif line.startswith("Recommends: "):
62 recommends = verregex.sub('', line.split(": ")[1])
63 for recommend in recommends.split(", "):
64 dep.append("%s [REC]" % recommend)
65 elif line.startswith("PackageArch: "):
66 pkgarch = line.split(": ")[1]
Patrick Williamsc124f4f2015-09-15 14:41:29 -050067
Patrick Williamsc0f7c042017-02-23 20:41:17 -060068 # When there is a blank line save the package information
69 elif not line:
70 # IPK doesn't include the filename
71 if not filename:
72 filename = "%s_%s_%s.ipk" % (pkg, ver, arch)
73 if pkg:
74 output[pkg] = {"arch":arch, "ver":ver,
75 "filename":filename, "deps": dep, "pkgarch":pkgarch }
76 pkg = ""
77 arch = ""
78 ver = ""
79 filename = ""
80 dep = []
81 pkgarch = ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -050082
Patrick Williamsc0f7c042017-02-23 20:41:17 -060083 if pkg:
84 if not filename:
85 filename = "%s_%s_%s.ipk" % (pkg, ver, arch)
86 output[pkg] = {"arch":arch, "ver":ver,
87 "filename":filename, "deps": dep }
88
89 return output
90
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080091def failed_postinsts_abort(pkgs, log_path):
92 bb.fatal("""Postinstall scriptlets of %s have failed. If the intention is to defer them to first boot,
93then please place them into pkg_postinst_ontarget_${PN} ().
94Deferring to first boot via 'exit 1' is no longer supported.
Brad Bishop316dfdd2018-06-25 12:45:53 -040095Details of the failure are in %s.""" %(pkgs, log_path))
Patrick Williamsc0f7c042017-02-23 20:41:17 -060096
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080097def generate_locale_archive(d, rootfs, target_arch, localedir):
98 # Pretty sure we don't need this for locale archive generation but
99 # keeping it to be safe...
100 locale_arch_options = { \
Brad Bishop19323692019-04-05 15:28:33 -0400101 "arc": ["--uint32-align=4", "--little-endian"],
102 "arceb": ["--uint32-align=4", "--big-endian"],
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800103 "arm": ["--uint32-align=4", "--little-endian"],
104 "armeb": ["--uint32-align=4", "--big-endian"],
105 "aarch64": ["--uint32-align=4", "--little-endian"],
106 "aarch64_be": ["--uint32-align=4", "--big-endian"],
107 "sh4": ["--uint32-align=4", "--big-endian"],
108 "powerpc": ["--uint32-align=4", "--big-endian"],
109 "powerpc64": ["--uint32-align=4", "--big-endian"],
110 "mips": ["--uint32-align=4", "--big-endian"],
111 "mipsisa32r6": ["--uint32-align=4", "--big-endian"],
112 "mips64": ["--uint32-align=4", "--big-endian"],
113 "mipsisa64r6": ["--uint32-align=4", "--big-endian"],
114 "mipsel": ["--uint32-align=4", "--little-endian"],
115 "mipsisa32r6el": ["--uint32-align=4", "--little-endian"],
116 "mips64el": ["--uint32-align=4", "--little-endian"],
117 "mipsisa64r6el": ["--uint32-align=4", "--little-endian"],
118 "riscv64": ["--uint32-align=4", "--little-endian"],
119 "riscv32": ["--uint32-align=4", "--little-endian"],
120 "i586": ["--uint32-align=4", "--little-endian"],
121 "i686": ["--uint32-align=4", "--little-endian"],
122 "x86_64": ["--uint32-align=4", "--little-endian"]
123 }
124 if target_arch in locale_arch_options:
125 arch_options = locale_arch_options[target_arch]
126 else:
127 bb.error("locale_arch_options not found for target_arch=" + target_arch)
128 bb.fatal("unknown arch:" + target_arch + " for locale_arch_options")
129
130 # Need to set this so cross-localedef knows where the archive is
131 env = dict(os.environ)
132 env["LOCALEARCHIVE"] = oe.path.join(localedir, "locale-archive")
133
134 for name in os.listdir(localedir):
135 path = os.path.join(localedir, name)
136 if os.path.isdir(path):
137 cmd = ["cross-localedef", "--verbose"]
138 cmd += arch_options
139 cmd += ["--add-to-archive", path]
140 subprocess.check_output(cmd, env=env, stderr=subprocess.STDOUT)
141
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600142class Indexer(object, metaclass=ABCMeta):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500143 def __init__(self, d, deploy_dir):
144 self.d = d
145 self.deploy_dir = deploy_dir
146
147 @abstractmethod
148 def write_index(self):
149 pass
150
151
152class RpmIndexer(Indexer):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500153 def write_index(self):
Brad Bishop316dfdd2018-06-25 12:45:53 -0400154 self.do_write_index(self.deploy_dir)
155
156 def do_write_index(self, deploy_dir):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500157 if self.d.getVar('PACKAGE_FEED_SIGN') == '1':
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500158 signer = get_signer(self.d, self.d.getVar('PACKAGE_FEED_GPG_BACKEND'))
159 else:
160 signer = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500161
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500162 createrepo_c = bb.utils.which(os.environ['PATH'], "createrepo_c")
Brad Bishop316dfdd2018-06-25 12:45:53 -0400163 result = create_index("%s --update -q %s" % (createrepo_c, deploy_dir))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500164 if result:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500165 bb.fatal(result)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500166
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500167 # Sign repomd
168 if signer:
169 sig_type = self.d.getVar('PACKAGE_FEED_GPG_SIGNATURE_TYPE')
170 is_ascii_sig = (sig_type.upper() != "BIN")
Brad Bishop316dfdd2018-06-25 12:45:53 -0400171 signer.detach_sign(os.path.join(deploy_dir, 'repodata', 'repomd.xml'),
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500172 self.d.getVar('PACKAGE_FEED_GPG_NAME'),
173 self.d.getVar('PACKAGE_FEED_GPG_PASSPHRASE_FILE'),
174 armor=is_ascii_sig)
175
Brad Bishop316dfdd2018-06-25 12:45:53 -0400176class RpmSubdirIndexer(RpmIndexer):
177 def write_index(self):
178 bb.note("Generating package index for %s" %(self.deploy_dir))
179 self.do_write_index(self.deploy_dir)
180 for entry in os.walk(self.deploy_dir):
181 if os.path.samefile(self.deploy_dir, entry[0]):
182 for dir in entry[1]:
183 if dir != 'repodata':
184 dir_path = oe.path.join(self.deploy_dir, dir)
185 bb.note("Generating package index for %s" %(dir_path))
186 self.do_write_index(dir_path)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500187
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500188class OpkgIndexer(Indexer):
189 def write_index(self):
190 arch_vars = ["ALL_MULTILIB_PACKAGE_ARCHS",
191 "SDK_PACKAGE_ARCHS",
Brad Bishop316dfdd2018-06-25 12:45:53 -0400192 ]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500193
194 opkg_index_cmd = bb.utils.which(os.getenv('PATH'), "opkg-make-index")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500195 if self.d.getVar('PACKAGE_FEED_SIGN') == '1':
196 signer = get_signer(self.d, self.d.getVar('PACKAGE_FEED_GPG_BACKEND'))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500197 else:
198 signer = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500199
200 if not os.path.exists(os.path.join(self.deploy_dir, "Packages")):
201 open(os.path.join(self.deploy_dir, "Packages"), "w").close()
202
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500203 index_cmds = set()
204 index_sign_files = set()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500205 for arch_var in arch_vars:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500206 archs = self.d.getVar(arch_var)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500207 if archs is None:
208 continue
209
210 for arch in archs.split():
211 pkgs_dir = os.path.join(self.deploy_dir, arch)
212 pkgs_file = os.path.join(pkgs_dir, "Packages")
213
214 if not os.path.isdir(pkgs_dir):
215 continue
216
217 if not os.path.exists(pkgs_file):
218 open(pkgs_file, "w").close()
219
Brad Bishop1d80a2e2019-11-15 16:35:03 -0500220 index_cmds.add('%s --checksum md5 --checksum sha256 -r %s -p %s -m %s' %
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500221 (opkg_index_cmd, pkgs_file, pkgs_file, pkgs_dir))
222
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500223 index_sign_files.add(pkgs_file)
224
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500225 if len(index_cmds) == 0:
226 bb.note("There are no packages in %s!" % self.deploy_dir)
227 return
228
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800229 oe.utils.multiprocess_launch(create_index, index_cmds, self.d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500230
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500231 if signer:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500232 feed_sig_type = self.d.getVar('PACKAGE_FEED_GPG_SIGNATURE_TYPE')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500233 is_ascii_sig = (feed_sig_type.upper() != "BIN")
234 for f in index_sign_files:
235 signer.detach_sign(f,
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500236 self.d.getVar('PACKAGE_FEED_GPG_NAME'),
237 self.d.getVar('PACKAGE_FEED_GPG_PASSPHRASE_FILE'),
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500238 armor=is_ascii_sig)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500239
240
241class DpkgIndexer(Indexer):
242 def _create_configs(self):
243 bb.utils.mkdirhier(self.apt_conf_dir)
244 bb.utils.mkdirhier(os.path.join(self.apt_conf_dir, "lists", "partial"))
245 bb.utils.mkdirhier(os.path.join(self.apt_conf_dir, "apt.conf.d"))
246 bb.utils.mkdirhier(os.path.join(self.apt_conf_dir, "preferences.d"))
247
248 with open(os.path.join(self.apt_conf_dir, "preferences"),
249 "w") as prefs_file:
250 pass
251 with open(os.path.join(self.apt_conf_dir, "sources.list"),
252 "w+") as sources_file:
253 pass
254
255 with open(self.apt_conf_file, "w") as apt_conf:
256 with open(os.path.join(self.d.expand("${STAGING_ETCDIR_NATIVE}"),
257 "apt", "apt.conf.sample")) as apt_conf_sample:
258 for line in apt_conf_sample.read().split("\n"):
Brad Bishop19323692019-04-05 15:28:33 -0400259 line = re.sub(r"#ROOTFS#", "/dev/null", line)
260 line = re.sub(r"#APTCONF#", self.apt_conf_dir, line)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500261 apt_conf.write(line + "\n")
262
263 def write_index(self):
264 self.apt_conf_dir = os.path.join(self.d.expand("${APTCONF_TARGET}"),
265 "apt-ftparchive")
266 self.apt_conf_file = os.path.join(self.apt_conf_dir, "apt.conf")
267 self._create_configs()
268
269 os.environ['APT_CONFIG'] = self.apt_conf_file
270
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500271 pkg_archs = self.d.getVar('PACKAGE_ARCHS')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500272 if pkg_archs is not None:
273 arch_list = pkg_archs.split()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500274 sdk_pkg_archs = self.d.getVar('SDK_PACKAGE_ARCHS')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500275 if sdk_pkg_archs is not None:
276 for a in sdk_pkg_archs.split():
277 if a not in pkg_archs:
278 arch_list.append(a)
279
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500280 all_mlb_pkg_arch_list = (self.d.getVar('ALL_MULTILIB_PACKAGE_ARCHS') or "").split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500281 arch_list.extend(arch for arch in all_mlb_pkg_arch_list if arch not in arch_list)
282
283 apt_ftparchive = bb.utils.which(os.getenv('PATH'), "apt-ftparchive")
284 gzip = bb.utils.which(os.getenv('PATH'), "gzip")
285
286 index_cmds = []
287 deb_dirs_found = False
288 for arch in arch_list:
289 arch_dir = os.path.join(self.deploy_dir, arch)
290 if not os.path.isdir(arch_dir):
291 continue
292
293 cmd = "cd %s; PSEUDO_UNLOAD=1 %s packages . > Packages;" % (arch_dir, apt_ftparchive)
294
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500295 cmd += "%s -fcn Packages > Packages.gz;" % gzip
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500296
297 with open(os.path.join(arch_dir, "Release"), "w+") as release:
298 release.write("Label: %s\n" % arch)
299
300 cmd += "PSEUDO_UNLOAD=1 %s release . >> Release" % apt_ftparchive
Brad Bishop64c979e2019-11-04 13:55:29 -0500301
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500302 index_cmds.append(cmd)
303
304 deb_dirs_found = True
305
306 if not deb_dirs_found:
307 bb.note("There are no packages in %s" % self.deploy_dir)
308 return
309
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800310 oe.utils.multiprocess_launch(create_index, index_cmds, self.d)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500311 if self.d.getVar('PACKAGE_FEED_SIGN') == '1':
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500312 raise NotImplementedError('Package feed signing not implementd for dpkg')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500313
314
315
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600316class PkgsList(object, metaclass=ABCMeta):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500317 def __init__(self, d, rootfs_dir):
318 self.d = d
319 self.rootfs_dir = rootfs_dir
320
321 @abstractmethod
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500322 def list_pkgs(self):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500323 pass
324
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500325class RpmPkgsList(PkgsList):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500326 def list_pkgs(self):
Brad Bishop19323692019-04-05 15:28:33 -0400327 return RpmPM(self.d, self.rootfs_dir, self.d.getVar('TARGET_VENDOR'), needfeed=False).list_installed()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500328
329class OpkgPkgsList(PkgsList):
330 def __init__(self, d, rootfs_dir, config_file):
331 super(OpkgPkgsList, self).__init__(d, rootfs_dir)
332
333 self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg")
334 self.opkg_args = "-f %s -o %s " % (config_file, rootfs_dir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500335 self.opkg_args += self.d.getVar("OPKG_ARGS")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500336
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500337 def list_pkgs(self, format=None):
338 cmd = "%s %s status" % (self.opkg_cmd, self.opkg_args)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500339
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500340 # opkg returns success even when it printed some
341 # "Collected errors:" report to stderr. Mixing stderr into
342 # stdout then leads to random failures later on when
343 # parsing the output. To avoid this we need to collect both
344 # output streams separately and check for empty stderr.
345 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
346 cmd_output, cmd_stderr = p.communicate()
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600347 cmd_output = cmd_output.decode("utf-8")
348 cmd_stderr = cmd_stderr.decode("utf-8")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500349 if p.returncode or cmd_stderr:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500350 bb.fatal("Cannot get the installed packages list. Command '%s' "
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500351 "returned %d and stderr:\n%s" % (cmd, p.returncode, cmd_stderr))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500352
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600353 return opkg_query(cmd_output)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500354
355
356class DpkgPkgsList(PkgsList):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500357
358 def list_pkgs(self):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500359 cmd = [bb.utils.which(os.getenv('PATH'), "dpkg-query"),
360 "--admindir=%s/var/lib/dpkg" % self.rootfs_dir,
361 "-W"]
362
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500363 cmd.append("-f=Package: ${Package}\nArchitecture: ${PackageArch}\nVersion: ${Version}\nFile: ${Package}_${Version}_${Architecture}.deb\nDepends: ${Depends}\nRecommends: ${Recommends}\n\n")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500364
365 try:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600366 cmd_output = subprocess.check_output(cmd, stderr=subprocess.STDOUT).strip().decode("utf-8")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500367 except subprocess.CalledProcessError as e:
368 bb.fatal("Cannot get the installed packages list. Command '%s' "
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600369 "returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output.decode("utf-8")))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500370
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600371 return opkg_query(cmd_output)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500372
373
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600374class PackageManager(object, metaclass=ABCMeta):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500375 """
376 This is an abstract class. Do not instantiate this directly.
377 """
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500378
Brad Bishop316dfdd2018-06-25 12:45:53 -0400379 def __init__(self, d, target_rootfs):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500380 self.d = d
Brad Bishop316dfdd2018-06-25 12:45:53 -0400381 self.target_rootfs = target_rootfs
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500382 self.deploy_dir = None
383 self.deploy_lock = None
Brad Bishop316dfdd2018-06-25 12:45:53 -0400384 self._initialize_intercepts()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500385
Brad Bishop316dfdd2018-06-25 12:45:53 -0400386 def _initialize_intercepts(self):
387 bb.note("Initializing intercept dir for %s" % self.target_rootfs)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400388 # As there might be more than one instance of PackageManager operating at the same time
389 # we need to isolate the intercept_scripts directories from each other,
390 # hence the ugly hash digest in dir name.
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800391 self.intercepts_dir = os.path.join(self.d.getVar('WORKDIR'), "intercept_scripts-%s" %
392 (hashlib.sha256(self.target_rootfs.encode()).hexdigest()))
Brad Bishop316dfdd2018-06-25 12:45:53 -0400393
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800394 postinst_intercepts = (self.d.getVar("POSTINST_INTERCEPTS") or "").split()
395 if not postinst_intercepts:
396 postinst_intercepts_path = self.d.getVar("POSTINST_INTERCEPTS_PATH")
397 if not postinst_intercepts_path:
398 postinst_intercepts_path = self.d.getVar("POSTINST_INTERCEPTS_DIR") or self.d.expand("${COREBASE}/scripts/postinst-intercepts")
399 postinst_intercepts = oe.path.which_wild('*', postinst_intercepts_path)
400
401 bb.debug(1, 'Collected intercepts:\n%s' % ''.join(' %s\n' % i for i in postinst_intercepts))
Brad Bishop316dfdd2018-06-25 12:45:53 -0400402 bb.utils.remove(self.intercepts_dir, True)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800403 bb.utils.mkdirhier(self.intercepts_dir)
404 for intercept in postinst_intercepts:
405 bb.utils.copyfile(intercept, os.path.join(self.intercepts_dir, os.path.basename(intercept)))
Brad Bishop316dfdd2018-06-25 12:45:53 -0400406
407 @abstractmethod
408 def _handle_intercept_failure(self, failed_script):
409 pass
410
411 def _postpone_to_first_boot(self, postinst_intercept_hook):
412 with open(postinst_intercept_hook) as intercept:
413 registered_pkgs = None
414 for line in intercept.read().split("\n"):
Brad Bishop19323692019-04-05 15:28:33 -0400415 m = re.match(r"^##PKGS:(.*)", line)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400416 if m is not None:
417 registered_pkgs = m.group(1).strip()
418 break
419
420 if registered_pkgs is not None:
421 bb.note("If an image is being built, the postinstalls for the following packages "
422 "will be postponed for first boot: %s" %
423 registered_pkgs)
424
425 # call the backend dependent handler
426 self._handle_intercept_failure(registered_pkgs)
427
428
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800429 def run_intercepts(self, populate_sdk=None):
Brad Bishop316dfdd2018-06-25 12:45:53 -0400430 intercepts_dir = self.intercepts_dir
431
432 bb.note("Running intercept scripts:")
433 os.environ['D'] = self.target_rootfs
434 os.environ['STAGING_DIR_NATIVE'] = self.d.getVar('STAGING_DIR_NATIVE')
435 for script in os.listdir(intercepts_dir):
436 script_full = os.path.join(intercepts_dir, script)
437
438 if script == "postinst_intercept" or not os.access(script_full, os.X_OK):
439 continue
440
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800441 # we do not want to run any multilib variant of this
442 if script.startswith("delay_to_first_boot"):
Brad Bishop316dfdd2018-06-25 12:45:53 -0400443 self._postpone_to_first_boot(script_full)
444 continue
445
Brad Bishop19323692019-04-05 15:28:33 -0400446 if populate_sdk == 'host' and self.d.getVar('SDK_OS') == 'mingw32':
447 bb.note("The postinstall intercept hook '%s' could not be executed due to missing wine support, details in %s/log.do_%s"
448 % (script, self.d.getVar('T'), self.d.getVar('BB_CURRENTTASK')))
449 continue
450
Brad Bishop316dfdd2018-06-25 12:45:53 -0400451 bb.note("> Executing %s intercept ..." % script)
452
453 try:
454 output = subprocess.check_output(script_full, stderr=subprocess.STDOUT)
455 if output: bb.note(output.decode("utf-8"))
456 except subprocess.CalledProcessError as e:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400457 bb.note("Exit code %d. Output:\n%s" % (e.returncode, e.output.decode("utf-8")))
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800458 if populate_sdk == 'host':
Brad Bishop19323692019-04-05 15:28:33 -0400459 bb.fatal("The postinstall intercept hook '%s' failed, details in %s/log.do_%s" % (script, self.d.getVar('T'), self.d.getVar('BB_CURRENTTASK')))
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800460 elif populate_sdk == 'target':
461 if "qemuwrapper: qemu usermode is not supported" in e.output.decode("utf-8"):
Brad Bishop19323692019-04-05 15:28:33 -0400462 bb.note("The postinstall intercept hook '%s' could not be executed due to missing qemu usermode support, details in %s/log.do_%s"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800463 % (script, self.d.getVar('T'), self.d.getVar('BB_CURRENTTASK')))
464 else:
465 bb.fatal("The postinstall intercept hook '%s' failed, details in %s/log.do_%s" % (script, self.d.getVar('T'), self.d.getVar('BB_CURRENTTASK')))
466 else:
467 if "qemuwrapper: qemu usermode is not supported" in e.output.decode("utf-8"):
468 bb.note("The postinstall intercept hook '%s' could not be executed due to missing qemu usermode support, details in %s/log.do_%s"
469 % (script, self.d.getVar('T'), self.d.getVar('BB_CURRENTTASK')))
470 self._postpone_to_first_boot(script_full)
471 else:
472 bb.fatal("The postinstall intercept hook '%s' failed, details in %s/log.do_%s" % (script, self.d.getVar('T'), self.d.getVar('BB_CURRENTTASK')))
Brad Bishop316dfdd2018-06-25 12:45:53 -0400473
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500474 @abstractmethod
475 def update(self):
Brad Bishop316dfdd2018-06-25 12:45:53 -0400476 """
477 Update the package manager package database.
478 """
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500479 pass
480
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500481 @abstractmethod
482 def install(self, pkgs, attempt_only=False):
Brad Bishop316dfdd2018-06-25 12:45:53 -0400483 """
484 Install a list of packages. 'pkgs' is a list object. If 'attempt_only' is
485 True, installation failures are ignored.
486 """
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500487 pass
488
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500489 @abstractmethod
490 def remove(self, pkgs, with_dependencies=True):
Brad Bishop316dfdd2018-06-25 12:45:53 -0400491 """
492 Remove a list of packages. 'pkgs' is a list object. If 'with_dependencies'
493 is False, then any dependencies are left in place.
494 """
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500495 pass
496
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500497 @abstractmethod
498 def write_index(self):
Brad Bishop316dfdd2018-06-25 12:45:53 -0400499 """
500 This function creates the index files
501 """
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500502 pass
503
504 @abstractmethod
505 def remove_packaging_data(self):
506 pass
507
508 @abstractmethod
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500509 def list_installed(self):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500510 pass
511
512 @abstractmethod
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500513 def extract(self, pkg):
Brad Bishop316dfdd2018-06-25 12:45:53 -0400514 """
515 Returns the path to a tmpdir where resides the contents of a package.
516 Deleting the tmpdir is responsability of the caller.
517 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500518 pass
519
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500520 @abstractmethod
521 def insert_feeds_uris(self, feed_uris, feed_base_paths, feed_archs):
Brad Bishop316dfdd2018-06-25 12:45:53 -0400522 """
523 Add remote package feeds into repository manager configuration. The parameters
524 for the feeds are set by feed_uris, feed_base_paths and feed_archs.
525 See http://www.yoctoproject.org/docs/current/ref-manual/ref-manual.html#var-PACKAGE_FEED_URIS
526 for their description.
527 """
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500528 pass
529
Brad Bishop00111322018-04-01 22:23:53 -0400530 def install_glob(self, globs, sdk=False):
Brad Bishop316dfdd2018-06-25 12:45:53 -0400531 """
532 Install all packages that match a glob.
533 """
Brad Bishop00111322018-04-01 22:23:53 -0400534 # TODO don't have sdk here but have a property on the superclass
535 # (and respect in install_complementary)
536 if sdk:
537 pkgdatadir = self.d.expand("${TMPDIR}/pkgdata/${SDK_SYS}")
538 else:
539 pkgdatadir = self.d.getVar("PKGDATA_DIR")
540
541 try:
542 bb.note("Installing globbed packages...")
543 cmd = ["oe-pkgdata-util", "-p", pkgdatadir, "list-pkgs", globs]
544 pkgs = subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode("utf-8")
545 self.install(pkgs.split(), attempt_only=True)
546 except subprocess.CalledProcessError as e:
547 # Return code 1 means no packages matched
548 if e.returncode != 1:
549 bb.fatal("Could not compute globbed packages list. Command "
550 "'%s' returned %d:\n%s" %
551 (' '.join(cmd), e.returncode, e.output.decode("utf-8")))
552
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500553 def install_complementary(self, globs=None):
Brad Bishop316dfdd2018-06-25 12:45:53 -0400554 """
555 Install complementary packages based upon the list of currently installed
556 packages e.g. locales, *-dev, *-dbg, etc. This will only attempt to install
557 these packages, if they don't exist then no error will occur. Note: every
558 backend needs to call this function explicitly after the normal package
559 installation
560 """
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500561 if globs is None:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500562 globs = self.d.getVar('IMAGE_INSTALL_COMPLEMENTARY')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500563 split_linguas = set()
564
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500565 for translation in self.d.getVar('IMAGE_LINGUAS').split():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500566 split_linguas.add(translation)
567 split_linguas.add(translation.split('-')[0])
568
569 split_linguas = sorted(split_linguas)
570
571 for lang in split_linguas:
572 globs += " *-locale-%s" % lang
Brad Bishop64c979e2019-11-04 13:55:29 -0500573 for complementary_linguas in (self.d.getVar('IMAGE_LINGUAS_COMPLEMENTARY') or "").split():
574 globs += (" " + complementary_linguas) % lang
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500575
576 if globs is None:
577 return
578
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500579 # we need to write the list of installed packages to a file because the
580 # oe-pkgdata-util reads it from a file
581 with tempfile.NamedTemporaryFile(mode="w+", prefix="installed-pkgs") as installed_pkgs:
582 pkgs = self.list_installed()
583 output = oe.utils.format_pkg_list(pkgs, "arch")
584 installed_pkgs.write(output)
585 installed_pkgs.flush()
586
Brad Bishop00111322018-04-01 22:23:53 -0400587 cmd = ["oe-pkgdata-util",
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500588 "-p", self.d.getVar('PKGDATA_DIR'), "glob", installed_pkgs.name,
589 globs]
590 exclude = self.d.getVar('PACKAGE_EXCLUDE_COMPLEMENTARY')
591 if exclude:
592 cmd.extend(['--exclude=' + '|'.join(exclude.split())])
593 try:
594 bb.note("Installing complementary packages ...")
595 bb.note('Running %s' % cmd)
596 complementary_pkgs = subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode("utf-8")
Brad Bishop00111322018-04-01 22:23:53 -0400597 self.install(complementary_pkgs.split(), attempt_only=True)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500598 except subprocess.CalledProcessError as e:
599 bb.fatal("Could not compute complementary packages list. Command "
600 "'%s' returned %d:\n%s" %
601 (' '.join(cmd), e.returncode, e.output.decode("utf-8")))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500602
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800603 target_arch = self.d.getVar('TARGET_ARCH')
604 localedir = oe.path.join(self.target_rootfs, self.d.getVar("libdir"), "locale")
605 if os.path.exists(localedir) and os.listdir(localedir):
606 generate_locale_archive(self.d, self.target_rootfs, target_arch, localedir)
607 # And now delete the binary locales
608 self.remove(fnmatch.filter(self.list_installed(), "glibc-binary-localedata-*"), False)
609
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500610 def deploy_dir_lock(self):
611 if self.deploy_dir is None:
612 raise RuntimeError("deploy_dir is not set!")
613
614 lock_file_name = os.path.join(self.deploy_dir, "deploy.lock")
615
616 self.deploy_lock = bb.utils.lockfile(lock_file_name)
617
618 def deploy_dir_unlock(self):
619 if self.deploy_lock is None:
620 return
621
622 bb.utils.unlockfile(self.deploy_lock)
623
624 self.deploy_lock = None
625
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500626 def construct_uris(self, uris, base_paths):
Brad Bishop316dfdd2018-06-25 12:45:53 -0400627 """
628 Construct URIs based on the following pattern: uri/base_path where 'uri'
629 and 'base_path' correspond to each element of the corresponding array
630 argument leading to len(uris) x len(base_paths) elements on the returned
631 array
632 """
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500633 def _append(arr1, arr2, sep='/'):
634 res = []
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600635 narr1 = [a.rstrip(sep) for a in arr1]
636 narr2 = [a.rstrip(sep).lstrip(sep) for a in arr2]
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500637 for a1 in narr1:
638 if arr2:
639 for a2 in narr2:
640 res.append("%s%s%s" % (a1, sep, a2))
641 else:
642 res.append(a1)
643 return res
644 return _append(uris, base_paths)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500645
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800646def create_packages_dir(d, subrepo_dir, deploydir, taskname, filterbydependencies):
Brad Bishop316dfdd2018-06-25 12:45:53 -0400647 """
648 Go through our do_package_write_X dependencies and hardlink the packages we depend
649 upon into the repo directory. This prevents us seeing other packages that may
650 have been built that we don't depend upon and also packages for architectures we don't
651 support.
652 """
653 import errno
654
655 taskdepdata = d.getVar("BB_TASKDEPDATA", False)
656 mytaskname = d.getVar("BB_RUNTASK")
657 pn = d.getVar("PN")
658 seendirs = set()
659 multilibs = {}
Brad Bishop64c979e2019-11-04 13:55:29 -0500660
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800661 bb.utils.remove(subrepo_dir, recurse=True)
662 bb.utils.mkdirhier(subrepo_dir)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400663
664 # Detect bitbake -b usage
665 nodeps = d.getVar("BB_LIMITEDDEPS") or False
666 if nodeps or not filterbydependencies:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800667 oe.path.symlink(deploydir, subrepo_dir, True)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400668 return
669
670 start = None
671 for dep in taskdepdata:
672 data = taskdepdata[dep]
673 if data[1] == mytaskname and data[0] == pn:
674 start = dep
675 break
676 if start is None:
677 bb.fatal("Couldn't find ourself in BB_TASKDEPDATA?")
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800678 pkgdeps = set()
Brad Bishop316dfdd2018-06-25 12:45:53 -0400679 start = [start]
680 seen = set(start)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800681 # Support direct dependencies (do_rootfs -> do_package_write_X)
682 # or indirect dependencies within PN (do_populate_sdk_ext -> do_rootfs -> do_package_write_X)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400683 while start:
684 next = []
685 for dep2 in start:
686 for dep in taskdepdata[dep2][3]:
687 if taskdepdata[dep][0] != pn:
688 if "do_" + taskname in dep:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800689 pkgdeps.add(dep)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400690 elif dep not in seen:
691 next.append(dep)
692 seen.add(dep)
693 start = next
694
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800695 for dep in pkgdeps:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400696 c = taskdepdata[dep][0]
697 manifest, d2 = oe.sstatesig.find_sstate_manifest(c, taskdepdata[dep][2], taskname, d, multilibs)
698 if not manifest:
699 bb.fatal("No manifest generated from: %s in %s" % (c, taskdepdata[dep][2]))
700 if not os.path.exists(manifest):
701 continue
702 with open(manifest, "r") as f:
703 for l in f:
704 l = l.strip()
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800705 deploydir = os.path.normpath(deploydir)
706 if bb.data.inherits_class('packagefeed-stability', d):
707 dest = l.replace(deploydir + "-prediff", "")
708 else:
709 dest = l.replace(deploydir, "")
710 dest = subrepo_dir + dest
Brad Bishop316dfdd2018-06-25 12:45:53 -0400711 if l.endswith("/"):
712 if dest not in seendirs:
713 bb.utils.mkdirhier(dest)
714 seendirs.add(dest)
715 continue
716 # Try to hardlink the file, copy if that fails
717 destdir = os.path.dirname(dest)
718 if destdir not in seendirs:
719 bb.utils.mkdirhier(destdir)
720 seendirs.add(destdir)
721 try:
722 os.link(l, dest)
723 except OSError as err:
724 if err.errno == errno.EXDEV:
725 bb.utils.copyfile(l, dest)
726 else:
727 raise
728
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500729class RpmPM(PackageManager):
730 def __init__(self,
731 d,
732 target_rootfs,
733 target_vendor,
734 task_name='target',
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500735 arch_var=None,
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500736 os_var=None,
Brad Bishop316dfdd2018-06-25 12:45:53 -0400737 rpm_repo_workdir="oe-rootfs-repo",
Brad Bishop19323692019-04-05 15:28:33 -0400738 filterbydependencies=True,
739 needfeed=True):
Brad Bishop316dfdd2018-06-25 12:45:53 -0400740 super(RpmPM, self).__init__(d, target_rootfs)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500741 self.target_vendor = target_vendor
742 self.task_name = task_name
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500743 if arch_var == None:
744 self.archs = self.d.getVar('ALL_MULTILIB_PACKAGE_ARCHS').replace("-","_")
745 else:
746 self.archs = self.d.getVar(arch_var).replace("-","_")
747 if task_name == "host":
748 self.primary_arch = self.d.getVar('SDK_ARCH')
749 else:
750 self.primary_arch = self.d.getVar('MACHINE_ARCH')
751
Brad Bishop19323692019-04-05 15:28:33 -0400752 if needfeed:
753 self.rpm_repo_dir = oe.path.join(self.d.getVar('WORKDIR'), rpm_repo_workdir)
754 create_packages_dir(self.d, oe.path.join(self.rpm_repo_dir, "rpm"), d.getVar("DEPLOY_DIR_RPM"), "package_write_rpm", filterbydependencies)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500755
756 self.saved_packaging_data = self.d.expand('${T}/saved_packaging_data/%s' % self.task_name)
757 if not os.path.exists(self.d.expand('${T}/saved_packaging_data')):
758 bb.utils.mkdirhier(self.d.expand('${T}/saved_packaging_data'))
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800759 self.packaging_data_dirs = ['etc/rpm', 'etc/rpmrc', 'etc/dnf', 'var/lib/rpm', 'var/lib/dnf', 'var/cache/dnf']
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500760 self.solution_manifest = self.d.expand('${T}/saved/%s_solution' %
761 self.task_name)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500762 if not os.path.exists(self.d.expand('${T}/saved')):
763 bb.utils.mkdirhier(self.d.expand('${T}/saved'))
764
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500765 def _configure_dnf(self):
766 # libsolv handles 'noarch' internally, we don't need to specify it explicitly
767 archs = [i for i in reversed(self.archs.split()) if i not in ["any", "all", "noarch"]]
768 # This prevents accidental matching against libsolv's built-in policies
769 if len(archs) <= 1:
770 archs = archs + ["bogusarch"]
771 confdir = "%s/%s" %(self.target_rootfs, "etc/dnf/vars/")
772 bb.utils.mkdirhier(confdir)
773 open(confdir + "arch", 'w').write(":".join(archs))
774 distro_codename = self.d.getVar('DISTRO_CODENAME')
775 open(confdir + "releasever", 'w').write(distro_codename if distro_codename is not None else '')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500776
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500777 open(oe.path.join(self.target_rootfs, "etc/dnf/dnf.conf"), 'w').write("")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500778
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500779
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500780 def _configure_rpm(self):
781 # We need to configure rpm to use our primary package architecture as the installation architecture,
782 # and to make it compatible with other package architectures that we use.
783 # Otherwise it will refuse to proceed with packages installation.
784 platformconfdir = "%s/%s" %(self.target_rootfs, "etc/rpm/")
785 rpmrcconfdir = "%s/%s" %(self.target_rootfs, "etc/")
786 bb.utils.mkdirhier(platformconfdir)
787 open(platformconfdir + "platform", 'w').write("%s-pc-linux" % self.primary_arch)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800788 with open(rpmrcconfdir + "rpmrc", 'w') as f:
789 f.write("arch_compat: %s: %s\n" % (self.primary_arch, self.archs if len(self.archs) > 0 else self.primary_arch))
790 f.write("buildarch_compat: %s: noarch\n" % self.primary_arch)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500791
792 open(platformconfdir + "macros", 'w').write("%_transaction_color 7\n")
793 if self.d.getVar('RPM_PREFER_ELF_ARCH'):
794 open(platformconfdir + "macros", 'a').write("%%_prefer_color %s" % (self.d.getVar('RPM_PREFER_ELF_ARCH')))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500795 else:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500796 open(platformconfdir + "macros", 'a').write("%_prefer_color 7")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500797
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500798 if self.d.getVar('RPM_SIGN_PACKAGES') == '1':
799 signer = get_signer(self.d, self.d.getVar('RPM_GPG_BACKEND'))
800 pubkey_path = oe.path.join(self.d.getVar('B'), 'rpm-key')
801 signer.export_pubkey(pubkey_path, self.d.getVar('RPM_GPG_NAME'))
802 rpm_bin = bb.utils.which(os.getenv('PATH'), "rpmkeys")
803 cmd = [rpm_bin, '--root=%s' % self.target_rootfs, '--import', pubkey_path]
804 try:
805 subprocess.check_output(cmd, stderr=subprocess.STDOUT)
806 except subprocess.CalledProcessError as e:
807 bb.fatal("Importing GPG key failed. Command '%s' "
808 "returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output.decode("utf-8")))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500809
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500810 def create_configs(self):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500811 self._configure_dnf()
812 self._configure_rpm()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500813
814 def write_index(self):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500815 lockfilename = self.d.getVar('DEPLOY_DIR_RPM') + "/rpm.lock"
816 lf = bb.utils.lockfile(lockfilename, False)
817 RpmIndexer(self.d, self.rpm_repo_dir).write_index()
818 bb.utils.unlockfile(lf)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500819
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500820 def insert_feeds_uris(self, feed_uris, feed_base_paths, feed_archs):
821 from urllib.parse import urlparse
822
823 if feed_uris == "":
824 return
825
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500826 gpg_opts = ''
827 if self.d.getVar('PACKAGE_FEED_SIGN') == '1':
828 gpg_opts += 'repo_gpgcheck=1\n'
829 gpg_opts += 'gpgkey=file://%s/pki/packagefeed-gpg/PACKAGEFEED-GPG-KEY-%s-%s\n' % (self.d.getVar('sysconfdir'), self.d.getVar('DISTRO'), self.d.getVar('DISTRO_CODENAME'))
830
Brad Bishop316dfdd2018-06-25 12:45:53 -0400831 if self.d.getVar('RPM_SIGN_PACKAGES') != '1':
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500832 gpg_opts += 'gpgcheck=0\n'
833
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500834 bb.utils.mkdirhier(oe.path.join(self.target_rootfs, "etc", "yum.repos.d"))
835 remote_uris = self.construct_uris(feed_uris.split(), feed_base_paths.split())
836 for uri in remote_uris:
837 repo_base = "oe-remote-repo" + "-".join(urlparse(uri).path.split("/"))
838 if feed_archs is not None:
839 for arch in feed_archs.split():
840 repo_uri = uri + "/" + arch
841 repo_id = "oe-remote-repo" + "-".join(urlparse(repo_uri).path.split("/"))
842 repo_name = "OE Remote Repo:" + " ".join(urlparse(repo_uri).path.split("/"))
843 open(oe.path.join(self.target_rootfs, "etc", "yum.repos.d", repo_base + ".repo"), 'a').write(
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500844 "[%s]\nname=%s\nbaseurl=%s\n%s\n" % (repo_id, repo_name, repo_uri, gpg_opts))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500845 else:
846 repo_name = "OE Remote Repo:" + " ".join(urlparse(uri).path.split("/"))
847 repo_uri = uri
848 open(oe.path.join(self.target_rootfs, "etc", "yum.repos.d", repo_base + ".repo"), 'w').write(
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500849 "[%s]\nname=%s\nbaseurl=%s\n%s" % (repo_base, repo_name, repo_uri, gpg_opts))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500850
851 def _prepare_pkg_transaction(self):
852 os.environ['D'] = self.target_rootfs
853 os.environ['OFFLINE_ROOT'] = self.target_rootfs
854 os.environ['IPKG_OFFLINE_ROOT'] = self.target_rootfs
855 os.environ['OPKG_OFFLINE_ROOT'] = self.target_rootfs
Brad Bishop316dfdd2018-06-25 12:45:53 -0400856 os.environ['INTERCEPT_DIR'] = self.intercepts_dir
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500857 os.environ['NATIVE_ROOT'] = self.d.getVar('STAGING_DIR_NATIVE')
858
859
860 def install(self, pkgs, attempt_only = False):
861 if len(pkgs) == 0:
862 return
863 self._prepare_pkg_transaction()
864
865 bad_recommendations = self.d.getVar('BAD_RECOMMENDATIONS')
866 package_exclude = self.d.getVar('PACKAGE_EXCLUDE')
867 exclude_pkgs = (bad_recommendations.split() if bad_recommendations else []) + (package_exclude.split() if package_exclude else [])
868
869 output = self._invoke_dnf((["--skip-broken"] if attempt_only else []) +
870 (["-x", ",".join(exclude_pkgs)] if len(exclude_pkgs) > 0 else []) +
871 (["--setopt=install_weak_deps=False"] if self.d.getVar('NO_RECOMMENDATIONS') == "1" else []) +
872 (["--nogpgcheck"] if self.d.getVar('RPM_SIGN_PACKAGES') != '1' else ["--setopt=gpgcheck=True"]) +
873 ["install"] +
874 pkgs)
875
876 failed_scriptlets_pkgnames = collections.OrderedDict()
877 for line in output.splitlines():
Brad Bishop19323692019-04-05 15:28:33 -0400878 if line.startswith("Error in POSTIN scriptlet in rpm package"):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500879 failed_scriptlets_pkgnames[line.split()[-1]] = True
880
Brad Bishop316dfdd2018-06-25 12:45:53 -0400881 if len(failed_scriptlets_pkgnames) > 0:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800882 failed_postinsts_abort(list(failed_scriptlets_pkgnames.keys()), self.d.expand("${T}/log.do_${BB_CURRENTTASK}"))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500883
884 def remove(self, pkgs, with_dependencies = True):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800885 if not pkgs:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500886 return
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800887
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500888 self._prepare_pkg_transaction()
889
890 if with_dependencies:
891 self._invoke_dnf(["remove"] + pkgs)
892 else:
893 cmd = bb.utils.which(os.getenv('PATH'), "rpm")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500894 args = ["-e", "-v", "--nodeps", "--root=%s" %self.target_rootfs]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500895
896 try:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500897 bb.note("Running %s" % ' '.join([cmd] + args + pkgs))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500898 output = subprocess.check_output([cmd] + args + pkgs, stderr=subprocess.STDOUT).decode("utf-8")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500899 bb.note(output)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500900 except subprocess.CalledProcessError as e:
901 bb.fatal("Could not invoke rpm. Command "
902 "'%s' returned %d:\n%s" % (' '.join([cmd] + args + pkgs), e.returncode, e.output.decode("utf-8")))
903
904 def upgrade(self):
905 self._prepare_pkg_transaction()
906 self._invoke_dnf(["upgrade"])
907
908 def autoremove(self):
909 self._prepare_pkg_transaction()
910 self._invoke_dnf(["autoremove"])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500911
912 def remove_packaging_data(self):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500913 self._invoke_dnf(["clean", "all"])
914 for dir in self.packaging_data_dirs:
915 bb.utils.remove(oe.path.join(self.target_rootfs, dir), True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500916
917 def backup_packaging_data(self):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500918 # Save the packaging dirs for increment rpm image generation
919 if os.path.exists(self.saved_packaging_data):
920 bb.utils.remove(self.saved_packaging_data, True)
921 for i in self.packaging_data_dirs:
922 source_dir = oe.path.join(self.target_rootfs, i)
923 target_dir = oe.path.join(self.saved_packaging_data, i)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800924 if os.path.isdir(source_dir):
925 shutil.copytree(source_dir, target_dir, symlinks=True)
926 elif os.path.isfile(source_dir):
927 shutil.copy2(source_dir, target_dir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500928
929 def recovery_packaging_data(self):
930 # Move the rpmlib back
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500931 if os.path.exists(self.saved_packaging_data):
932 for i in self.packaging_data_dirs:
933 target_dir = oe.path.join(self.target_rootfs, i)
934 if os.path.exists(target_dir):
935 bb.utils.remove(target_dir, True)
936 source_dir = oe.path.join(self.saved_packaging_data, i)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800937 if os.path.isdir(source_dir):
938 shutil.copytree(source_dir, target_dir, symlinks=True)
939 elif os.path.isfile(source_dir):
940 shutil.copy2(source_dir, target_dir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500941
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500942 def list_installed(self):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500943 output = self._invoke_dnf(["repoquery", "--installed", "--queryformat", "Package: %{name} %{arch} %{version} %{name}-%{version}-%{release}.%{arch}.rpm\nDependencies:\n%{requires}\nRecommendations:\n%{recommends}\nDependenciesEndHere:\n"],
944 print_output = False)
945 packages = {}
946 current_package = None
947 current_deps = None
948 current_state = "initial"
949 for line in output.splitlines():
950 if line.startswith("Package:"):
951 package_info = line.split(" ")[1:]
952 current_package = package_info[0]
953 package_arch = package_info[1]
954 package_version = package_info[2]
955 package_rpm = package_info[3]
956 packages[current_package] = {"arch":package_arch, "ver":package_version, "filename":package_rpm}
957 current_deps = []
958 elif line.startswith("Dependencies:"):
959 current_state = "dependencies"
960 elif line.startswith("Recommendations"):
961 current_state = "recommendations"
962 elif line.startswith("DependenciesEndHere:"):
963 current_state = "initial"
964 packages[current_package]["deps"] = current_deps
965 elif len(line) > 0:
966 if current_state == "dependencies":
967 current_deps.append(line)
968 elif current_state == "recommendations":
969 current_deps.append("%s [REC]" % line)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500970
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500971 return packages
972
973 def update(self):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500974 self._invoke_dnf(["makecache", "--refresh"])
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500975
976 def _invoke_dnf(self, dnf_args, fatal = True, print_output = True ):
977 os.environ['RPM_ETCCONFIGDIR'] = self.target_rootfs
978
979 dnf_cmd = bb.utils.which(os.getenv('PATH'), "dnf")
Brad Bishop19323692019-04-05 15:28:33 -0400980 standard_dnf_args = ["-v", "--rpmverbosity=info", "-y",
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500981 "-c", oe.path.join(self.target_rootfs, "etc/dnf/dnf.conf"),
982 "--setopt=reposdir=%s" %(oe.path.join(self.target_rootfs, "etc/yum.repos.d")),
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500983 "--installroot=%s" % (self.target_rootfs),
984 "--setopt=logdir=%s" % (self.d.getVar('T'))
985 ]
Brad Bishop19323692019-04-05 15:28:33 -0400986 if hasattr(self, "rpm_repo_dir"):
987 standard_dnf_args.append("--repofrompath=oe-repo,%s" % (self.rpm_repo_dir))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500988 cmd = [dnf_cmd] + standard_dnf_args + dnf_args
Brad Bishop316dfdd2018-06-25 12:45:53 -0400989 bb.note('Running %s' % ' '.join(cmd))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500990 try:
991 output = subprocess.check_output(cmd,stderr=subprocess.STDOUT).decode("utf-8")
992 if print_output:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800993 bb.debug(1, output)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500994 return output
995 except subprocess.CalledProcessError as e:
996 if print_output:
997 (bb.note, bb.fatal)[fatal]("Could not invoke dnf. Command "
998 "'%s' returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output.decode("utf-8")))
999 else:
1000 (bb.note, bb.fatal)[fatal]("Could not invoke dnf. Command "
1001 "'%s' returned %d:" % (' '.join(cmd), e.returncode))
1002 return e.output.decode("utf-8")
1003
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001004 def dump_install_solution(self, pkgs):
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001005 open(self.solution_manifest, 'w').write(" ".join(pkgs))
1006 return pkgs
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001007
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001008 def load_old_install_solution(self):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001009 if not os.path.exists(self.solution_manifest):
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001010 return []
Brad Bishop64c979e2019-11-04 13:55:29 -05001011 with open(self.solution_manifest, 'r') as fd:
1012 return fd.read().split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001013
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001014 def _script_num_prefix(self, path):
1015 files = os.listdir(path)
1016 numbers = set()
1017 numbers.add(99)
1018 for f in files:
1019 numbers.add(int(f.split("-")[0]))
1020 return max(numbers) + 1
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001021
1022 def save_rpmpostinst(self, pkg):
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001023 bb.note("Saving postinstall script of %s" % (pkg))
1024 cmd = bb.utils.which(os.getenv('PATH'), "rpm")
1025 args = ["-q", "--root=%s" % self.target_rootfs, "--queryformat", "%{postin}", pkg]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001026
1027 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001028 output = subprocess.check_output([cmd] + args,stderr=subprocess.STDOUT).decode("utf-8")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001029 except subprocess.CalledProcessError as e:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001030 bb.fatal("Could not invoke rpm. Command "
1031 "'%s' returned %d:\n%s" % (' '.join([cmd] + args), e.returncode, e.output.decode("utf-8")))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001032
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001033 # may need to prepend #!/bin/sh to output
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001034
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001035 target_path = oe.path.join(self.target_rootfs, self.d.expand('${sysconfdir}/rpm-postinsts/'))
1036 bb.utils.mkdirhier(target_path)
1037 num = self._script_num_prefix(target_path)
1038 saved_script_name = oe.path.join(target_path, "%d-%s" % (num, pkg))
1039 open(saved_script_name, 'w').write(output)
1040 os.chmod(saved_script_name, 0o755)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001041
Brad Bishop316dfdd2018-06-25 12:45:53 -04001042 def _handle_intercept_failure(self, registered_pkgs):
1043 rpm_postinsts_dir = self.target_rootfs + self.d.expand('${sysconfdir}/rpm-postinsts/')
1044 bb.utils.mkdirhier(rpm_postinsts_dir)
1045
1046 # Save the package postinstalls in /etc/rpm-postinsts
1047 for pkg in registered_pkgs.split():
1048 self.save_rpmpostinst(pkg)
1049
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001050 def extract(self, pkg):
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001051 output = self._invoke_dnf(["repoquery", "--queryformat", "%{location}", pkg])
1052 pkg_name = output.splitlines()[-1]
1053 if not pkg_name.endswith(".rpm"):
1054 bb.fatal("dnf could not find package %s in repository: %s" %(pkg, output))
1055 pkg_path = oe.path.join(self.rpm_repo_dir, pkg_name)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001056
1057 cpio_cmd = bb.utils.which(os.getenv("PATH"), "cpio")
1058 rpm2cpio_cmd = bb.utils.which(os.getenv("PATH"), "rpm2cpio")
1059
1060 if not os.path.isfile(pkg_path):
1061 bb.fatal("Unable to extract package for '%s'."
1062 "File %s doesn't exists" % (pkg, pkg_path))
1063
1064 tmp_dir = tempfile.mkdtemp()
1065 current_dir = os.getcwd()
1066 os.chdir(tmp_dir)
1067
1068 try:
1069 cmd = "%s %s | %s -idmv" % (rpm2cpio_cmd, pkg_path, cpio_cmd)
1070 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
1071 except subprocess.CalledProcessError as e:
1072 bb.utils.remove(tmp_dir, recurse=True)
1073 bb.fatal("Unable to extract %s package. Command '%s' "
1074 "returned %d:\n%s" % (pkg_path, cmd, e.returncode, e.output.decode("utf-8")))
1075 except OSError as e:
1076 bb.utils.remove(tmp_dir, recurse=True)
1077 bb.fatal("Unable to extract %s package. Command '%s' "
1078 "returned %d:\n%s at %s" % (pkg_path, cmd, e.errno, e.strerror, e.filename))
1079
1080 bb.note("Extracted %s to %s" % (pkg_path, tmp_dir))
1081 os.chdir(current_dir)
1082
1083 return tmp_dir
1084
1085
1086class OpkgDpkgPM(PackageManager):
Brad Bishop316dfdd2018-06-25 12:45:53 -04001087 def __init__(self, d, target_rootfs):
1088 """
1089 This is an abstract class. Do not instantiate this directly.
1090 """
1091 super(OpkgDpkgPM, self).__init__(d, target_rootfs)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001092
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001093 def package_info(self, pkg, cmd):
Brad Bishop316dfdd2018-06-25 12:45:53 -04001094 """
1095 Returns a dictionary with the package info.
1096
1097 This method extracts the common parts for Opkg and Dpkg
1098 """
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001099
1100 try:
1101 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8")
1102 except subprocess.CalledProcessError as e:
1103 bb.fatal("Unable to list available packages. Command '%s' "
1104 "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8")))
1105 return opkg_query(output)
1106
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001107 def extract(self, pkg, pkg_info):
Brad Bishop316dfdd2018-06-25 12:45:53 -04001108 """
1109 Returns the path to a tmpdir where resides the contents of a package.
1110
1111 Deleting the tmpdir is responsability of the caller.
1112
1113 This method extracts the common parts for Opkg and Dpkg
1114 """
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001115
1116 ar_cmd = bb.utils.which(os.getenv("PATH"), "ar")
1117 tar_cmd = bb.utils.which(os.getenv("PATH"), "tar")
1118 pkg_path = pkg_info[pkg]["filepath"]
1119
1120 if not os.path.isfile(pkg_path):
1121 bb.fatal("Unable to extract package for '%s'."
1122 "File %s doesn't exists" % (pkg, pkg_path))
1123
1124 tmp_dir = tempfile.mkdtemp()
1125 current_dir = os.getcwd()
1126 os.chdir(tmp_dir)
Brad Bishop19323692019-04-05 15:28:33 -04001127 data_tar = 'data.tar.xz'
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001128
1129 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001130 cmd = [ar_cmd, 'x', pkg_path]
1131 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
1132 cmd = [tar_cmd, 'xf', data_tar]
1133 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001134 except subprocess.CalledProcessError as e:
1135 bb.utils.remove(tmp_dir, recurse=True)
1136 bb.fatal("Unable to extract %s package. Command '%s' "
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001137 "returned %d:\n%s" % (pkg_path, ' '.join(cmd), e.returncode, e.output.decode("utf-8")))
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001138 except OSError as e:
1139 bb.utils.remove(tmp_dir, recurse=True)
1140 bb.fatal("Unable to extract %s package. Command '%s' "
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001141 "returned %d:\n%s at %s" % (pkg_path, ' '.join(cmd), e.errno, e.strerror, e.filename))
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001142
1143 bb.note("Extracted %s to %s" % (pkg_path, tmp_dir))
1144 bb.utils.remove(os.path.join(tmp_dir, "debian-binary"))
1145 bb.utils.remove(os.path.join(tmp_dir, "control.tar.gz"))
1146 os.chdir(current_dir)
1147
1148 return tmp_dir
1149
Brad Bishop316dfdd2018-06-25 12:45:53 -04001150 def _handle_intercept_failure(self, registered_pkgs):
1151 self.mark_packages("unpacked", registered_pkgs.split())
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001152
1153class OpkgPM(OpkgDpkgPM):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001154 def __init__(self, d, target_rootfs, config_file, archs, task_name='target', ipk_repo_workdir="oe-rootfs-repo", filterbydependencies=True, prepare_index=True):
Brad Bishop316dfdd2018-06-25 12:45:53 -04001155 super(OpkgPM, self).__init__(d, target_rootfs)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001156
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001157 self.config_file = config_file
1158 self.pkg_archs = archs
1159 self.task_name = task_name
1160
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001161 self.deploy_dir = oe.path.join(self.d.getVar('WORKDIR'), ipk_repo_workdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001162 self.deploy_lock_file = os.path.join(self.deploy_dir, "deploy.lock")
1163 self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg")
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001164 self.opkg_args = "--volatile-cache -f %s -t %s -o %s " % (self.config_file, self.d.expand('${T}/ipktemp/') ,target_rootfs)
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001165 self.opkg_args += self.d.getVar("OPKG_ARGS")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001166
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001167 if prepare_index:
1168 create_packages_dir(self.d, self.deploy_dir, d.getVar("DEPLOY_DIR_IPK"), "package_write_ipk", filterbydependencies)
1169
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001170 opkg_lib_dir = self.d.getVar('OPKGLIBDIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001171 if opkg_lib_dir[0] == "/":
1172 opkg_lib_dir = opkg_lib_dir[1:]
1173
1174 self.opkg_dir = os.path.join(target_rootfs, opkg_lib_dir, "opkg")
1175
1176 bb.utils.mkdirhier(self.opkg_dir)
1177
1178 self.saved_opkg_dir = self.d.expand('${T}/saved/%s' % self.task_name)
1179 if not os.path.exists(self.d.expand('${T}/saved')):
1180 bb.utils.mkdirhier(self.d.expand('${T}/saved'))
1181
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001182 self.from_feeds = (self.d.getVar('BUILD_IMAGES_FROM_FEEDS') or "") == "1"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001183 if self.from_feeds:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001184 self._create_custom_config()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001185 else:
1186 self._create_config()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001187
1188 self.indexer = OpkgIndexer(self.d, self.deploy_dir)
1189
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001190 def mark_packages(self, status_tag, packages=None):
Brad Bishop316dfdd2018-06-25 12:45:53 -04001191 """
1192 This function will change a package's status in /var/lib/opkg/status file.
1193 If 'packages' is None then the new_status will be applied to all
1194 packages
1195 """
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001196 status_file = os.path.join(self.opkg_dir, "status")
1197
1198 with open(status_file, "r") as sf:
1199 with open(status_file + ".tmp", "w+") as tmp_sf:
1200 if packages is None:
1201 tmp_sf.write(re.sub(r"Package: (.*?)\n((?:[^\n]+\n)*?)Status: (.*)(?:unpacked|installed)",
1202 r"Package: \1\n\2Status: \3%s" % status_tag,
1203 sf.read()))
1204 else:
1205 if type(packages).__name__ != "list":
1206 raise TypeError("'packages' should be a list object")
1207
1208 status = sf.read()
1209 for pkg in packages:
1210 status = re.sub(r"Package: %s\n((?:[^\n]+\n)*?)Status: (.*)(?:unpacked|installed)" % pkg,
1211 r"Package: %s\n\1Status: \2%s" % (pkg, status_tag),
1212 status)
1213
1214 tmp_sf.write(status)
1215
1216 os.rename(status_file + ".tmp", status_file)
1217
1218 def _create_custom_config(self):
1219 bb.note("Building from feeds activated!")
1220
1221 with open(self.config_file, "w+") as config_file:
1222 priority = 1
1223 for arch in self.pkg_archs.split():
1224 config_file.write("arch %s %d\n" % (arch, priority))
1225 priority += 5
1226
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001227 for line in (self.d.getVar('IPK_FEED_URIS') or "").split():
Brad Bishop19323692019-04-05 15:28:33 -04001228 feed_match = re.match(r"^[ \t]*(.*)##([^ \t]*)[ \t]*$", line)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001229
1230 if feed_match is not None:
1231 feed_name = feed_match.group(1)
1232 feed_uri = feed_match.group(2)
1233
1234 bb.note("Add %s feed with URL %s" % (feed_name, feed_uri))
1235
1236 config_file.write("src/gz %s %s\n" % (feed_name, feed_uri))
1237
1238 """
1239 Allow to use package deploy directory contents as quick devel-testing
1240 feed. This creates individual feed configs for each arch subdir of those
1241 specified as compatible for the current machine.
1242 NOTE: Development-helper feature, NOT a full-fledged feed.
1243 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001244 if (self.d.getVar('FEED_DEPLOYDIR_BASE_URI') or "") != "":
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001245 for arch in self.pkg_archs.split():
1246 cfg_file_name = os.path.join(self.target_rootfs,
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001247 self.d.getVar("sysconfdir"),
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001248 "opkg",
1249 "local-%s-feed.conf" % arch)
1250
1251 with open(cfg_file_name, "w+") as cfg_file:
1252 cfg_file.write("src/gz local-%s %s/%s" %
1253 (arch,
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001254 self.d.getVar('FEED_DEPLOYDIR_BASE_URI'),
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001255 arch))
1256
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001257 if self.d.getVar('OPKGLIBDIR') != '/var/lib':
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001258 # There is no command line option for this anymore, we need to add
1259 # info_dir and status_file to config file, if OPKGLIBDIR doesn't have
1260 # the default value of "/var/lib" as defined in opkg:
Brad Bishop37a0e4d2017-12-04 01:01:44 -05001261 # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_LISTS_DIR VARDIR "/lib/opkg/lists"
1262 # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_INFO_DIR VARDIR "/lib/opkg/info"
1263 # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_STATUS_FILE VARDIR "/lib/opkg/status"
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001264 cfg_file.write("option info_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'info'))
1265 cfg_file.write("option lists_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'lists'))
1266 cfg_file.write("option status_file %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'status'))
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001267
1268
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001269 def _create_config(self):
1270 with open(self.config_file, "w+") as config_file:
1271 priority = 1
1272 for arch in self.pkg_archs.split():
1273 config_file.write("arch %s %d\n" % (arch, priority))
1274 priority += 5
1275
1276 config_file.write("src oe file:%s\n" % self.deploy_dir)
1277
1278 for arch in self.pkg_archs.split():
1279 pkgs_dir = os.path.join(self.deploy_dir, arch)
1280 if os.path.isdir(pkgs_dir):
1281 config_file.write("src oe-%s file:%s\n" %
1282 (arch, pkgs_dir))
1283
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001284 if self.d.getVar('OPKGLIBDIR') != '/var/lib':
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001285 # There is no command line option for this anymore, we need to add
1286 # info_dir and status_file to config file, if OPKGLIBDIR doesn't have
1287 # the default value of "/var/lib" as defined in opkg:
Brad Bishop37a0e4d2017-12-04 01:01:44 -05001288 # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_LISTS_DIR VARDIR "/lib/opkg/lists"
1289 # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_INFO_DIR VARDIR "/lib/opkg/info"
1290 # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_STATUS_FILE VARDIR "/lib/opkg/status"
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001291 config_file.write("option info_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'info'))
1292 config_file.write("option lists_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'lists'))
1293 config_file.write("option status_file %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'status'))
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001294
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001295 def insert_feeds_uris(self, feed_uris, feed_base_paths, feed_archs):
1296 if feed_uris == "":
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001297 return
1298
1299 rootfs_config = os.path.join('%s/etc/opkg/base-feeds.conf'
1300 % self.target_rootfs)
1301
Brad Bishop96ff1982019-08-19 13:50:42 -04001302 os.makedirs('%s/etc/opkg' % self.target_rootfs, exist_ok=True)
1303
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001304 feed_uris = self.construct_uris(feed_uris.split(), feed_base_paths.split())
1305 archs = self.pkg_archs.split() if feed_archs is None else feed_archs.split()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001306
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001307 with open(rootfs_config, "w+") as config_file:
1308 uri_iterator = 0
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001309 for uri in feed_uris:
1310 if archs:
1311 for arch in archs:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001312 if (feed_archs is None) and (not os.path.exists(oe.path.join(self.deploy_dir, arch))):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001313 continue
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001314 bb.note('Adding opkg feed url-%s-%d (%s)' %
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001315 (arch, uri_iterator, uri))
1316 config_file.write("src/gz uri-%s-%d %s/%s\n" %
1317 (arch, uri_iterator, uri, arch))
1318 else:
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001319 bb.note('Adding opkg feed url-%d (%s)' %
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001320 (uri_iterator, uri))
1321 config_file.write("src/gz uri-%d %s\n" %
1322 (uri_iterator, uri))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001323
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001324 uri_iterator += 1
1325
1326 def update(self):
1327 self.deploy_dir_lock()
1328
1329 cmd = "%s %s update" % (self.opkg_cmd, self.opkg_args)
1330
1331 try:
1332 subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT)
1333 except subprocess.CalledProcessError as e:
1334 self.deploy_dir_unlock()
1335 bb.fatal("Unable to update the package index files. Command '%s' "
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001336 "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8")))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001337
1338 self.deploy_dir_unlock()
1339
1340 def install(self, pkgs, attempt_only=False):
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001341 if not pkgs:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001342 return
1343
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001344 cmd = "%s %s" % (self.opkg_cmd, self.opkg_args)
1345 for exclude in (self.d.getVar("PACKAGE_EXCLUDE") or "").split():
1346 cmd += " --add-exclude %s" % exclude
Brad Bishop19323692019-04-05 15:28:33 -04001347 for bad_recommendation in (self.d.getVar("BAD_RECOMMENDATIONS") or "").split():
1348 cmd += " --add-ignore-recommends %s" % bad_recommendation
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001349 cmd += " install "
1350 cmd += " ".join(pkgs)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001351
1352 os.environ['D'] = self.target_rootfs
1353 os.environ['OFFLINE_ROOT'] = self.target_rootfs
1354 os.environ['IPKG_OFFLINE_ROOT'] = self.target_rootfs
1355 os.environ['OPKG_OFFLINE_ROOT'] = self.target_rootfs
Brad Bishop316dfdd2018-06-25 12:45:53 -04001356 os.environ['INTERCEPT_DIR'] = self.intercepts_dir
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001357 os.environ['NATIVE_ROOT'] = self.d.getVar('STAGING_DIR_NATIVE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001358
1359 try:
1360 bb.note("Installing the following packages: %s" % ' '.join(pkgs))
1361 bb.note(cmd)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001362 output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT).decode("utf-8")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001363 bb.note(output)
Brad Bishop316dfdd2018-06-25 12:45:53 -04001364 failed_pkgs = []
1365 for line in output.split('\n'):
1366 if line.endswith("configuration required on target."):
1367 bb.warn(line)
1368 failed_pkgs.append(line.split(".")[0])
1369 if failed_pkgs:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001370 failed_postinsts_abort(failed_pkgs, self.d.expand("${T}/log.do_${BB_CURRENTTASK}"))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001371 except subprocess.CalledProcessError as e:
Brad Bishop00111322018-04-01 22:23:53 -04001372 (bb.fatal, bb.warn)[attempt_only]("Unable to install packages. "
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001373 "Command '%s' returned %d:\n%s" %
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001374 (cmd, e.returncode, e.output.decode("utf-8")))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001375
1376 def remove(self, pkgs, with_dependencies=True):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001377 if not pkgs:
1378 return
1379
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001380 if with_dependencies:
Brad Bishop37a0e4d2017-12-04 01:01:44 -05001381 cmd = "%s %s --force-remove --force-removal-of-dependent-packages remove %s" % \
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001382 (self.opkg_cmd, self.opkg_args, ' '.join(pkgs))
1383 else:
1384 cmd = "%s %s --force-depends remove %s" % \
1385 (self.opkg_cmd, self.opkg_args, ' '.join(pkgs))
1386
1387 try:
1388 bb.note(cmd)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001389 output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT).decode("utf-8")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001390 bb.note(output)
1391 except subprocess.CalledProcessError as e:
1392 bb.fatal("Unable to remove packages. Command '%s' "
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001393 "returned %d:\n%s" % (e.cmd, e.returncode, e.output.decode("utf-8")))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001394
1395 def write_index(self):
1396 self.deploy_dir_lock()
1397
1398 result = self.indexer.write_index()
1399
1400 self.deploy_dir_unlock()
1401
1402 if result is not None:
1403 bb.fatal(result)
1404
1405 def remove_packaging_data(self):
1406 bb.utils.remove(self.opkg_dir, True)
1407 # create the directory back, it's needed by PM lock
1408 bb.utils.mkdirhier(self.opkg_dir)
1409
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001410 def remove_lists(self):
1411 if not self.from_feeds:
1412 bb.utils.remove(os.path.join(self.opkg_dir, "lists"), True)
1413
1414 def list_installed(self):
1415 return OpkgPkgsList(self.d, self.target_rootfs, self.config_file).list_pkgs()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001416
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001417 def dummy_install(self, pkgs):
Brad Bishop316dfdd2018-06-25 12:45:53 -04001418 """
1419 The following function dummy installs pkgs and returns the log of output.
1420 """
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001421 if len(pkgs) == 0:
1422 return
1423
1424 # Create an temp dir as opkg root for dummy installation
1425 temp_rootfs = self.d.expand('${T}/opkg')
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001426 opkg_lib_dir = self.d.getVar('OPKGLIBDIR')
Brad Bishop37a0e4d2017-12-04 01:01:44 -05001427 if opkg_lib_dir[0] == "/":
1428 opkg_lib_dir = opkg_lib_dir[1:]
1429 temp_opkg_dir = os.path.join(temp_rootfs, opkg_lib_dir, 'opkg')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001430 bb.utils.mkdirhier(temp_opkg_dir)
1431
1432 opkg_args = "-f %s -o %s " % (self.config_file, temp_rootfs)
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001433 opkg_args += self.d.getVar("OPKG_ARGS")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001434
1435 cmd = "%s %s update" % (self.opkg_cmd, opkg_args)
1436 try:
1437 subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
1438 except subprocess.CalledProcessError as e:
1439 bb.fatal("Unable to update. Command '%s' "
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001440 "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8")))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001441
1442 # Dummy installation
1443 cmd = "%s %s --noaction install %s " % (self.opkg_cmd,
1444 opkg_args,
1445 ' '.join(pkgs))
1446 try:
1447 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
1448 except subprocess.CalledProcessError as e:
1449 bb.fatal("Unable to dummy install packages. Command '%s' "
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001450 "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8")))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001451
1452 bb.utils.remove(temp_rootfs, True)
1453
1454 return output
1455
1456 def backup_packaging_data(self):
1457 # Save the opkglib for increment ipk image generation
1458 if os.path.exists(self.saved_opkg_dir):
1459 bb.utils.remove(self.saved_opkg_dir, True)
1460 shutil.copytree(self.opkg_dir,
1461 self.saved_opkg_dir,
1462 symlinks=True)
1463
1464 def recover_packaging_data(self):
1465 # Move the opkglib back
1466 if os.path.exists(self.saved_opkg_dir):
1467 if os.path.exists(self.opkg_dir):
1468 bb.utils.remove(self.opkg_dir, True)
1469
1470 bb.note('Recover packaging data')
1471 shutil.copytree(self.saved_opkg_dir,
1472 self.opkg_dir,
1473 symlinks=True)
1474
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001475 def package_info(self, pkg):
Brad Bishop316dfdd2018-06-25 12:45:53 -04001476 """
1477 Returns a dictionary with the package info.
1478 """
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001479 cmd = "%s %s info %s" % (self.opkg_cmd, self.opkg_args, pkg)
1480 pkg_info = super(OpkgPM, self).package_info(pkg, cmd)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001481
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001482 pkg_arch = pkg_info[pkg]["arch"]
1483 pkg_filename = pkg_info[pkg]["filename"]
1484 pkg_info[pkg]["filepath"] = \
1485 os.path.join(self.deploy_dir, pkg_arch, pkg_filename)
1486
1487 return pkg_info
1488
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001489 def extract(self, pkg):
Brad Bishop316dfdd2018-06-25 12:45:53 -04001490 """
1491 Returns the path to a tmpdir where resides the contents of a package.
1492
1493 Deleting the tmpdir is responsability of the caller.
1494 """
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001495 pkg_info = self.package_info(pkg)
1496 if not pkg_info:
1497 bb.fatal("Unable to get information for package '%s' while "
1498 "trying to extract the package." % pkg)
1499
1500 tmp_dir = super(OpkgPM, self).extract(pkg, pkg_info)
Brad Bishop19323692019-04-05 15:28:33 -04001501 bb.utils.remove(os.path.join(tmp_dir, "data.tar.xz"))
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001502
1503 return tmp_dir
1504
1505class DpkgPM(OpkgDpkgPM):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001506 def __init__(self, d, target_rootfs, archs, base_archs, apt_conf_dir=None, deb_repo_workdir="oe-rootfs-repo", filterbydependencies=True):
Brad Bishop316dfdd2018-06-25 12:45:53 -04001507 super(DpkgPM, self).__init__(d, target_rootfs)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001508 self.deploy_dir = oe.path.join(self.d.getVar('WORKDIR'), deb_repo_workdir)
1509
1510 create_packages_dir(self.d, self.deploy_dir, d.getVar("DEPLOY_DIR_DEB"), "package_write_deb", filterbydependencies)
1511
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001512 if apt_conf_dir is None:
1513 self.apt_conf_dir = self.d.expand("${APTCONF_TARGET}/apt")
1514 else:
1515 self.apt_conf_dir = apt_conf_dir
1516 self.apt_conf_file = os.path.join(self.apt_conf_dir, "apt.conf")
1517 self.apt_get_cmd = bb.utils.which(os.getenv('PATH'), "apt-get")
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001518 self.apt_cache_cmd = bb.utils.which(os.getenv('PATH'), "apt-cache")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001519
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001520 self.apt_args = d.getVar("APT_ARGS")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001521
1522 self.all_arch_list = archs.split()
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001523 all_mlb_pkg_arch_list = (self.d.getVar('ALL_MULTILIB_PACKAGE_ARCHS') or "").split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001524 self.all_arch_list.extend(arch for arch in all_mlb_pkg_arch_list if arch not in self.all_arch_list)
1525
1526 self._create_configs(archs, base_archs)
1527
1528 self.indexer = DpkgIndexer(self.d, self.deploy_dir)
1529
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001530 def mark_packages(self, status_tag, packages=None):
Brad Bishop316dfdd2018-06-25 12:45:53 -04001531 """
1532 This function will change a package's status in /var/lib/dpkg/status file.
1533 If 'packages' is None then the new_status will be applied to all
1534 packages
1535 """
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001536 status_file = self.target_rootfs + "/var/lib/dpkg/status"
1537
1538 with open(status_file, "r") as sf:
1539 with open(status_file + ".tmp", "w+") as tmp_sf:
1540 if packages is None:
1541 tmp_sf.write(re.sub(r"Package: (.*?)\n((?:[^\n]+\n)*?)Status: (.*)(?:unpacked|installed)",
1542 r"Package: \1\n\2Status: \3%s" % status_tag,
1543 sf.read()))
1544 else:
1545 if type(packages).__name__ != "list":
1546 raise TypeError("'packages' should be a list object")
1547
1548 status = sf.read()
1549 for pkg in packages:
1550 status = re.sub(r"Package: %s\n((?:[^\n]+\n)*?)Status: (.*)(?:unpacked|installed)" % pkg,
1551 r"Package: %s\n\1Status: \2%s" % (pkg, status_tag),
1552 status)
1553
1554 tmp_sf.write(status)
1555
1556 os.rename(status_file + ".tmp", status_file)
1557
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001558 def run_pre_post_installs(self, package_name=None):
Brad Bishop316dfdd2018-06-25 12:45:53 -04001559 """
1560 Run the pre/post installs for package "package_name". If package_name is
1561 None, then run all pre/post install scriptlets.
1562 """
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001563 info_dir = self.target_rootfs + "/var/lib/dpkg/info"
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001564 ControlScript = collections.namedtuple("ControlScript", ["suffix", "name", "argument"])
1565 control_scripts = [
1566 ControlScript(".preinst", "Preinstall", "install"),
1567 ControlScript(".postinst", "Postinstall", "configure")]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001568 status_file = self.target_rootfs + "/var/lib/dpkg/status"
1569 installed_pkgs = []
1570
1571 with open(status_file, "r") as status:
1572 for line in status.read().split('\n'):
Brad Bishop19323692019-04-05 15:28:33 -04001573 m = re.match(r"^Package: (.*)", line)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001574 if m is not None:
1575 installed_pkgs.append(m.group(1))
1576
1577 if package_name is not None and not package_name in installed_pkgs:
1578 return
1579
1580 os.environ['D'] = self.target_rootfs
1581 os.environ['OFFLINE_ROOT'] = self.target_rootfs
1582 os.environ['IPKG_OFFLINE_ROOT'] = self.target_rootfs
1583 os.environ['OPKG_OFFLINE_ROOT'] = self.target_rootfs
Brad Bishop316dfdd2018-06-25 12:45:53 -04001584 os.environ['INTERCEPT_DIR'] = self.intercepts_dir
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001585 os.environ['NATIVE_ROOT'] = self.d.getVar('STAGING_DIR_NATIVE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001586
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001587 for pkg_name in installed_pkgs:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001588 for control_script in control_scripts:
1589 p_full = os.path.join(info_dir, pkg_name + control_script.suffix)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001590 if os.path.exists(p_full):
1591 try:
1592 bb.note("Executing %s for package: %s ..." %
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001593 (control_script.name.lower(), pkg_name))
1594 output = subprocess.check_output([p_full, control_script.argument],
1595 stderr=subprocess.STDOUT).decode("utf-8")
1596 bb.note(output)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001597 except subprocess.CalledProcessError as e:
Brad Bishop316dfdd2018-06-25 12:45:53 -04001598 bb.warn("%s for package %s failed with %d:\n%s" %
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001599 (control_script.name, pkg_name, e.returncode,
1600 e.output.decode("utf-8")))
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001601 failed_postinsts_abort([pkg_name], self.d.expand("${T}/log.do_${BB_CURRENTTASK}"))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001602
1603 def update(self):
1604 os.environ['APT_CONFIG'] = self.apt_conf_file
1605
1606 self.deploy_dir_lock()
1607
1608 cmd = "%s update" % self.apt_get_cmd
1609
1610 try:
1611 subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT)
1612 except subprocess.CalledProcessError as e:
1613 bb.fatal("Unable to update the package index files. Command '%s' "
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001614 "returned %d:\n%s" % (e.cmd, e.returncode, e.output.decode("utf-8")))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001615
1616 self.deploy_dir_unlock()
1617
1618 def install(self, pkgs, attempt_only=False):
1619 if attempt_only and len(pkgs) == 0:
1620 return
1621
1622 os.environ['APT_CONFIG'] = self.apt_conf_file
1623
1624 cmd = "%s %s install --force-yes --allow-unauthenticated %s" % \
1625 (self.apt_get_cmd, self.apt_args, ' '.join(pkgs))
1626
1627 try:
1628 bb.note("Installing the following packages: %s" % ' '.join(pkgs))
1629 subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT)
1630 except subprocess.CalledProcessError as e:
Brad Bishop00111322018-04-01 22:23:53 -04001631 (bb.fatal, bb.warn)[attempt_only]("Unable to install packages. "
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001632 "Command '%s' returned %d:\n%s" %
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001633 (cmd, e.returncode, e.output.decode("utf-8")))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001634
1635 # rename *.dpkg-new files/dirs
1636 for root, dirs, files in os.walk(self.target_rootfs):
1637 for dir in dirs:
Brad Bishop19323692019-04-05 15:28:33 -04001638 new_dir = re.sub(r"\.dpkg-new", "", dir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001639 if dir != new_dir:
1640 os.rename(os.path.join(root, dir),
1641 os.path.join(root, new_dir))
1642
1643 for file in files:
Brad Bishop19323692019-04-05 15:28:33 -04001644 new_file = re.sub(r"\.dpkg-new", "", file)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001645 if file != new_file:
1646 os.rename(os.path.join(root, file),
1647 os.path.join(root, new_file))
1648
1649
1650 def remove(self, pkgs, with_dependencies=True):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001651 if not pkgs:
1652 return
1653
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001654 if with_dependencies:
1655 os.environ['APT_CONFIG'] = self.apt_conf_file
1656 cmd = "%s purge %s" % (self.apt_get_cmd, ' '.join(pkgs))
1657 else:
1658 cmd = "%s --admindir=%s/var/lib/dpkg --instdir=%s" \
1659 " -P --force-depends %s" % \
1660 (bb.utils.which(os.getenv('PATH'), "dpkg"),
1661 self.target_rootfs, self.target_rootfs, ' '.join(pkgs))
1662
1663 try:
1664 subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT)
1665 except subprocess.CalledProcessError as e:
1666 bb.fatal("Unable to remove packages. Command '%s' "
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001667 "returned %d:\n%s" % (e.cmd, e.returncode, e.output.decode("utf-8")))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001668
1669 def write_index(self):
1670 self.deploy_dir_lock()
1671
1672 result = self.indexer.write_index()
1673
1674 self.deploy_dir_unlock()
1675
1676 if result is not None:
1677 bb.fatal(result)
1678
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001679 def insert_feeds_uris(self, feed_uris, feed_base_paths, feed_archs):
1680 if feed_uris == "":
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001681 return
1682
1683 sources_conf = os.path.join("%s/etc/apt/sources.list"
1684 % self.target_rootfs)
1685 arch_list = []
1686
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001687 if feed_archs is None:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001688 for arch in self.all_arch_list:
1689 if not os.path.exists(os.path.join(self.deploy_dir, arch)):
1690 continue
1691 arch_list.append(arch)
1692 else:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001693 arch_list = feed_archs.split()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001694
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001695 feed_uris = self.construct_uris(feed_uris.split(), feed_base_paths.split())
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001696
1697 with open(sources_conf, "w+") as sources_file:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001698 for uri in feed_uris:
1699 if arch_list:
1700 for arch in arch_list:
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001701 bb.note('Adding dpkg channel at (%s)' % uri)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001702 sources_file.write("deb %s/%s ./\n" %
1703 (uri, arch))
1704 else:
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001705 bb.note('Adding dpkg channel at (%s)' % uri)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001706 sources_file.write("deb %s ./\n" % uri)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001707
1708 def _create_configs(self, archs, base_archs):
Brad Bishop19323692019-04-05 15:28:33 -04001709 base_archs = re.sub(r"_", r"-", base_archs)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001710
1711 if os.path.exists(self.apt_conf_dir):
1712 bb.utils.remove(self.apt_conf_dir, True)
1713
1714 bb.utils.mkdirhier(self.apt_conf_dir)
1715 bb.utils.mkdirhier(self.apt_conf_dir + "/lists/partial/")
1716 bb.utils.mkdirhier(self.apt_conf_dir + "/apt.conf.d/")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001717 bb.utils.mkdirhier(self.apt_conf_dir + "/preferences.d/")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001718
1719 arch_list = []
1720 for arch in self.all_arch_list:
1721 if not os.path.exists(os.path.join(self.deploy_dir, arch)):
1722 continue
1723 arch_list.append(arch)
1724
1725 with open(os.path.join(self.apt_conf_dir, "preferences"), "w+") as prefs_file:
1726 priority = 801
1727 for arch in arch_list:
1728 prefs_file.write(
1729 "Package: *\n"
1730 "Pin: release l=%s\n"
1731 "Pin-Priority: %d\n\n" % (arch, priority))
1732
1733 priority += 5
1734
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001735 pkg_exclude = self.d.getVar('PACKAGE_EXCLUDE') or ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001736 for pkg in pkg_exclude.split():
1737 prefs_file.write(
1738 "Package: %s\n"
1739 "Pin: release *\n"
1740 "Pin-Priority: -1\n\n" % pkg)
1741
1742 arch_list.reverse()
1743
1744 with open(os.path.join(self.apt_conf_dir, "sources.list"), "w+") as sources_file:
1745 for arch in arch_list:
1746 sources_file.write("deb file:%s/ ./\n" %
1747 os.path.join(self.deploy_dir, arch))
1748
1749 base_arch_list = base_archs.split()
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001750 multilib_variants = self.d.getVar("MULTILIB_VARIANTS");
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001751 for variant in multilib_variants.split():
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001752 localdata = bb.data.createCopy(self.d)
1753 variant_tune = localdata.getVar("DEFAULTTUNE_virtclass-multilib-" + variant, False)
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001754 orig_arch = localdata.getVar("DPKG_ARCH")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001755 localdata.setVar("DEFAULTTUNE", variant_tune)
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001756 variant_arch = localdata.getVar("DPKG_ARCH")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001757 if variant_arch not in base_arch_list:
1758 base_arch_list.append(variant_arch)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001759
1760 with open(self.apt_conf_file, "w+") as apt_conf:
1761 with open(self.d.expand("${STAGING_ETCDIR_NATIVE}/apt/apt.conf.sample")) as apt_conf_sample:
1762 for line in apt_conf_sample.read().split("\n"):
Brad Bishop19323692019-04-05 15:28:33 -04001763 match_arch = re.match(r" Architecture \".*\";$", line)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001764 architectures = ""
1765 if match_arch:
1766 for base_arch in base_arch_list:
1767 architectures += "\"%s\";" % base_arch
1768 apt_conf.write(" Architectures {%s};\n" % architectures);
1769 apt_conf.write(" Architecture \"%s\";\n" % base_archs)
1770 else:
Brad Bishop19323692019-04-05 15:28:33 -04001771 line = re.sub(r"#ROOTFS#", self.target_rootfs, line)
1772 line = re.sub(r"#APTCONF#", self.apt_conf_dir, line)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001773 apt_conf.write(line + "\n")
1774
1775 target_dpkg_dir = "%s/var/lib/dpkg" % self.target_rootfs
1776 bb.utils.mkdirhier(os.path.join(target_dpkg_dir, "info"))
1777
1778 bb.utils.mkdirhier(os.path.join(target_dpkg_dir, "updates"))
1779
1780 if not os.path.exists(os.path.join(target_dpkg_dir, "status")):
1781 open(os.path.join(target_dpkg_dir, "status"), "w+").close()
1782 if not os.path.exists(os.path.join(target_dpkg_dir, "available")):
1783 open(os.path.join(target_dpkg_dir, "available"), "w+").close()
1784
1785 def remove_packaging_data(self):
1786 bb.utils.remove(os.path.join(self.target_rootfs,
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001787 self.d.getVar('opkglibdir')), True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001788 bb.utils.remove(self.target_rootfs + "/var/lib/dpkg/", True)
1789
1790 def fix_broken_dependencies(self):
1791 os.environ['APT_CONFIG'] = self.apt_conf_file
1792
1793 cmd = "%s %s -f install" % (self.apt_get_cmd, self.apt_args)
1794
1795 try:
1796 subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT)
1797 except subprocess.CalledProcessError as e:
1798 bb.fatal("Cannot fix broken dependencies. Command '%s' "
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001799 "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8")))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001800
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001801 def list_installed(self):
1802 return DpkgPkgsList(self.d, self.target_rootfs).list_pkgs()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001803
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001804 def package_info(self, pkg):
Brad Bishop316dfdd2018-06-25 12:45:53 -04001805 """
1806 Returns a dictionary with the package info.
1807 """
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001808 cmd = "%s show %s" % (self.apt_cache_cmd, pkg)
1809 pkg_info = super(DpkgPM, self).package_info(pkg, cmd)
1810
1811 pkg_arch = pkg_info[pkg]["pkgarch"]
1812 pkg_filename = pkg_info[pkg]["filename"]
1813 pkg_info[pkg]["filepath"] = \
1814 os.path.join(self.deploy_dir, pkg_arch, pkg_filename)
1815
1816 return pkg_info
1817
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001818 def extract(self, pkg):
Brad Bishop316dfdd2018-06-25 12:45:53 -04001819 """
1820 Returns the path to a tmpdir where resides the contents of a package.
1821
1822 Deleting the tmpdir is responsability of the caller.
1823 """
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001824 pkg_info = self.package_info(pkg)
1825 if not pkg_info:
1826 bb.fatal("Unable to get information for package '%s' while "
1827 "trying to extract the package." % pkg)
1828
1829 tmp_dir = super(DpkgPM, self).extract(pkg, pkg_info)
1830 bb.utils.remove(os.path.join(tmp_dir, "data.tar.xz"))
1831
1832 return tmp_dir
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001833
1834def generate_index_files(d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001835 classes = d.getVar('PACKAGE_CLASSES').replace("package_", "").split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001836
1837 indexer_map = {
Brad Bishop316dfdd2018-06-25 12:45:53 -04001838 "rpm": (RpmSubdirIndexer, d.getVar('DEPLOY_DIR_RPM')),
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001839 "ipk": (OpkgIndexer, d.getVar('DEPLOY_DIR_IPK')),
1840 "deb": (DpkgIndexer, d.getVar('DEPLOY_DIR_DEB'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001841 }
1842
1843 result = None
1844
1845 for pkg_class in classes:
1846 if not pkg_class in indexer_map:
1847 continue
1848
1849 if os.path.exists(indexer_map[pkg_class][1]):
1850 result = indexer_map[pkg_class][0](d, indexer_map[pkg_class][1]).write_index()
1851
1852 if result is not None:
1853 bb.fatal(result)