blob: 247bdc7bbfe3beb31026977fc470d0d815b11659 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001#
2# Packaging process
3#
4# Executive summary: This class iterates over the functions listed in PACKAGEFUNCS
5# Taking D and splitting it up into the packages listed in PACKAGES, placing the
6# resulting output in PKGDEST.
7#
8# There are the following default steps but PACKAGEFUNCS can be extended:
9#
Andrew Geisslerc9f78652020-09-18 14:11:35 -050010# a) package_convert_pr_autoinc - convert AUTOINC in PKGV to ${PRSERV_PV_AUTOINC}
Patrick Williamsc124f4f2015-09-15 14:41:29 -050011#
12# b) perform_packagecopy - Copy D into PKGD
13#
14# c) package_do_split_locales - Split out the locale files, updates FILES and PACKAGES
15#
16# d) split_and_strip_files - split the files into runtime and debug and strip them.
17# Debug files include debug info split, and associated sources that end up in -dbg packages
18#
19# e) fixup_perms - Fix up permissions in the package before we split it.
20#
21# f) populate_packages - Split the files in PKGD into separate packages in PKGDEST/<pkgname>
22# Also triggers the binary stripping code to put files in -dbg packages.
23#
24# g) package_do_filedeps - Collect perfile run-time dependency metadata
25# The data is stores in FILER{PROVIDES,DEPENDS}_file_pkg variables with
26# a list of affected files in FILER{PROVIDES,DEPENDS}FLIST_pkg
27#
28# h) package_do_shlibs - Look at the shared libraries generated and autotmatically add any
Brad Bishop316dfdd2018-06-25 12:45:53 -040029# dependencies found. Also stores the package name so anyone else using this library
Patrick Williamsc124f4f2015-09-15 14:41:29 -050030# knows which package to depend on.
31#
32# i) package_do_pkgconfig - Keep track of which packages need and provide which .pc files
33#
34# j) read_shlibdeps - Reads the stored shlibs information into the metadata
35#
36# k) package_depchains - Adds automatic dependencies to -dbg and -dev packages
37#
38# l) emit_pkgdata - saves the packaging data into PKGDATA_DIR for use in later
39# packaging steps
40
41inherit packagedata
Patrick Williamsc124f4f2015-09-15 14:41:29 -050042inherit chrpath
Brad Bishop96ff1982019-08-19 13:50:42 -040043inherit package_pkgdata
Patrick Williamsc124f4f2015-09-15 14:41:29 -050044
45# Need the package_qa_handle_error() in insane.bbclass
46inherit insane
47
48PKGD = "${WORKDIR}/package"
49PKGDEST = "${WORKDIR}/packages-split"
50
51LOCALE_SECTION ?= ''
52
53ALL_MULTILIB_PACKAGE_ARCHS = "${@all_multilib_tune_values(d, 'PACKAGE_ARCHS')}"
54
55# rpm is used for the per-file dependency identification
Brad Bishop316dfdd2018-06-25 12:45:53 -040056# dwarfsrcfiles is used to determine the list of debug source files
57PACKAGE_DEPENDS += "rpm-native dwarfsrcfiles-native"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050058
Brad Bishop6e60e8b2018-02-01 10:27:11 -050059
60# If your postinstall can execute at rootfs creation time rather than on
61# target but depends on a native/cross tool in order to execute, you need to
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080062# list that tool in PACKAGE_WRITE_DEPS. Target package dependencies belong
Brad Bishop6e60e8b2018-02-01 10:27:11 -050063# in the package dependencies as normal, this is just for native/cross support
64# tools at rootfs build time.
65PACKAGE_WRITE_DEPS ??= ""
66
Patrick Williamsc124f4f2015-09-15 14:41:29 -050067def legitimize_package_name(s):
68 """
69 Make sure package names are legitimate strings
70 """
71 import re
72
73 def fixutf(m):
74 cp = m.group(1)
75 if cp:
Patrick Williamsc0f7c042017-02-23 20:41:17 -060076 return ('\\u%s' % cp).encode('latin-1').decode('unicode_escape')
Patrick Williamsc124f4f2015-09-15 14:41:29 -050077
78 # Handle unicode codepoints encoded as <U0123>, as in glibc locale files.
Brad Bishop19323692019-04-05 15:28:33 -040079 s = re.sub(r'<U([0-9A-Fa-f]{1,4})>', fixutf, s)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050080
81 # Remaining package name validity fixes
82 return s.lower().replace('_', '-').replace('@', '+').replace(',', '+').replace('/', '-')
83
84def do_split_packages(d, root, file_regex, output_pattern, description, postinst=None, recursive=False, hook=None, extra_depends=None, aux_files_pattern=None, postrm=None, allow_dirs=False, prepend=False, match_path=False, aux_files_pattern_verbatim=None, allow_links=False, summary=None):
85 """
86 Used in .bb files to split up dynamically generated subpackages of a
87 given package, usually plugins or modules.
88
89 Arguments:
90 root -- the path in which to search
91 file_regex -- regular expression to match searched files. Use
92 parentheses () to mark the part of this expression
93 that should be used to derive the module name (to be
94 substituted where %s is used in other function
95 arguments as noted below)
96 output_pattern -- pattern to use for the package names. Must include %s.
97 description -- description to set for each package. Must include %s.
98 postinst -- postinstall script to use for all packages (as a
99 string)
100 recursive -- True to perform a recursive search - default False
101 hook -- a hook function to be called for every match. The
102 function will be called with the following arguments
103 (in the order listed):
104 f: full path to the file/directory match
105 pkg: the package name
106 file_regex: as above
107 output_pattern: as above
108 modulename: the module name derived using file_regex
109 extra_depends -- extra runtime dependencies (RDEPENDS) to be set for
110 all packages. The default value of None causes a
111 dependency on the main package (${PN}) - if you do
112 not want this, pass '' for this parameter.
113 aux_files_pattern -- extra item(s) to be added to FILES for each
114 package. Can be a single string item or a list of
115 strings for multiple items. Must include %s.
116 postrm -- postrm script to use for all packages (as a string)
117 allow_dirs -- True allow directories to be matched - default False
118 prepend -- if True, prepend created packages to PACKAGES instead
119 of the default False which appends them
120 match_path -- match file_regex on the whole relative path to the
121 root rather than just the file name
122 aux_files_pattern_verbatim -- extra item(s) to be added to FILES for
123 each package, using the actual derived module name
124 rather than converting it to something legal for a
125 package name. Can be a single string item or a list
126 of strings for multiple items. Must include %s.
127 allow_links -- True to allow symlinks to be matched - default False
128 summary -- Summary to set for each package. Must include %s;
129 defaults to description if not set.
130
131 """
132
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500133 dvar = d.getVar('PKGD')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500134 root = d.expand(root)
135 output_pattern = d.expand(output_pattern)
136 extra_depends = d.expand(extra_depends)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500137
138 # If the root directory doesn't exist, don't error out later but silently do
139 # no splitting.
140 if not os.path.exists(dvar + root):
141 return []
142
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500143 ml = d.getVar("MLPREFIX")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500144 if ml:
145 if not output_pattern.startswith(ml):
146 output_pattern = ml + output_pattern
147
148 newdeps = []
149 for dep in (extra_depends or "").split():
150 if dep.startswith(ml):
151 newdeps.append(dep)
152 else:
153 newdeps.append(ml + dep)
154 if newdeps:
155 extra_depends = " ".join(newdeps)
156
157
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500158 packages = d.getVar('PACKAGES').split()
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600159 split_packages = set()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500160
161 if postinst:
162 postinst = '#!/bin/sh\n' + postinst + '\n'
163 if postrm:
164 postrm = '#!/bin/sh\n' + postrm + '\n'
165 if not recursive:
166 objs = os.listdir(dvar + root)
167 else:
168 objs = []
169 for walkroot, dirs, files in os.walk(dvar + root):
170 for file in files:
171 relpath = os.path.join(walkroot, file).replace(dvar + root + '/', '', 1)
172 if relpath:
173 objs.append(relpath)
174
175 if extra_depends == None:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500176 extra_depends = d.getVar("PN")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500177
178 if not summary:
179 summary = description
180
181 for o in sorted(objs):
182 import re, stat
183 if match_path:
184 m = re.match(file_regex, o)
185 else:
186 m = re.match(file_regex, os.path.basename(o))
187
188 if not m:
189 continue
190 f = os.path.join(dvar + root, o)
191 mode = os.lstat(f).st_mode
192 if not (stat.S_ISREG(mode) or (allow_links and stat.S_ISLNK(mode)) or (allow_dirs and stat.S_ISDIR(mode))):
193 continue
194 on = legitimize_package_name(m.group(1))
195 pkg = output_pattern % on
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600196 split_packages.add(pkg)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500197 if not pkg in packages:
198 if prepend:
199 packages = [pkg] + packages
200 else:
201 packages.append(pkg)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500202 oldfiles = d.getVar('FILES_' + pkg)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500203 newfile = os.path.join(root, o)
204 # These names will be passed through glob() so if the filename actually
205 # contains * or ? (rare, but possible) we need to handle that specially
206 newfile = newfile.replace('*', '[*]')
207 newfile = newfile.replace('?', '[?]')
208 if not oldfiles:
209 the_files = [newfile]
210 if aux_files_pattern:
211 if type(aux_files_pattern) is list:
212 for fp in aux_files_pattern:
213 the_files.append(fp % on)
214 else:
215 the_files.append(aux_files_pattern % on)
216 if aux_files_pattern_verbatim:
217 if type(aux_files_pattern_verbatim) is list:
218 for fp in aux_files_pattern_verbatim:
219 the_files.append(fp % m.group(1))
220 else:
221 the_files.append(aux_files_pattern_verbatim % m.group(1))
222 d.setVar('FILES_' + pkg, " ".join(the_files))
223 else:
224 d.setVar('FILES_' + pkg, oldfiles + " " + newfile)
225 if extra_depends != '':
226 d.appendVar('RDEPENDS_' + pkg, ' ' + extra_depends)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500227 if not d.getVar('DESCRIPTION_' + pkg):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500228 d.setVar('DESCRIPTION_' + pkg, description % on)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500229 if not d.getVar('SUMMARY_' + pkg):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500230 d.setVar('SUMMARY_' + pkg, summary % on)
231 if postinst:
232 d.setVar('pkg_postinst_' + pkg, postinst)
233 if postrm:
234 d.setVar('pkg_postrm_' + pkg, postrm)
235 if callable(hook):
236 hook(f, pkg, file_regex, output_pattern, m.group(1))
237
238 d.setVar('PACKAGES', ' '.join(packages))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600239 return list(split_packages)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500240
241PACKAGE_DEPENDS += "file-native"
242
243python () {
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500244 if d.getVar('PACKAGES') != '':
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500245 deps = ""
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500246 for dep in (d.getVar('PACKAGE_DEPENDS') or "").split():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500247 deps += " %s:do_populate_sysroot" % dep
Andrew Geissler82c905d2020-04-13 13:39:40 -0500248 if d.getVar('PACKAGE_MINIDEBUGINFO') == '1':
249 deps += ' xz-native:do_populate_sysroot'
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500250 d.appendVarFlag('do_package', 'depends', deps)
251
252 # shlibs requires any DEPENDS to have already packaged for the *.list files
253 d.appendVarFlag('do_package', 'deptask', " do_packagedata")
254}
255
256# Get a list of files from file vars by searching files under current working directory
257# The list contains symlinks, directories and normal files.
258def files_from_filevars(filevars):
259 import os,glob
260 cpath = oe.cachedpath.CachedPath()
261 files = []
262 for f in filevars:
263 if os.path.isabs(f):
264 f = '.' + f
265 if not f.startswith("./"):
266 f = './' + f
267 globbed = glob.glob(f)
268 if globbed:
269 if [ f ] != globbed:
270 files += globbed
271 continue
272 files.append(f)
273
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600274 symlink_paths = []
275 for ind, f in enumerate(files):
276 # Handle directory symlinks. Truncate path to the lowest level symlink
277 parent = ''
278 for dirname in f.split('/')[:-1]:
279 parent = os.path.join(parent, dirname)
280 if dirname == '.':
281 continue
282 if cpath.islink(parent):
283 bb.warn("FILES contains file '%s' which resides under a "
284 "directory symlink. Please fix the recipe and use the "
285 "real path for the file." % f[1:])
286 symlink_paths.append(f)
287 files[ind] = parent
288 f = parent
289 break
290
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500291 if not cpath.islink(f):
292 if cpath.isdir(f):
293 newfiles = [ os.path.join(f,x) for x in os.listdir(f) ]
294 if newfiles:
295 files += newfiles
296
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600297 return files, symlink_paths
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500298
299# Called in package_<rpm,ipk,deb>.bbclass to get the correct list of configuration files
300def get_conffiles(pkg, d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500301 pkgdest = d.getVar('PKGDEST')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500302 root = os.path.join(pkgdest, pkg)
303 cwd = os.getcwd()
304 os.chdir(root)
305
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500306 conffiles = d.getVar('CONFFILES_%s' % pkg);
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500307 if conffiles == None:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500308 conffiles = d.getVar('CONFFILES')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500309 if conffiles == None:
310 conffiles = ""
311 conffiles = conffiles.split()
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600312 conf_orig_list = files_from_filevars(conffiles)[0]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500313
314 # Remove links and directories from conf_orig_list to get conf_list which only contains normal files
315 conf_list = []
316 for f in conf_orig_list:
317 if os.path.isdir(f):
318 continue
319 if os.path.islink(f):
320 continue
321 if not os.path.exists(f):
322 continue
323 conf_list.append(f)
324
325 # Remove the leading './'
326 for i in range(0, len(conf_list)):
327 conf_list[i] = conf_list[i][1:]
328
329 os.chdir(cwd)
330 return conf_list
331
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500332def checkbuildpath(file, d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500333 tmpdir = d.getVar('TMPDIR')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500334 with open(file) as f:
335 file_content = f.read()
336 if tmpdir in file_content:
337 return True
338
339 return False
340
Brad Bishop316dfdd2018-06-25 12:45:53 -0400341def parse_debugsources_from_dwarfsrcfiles_output(dwarfsrcfiles_output):
342 debugfiles = {}
343
344 for line in dwarfsrcfiles_output.splitlines():
345 if line.startswith("\t"):
346 debugfiles[os.path.normpath(line.split()[0])] = ""
347
348 return debugfiles.keys()
349
Brad Bishop19323692019-04-05 15:28:33 -0400350def source_info(file, d, fatal=True):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800351 import subprocess
352
353 cmd = ["dwarfsrcfiles", file]
354 try:
355 output = subprocess.check_output(cmd, universal_newlines=True, stderr=subprocess.STDOUT)
356 retval = 0
357 except subprocess.CalledProcessError as exc:
358 output = exc.output
359 retval = exc.returncode
360
Brad Bishop316dfdd2018-06-25 12:45:53 -0400361 # 255 means a specific file wasn't fully parsed to get the debug file list, which is not a fatal failure
362 if retval != 0 and retval != 255:
363 msg = "dwarfsrcfiles failed with exit code %s (cmd was %s)%s" % (retval, cmd, ":\n%s" % output if output else "")
364 if fatal:
365 bb.fatal(msg)
366 bb.note(msg)
367
368 debugsources = parse_debugsources_from_dwarfsrcfiles_output(output)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400369
Brad Bishop19323692019-04-05 15:28:33 -0400370 return list(debugsources)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400371
Brad Bishop19323692019-04-05 15:28:33 -0400372def splitdebuginfo(file, dvar, debugdir, debuglibdir, debugappend, debugsrcdir, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500373 # Function to split a single file into two components, one is the stripped
374 # target system binary, the other contains any debugging information. The
375 # two files are linked to reference each other.
376 #
Brad Bishop19323692019-04-05 15:28:33 -0400377 # return a mapping of files:debugsources
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500378
379 import stat
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800380 import subprocess
381
382 src = file[len(dvar):]
383 dest = debuglibdir + os.path.dirname(src) + debugdir + "/" + os.path.basename(src) + debugappend
384 debugfile = dvar + dest
Brad Bishop19323692019-04-05 15:28:33 -0400385 sources = []
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800386
387 # Split the file...
388 bb.utils.mkdirhier(os.path.dirname(debugfile))
389 #bb.note("Split %s -> %s" % (file, debugfile))
390 # Only store off the hard link reference if we successfully split!
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500391
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500392 dvar = d.getVar('PKGD')
393 objcopy = d.getVar("OBJCOPY")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500394
395 # We ignore kernel modules, we don't generate debug info files.
396 if file.find("/lib/modules/") != -1 and file.endswith(".ko"):
Brad Bishop19323692019-04-05 15:28:33 -0400397 return (file, sources)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500398
399 newmode = None
400 if not os.access(file, os.W_OK) or os.access(file, os.R_OK):
401 origmode = os.stat(file)[stat.ST_MODE]
402 newmode = origmode | stat.S_IWRITE | stat.S_IREAD
403 os.chmod(file, newmode)
404
405 # We need to extract the debug src information here...
406 if debugsrcdir:
Brad Bishop19323692019-04-05 15:28:33 -0400407 sources = source_info(file, d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500408
409 bb.utils.mkdirhier(os.path.dirname(debugfile))
410
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800411 subprocess.check_output([objcopy, '--only-keep-debug', file, debugfile], stderr=subprocess.STDOUT)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500412
413 # Set the debuglink to have the view of the file path on the target
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800414 subprocess.check_output([objcopy, '--add-gnu-debuglink', debugfile, file], stderr=subprocess.STDOUT)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500415
416 if newmode:
417 os.chmod(file, origmode)
418
Brad Bishop19323692019-04-05 15:28:33 -0400419 return (file, sources)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500420
Andrew Geissler82c905d2020-04-13 13:39:40 -0500421def splitstaticdebuginfo(file, dvar, debugstaticdir, debugstaticlibdir, debugstaticappend, debugsrcdir, d):
422 # Unlike the function above, there is no way to split a static library
423 # two components. So to get similar results we will copy the unmodified
424 # static library (containing the debug symbols) into a new directory.
425 # We will then strip (preserving symbols) the static library in the
426 # typical location.
427 #
428 # return a mapping of files:debugsources
429
430 import stat
431 import shutil
432
433 src = file[len(dvar):]
434 dest = debugstaticlibdir + os.path.dirname(src) + debugstaticdir + "/" + os.path.basename(src) + debugstaticappend
435 debugfile = dvar + dest
436 sources = []
437
438 # Copy the file...
439 bb.utils.mkdirhier(os.path.dirname(debugfile))
440 #bb.note("Copy %s -> %s" % (file, debugfile))
441
442 dvar = d.getVar('PKGD')
443
444 newmode = None
445 if not os.access(file, os.W_OK) or os.access(file, os.R_OK):
446 origmode = os.stat(file)[stat.ST_MODE]
447 newmode = origmode | stat.S_IWRITE | stat.S_IREAD
448 os.chmod(file, newmode)
449
450 # We need to extract the debug src information here...
451 if debugsrcdir:
452 sources = source_info(file, d)
453
454 bb.utils.mkdirhier(os.path.dirname(debugfile))
455
456 # Copy the unmodified item to the debug directory
457 shutil.copy2(file, debugfile)
458
459 if newmode:
460 os.chmod(file, origmode)
461
462 return (file, sources)
463
464def inject_minidebuginfo(file, dvar, debugdir, debuglibdir, debugappend, debugsrcdir, d):
465 # Extract just the symbols from debuginfo into minidebuginfo,
466 # compress it with xz and inject it back into the binary in a .gnu_debugdata section.
467 # https://sourceware.org/gdb/onlinedocs/gdb/MiniDebugInfo.html
468
469 import subprocess
470
471 readelf = d.getVar('READELF')
472 nm = d.getVar('NM')
473 objcopy = d.getVar('OBJCOPY')
474
475 minidebuginfodir = d.expand('${WORKDIR}/minidebuginfo')
476
477 src = file[len(dvar):]
478 dest = debuglibdir + os.path.dirname(src) + debugdir + "/" + os.path.basename(src) + debugappend
479 debugfile = dvar + dest
480 minidebugfile = minidebuginfodir + src + '.minidebug'
481 bb.utils.mkdirhier(os.path.dirname(minidebugfile))
482
483 # If we didn't produce debuginfo for any reason, we can't produce minidebuginfo either
484 # so skip it.
485 if not os.path.exists(debugfile):
486 bb.debug(1, 'ELF file {} has no debuginfo, skipping minidebuginfo injection'.format(file))
487 return
488
489 # Find non-allocated PROGBITS, NOTE, and NOBITS sections in the debuginfo.
490 # We will exclude all of these from minidebuginfo to save space.
491 remove_section_names = []
492 for line in subprocess.check_output([readelf, '-W', '-S', debugfile], universal_newlines=True).splitlines():
493 fields = line.split()
494 if len(fields) < 8:
495 continue
496 name = fields[0]
497 type = fields[1]
498 flags = fields[7]
499 # .debug_ sections will be removed by objcopy -S so no need to explicitly remove them
500 if name.startswith('.debug_'):
501 continue
502 if 'A' not in flags and type in ['PROGBITS', 'NOTE', 'NOBITS']:
503 remove_section_names.append(name)
504
505 # List dynamic symbols in the binary. We can exclude these from minidebuginfo
506 # because they are always present in the binary.
507 dynsyms = set()
508 for line in subprocess.check_output([nm, '-D', file, '--format=posix', '--defined-only'], universal_newlines=True).splitlines():
509 dynsyms.add(line.split()[0])
510
511 # Find all function symbols from debuginfo which aren't in the dynamic symbols table.
512 # These are the ones we want to keep in minidebuginfo.
513 keep_symbols_file = minidebugfile + '.symlist'
514 found_any_symbols = False
515 with open(keep_symbols_file, 'w') as f:
516 for line in subprocess.check_output([nm, debugfile, '--format=sysv', '--defined-only'], universal_newlines=True).splitlines():
517 fields = line.split('|')
518 if len(fields) < 7:
519 continue
520 name = fields[0].strip()
521 type = fields[3].strip()
522 if type == 'FUNC' and name not in dynsyms:
523 f.write('{}\n'.format(name))
524 found_any_symbols = True
525
526 if not found_any_symbols:
527 bb.debug(1, 'ELF file {} contains no symbols, skipping minidebuginfo injection'.format(file))
528 return
529
530 bb.utils.remove(minidebugfile)
531 bb.utils.remove(minidebugfile + '.xz')
532
533 subprocess.check_call([objcopy, '-S'] +
534 ['--remove-section={}'.format(s) for s in remove_section_names] +
535 ['--keep-symbols={}'.format(keep_symbols_file), debugfile, minidebugfile])
536
537 subprocess.check_call(['xz', '--keep', minidebugfile])
538
539 subprocess.check_call([objcopy, '--add-section', '.gnu_debugdata={}.xz'.format(minidebugfile), file])
540
Brad Bishop19323692019-04-05 15:28:33 -0400541def copydebugsources(debugsrcdir, sources, d):
Brad Bishop316dfdd2018-06-25 12:45:53 -0400542 # The debug src information written out to sourcefile is further processed
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500543 # and copied to the destination here.
544
545 import stat
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800546 import subprocess
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500547
Brad Bishop19323692019-04-05 15:28:33 -0400548 if debugsrcdir and sources:
549 sourcefile = d.expand("${WORKDIR}/debugsources.list")
550 bb.utils.remove(sourcefile)
551
552 # filenames are null-separated - this is an artefact of the previous use
553 # of rpm's debugedit, which was writing them out that way, and the code elsewhere
554 # is still assuming that.
555 debuglistoutput = '\0'.join(sources) + '\0'
556 with open(sourcefile, 'a') as sf:
557 sf.write(debuglistoutput)
558
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500559 dvar = d.getVar('PKGD')
560 strip = d.getVar("STRIP")
561 objcopy = d.getVar("OBJCOPY")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500562 workdir = d.getVar("WORKDIR")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500563 workparentdir = os.path.dirname(os.path.dirname(workdir))
564 workbasedir = os.path.basename(os.path.dirname(workdir)) + "/" + os.path.basename(workdir)
565
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500566 # If build path exists in sourcefile, it means toolchain did not use
567 # -fdebug-prefix-map to compile
568 if checkbuildpath(sourcefile, d):
569 localsrc_prefix = workparentdir + "/"
570 else:
571 localsrc_prefix = "/usr/src/debug/"
572
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500573 nosuchdir = []
574 basepath = dvar
575 for p in debugsrcdir.split("/"):
576 basepath = basepath + "/" + p
577 if not cpath.exists(basepath):
578 nosuchdir.append(basepath)
579 bb.utils.mkdirhier(basepath)
580 cpath.updatecache(basepath)
581
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500582 # Ignore files from the recipe sysroots (target and native)
583 processdebugsrc = "LC_ALL=C ; sort -z -u '%s' | egrep -v -z '((<internal>|<built-in>)$|/.*recipe-sysroot.*/)' | "
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500584 # We need to ignore files that are not actually ours
585 # we do this by only paying attention to items from this package
586 processdebugsrc += "fgrep -zw '%s' | "
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500587 # Remove prefix in the source paths
588 processdebugsrc += "sed 's#%s##g' | "
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500589 processdebugsrc += "(cd '%s' ; cpio -pd0mlL --no-preserve-owner '%s%s' 2>/dev/null)"
590
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500591 cmd = processdebugsrc % (sourcefile, workbasedir, localsrc_prefix, workparentdir, dvar, debugsrcdir)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800592 try:
593 subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
594 except subprocess.CalledProcessError:
595 # Can "fail" if internal headers/transient sources are attempted
596 pass
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500597
598 # cpio seems to have a bug with -lL together and symbolic links are just copied, not dereferenced.
599 # Work around this by manually finding and copying any symbolic links that made it through.
Brad Bishop19323692019-04-05 15:28:33 -0400600 cmd = "find %s%s -type l -print0 -delete | sed s#%s%s/##g | (cd '%s' ; cpio -pd0mL --no-preserve-owner '%s%s')" % \
601 (dvar, debugsrcdir, dvar, debugsrcdir, workparentdir, dvar, debugsrcdir)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800602 subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500603
604 # The copy by cpio may have resulted in some empty directories! Remove these
605 cmd = "find %s%s -empty -type d -delete" % (dvar, debugsrcdir)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800606 subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500607
608 # Also remove debugsrcdir if its empty
609 for p in nosuchdir[::-1]:
610 if os.path.exists(p) and not os.listdir(p):
611 os.rmdir(p)
612
613#
614# Package data handling routines
615#
616
Andrew Geissler82c905d2020-04-13 13:39:40 -0500617def get_package_mapping (pkg, basepkg, d, depversions=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500618 import oe.packagedata
619
620 data = oe.packagedata.read_subpkgdata(pkg, d)
621 key = "PKG_%s" % pkg
622
623 if key in data:
624 # Have to avoid undoing the write_extra_pkgs(global_variants...)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800625 if bb.data.inherits_class('allarch', d) and not d.getVar('MULTILIB_VARIANTS') \
626 and data[key] == basepkg:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500627 return pkg
Andrew Geissler82c905d2020-04-13 13:39:40 -0500628 if depversions == []:
629 # Avoid returning a mapping if the renamed package rprovides its original name
630 rprovkey = "RPROVIDES_%s" % pkg
631 if rprovkey in data:
632 if pkg in bb.utils.explode_dep_versions2(data[rprovkey]):
633 bb.note("%s rprovides %s, not replacing the latter" % (data[key], pkg))
634 return pkg
635 # Do map to rewritten package name
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500636 return data[key]
637
638 return pkg
639
640def get_package_additional_metadata (pkg_type, d):
641 base_key = "PACKAGE_ADD_METADATA"
642 for key in ("%s_%s" % (base_key, pkg_type.upper()), base_key):
643 if d.getVar(key, False) is None:
644 continue
645 d.setVarFlag(key, "type", "list")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500646 if d.getVarFlag(key, "separator") is None:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500647 d.setVarFlag(key, "separator", "\\n")
648 metadata_fields = [field.strip() for field in oe.data.typed_value(key, d)]
649 return "\n".join(metadata_fields).strip()
650
651def runtime_mapping_rename (varname, pkg, d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500652 #bb.note("%s before: %s" % (varname, d.getVar(varname)))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500653
654 new_depends = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500655 deps = bb.utils.explode_dep_versions2(d.getVar(varname) or "")
Andrew Geissler82c905d2020-04-13 13:39:40 -0500656 for depend, depversions in deps.items():
657 new_depend = get_package_mapping(depend, pkg, d, depversions)
658 if depend != new_depend:
659 bb.note("package name mapping done: %s -> %s" % (depend, new_depend))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500660 new_depends[new_depend] = deps[depend]
661
662 d.setVar(varname, bb.utils.join_deps(new_depends, commasep=False))
663
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500664 #bb.note("%s after: %s" % (varname, d.getVar(varname)))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500665
666#
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500667# Used by do_packagedata (and possibly other routines post do_package)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500668#
669
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500670package_get_auto_pr[vardepsexclude] = "BB_TASKDEPDATA"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500671python package_get_auto_pr() {
672 import oe.prservice
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500673
674 def get_do_package_hash(pn):
675 if d.getVar("BB_RUNTASK") != "do_package":
676 taskdepdata = d.getVar("BB_TASKDEPDATA", False)
677 for dep in taskdepdata:
678 if taskdepdata[dep][1] == "do_package" and taskdepdata[dep][0] == pn:
679 return taskdepdata[dep][6]
680 return None
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500681
682 # Support per recipe PRSERV_HOST
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500683 pn = d.getVar('PN')
684 host = d.getVar("PRSERV_HOST_" + pn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500685 if not (host is None):
686 d.setVar("PRSERV_HOST", host)
687
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500688 pkgv = d.getVar("PKGV")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500689
690 # PR Server not active, handle AUTOINC
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500691 if not d.getVar('PRSERV_HOST'):
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500692 d.setVar("PRSERV_PV_AUTOINC", "0")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500693 return
694
695 auto_pr = None
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500696 pv = d.getVar("PV")
697 version = d.getVar("PRAUTOINX")
698 pkgarch = d.getVar("PACKAGE_ARCH")
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500699 checksum = get_do_package_hash(pn)
700
701 # If do_package isn't in the dependencies, we can't get the checksum...
702 if not checksum:
703 bb.warn('Task %s requested do_package unihash, but it was not available.' % d.getVar('BB_RUNTASK'))
704 #taskdepdata = d.getVar("BB_TASKDEPDATA", False)
705 #for dep in taskdepdata:
706 # bb.warn('%s:%s = %s' % (taskdepdata[dep][0], taskdepdata[dep][1], taskdepdata[dep][6]))
707 return
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500708
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500709 if d.getVar('PRSERV_LOCKDOWN'):
710 auto_pr = d.getVar('PRAUTO_' + version + '_' + pkgarch) or d.getVar('PRAUTO_' + version) or None
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500711 if auto_pr is None:
712 bb.fatal("Can NOT get PRAUTO from lockdown exported file")
713 d.setVar('PRAUTO',str(auto_pr))
714 return
715
716 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500717 conn = d.getVar("__PRSERV_CONN")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500718 if conn is None:
719 conn = oe.prservice.prserv_make_conn(d)
720 if conn is not None:
721 if "AUTOINC" in pkgv:
722 srcpv = bb.fetch2.get_srcrev(d)
723 base_ver = "AUTOINC-%s" % version[:version.find(srcpv)]
724 value = conn.getPR(base_ver, pkgarch, srcpv)
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500725 d.setVar("PRSERV_PV_AUTOINC", str(value))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500726
727 auto_pr = conn.getPR(version, pkgarch, checksum)
728 except Exception as e:
729 bb.fatal("Can NOT get PRAUTO, exception %s" % str(e))
730 if auto_pr is None:
731 bb.fatal("Can NOT get PRAUTO from remote PR service")
732 d.setVar('PRAUTO',str(auto_pr))
733}
734
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500735#
736# Package functions suitable for inclusion in PACKAGEFUNCS
737#
738
739python package_convert_pr_autoinc() {
740 pkgv = d.getVar("PKGV")
741
742 # Adjust pkgv as necessary...
743 if 'AUTOINC' in pkgv:
744 d.setVar("PKGV", pkgv.replace("AUTOINC", "${PRSERV_PV_AUTOINC}"))
745
746 # Change PRSERV_PV_AUTOINC and EXTENDPRAUTO usage to special values
747 d.setVar('PRSERV_PV_AUTOINC', '@PRSERV_PV_AUTOINC@')
748 d.setVar('EXTENDPRAUTO', '@EXTENDPRAUTO@')
749}
750
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500751LOCALEBASEPN ??= "${PN}"
752
753python package_do_split_locales() {
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500754 if (d.getVar('PACKAGE_NO_LOCALE') == '1'):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500755 bb.debug(1, "package requested not splitting locales")
756 return
757
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500758 packages = (d.getVar('PACKAGES') or "").split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500759
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500760 datadir = d.getVar('datadir')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500761 if not datadir:
762 bb.note("datadir not defined")
763 return
764
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500765 dvar = d.getVar('PKGD')
766 pn = d.getVar('LOCALEBASEPN')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500767
768 if pn + '-locale' in packages:
769 packages.remove(pn + '-locale')
770
771 localedir = os.path.join(dvar + datadir, 'locale')
772
773 if not cpath.isdir(localedir):
774 bb.debug(1, "No locale files in this package")
775 return
776
777 locales = os.listdir(localedir)
778
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500779 summary = d.getVar('SUMMARY') or pn
780 description = d.getVar('DESCRIPTION') or ""
781 locale_section = d.getVar('LOCALE_SECTION')
782 mlprefix = d.getVar('MLPREFIX') or ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500783 for l in sorted(locales):
784 ln = legitimize_package_name(l)
785 pkg = pn + '-locale-' + ln
786 packages.append(pkg)
787 d.setVar('FILES_' + pkg, os.path.join(datadir, 'locale', l))
788 d.setVar('RRECOMMENDS_' + pkg, '%svirtual-locale-%s' % (mlprefix, ln))
789 d.setVar('RPROVIDES_' + pkg, '%s-locale %s%s-translation' % (pn, mlprefix, ln))
790 d.setVar('SUMMARY_' + pkg, '%s - %s translations' % (summary, l))
791 d.setVar('DESCRIPTION_' + pkg, '%s This package contains language translation files for the %s locale.' % (description, l))
792 if locale_section:
793 d.setVar('SECTION_' + pkg, locale_section)
794
795 d.setVar('PACKAGES', ' '.join(packages))
796
797 # Disabled by RP 18/06/07
798 # Wildcards aren't supported in debian
799 # They break with ipkg since glibc-locale* will mean that
800 # glibc-localedata-translit* won't install as a dependency
801 # for some other package which breaks meta-toolchain
802 # Probably breaks since virtual-locale- isn't provided anywhere
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500803 #rdep = (d.getVar('RDEPENDS_%s' % pn) or "").split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500804 #rdep.append('%s-locale*' % pn)
805 #d.setVar('RDEPENDS_%s' % pn, ' '.join(rdep))
806}
807
808python perform_packagecopy () {
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800809 import subprocess
810
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500811 dest = d.getVar('D')
812 dvar = d.getVar('PKGD')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500813
814 # Start by package population by taking a copy of the installed
815 # files to operate on
816 # Preserve sparse files and hard links
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800817 cmd = 'tar -cf - -C %s -p -S . | tar -xf - -C %s' % (dest, dvar)
818 subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500819
820 # replace RPATHs for the nativesdk binaries, to make them relocatable
821 if bb.data.inherits_class('nativesdk', d) or bb.data.inherits_class('cross-canadian', d):
822 rpath_replace (dvar, d)
823}
824perform_packagecopy[cleandirs] = "${PKGD}"
825perform_packagecopy[dirs] = "${PKGD}"
826
827# We generate a master list of directories to process, we start by
828# seeding this list with reasonable defaults, then load from
829# the fs-perms.txt files
830python fixup_perms () {
831 import pwd, grp
832
833 # init using a string with the same format as a line as documented in
834 # the fs-perms.txt file
835 # <path> <mode> <uid> <gid> <walk> <fmode> <fuid> <fgid>
836 # <path> link <link target>
837 #
838 # __str__ can be used to print out an entry in the input format
839 #
840 # if fs_perms_entry.path is None:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400841 # an error occurred
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500842 # if fs_perms_entry.link, you can retrieve:
843 # fs_perms_entry.path = path
844 # fs_perms_entry.link = target of link
845 # if not fs_perms_entry.link, you can retrieve:
846 # fs_perms_entry.path = path
847 # fs_perms_entry.mode = expected dir mode or None
848 # fs_perms_entry.uid = expected uid or -1
849 # fs_perms_entry.gid = expected gid or -1
850 # fs_perms_entry.walk = 'true' or something else
851 # fs_perms_entry.fmode = expected file mode or None
852 # fs_perms_entry.fuid = expected file uid or -1
853 # fs_perms_entry_fgid = expected file gid or -1
854 class fs_perms_entry():
855 def __init__(self, line):
856 lsplit = line.split()
857 if len(lsplit) == 3 and lsplit[1].lower() == "link":
858 self._setlink(lsplit[0], lsplit[2])
859 elif len(lsplit) == 8:
860 self._setdir(lsplit[0], lsplit[1], lsplit[2], lsplit[3], lsplit[4], lsplit[5], lsplit[6], lsplit[7])
861 else:
862 msg = "Fixup Perms: invalid config line %s" % line
863 package_qa_handle_error("perm-config", msg, d)
864 self.path = None
865 self.link = None
866
867 def _setdir(self, path, mode, uid, gid, walk, fmode, fuid, fgid):
868 self.path = os.path.normpath(path)
869 self.link = None
870 self.mode = self._procmode(mode)
871 self.uid = self._procuid(uid)
872 self.gid = self._procgid(gid)
873 self.walk = walk.lower()
874 self.fmode = self._procmode(fmode)
875 self.fuid = self._procuid(fuid)
876 self.fgid = self._procgid(fgid)
877
878 def _setlink(self, path, link):
879 self.path = os.path.normpath(path)
880 self.link = link
881
882 def _procmode(self, mode):
883 if not mode or (mode and mode == "-"):
884 return None
885 else:
886 return int(mode,8)
887
888 # Note uid/gid -1 has special significance in os.lchown
889 def _procuid(self, uid):
890 if uid is None or uid == "-":
891 return -1
892 elif uid.isdigit():
893 return int(uid)
894 else:
895 return pwd.getpwnam(uid).pw_uid
896
897 def _procgid(self, gid):
898 if gid is None or gid == "-":
899 return -1
900 elif gid.isdigit():
901 return int(gid)
902 else:
903 return grp.getgrnam(gid).gr_gid
904
905 # Use for debugging the entries
906 def __str__(self):
907 if self.link:
908 return "%s link %s" % (self.path, self.link)
909 else:
910 mode = "-"
911 if self.mode:
912 mode = "0%o" % self.mode
913 fmode = "-"
914 if self.fmode:
915 fmode = "0%o" % self.fmode
916 uid = self._mapugid(self.uid)
917 gid = self._mapugid(self.gid)
918 fuid = self._mapugid(self.fuid)
919 fgid = self._mapugid(self.fgid)
920 return "%s %s %s %s %s %s %s %s" % (self.path, mode, uid, gid, self.walk, fmode, fuid, fgid)
921
922 def _mapugid(self, id):
923 if id is None or id == -1:
924 return "-"
925 else:
926 return "%d" % id
927
928 # Fix the permission, owner and group of path
929 def fix_perms(path, mode, uid, gid, dir):
930 if mode and not os.path.islink(path):
931 #bb.note("Fixup Perms: chmod 0%o %s" % (mode, dir))
932 os.chmod(path, mode)
933 # -1 is a special value that means don't change the uid/gid
934 # if they are BOTH -1, don't bother to lchown
935 if not (uid == -1 and gid == -1):
936 #bb.note("Fixup Perms: lchown %d:%d %s" % (uid, gid, dir))
937 os.lchown(path, uid, gid)
938
939 # Return a list of configuration files based on either the default
940 # files/fs-perms.txt or the contents of FILESYSTEM_PERMS_TABLES
941 # paths are resolved via BBPATH
942 def get_fs_perms_list(d):
943 str = ""
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500944 bbpath = d.getVar('BBPATH')
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500945 fs_perms_tables = d.getVar('FILESYSTEM_PERMS_TABLES') or ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500946 for conf_file in fs_perms_tables.split():
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800947 confpath = bb.utils.which(bbpath, conf_file)
948 if confpath:
949 str += " %s" % bb.utils.which(bbpath, conf_file)
950 else:
951 bb.warn("cannot find %s specified in FILESYSTEM_PERMS_TABLES" % conf_file)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500952 return str
953
954
955
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500956 dvar = d.getVar('PKGD')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500957
958 fs_perms_table = {}
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500959 fs_link_table = {}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500960
961 # By default all of the standard directories specified in
962 # bitbake.conf will get 0755 root:root.
963 target_path_vars = [ 'base_prefix',
964 'prefix',
965 'exec_prefix',
966 'base_bindir',
967 'base_sbindir',
968 'base_libdir',
969 'datadir',
970 'sysconfdir',
971 'servicedir',
972 'sharedstatedir',
973 'localstatedir',
974 'infodir',
975 'mandir',
976 'docdir',
977 'bindir',
978 'sbindir',
979 'libexecdir',
980 'libdir',
981 'includedir',
982 'oldincludedir' ]
983
984 for path in target_path_vars:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500985 dir = d.getVar(path) or ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500986 if dir == "":
987 continue
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500988 fs_perms_table[dir] = fs_perms_entry(d.expand("%s 0755 root root false - - -" % (dir)))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500989
990 # Now we actually load from the configuration files
991 for conf in get_fs_perms_list(d).split():
Brad Bishop64c979e2019-11-04 13:55:29 -0500992 if not os.path.exists(conf):
993 continue
994 with open(conf) as f:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500995 for line in f:
996 if line.startswith('#'):
997 continue
998 lsplit = line.split()
999 if len(lsplit) == 0:
1000 continue
1001 if len(lsplit) != 8 and not (len(lsplit) == 3 and lsplit[1].lower() == "link"):
1002 msg = "Fixup perms: %s invalid line: %s" % (conf, line)
1003 package_qa_handle_error("perm-line", msg, d)
1004 continue
1005 entry = fs_perms_entry(d.expand(line))
1006 if entry and entry.path:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001007 if entry.link:
1008 fs_link_table[entry.path] = entry
1009 if entry.path in fs_perms_table:
1010 fs_perms_table.pop(entry.path)
1011 else:
1012 fs_perms_table[entry.path] = entry
1013 if entry.path in fs_link_table:
1014 fs_link_table.pop(entry.path)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001015
1016 # Debug -- list out in-memory table
1017 #for dir in fs_perms_table:
1018 # bb.note("Fixup Perms: %s: %s" % (dir, str(fs_perms_table[dir])))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001019 #for link in fs_link_table:
1020 # bb.note("Fixup Perms: %s: %s" % (link, str(fs_link_table[link])))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001021
1022 # We process links first, so we can go back and fixup directory ownership
1023 # for any newly created directories
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001024 # Process in sorted order so /run gets created before /run/lock, etc.
1025 for entry in sorted(fs_link_table.values(), key=lambda x: x.link):
1026 link = entry.link
1027 dir = entry.path
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001028 origin = dvar + dir
1029 if not (cpath.exists(origin) and cpath.isdir(origin) and not cpath.islink(origin)):
1030 continue
1031
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001032 if link[0] == "/":
1033 target = dvar + link
1034 ptarget = link
1035 else:
1036 target = os.path.join(os.path.dirname(origin), link)
1037 ptarget = os.path.join(os.path.dirname(dir), link)
1038 if os.path.exists(target):
1039 msg = "Fixup Perms: Unable to correct directory link, target already exists: %s -> %s" % (dir, ptarget)
1040 package_qa_handle_error("perm-link", msg, d)
1041 continue
1042
1043 # Create path to move directory to, move it, and then setup the symlink
1044 bb.utils.mkdirhier(os.path.dirname(target))
1045 #bb.note("Fixup Perms: Rename %s -> %s" % (dir, ptarget))
1046 os.rename(origin, target)
1047 #bb.note("Fixup Perms: Link %s -> %s" % (dir, link))
1048 os.symlink(link, origin)
1049
1050 for dir in fs_perms_table:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001051 origin = dvar + dir
1052 if not (cpath.exists(origin) and cpath.isdir(origin)):
1053 continue
1054
1055 fix_perms(origin, fs_perms_table[dir].mode, fs_perms_table[dir].uid, fs_perms_table[dir].gid, dir)
1056
1057 if fs_perms_table[dir].walk == 'true':
1058 for root, dirs, files in os.walk(origin):
1059 for dr in dirs:
1060 each_dir = os.path.join(root, dr)
1061 fix_perms(each_dir, fs_perms_table[dir].mode, fs_perms_table[dir].uid, fs_perms_table[dir].gid, dir)
1062 for f in files:
1063 each_file = os.path.join(root, f)
1064 fix_perms(each_file, fs_perms_table[dir].fmode, fs_perms_table[dir].fuid, fs_perms_table[dir].fgid, dir)
1065}
1066
1067python split_and_strip_files () {
1068 import stat, errno
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001069 import subprocess
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001070
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001071 dvar = d.getVar('PKGD')
1072 pn = d.getVar('PN')
Andrew Geisslerb7d28612020-07-24 16:15:54 -05001073 hostos = d.getVar('HOST_OS')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001074
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001075 oldcwd = os.getcwd()
1076 os.chdir(dvar)
1077
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001078 # We default to '.debug' style
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001079 if d.getVar('PACKAGE_DEBUG_SPLIT_STYLE') == 'debug-file-directory':
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001080 # Single debug-file-directory style debug info
1081 debugappend = ".debug"
Andrew Geissler82c905d2020-04-13 13:39:40 -05001082 debugstaticappend = ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001083 debugdir = ""
Andrew Geissler82c905d2020-04-13 13:39:40 -05001084 debugstaticdir = ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001085 debuglibdir = "/usr/lib/debug"
Andrew Geissler82c905d2020-04-13 13:39:40 -05001086 debugstaticlibdir = "/usr/lib/debug-static"
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001087 debugsrcdir = "/usr/src/debug"
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001088 elif d.getVar('PACKAGE_DEBUG_SPLIT_STYLE') == 'debug-without-src':
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001089 # Original OE-core, a.k.a. ".debug", style debug info, but without sources in /usr/src/debug
1090 debugappend = ""
Andrew Geissler82c905d2020-04-13 13:39:40 -05001091 debugstaticappend = ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001092 debugdir = "/.debug"
Andrew Geissler82c905d2020-04-13 13:39:40 -05001093 debugstaticdir = "/.debug-static"
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001094 debuglibdir = ""
Andrew Geissler82c905d2020-04-13 13:39:40 -05001095 debugstaticlibdir = ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001096 debugsrcdir = ""
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001097 elif d.getVar('PACKAGE_DEBUG_SPLIT_STYLE') == 'debug-with-srcpkg':
1098 debugappend = ""
Andrew Geissler82c905d2020-04-13 13:39:40 -05001099 debugstaticappend = ""
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001100 debugdir = "/.debug"
Andrew Geissler82c905d2020-04-13 13:39:40 -05001101 debugstaticdir = "/.debug-static"
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001102 debuglibdir = ""
Andrew Geissler82c905d2020-04-13 13:39:40 -05001103 debugstaticlibdir = ""
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001104 debugsrcdir = "/usr/src/debug"
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001105 else:
1106 # Original OE-core, a.k.a. ".debug", style debug info
1107 debugappend = ""
Andrew Geissler82c905d2020-04-13 13:39:40 -05001108 debugstaticappend = ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001109 debugdir = "/.debug"
Andrew Geissler82c905d2020-04-13 13:39:40 -05001110 debugstaticdir = "/.debug-static"
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001111 debuglibdir = ""
Andrew Geissler82c905d2020-04-13 13:39:40 -05001112 debugstaticlibdir = ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001113 debugsrcdir = "/usr/src/debug"
1114
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001115 #
1116 # First lets figure out all of the files we may have to process ... do this only once!
1117 #
1118 elffiles = {}
1119 symlinks = {}
1120 kernmods = []
Brad Bishop316dfdd2018-06-25 12:45:53 -04001121 staticlibs = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001122 inodes = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001123 libdir = os.path.abspath(dvar + os.sep + d.getVar("libdir"))
1124 baselibdir = os.path.abspath(dvar + os.sep + d.getVar("base_libdir"))
Brad Bishop316dfdd2018-06-25 12:45:53 -04001125 skipfiles = (d.getVar("INHIBIT_PACKAGE_STRIP_FILES") or "").split()
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001126 if (d.getVar('INHIBIT_PACKAGE_STRIP') != '1' or \
1127 d.getVar('INHIBIT_PACKAGE_DEBUG_SPLIT') != '1'):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001128 checkelf = {}
1129 checkelflinks = {}
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001130 for root, dirs, files in cpath.walk(dvar):
1131 for f in files:
1132 file = os.path.join(root, f)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001133
1134 # Skip debug files
1135 if debugappend and file.endswith(debugappend):
1136 continue
1137 if debugdir and debugdir in os.path.dirname(file[len(dvar):]):
1138 continue
1139
Brad Bishop316dfdd2018-06-25 12:45:53 -04001140 if file in skipfiles:
1141 continue
1142
Andrew Geissler82c905d2020-04-13 13:39:40 -05001143 if file.endswith(".ko") and file.find("/lib/modules/") != -1:
1144 kernmods.append(file)
1145 continue
1146 if oe.package.is_static_lib(file):
1147 staticlibs.append(file)
1148 continue
1149
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001150 try:
1151 ltarget = cpath.realpath(file, dvar, False)
1152 s = cpath.lstat(ltarget)
1153 except OSError as e:
1154 (err, strerror) = e.args
1155 if err != errno.ENOENT:
1156 raise
1157 # Skip broken symlinks
1158 continue
1159 if not s:
1160 continue
Brad Bishop316dfdd2018-06-25 12:45:53 -04001161 # Check its an executable
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001162 if (s[stat.ST_MODE] & stat.S_IXUSR) or (s[stat.ST_MODE] & stat.S_IXGRP) or (s[stat.ST_MODE] & stat.S_IXOTH) \
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001163 or ((file.startswith(libdir) or file.startswith(baselibdir)) and (".so" in f or ".node" in f)):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001164
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001165 if cpath.islink(file):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001166 checkelflinks[file] = ltarget
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001167 continue
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001168 # Use a reference of device ID and inode number to identify files
1169 file_reference = "%d_%d" % (s.st_dev, s.st_ino)
1170 checkelf[file] = (file, file_reference)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001171
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001172 results = oe.utils.multiprocess_launch(oe.package.is_elf, checkelflinks.values(), d)
1173 results_map = {}
1174 for (ltarget, elf_file) in results:
1175 results_map[ltarget] = elf_file
1176 for file in checkelflinks:
1177 ltarget = checkelflinks[file]
1178 # If it's a symlink, and points to an ELF file, we capture the readlink target
1179 if results_map[ltarget]:
1180 target = os.readlink(file)
1181 #bb.note("Sym: %s (%d)" % (ltarget, results_map[ltarget]))
1182 symlinks[file] = target
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001183
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001184 results = oe.utils.multiprocess_launch(oe.package.is_elf, checkelf.keys(), d)
Brad Bishop15ae2502019-06-18 21:44:24 -04001185
1186 # Sort results by file path. This ensures that the files are always
1187 # processed in the same order, which is important to make sure builds
1188 # are reproducible when dealing with hardlinks
1189 results.sort(key=lambda x: x[0])
1190
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001191 for (file, elf_file) in results:
1192 # It's a file (or hardlink), not a link
1193 # ...but is it ELF, and is it already stripped?
1194 if elf_file & 1:
1195 if elf_file & 2:
1196 if 'already-stripped' in (d.getVar('INSANE_SKIP_' + pn) or "").split():
1197 bb.note("Skipping file %s from %s for already-stripped QA test" % (file[len(dvar):], pn))
1198 else:
1199 msg = "File '%s' from %s was already stripped, this will prevent future debugging!" % (file[len(dvar):], pn)
1200 package_qa_handle_error("already-stripped", msg, d)
1201 continue
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001202
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001203 # At this point we have an unstripped elf file. We need to:
1204 # a) Make sure any file we strip is not hardlinked to anything else outside this tree
1205 # b) Only strip any hardlinked file once (no races)
1206 # c) Track any hardlinks between files so that we can reconstruct matching debug file hardlinks
1207
1208 # Use a reference of device ID and inode number to identify files
1209 file_reference = checkelf[file][1]
1210 if file_reference in inodes:
1211 os.unlink(file)
1212 os.link(inodes[file_reference][0], file)
1213 inodes[file_reference].append(file)
1214 else:
1215 inodes[file_reference] = [file]
1216 # break hardlink
1217 bb.utils.break_hardlinks(file)
1218 elffiles[file] = elf_file
1219 # Modified the file so clear the cache
1220 cpath.updatecache(file)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001221
1222 #
1223 # First lets process debug splitting
1224 #
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001225 if (d.getVar('INHIBIT_PACKAGE_DEBUG_SPLIT') != '1'):
Brad Bishop19323692019-04-05 15:28:33 -04001226 results = oe.utils.multiprocess_launch(splitdebuginfo, list(elffiles), d, extraargs=(dvar, debugdir, debuglibdir, debugappend, debugsrcdir, d))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001227
Andrew Geisslerb7d28612020-07-24 16:15:54 -05001228 if debugsrcdir and not hostos.startswith("mingw"):
Andrew Geissler82c905d2020-04-13 13:39:40 -05001229 if (d.getVar('PACKAGE_DEBUG_STATIC_SPLIT') == '1'):
1230 results = oe.utils.multiprocess_launch(splitstaticdebuginfo, staticlibs, d, extraargs=(dvar, debugstaticdir, debugstaticlibdir, debugstaticappend, debugsrcdir, d))
1231 else:
1232 for file in staticlibs:
1233 results.append( (file,source_info(file, d)) )
Brad Bishop19323692019-04-05 15:28:33 -04001234
1235 sources = set()
1236 for r in results:
1237 sources.update(r[1])
Brad Bishop316dfdd2018-06-25 12:45:53 -04001238
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001239 # Hardlink our debug symbols to the other hardlink copies
1240 for ref in inodes:
1241 if len(inodes[ref]) == 1:
1242 continue
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001243
1244 target = inodes[ref][0][len(dvar):]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001245 for file in inodes[ref][1:]:
1246 src = file[len(dvar):]
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001247 dest = debuglibdir + os.path.dirname(src) + debugdir + "/" + os.path.basename(target) + debugappend
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001248 fpath = dvar + dest
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001249 ftarget = dvar + debuglibdir + os.path.dirname(target) + debugdir + "/" + os.path.basename(target) + debugappend
1250 bb.utils.mkdirhier(os.path.dirname(fpath))
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001251 # Only one hardlink of separated debug info file in each directory
1252 if not os.access(fpath, os.R_OK):
1253 #bb.note("Link %s -> %s" % (fpath, ftarget))
1254 os.link(ftarget, fpath)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001255
1256 # Create symlinks for all cases we were able to split symbols
1257 for file in symlinks:
1258 src = file[len(dvar):]
1259 dest = debuglibdir + os.path.dirname(src) + debugdir + "/" + os.path.basename(src) + debugappend
1260 fpath = dvar + dest
1261 # Skip it if the target doesn't exist
1262 try:
1263 s = os.stat(fpath)
1264 except OSError as e:
1265 (err, strerror) = e.args
1266 if err != errno.ENOENT:
1267 raise
1268 continue
1269
1270 ltarget = symlinks[file]
1271 lpath = os.path.dirname(ltarget)
1272 lbase = os.path.basename(ltarget)
1273 ftarget = ""
1274 if lpath and lpath != ".":
1275 ftarget += lpath + debugdir + "/"
1276 ftarget += lbase + debugappend
1277 if lpath.startswith(".."):
1278 ftarget = os.path.join("..", ftarget)
1279 bb.utils.mkdirhier(os.path.dirname(fpath))
1280 #bb.note("Symlink %s -> %s" % (fpath, ftarget))
1281 os.symlink(ftarget, fpath)
1282
1283 # Process the debugsrcdir if requested...
1284 # This copies and places the referenced sources for later debugging...
Brad Bishop19323692019-04-05 15:28:33 -04001285 copydebugsources(debugsrcdir, sources, d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001286 #
1287 # End of debug splitting
1288 #
1289
1290 #
1291 # Now lets go back over things and strip them
1292 #
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001293 if (d.getVar('INHIBIT_PACKAGE_STRIP') != '1'):
1294 strip = d.getVar("STRIP")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001295 sfiles = []
1296 for file in elffiles:
1297 elf_file = int(elffiles[file])
1298 #bb.note("Strip %s" % file)
1299 sfiles.append((file, elf_file, strip))
1300 for f in kernmods:
1301 sfiles.append((f, 16, strip))
Andrew Geissler82c905d2020-04-13 13:39:40 -05001302 if (d.getVar('PACKAGE_STRIP_STATIC') == '1' or d.getVar('PACKAGE_DEBUG_STATIC_SPLIT') == '1'):
1303 for f in staticlibs:
1304 sfiles.append((f, 16, strip))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001305
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001306 oe.utils.multiprocess_launch(oe.package.runstrip, sfiles, d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001307
Andrew Geissler82c905d2020-04-13 13:39:40 -05001308 # Build "minidebuginfo" and reinject it back into the stripped binaries
1309 if d.getVar('PACKAGE_MINIDEBUGINFO') == '1':
1310 oe.utils.multiprocess_launch(inject_minidebuginfo, list(elffiles), d,
1311 extraargs=(dvar, debugdir, debuglibdir, debugappend, debugsrcdir, d))
1312
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001313 #
1314 # End of strip
1315 #
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001316 os.chdir(oldcwd)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001317}
1318
1319python populate_packages () {
1320 import glob, re
1321
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001322 workdir = d.getVar('WORKDIR')
1323 outdir = d.getVar('DEPLOY_DIR')
1324 dvar = d.getVar('PKGD')
Brad Bishop19323692019-04-05 15:28:33 -04001325 packages = d.getVar('PACKAGES').split()
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001326 pn = d.getVar('PN')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001327
1328 bb.utils.mkdirhier(outdir)
1329 os.chdir(dvar)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001330
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001331 autodebug = not (d.getVar("NOAUTOPACKAGEDEBUG") or False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001332
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001333 split_source_package = (d.getVar('PACKAGE_DEBUG_SPLIT_STYLE') == 'debug-with-srcpkg')
1334
Brad Bishop19323692019-04-05 15:28:33 -04001335 # If debug-with-srcpkg mode is enabled then add the source package if it
1336 # doesn't exist and add the source file contents to the source package.
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001337 if split_source_package:
1338 src_package_name = ('%s-src' % d.getVar('PN'))
Brad Bishop19323692019-04-05 15:28:33 -04001339 if not src_package_name in packages:
1340 packages.append(src_package_name)
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001341 d.setVar('FILES_%s' % src_package_name, '/usr/src/debug')
1342
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001343 # Sanity check PACKAGES for duplicates
Brad Bishop316dfdd2018-06-25 12:45:53 -04001344 # Sanity should be moved to sanity.bbclass once we have the infrastructure
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001345 package_dict = {}
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001346
Brad Bishop19323692019-04-05 15:28:33 -04001347 for i, pkg in enumerate(packages):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001348 if pkg in package_dict:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001349 msg = "%s is listed in PACKAGES multiple times, this leads to packaging errors." % pkg
1350 package_qa_handle_error("packages-list", msg, d)
Brad Bishop19323692019-04-05 15:28:33 -04001351 # Ensure the source package gets the chance to pick up the source files
1352 # before the debug package by ordering it first in PACKAGES. Whether it
1353 # actually picks up any source files is controlled by
1354 # PACKAGE_DEBUG_SPLIT_STYLE.
1355 elif pkg.endswith("-src"):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001356 package_dict[pkg] = (10, i)
1357 elif autodebug and pkg.endswith("-dbg"):
1358 package_dict[pkg] = (30, i)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001359 else:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001360 package_dict[pkg] = (50, i)
Brad Bishop19323692019-04-05 15:28:33 -04001361 packages = sorted(package_dict.keys(), key=package_dict.get)
1362 d.setVar('PACKAGES', ' '.join(packages))
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001363 pkgdest = d.getVar('PKGDEST')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001364
1365 seen = []
1366
1367 # os.mkdir masks the permissions with umask so we have to unset it first
1368 oldumask = os.umask(0)
1369
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001370 debug = []
1371 for root, dirs, files in cpath.walk(dvar):
1372 dir = root[len(dvar):]
1373 if not dir:
1374 dir = os.sep
1375 for f in (files + dirs):
1376 path = "." + os.path.join(dir, f)
Andrew Geissler82c905d2020-04-13 13:39:40 -05001377 if "/.debug/" in path or "/.debug-static/" in path or path.endswith("/.debug"):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001378 debug.append(path)
1379
Brad Bishop19323692019-04-05 15:28:33 -04001380 for pkg in packages:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001381 root = os.path.join(pkgdest, pkg)
1382 bb.utils.mkdirhier(root)
1383
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001384 filesvar = d.getVar('FILES_%s' % pkg) or ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001385 if "//" in filesvar:
1386 msg = "FILES variable for package %s contains '//' which is invalid. Attempting to fix this but you should correct the metadata.\n" % pkg
1387 package_qa_handle_error("files-invalid", msg, d)
1388 filesvar.replace("//", "/")
1389
1390 origfiles = filesvar.split()
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001391 files, symlink_paths = files_from_filevars(origfiles)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001392
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001393 if autodebug and pkg.endswith("-dbg"):
1394 files.extend(debug)
1395
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001396 for file in files:
1397 if (not cpath.islink(file)) and (not cpath.exists(file)):
1398 continue
1399 if file in seen:
1400 continue
1401 seen.append(file)
1402
1403 def mkdir(src, dest, p):
1404 src = os.path.join(src, p)
1405 dest = os.path.join(dest, p)
1406 fstat = cpath.stat(src)
Brad Bishop96ff1982019-08-19 13:50:42 -04001407 os.mkdir(dest)
1408 os.chmod(dest, fstat.st_mode)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001409 os.chown(dest, fstat.st_uid, fstat.st_gid)
1410 if p not in seen:
1411 seen.append(p)
1412 cpath.updatecache(dest)
1413
1414 def mkdir_recurse(src, dest, paths):
1415 if cpath.exists(dest + '/' + paths):
1416 return
1417 while paths.startswith("./"):
1418 paths = paths[2:]
1419 p = "."
1420 for c in paths.split("/"):
1421 p = os.path.join(p, c)
1422 if not cpath.exists(os.path.join(dest, p)):
1423 mkdir(src, dest, p)
1424
1425 if cpath.isdir(file) and not cpath.islink(file):
1426 mkdir_recurse(dvar, root, file)
1427 continue
1428
1429 mkdir_recurse(dvar, root, os.path.dirname(file))
1430 fpath = os.path.join(root,file)
1431 if not cpath.islink(file):
1432 os.link(file, fpath)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001433 continue
1434 ret = bb.utils.copyfile(file, fpath)
1435 if ret is False or ret == 0:
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001436 bb.fatal("File population failed")
1437
1438 # Check if symlink paths exist
1439 for file in symlink_paths:
1440 if not os.path.exists(os.path.join(root,file)):
1441 bb.fatal("File '%s' cannot be packaged into '%s' because its "
1442 "parent directory structure does not exist. One of "
1443 "its parent directories is a symlink whose target "
1444 "directory is not included in the package." %
1445 (file, pkg))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001446
1447 os.umask(oldumask)
1448 os.chdir(workdir)
1449
1450 # Handle LICENSE_EXCLUSION
1451 package_list = []
Brad Bishop19323692019-04-05 15:28:33 -04001452 for pkg in packages:
Andrew Geissler82c905d2020-04-13 13:39:40 -05001453 licenses = d.getVar('LICENSE_EXCLUSION-' + pkg)
1454 if licenses:
1455 msg = "Excluding %s from packaging as it has incompatible license(s): %s" % (pkg, licenses)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001456 package_qa_handle_error("incompatible-license", msg, d)
1457 else:
1458 package_list.append(pkg)
1459 d.setVar('PACKAGES', ' '.join(package_list))
1460
1461 unshipped = []
1462 for root, dirs, files in cpath.walk(dvar):
1463 dir = root[len(dvar):]
1464 if not dir:
1465 dir = os.sep
1466 for f in (files + dirs):
1467 path = os.path.join(dir, f)
1468 if ('.' + path) not in seen:
1469 unshipped.append(path)
1470
1471 if unshipped != []:
1472 msg = pn + ": Files/directories were installed but not shipped in any package:"
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001473 if "installed-vs-shipped" in (d.getVar('INSANE_SKIP_' + pn) or "").split():
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001474 bb.note("Package %s skipping QA tests: installed-vs-shipped" % pn)
1475 else:
1476 for f in unshipped:
1477 msg = msg + "\n " + f
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001478 msg = msg + "\nPlease set FILES such that these items are packaged. Alternatively if they are unneeded, avoid installing them or delete them within do_install.\n"
1479 msg = msg + "%s: %d installed and not shipped files." % (pn, len(unshipped))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001480 package_qa_handle_error("installed-vs-shipped", msg, d)
1481}
1482populate_packages[dirs] = "${D}"
1483
1484python package_fixsymlinks () {
1485 import errno
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001486 pkgdest = d.getVar('PKGDEST')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001487 packages = d.getVar("PACKAGES", False).split()
1488
1489 dangling_links = {}
1490 pkg_files = {}
1491 for pkg in packages:
1492 dangling_links[pkg] = []
1493 pkg_files[pkg] = []
1494 inst_root = os.path.join(pkgdest, pkg)
1495 for path in pkgfiles[pkg]:
1496 rpath = path[len(inst_root):]
1497 pkg_files[pkg].append(rpath)
1498 rtarget = cpath.realpath(path, inst_root, True, assume_dir = True)
1499 if not cpath.lexists(rtarget):
1500 dangling_links[pkg].append(os.path.normpath(rtarget[len(inst_root):]))
1501
1502 newrdepends = {}
1503 for pkg in dangling_links:
1504 for l in dangling_links[pkg]:
1505 found = False
1506 bb.debug(1, "%s contains dangling link %s" % (pkg, l))
1507 for p in packages:
1508 if l in pkg_files[p]:
1509 found = True
1510 bb.debug(1, "target found in %s" % p)
1511 if p == pkg:
1512 break
1513 if pkg not in newrdepends:
1514 newrdepends[pkg] = []
1515 newrdepends[pkg].append(p)
1516 break
1517 if found == False:
1518 bb.note("%s contains dangling symlink to %s" % (pkg, l))
1519
1520 for pkg in newrdepends:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001521 rdepends = bb.utils.explode_dep_versions2(d.getVar('RDEPENDS_' + pkg) or "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001522 for p in newrdepends[pkg]:
1523 if p not in rdepends:
1524 rdepends[p] = []
1525 d.setVar('RDEPENDS_' + pkg, bb.utils.join_deps(rdepends, commasep=False))
1526}
1527
1528
1529python package_package_name_hook() {
1530 """
1531 A package_name_hook function can be used to rewrite the package names by
1532 changing PKG. For an example, see debian.bbclass.
1533 """
1534 pass
1535}
1536
1537EXPORT_FUNCTIONS package_name_hook
1538
1539
1540PKGDESTWORK = "${WORKDIR}/pkgdata"
1541
Andrew Geissler1e34c2d2020-05-29 16:02:59 -05001542PKGDATA_VARS = "PN PE PV PR PKGE PKGV PKGR LICENSE DESCRIPTION SUMMARY RDEPENDS RPROVIDES RRECOMMENDS RSUGGESTS RREPLACES RCONFLICTS SECTION PKG ALLOW_EMPTY FILES CONFFILES FILES_INFO PACKAGE_ADD_METADATA pkg_postinst pkg_postrm pkg_preinst pkg_prerm"
Brad Bishop15ae2502019-06-18 21:44:24 -04001543
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001544python emit_pkgdata() {
1545 from glob import glob
1546 import json
1547
Brad Bishop316dfdd2018-06-25 12:45:53 -04001548 def process_postinst_on_target(pkg, mlprefix):
Brad Bishop96ff1982019-08-19 13:50:42 -04001549 pkgval = d.getVar('PKG_%s' % pkg)
1550 if pkgval is None:
1551 pkgval = pkg
1552
Brad Bishop316dfdd2018-06-25 12:45:53 -04001553 defer_fragment = """
1554if [ -n "$D" ]; then
1555 $INTERCEPT_DIR/postinst_intercept delay_to_first_boot %s mlprefix=%s
1556 exit 0
1557fi
Brad Bishop96ff1982019-08-19 13:50:42 -04001558""" % (pkgval, mlprefix)
Brad Bishop316dfdd2018-06-25 12:45:53 -04001559
1560 postinst = d.getVar('pkg_postinst_%s' % pkg)
1561 postinst_ontarget = d.getVar('pkg_postinst_ontarget_%s' % pkg)
1562
1563 if postinst_ontarget:
1564 bb.debug(1, 'adding deferred pkg_postinst_ontarget() to pkg_postinst() for %s' % pkg)
1565 if not postinst:
1566 postinst = '#!/bin/sh\n'
1567 postinst += defer_fragment
1568 postinst += postinst_ontarget
1569 d.setVar('pkg_postinst_%s' % pkg, postinst)
1570
1571 def add_set_e_to_scriptlets(pkg):
1572 for scriptlet_name in ('pkg_preinst', 'pkg_postinst', 'pkg_prerm', 'pkg_postrm'):
1573 scriptlet = d.getVar('%s_%s' % (scriptlet_name, pkg))
1574 if scriptlet:
1575 scriptlet_split = scriptlet.split('\n')
1576 if scriptlet_split[0].startswith("#!"):
1577 scriptlet = scriptlet_split[0] + "\nset -e\n" + "\n".join(scriptlet_split[1:])
1578 else:
1579 scriptlet = "set -e\n" + "\n".join(scriptlet_split[0:])
1580 d.setVar('%s_%s' % (scriptlet_name, pkg), scriptlet)
1581
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001582 def write_if_exists(f, pkg, var):
1583 def encode(str):
1584 import codecs
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001585 c = codecs.getencoder("unicode_escape")
1586 return c(str)[0].decode("latin1")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001587
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001588 val = d.getVar('%s_%s' % (var, pkg))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001589 if val:
1590 f.write('%s_%s: %s\n' % (var, pkg, encode(val)))
1591 return val
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001592 val = d.getVar('%s' % (var))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001593 if val:
1594 f.write('%s: %s\n' % (var, encode(val)))
1595 return val
1596
1597 def write_extra_pkgs(variants, pn, packages, pkgdatadir):
1598 for variant in variants:
1599 with open("%s/%s-%s" % (pkgdatadir, variant, pn), 'w') as fd:
1600 fd.write("PACKAGES: %s\n" % ' '.join(
1601 map(lambda pkg: '%s-%s' % (variant, pkg), packages.split())))
1602
1603 def write_extra_runtime_pkgs(variants, packages, pkgdatadir):
1604 for variant in variants:
1605 for pkg in packages.split():
1606 ml_pkg = "%s-%s" % (variant, pkg)
1607 subdata_file = "%s/runtime/%s" % (pkgdatadir, ml_pkg)
1608 with open(subdata_file, 'w') as fd:
1609 fd.write("PKG_%s: %s" % (ml_pkg, pkg))
1610
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001611 packages = d.getVar('PACKAGES')
1612 pkgdest = d.getVar('PKGDEST')
1613 pkgdatadir = d.getVar('PKGDESTWORK')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001614
Brad Bishop64c979e2019-11-04 13:55:29 -05001615 data_file = pkgdatadir + d.expand("/${PN}")
1616 with open(data_file, 'w') as fd:
1617 fd.write("PACKAGES: %s\n" % packages)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001618
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001619 pn = d.getVar('PN')
1620 global_variants = (d.getVar('MULTILIB_GLOBAL_VARIANTS') or "").split()
1621 variants = (d.getVar('MULTILIB_VARIANTS') or "").split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001622
1623 if bb.data.inherits_class('kernel', d) or bb.data.inherits_class('module-base', d):
1624 write_extra_pkgs(variants, pn, packages, pkgdatadir)
1625
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001626 if bb.data.inherits_class('allarch', d) and not variants \
1627 and not bb.data.inherits_class('packagegroup', d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001628 write_extra_pkgs(global_variants, pn, packages, pkgdatadir)
1629
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001630 workdir = d.getVar('WORKDIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001631
1632 for pkg in packages.split():
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001633 pkgval = d.getVar('PKG_%s' % pkg)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001634 if pkgval is None:
1635 pkgval = pkg
1636 d.setVar('PKG_%s' % pkg, pkg)
1637
1638 pkgdestpkg = os.path.join(pkgdest, pkg)
1639 files = {}
1640 total_size = 0
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001641 seen = set()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001642 for f in pkgfiles[pkg]:
1643 relpth = os.path.relpath(f, pkgdestpkg)
1644 fstat = os.lstat(f)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001645 files[os.sep + relpth] = fstat.st_size
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001646 if fstat.st_ino not in seen:
1647 seen.add(fstat.st_ino)
1648 total_size += fstat.st_size
Brad Bishop19323692019-04-05 15:28:33 -04001649 d.setVar('FILES_INFO', json.dumps(files, sort_keys=True))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001650
Brad Bishop316dfdd2018-06-25 12:45:53 -04001651 process_postinst_on_target(pkg, d.getVar("MLPREFIX"))
1652 add_set_e_to_scriptlets(pkg)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001653
Brad Bishop15ae2502019-06-18 21:44:24 -04001654 subdata_file = pkgdatadir + "/runtime/%s" % pkg
1655 with open(subdata_file, 'w') as sf:
1656 for var in (d.getVar('PKGDATA_VARS') or "").split():
1657 val = write_if_exists(sf, pkg, var)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001658
Brad Bishop15ae2502019-06-18 21:44:24 -04001659 write_if_exists(sf, pkg, 'FILERPROVIDESFLIST')
1660 for dfile in (d.getVar('FILERPROVIDESFLIST_' + pkg) or "").split():
1661 write_if_exists(sf, pkg, 'FILERPROVIDES_' + dfile)
1662
1663 write_if_exists(sf, pkg, 'FILERDEPENDSFLIST')
1664 for dfile in (d.getVar('FILERDEPENDSFLIST_' + pkg) or "").split():
1665 write_if_exists(sf, pkg, 'FILERDEPENDS_' + dfile)
1666
1667 sf.write('%s_%s: %d\n' % ('PKGSIZE', pkg, total_size))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001668
1669 # Symlinks needed for rprovides lookup
Brad Bishop15ae2502019-06-18 21:44:24 -04001670 rprov = d.getVar('RPROVIDES_%s' % pkg) or d.getVar('RPROVIDES')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001671 if rprov:
Andrew Geisslerc9f78652020-09-18 14:11:35 -05001672 for p in bb.utils.explode_deps(rprov):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001673 subdata_sym = pkgdatadir + "/runtime-rprovides/%s/%s" % (p, pkg)
1674 bb.utils.mkdirhier(os.path.dirname(subdata_sym))
1675 oe.path.symlink("../../runtime/%s" % pkg, subdata_sym, True)
1676
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001677 allow_empty = d.getVar('ALLOW_EMPTY_%s' % pkg)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001678 if not allow_empty:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001679 allow_empty = d.getVar('ALLOW_EMPTY')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001680 root = "%s/%s" % (pkgdest, pkg)
1681 os.chdir(root)
1682 g = glob('*')
1683 if g or allow_empty == "1":
1684 # Symlinks needed for reverse lookups (from the final package name)
1685 subdata_sym = pkgdatadir + "/runtime-reverse/%s" % pkgval
1686 oe.path.symlink("../runtime/%s" % pkg, subdata_sym, True)
1687
1688 packagedfile = pkgdatadir + '/runtime/%s.packaged' % pkg
1689 open(packagedfile, 'w').close()
1690
1691 if bb.data.inherits_class('kernel', d) or bb.data.inherits_class('module-base', d):
1692 write_extra_runtime_pkgs(variants, packages, pkgdatadir)
1693
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001694 if bb.data.inherits_class('allarch', d) and not variants \
1695 and not bb.data.inherits_class('packagegroup', d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001696 write_extra_runtime_pkgs(global_variants, packages, pkgdatadir)
1697
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001698}
1699emit_pkgdata[dirs] = "${PKGDESTWORK}/runtime ${PKGDESTWORK}/runtime-reverse ${PKGDESTWORK}/runtime-rprovides"
1700
1701ldconfig_postinst_fragment() {
1702if [ x"$D" = "x" ]; then
1703 if [ -x /sbin/ldconfig ]; then /sbin/ldconfig ; fi
1704fi
1705}
1706
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001707RPMDEPS = "${STAGING_LIBDIR_NATIVE}/rpm/rpmdeps --alldeps"
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001708
1709# Collect perfile run-time dependency metadata
1710# Output:
1711# FILERPROVIDESFLIST_pkg - list of all files w/ deps
1712# FILERPROVIDES_filepath_pkg - per file dep
1713#
1714# FILERDEPENDSFLIST_pkg - list of all files w/ deps
1715# FILERDEPENDS_filepath_pkg - per file dep
1716
1717python package_do_filedeps() {
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001718 if d.getVar('SKIP_FILEDEPS') == '1':
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001719 return
1720
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001721 pkgdest = d.getVar('PKGDEST')
1722 packages = d.getVar('PACKAGES')
1723 rpmdeps = d.getVar('RPMDEPS')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001724
1725 def chunks(files, n):
1726 return [files[i:i+n] for i in range(0, len(files), n)]
1727
1728 pkglist = []
1729 for pkg in packages.split():
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001730 if d.getVar('SKIP_FILEDEPS_' + pkg) == '1':
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001731 continue
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001732 if pkg.endswith('-dbg') or pkg.endswith('-doc') or pkg.find('-locale-') != -1 or pkg.find('-localedata-') != -1 or pkg.find('-gconv-') != -1 or pkg.find('-charmap-') != -1 or pkg.startswith('kernel-module-') or pkg.endswith('-src'):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001733 continue
1734 for files in chunks(pkgfiles[pkg], 100):
1735 pkglist.append((pkg, files, rpmdeps, pkgdest))
1736
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001737 processed = oe.utils.multiprocess_launch(oe.package.filedeprunner, pkglist, d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001738
1739 provides_files = {}
1740 requires_files = {}
1741
1742 for result in processed:
1743 (pkg, provides, requires) = result
1744
1745 if pkg not in provides_files:
1746 provides_files[pkg] = []
1747 if pkg not in requires_files:
1748 requires_files[pkg] = []
1749
Brad Bishop19323692019-04-05 15:28:33 -04001750 for file in sorted(provides):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001751 provides_files[pkg].append(file)
1752 key = "FILERPROVIDES_" + file + "_" + pkg
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001753 d.appendVar(key, " " + " ".join(provides[file]))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001754
Brad Bishop19323692019-04-05 15:28:33 -04001755 for file in sorted(requires):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001756 requires_files[pkg].append(file)
1757 key = "FILERDEPENDS_" + file + "_" + pkg
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001758 d.appendVar(key, " " + " ".join(requires[file]))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001759
1760 for pkg in requires_files:
1761 d.setVar("FILERDEPENDSFLIST_" + pkg, " ".join(requires_files[pkg]))
1762 for pkg in provides_files:
1763 d.setVar("FILERPROVIDESFLIST_" + pkg, " ".join(provides_files[pkg]))
1764}
1765
Brad Bishop96ff1982019-08-19 13:50:42 -04001766SHLIBSDIRS = "${WORKDIR_PKGDATA}/${MLPREFIX}shlibs2"
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001767SHLIBSWORKDIR = "${PKGDESTWORK}/${MLPREFIX}shlibs2"
1768
1769python package_do_shlibs() {
Brad Bishop00e122a2019-10-05 11:10:57 -04001770 import itertools
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001771 import re, pipes
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001772 import subprocess
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001773
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001774 exclude_shlibs = d.getVar('EXCLUDE_FROM_SHLIBS', False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001775 if exclude_shlibs:
1776 bb.note("not generating shlibs")
1777 return
1778
Brad Bishop19323692019-04-05 15:28:33 -04001779 lib_re = re.compile(r"^.*\.so")
1780 libdir_re = re.compile(r".*/%s$" % d.getVar('baselib'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001781
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001782 packages = d.getVar('PACKAGES')
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001783
1784 shlib_pkgs = []
1785 exclusion_list = d.getVar("EXCLUDE_PACKAGES_FROM_SHLIBS")
1786 if exclusion_list:
1787 for pkg in packages.split():
1788 if pkg not in exclusion_list.split():
1789 shlib_pkgs.append(pkg)
1790 else:
1791 bb.note("not generating shlibs for %s" % pkg)
1792 else:
1793 shlib_pkgs = packages.split()
1794
Andrew Geisslerb7d28612020-07-24 16:15:54 -05001795 hostos = d.getVar('HOST_OS')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001796
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001797 workdir = d.getVar('WORKDIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001798
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001799 ver = d.getVar('PKGV')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001800 if not ver:
1801 msg = "PKGV not defined"
1802 package_qa_handle_error("pkgv-undefined", msg, d)
1803 return
1804
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001805 pkgdest = d.getVar('PKGDEST')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001806
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001807 shlibswork_dir = d.getVar('SHLIBSWORKDIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001808
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001809 def linux_so(file, pkg, pkgver, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001810 needs_ldconfig = False
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001811 needed = set()
1812 sonames = set()
1813 renames = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001814 ldir = os.path.dirname(file).replace(pkgdest + "/" + pkg, '')
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001815 cmd = d.getVar('OBJDUMP') + " -p " + pipes.quote(file) + " 2>/dev/null"
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001816 fd = os.popen(cmd)
1817 lines = fd.readlines()
1818 fd.close()
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001819 rpath = tuple()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001820 for l in lines:
Brad Bishop19323692019-04-05 15:28:33 -04001821 m = re.match(r"\s+RPATH\s+([^\s]*)", l)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001822 if m:
1823 rpaths = m.group(1).replace("$ORIGIN", ldir).split(":")
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001824 rpath = tuple(map(os.path.normpath, rpaths))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001825 for l in lines:
Brad Bishop19323692019-04-05 15:28:33 -04001826 m = re.match(r"\s+NEEDED\s+([^\s]*)", l)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001827 if m:
1828 dep = m.group(1)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001829 if dep not in needed:
1830 needed.add((dep, file, rpath))
Brad Bishop19323692019-04-05 15:28:33 -04001831 m = re.match(r"\s+SONAME\s+([^\s]*)", l)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001832 if m:
1833 this_soname = m.group(1)
1834 prov = (this_soname, ldir, pkgver)
1835 if not prov in sonames:
1836 # if library is private (only used by package) then do not build shlib for it
Brad Bishop79641f22019-09-10 07:20:22 -04001837 import fnmatch
1838 if not private_libs or len([i for i in private_libs if fnmatch.fnmatch(this_soname, i)]) == 0:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001839 sonames.add(prov)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001840 if libdir_re.match(os.path.dirname(file)):
1841 needs_ldconfig = True
1842 if snap_symlinks and (os.path.basename(file) != this_soname):
1843 renames.append((file, os.path.join(os.path.dirname(file), this_soname)))
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001844 return (needs_ldconfig, needed, sonames, renames)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001845
1846 def darwin_so(file, needed, sonames, renames, pkgver):
1847 if not os.path.exists(file):
1848 return
1849 ldir = os.path.dirname(file).replace(pkgdest + "/" + pkg, '')
1850
1851 def get_combinations(base):
1852 #
1853 # Given a base library name, find all combinations of this split by "." and "-"
1854 #
1855 combos = []
1856 options = base.split(".")
1857 for i in range(1, len(options) + 1):
1858 combos.append(".".join(options[0:i]))
1859 options = base.split("-")
1860 for i in range(1, len(options) + 1):
1861 combos.append("-".join(options[0:i]))
1862 return combos
1863
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001864 if (file.endswith('.dylib') or file.endswith('.so')) and not pkg.endswith('-dev') and not pkg.endswith('-dbg') and not pkg.endswith('-src'):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001865 # Drop suffix
1866 name = os.path.basename(file).rsplit(".",1)[0]
1867 # Find all combinations
1868 combos = get_combinations(name)
1869 for combo in combos:
1870 if not combo in sonames:
1871 prov = (combo, ldir, pkgver)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001872 sonames.add(prov)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001873 if file.endswith('.dylib') or file.endswith('.so'):
1874 rpath = []
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001875 p = subprocess.Popen([d.expand("${HOST_PREFIX}otool"), '-l', file], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001876 out, err = p.communicate()
1877 # If returned successfully, process stdout for results
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001878 if p.returncode == 0:
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001879 for l in out.split("\n"):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001880 l = l.strip()
1881 if l.startswith('path '):
1882 rpath.append(l.split()[1])
1883
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001884 p = subprocess.Popen([d.expand("${HOST_PREFIX}otool"), '-L', file], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001885 out, err = p.communicate()
1886 # If returned successfully, process stdout for results
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001887 if p.returncode == 0:
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001888 for l in out.split("\n"):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001889 l = l.strip()
1890 if not l or l.endswith(":"):
1891 continue
1892 if "is not an object file" in l:
1893 continue
1894 name = os.path.basename(l.split()[0]).rsplit(".", 1)[0]
1895 if name and name not in needed[pkg]:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001896 needed[pkg].add((name, file, tuple()))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001897
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001898 def mingw_dll(file, needed, sonames, renames, pkgver):
1899 if not os.path.exists(file):
1900 return
1901
1902 if file.endswith(".dll"):
1903 # assume all dlls are shared objects provided by the package
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001904 sonames.add((os.path.basename(file), os.path.dirname(file).replace(pkgdest + "/" + pkg, ''), pkgver))
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001905
1906 if (file.endswith(".dll") or file.endswith(".exe")):
1907 # use objdump to search for "DLL Name: .*\.dll"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001908 p = subprocess.Popen([d.expand("${HOST_PREFIX}objdump"), "-p", file], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001909 out, err = p.communicate()
1910 # process the output, grabbing all .dll names
1911 if p.returncode == 0:
Brad Bishop19323692019-04-05 15:28:33 -04001912 for m in re.finditer(r"DLL Name: (.*?\.dll)$", out.decode(), re.MULTILINE | re.IGNORECASE):
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001913 dllname = m.group(1)
1914 if dllname:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001915 needed[pkg].add((dllname, file, tuple()))
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001916
1917 if d.getVar('PACKAGE_SNAP_LIB_SYMLINKS') == "1":
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001918 snap_symlinks = True
1919 else:
1920 snap_symlinks = False
1921
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001922 needed = {}
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001923
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001924 shlib_provider = oe.package.read_shlib_providers(d)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001925
1926 for pkg in shlib_pkgs:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001927 private_libs = d.getVar('PRIVATE_LIBS_' + pkg) or d.getVar('PRIVATE_LIBS') or ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001928 private_libs = private_libs.split()
1929 needs_ldconfig = False
1930 bb.debug(2, "calculating shlib provides for %s" % pkg)
1931
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001932 pkgver = d.getVar('PKGV_' + pkg)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001933 if not pkgver:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001934 pkgver = d.getVar('PV_' + pkg)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001935 if not pkgver:
1936 pkgver = ver
1937
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001938 needed[pkg] = set()
1939 sonames = set()
1940 renames = []
1941 linuxlist = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001942 for file in pkgfiles[pkg]:
1943 soname = None
1944 if cpath.islink(file):
1945 continue
Andrew Geisslerb7d28612020-07-24 16:15:54 -05001946 if hostos == "darwin" or hostos == "darwin8":
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001947 darwin_so(file, needed, sonames, renames, pkgver)
Andrew Geisslerb7d28612020-07-24 16:15:54 -05001948 elif hostos.startswith("mingw"):
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001949 mingw_dll(file, needed, sonames, renames, pkgver)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001950 elif os.access(file, os.X_OK) or lib_re.match(file):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001951 linuxlist.append(file)
1952
1953 if linuxlist:
1954 results = oe.utils.multiprocess_launch(linux_so, linuxlist, d, extraargs=(pkg, pkgver, d))
1955 for r in results:
1956 ldconfig = r[0]
1957 needed[pkg] |= r[1]
1958 sonames |= r[2]
1959 renames.extend(r[3])
1960 needs_ldconfig = needs_ldconfig or ldconfig
1961
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001962 for (old, new) in renames:
1963 bb.note("Renaming %s to %s" % (old, new))
1964 os.rename(old, new)
1965 pkgfiles[pkg].remove(old)
Brad Bishop64c979e2019-11-04 13:55:29 -05001966
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001967 shlibs_file = os.path.join(shlibswork_dir, pkg + ".list")
1968 if len(sonames):
Brad Bishop64c979e2019-11-04 13:55:29 -05001969 with open(shlibs_file, 'w') as fd:
Andrew Geissler635e0e42020-08-21 15:58:33 -05001970 for s in sorted(sonames):
Brad Bishop64c979e2019-11-04 13:55:29 -05001971 if s[0] in shlib_provider and s[1] in shlib_provider[s[0]]:
1972 (old_pkg, old_pkgver) = shlib_provider[s[0]][s[1]]
1973 if old_pkg != pkg:
1974 bb.warn('%s-%s was registered as shlib provider for %s, changing it to %s-%s because it was built later' % (old_pkg, old_pkgver, s[0], pkg, pkgver))
1975 bb.debug(1, 'registering %s-%s as shlib provider for %s' % (pkg, pkgver, s[0]))
1976 fd.write(s[0] + ':' + s[1] + ':' + s[2] + '\n')
1977 if s[0] not in shlib_provider:
1978 shlib_provider[s[0]] = {}
1979 shlib_provider[s[0]][s[1]] = (pkg, pkgver)
Brad Bishop1d80a2e2019-11-15 16:35:03 -05001980 if needs_ldconfig:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001981 bb.debug(1, 'adding ldconfig call to postinst for %s' % pkg)
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001982 postinst = d.getVar('pkg_postinst_%s' % pkg)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001983 if not postinst:
1984 postinst = '#!/bin/sh\n'
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001985 postinst += d.getVar('ldconfig_postinst_fragment')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001986 d.setVar('pkg_postinst_%s' % pkg, postinst)
1987 bb.debug(1, 'LIBNAMES: pkg %s sonames %s' % (pkg, sonames))
1988
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001989 assumed_libs = d.getVar('ASSUME_SHLIBS')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001990 if assumed_libs:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001991 libdir = d.getVar("libdir")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001992 for e in assumed_libs.split():
1993 l, dep_pkg = e.split(":")
1994 lib_ver = None
1995 dep_pkg = dep_pkg.rsplit("_", 1)
1996 if len(dep_pkg) == 2:
1997 lib_ver = dep_pkg[1]
1998 dep_pkg = dep_pkg[0]
1999 if l not in shlib_provider:
2000 shlib_provider[l] = {}
2001 shlib_provider[l][libdir] = (dep_pkg, lib_ver)
2002
Brad Bishop6e60e8b2018-02-01 10:27:11 -05002003 libsearchpath = [d.getVar('libdir'), d.getVar('base_libdir')]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002004
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08002005 for pkg in shlib_pkgs:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002006 bb.debug(2, "calculating shlib requirements for %s" % pkg)
2007
Brad Bishop316dfdd2018-06-25 12:45:53 -04002008 private_libs = d.getVar('PRIVATE_LIBS_' + pkg) or d.getVar('PRIVATE_LIBS') or ""
2009 private_libs = private_libs.split()
2010
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002011 deps = list()
2012 for n in needed[pkg]:
2013 # if n is in private libraries, don't try to search provider for it
2014 # this could cause problem in case some abc.bb provides private
2015 # /opt/abc/lib/libfoo.so.1 and contains /usr/bin/abc depending on system library libfoo.so.1
2016 # but skipping it is still better alternative than providing own
2017 # version and then adding runtime dependency for the same system library
Brad Bishop79641f22019-09-10 07:20:22 -04002018 import fnmatch
2019 if private_libs and len([i for i in private_libs if fnmatch.fnmatch(n[0], i)]) > 0:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002020 bb.debug(2, '%s: Dependency %s covered by PRIVATE_LIBS' % (pkg, n[0]))
2021 continue
2022 if n[0] in shlib_provider.keys():
Brad Bishop00e122a2019-10-05 11:10:57 -04002023 shlib_provider_map = shlib_provider[n[0]]
2024 matches = set()
2025 for p in itertools.chain(list(n[2]), sorted(shlib_provider_map.keys()), libsearchpath):
2026 if p in shlib_provider_map:
2027 matches.add(p)
2028 if len(matches) > 1:
2029 matchpkgs = ', '.join([shlib_provider_map[match][0] for match in matches])
2030 bb.error("%s: Multiple shlib providers for %s: %s (used by files: %s)" % (pkg, n[0], matchpkgs, n[1]))
2031 elif len(matches) == 1:
2032 (dep_pkg, ver_needed) = shlib_provider_map[matches.pop()]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002033
2034 bb.debug(2, '%s: Dependency %s requires package %s (used by files: %s)' % (pkg, n[0], dep_pkg, n[1]))
2035
2036 if dep_pkg == pkg:
2037 continue
2038
2039 if ver_needed:
2040 dep = "%s (>= %s)" % (dep_pkg, ver_needed)
2041 else:
2042 dep = dep_pkg
2043 if not dep in deps:
2044 deps.append(dep)
2045 continue
2046 bb.note("Couldn't find shared library provider for %s, used by files: %s" % (n[0], n[1]))
2047
2048 deps_file = os.path.join(pkgdest, pkg + ".shlibdeps")
2049 if os.path.exists(deps_file):
2050 os.remove(deps_file)
Brad Bishop64c979e2019-11-04 13:55:29 -05002051 if deps:
2052 with open(deps_file, 'w') as fd:
2053 for dep in sorted(deps):
2054 fd.write(dep + '\n')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002055}
2056
2057python package_do_pkgconfig () {
2058 import re
2059
Brad Bishop6e60e8b2018-02-01 10:27:11 -05002060 packages = d.getVar('PACKAGES')
2061 workdir = d.getVar('WORKDIR')
2062 pkgdest = d.getVar('PKGDEST')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002063
Brad Bishop6e60e8b2018-02-01 10:27:11 -05002064 shlibs_dirs = d.getVar('SHLIBSDIRS').split()
2065 shlibswork_dir = d.getVar('SHLIBSWORKDIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002066
Brad Bishop19323692019-04-05 15:28:33 -04002067 pc_re = re.compile(r'(.*)\.pc$')
2068 var_re = re.compile(r'(.*)=(.*)')
2069 field_re = re.compile(r'(.*): (.*)')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002070
2071 pkgconfig_provided = {}
2072 pkgconfig_needed = {}
2073 for pkg in packages.split():
2074 pkgconfig_provided[pkg] = []
2075 pkgconfig_needed[pkg] = []
2076 for file in pkgfiles[pkg]:
2077 m = pc_re.match(file)
2078 if m:
2079 pd = bb.data.init()
2080 name = m.group(1)
2081 pkgconfig_provided[pkg].append(name)
2082 if not os.access(file, os.R_OK):
2083 continue
Brad Bishop64c979e2019-11-04 13:55:29 -05002084 with open(file, 'r') as f:
2085 lines = f.readlines()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002086 for l in lines:
2087 m = var_re.match(l)
2088 if m:
2089 name = m.group(1)
2090 val = m.group(2)
2091 pd.setVar(name, pd.expand(val))
2092 continue
2093 m = field_re.match(l)
2094 if m:
2095 hdr = m.group(1)
Brad Bishop6e60e8b2018-02-01 10:27:11 -05002096 exp = pd.expand(m.group(2))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002097 if hdr == 'Requires':
2098 pkgconfig_needed[pkg] += exp.replace(',', ' ').split()
2099
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002100 for pkg in packages.split():
2101 pkgs_file = os.path.join(shlibswork_dir, pkg + ".pclist")
2102 if pkgconfig_provided[pkg] != []:
Brad Bishop64c979e2019-11-04 13:55:29 -05002103 with open(pkgs_file, 'w') as f:
2104 for p in pkgconfig_provided[pkg]:
2105 f.write('%s\n' % p)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002106
2107 # Go from least to most specific since the last one found wins
2108 for dir in reversed(shlibs_dirs):
2109 if not os.path.exists(dir):
2110 continue
Brad Bishop08902b02019-08-20 09:16:51 -04002111 for file in sorted(os.listdir(dir)):
Brad Bishop19323692019-04-05 15:28:33 -04002112 m = re.match(r'^(.*)\.pclist$', file)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002113 if m:
2114 pkg = m.group(1)
Brad Bishop64c979e2019-11-04 13:55:29 -05002115 with open(os.path.join(dir, file)) as fd:
2116 lines = fd.readlines()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002117 pkgconfig_provided[pkg] = []
2118 for l in lines:
2119 pkgconfig_provided[pkg].append(l.rstrip())
2120
2121 for pkg in packages.split():
2122 deps = []
2123 for n in pkgconfig_needed[pkg]:
2124 found = False
2125 for k in pkgconfig_provided.keys():
2126 if n in pkgconfig_provided[k]:
2127 if k != pkg and not (k in deps):
2128 deps.append(k)
2129 found = True
2130 if found == False:
2131 bb.note("couldn't find pkgconfig module '%s' in any package" % n)
2132 deps_file = os.path.join(pkgdest, pkg + ".pcdeps")
2133 if len(deps):
Brad Bishop64c979e2019-11-04 13:55:29 -05002134 with open(deps_file, 'w') as fd:
2135 for dep in deps:
2136 fd.write(dep + '\n')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002137}
2138
2139def read_libdep_files(d):
2140 pkglibdeps = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -05002141 packages = d.getVar('PACKAGES').split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002142 for pkg in packages:
2143 pkglibdeps[pkg] = {}
2144 for extension in ".shlibdeps", ".pcdeps", ".clilibdeps":
2145 depsfile = d.expand("${PKGDEST}/" + pkg + extension)
2146 if os.access(depsfile, os.R_OK):
Brad Bishop64c979e2019-11-04 13:55:29 -05002147 with open(depsfile) as fd:
2148 lines = fd.readlines()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002149 for l in lines:
2150 l.rstrip()
2151 deps = bb.utils.explode_dep_versions2(l)
2152 for dep in deps:
2153 if not dep in pkglibdeps[pkg]:
2154 pkglibdeps[pkg][dep] = deps[dep]
2155 return pkglibdeps
2156
2157python read_shlibdeps () {
2158 pkglibdeps = read_libdep_files(d)
2159
Brad Bishop6e60e8b2018-02-01 10:27:11 -05002160 packages = d.getVar('PACKAGES').split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002161 for pkg in packages:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05002162 rdepends = bb.utils.explode_dep_versions2(d.getVar('RDEPENDS_' + pkg) or "")
Brad Bishop19323692019-04-05 15:28:33 -04002163 for dep in sorted(pkglibdeps[pkg]):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002164 # Add the dep if it's not already there, or if no comparison is set
2165 if dep not in rdepends:
2166 rdepends[dep] = []
2167 for v in pkglibdeps[pkg][dep]:
2168 if v not in rdepends[dep]:
2169 rdepends[dep].append(v)
2170 d.setVar('RDEPENDS_' + pkg, bb.utils.join_deps(rdepends, commasep=False))
2171}
2172
2173python package_depchains() {
2174 """
2175 For a given set of prefix and postfix modifiers, make those packages
2176 RRECOMMENDS on the corresponding packages for its RDEPENDS.
2177
2178 Example: If package A depends upon package B, and A's .bb emits an
2179 A-dev package, this would make A-dev Recommends: B-dev.
2180
2181 If only one of a given suffix is specified, it will take the RRECOMMENDS
2182 based on the RDEPENDS of *all* other packages. If more than one of a given
2183 suffix is specified, its will only use the RDEPENDS of the single parent
2184 package.
2185 """
2186
Brad Bishop6e60e8b2018-02-01 10:27:11 -05002187 packages = d.getVar('PACKAGES')
2188 postfixes = (d.getVar('DEPCHAIN_POST') or '').split()
2189 prefixes = (d.getVar('DEPCHAIN_PRE') or '').split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002190
2191 def pkg_adddeprrecs(pkg, base, suffix, getname, depends, d):
2192
2193 #bb.note('depends for %s is %s' % (base, depends))
Brad Bishop6e60e8b2018-02-01 10:27:11 -05002194 rreclist = bb.utils.explode_dep_versions2(d.getVar('RRECOMMENDS_' + pkg) or "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002195
Brad Bishop19323692019-04-05 15:28:33 -04002196 for depend in sorted(depends):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002197 if depend.find('-native') != -1 or depend.find('-cross') != -1 or depend.startswith('virtual/'):
2198 #bb.note("Skipping %s" % depend)
2199 continue
2200 if depend.endswith('-dev'):
2201 depend = depend[:-4]
2202 if depend.endswith('-dbg'):
2203 depend = depend[:-4]
2204 pkgname = getname(depend, suffix)
2205 #bb.note("Adding %s for %s" % (pkgname, depend))
2206 if pkgname not in rreclist and pkgname != pkg:
2207 rreclist[pkgname] = []
2208
2209 #bb.note('setting: RRECOMMENDS_%s=%s' % (pkg, ' '.join(rreclist)))
2210 d.setVar('RRECOMMENDS_%s' % pkg, bb.utils.join_deps(rreclist, commasep=False))
2211
2212 def pkg_addrrecs(pkg, base, suffix, getname, rdepends, d):
2213
2214 #bb.note('rdepends for %s is %s' % (base, rdepends))
Brad Bishop6e60e8b2018-02-01 10:27:11 -05002215 rreclist = bb.utils.explode_dep_versions2(d.getVar('RRECOMMENDS_' + pkg) or "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002216
Brad Bishop19323692019-04-05 15:28:33 -04002217 for depend in sorted(rdepends):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002218 if depend.find('virtual-locale-') != -1:
2219 #bb.note("Skipping %s" % depend)
2220 continue
2221 if depend.endswith('-dev'):
2222 depend = depend[:-4]
2223 if depend.endswith('-dbg'):
2224 depend = depend[:-4]
2225 pkgname = getname(depend, suffix)
2226 #bb.note("Adding %s for %s" % (pkgname, depend))
2227 if pkgname not in rreclist and pkgname != pkg:
2228 rreclist[pkgname] = []
2229
2230 #bb.note('setting: RRECOMMENDS_%s=%s' % (pkg, ' '.join(rreclist)))
2231 d.setVar('RRECOMMENDS_%s' % pkg, bb.utils.join_deps(rreclist, commasep=False))
2232
2233 def add_dep(list, dep):
2234 if dep not in list:
2235 list.append(dep)
2236
2237 depends = []
Brad Bishop6e60e8b2018-02-01 10:27:11 -05002238 for dep in bb.utils.explode_deps(d.getVar('DEPENDS') or ""):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002239 add_dep(depends, dep)
2240
2241 rdepends = []
2242 for pkg in packages.split():
Brad Bishop6e60e8b2018-02-01 10:27:11 -05002243 for dep in bb.utils.explode_deps(d.getVar('RDEPENDS_' + pkg) or ""):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002244 add_dep(rdepends, dep)
2245
2246 #bb.note('rdepends is %s' % rdepends)
2247
2248 def post_getname(name, suffix):
2249 return '%s%s' % (name, suffix)
2250 def pre_getname(name, suffix):
2251 return '%s%s' % (suffix, name)
2252
2253 pkgs = {}
2254 for pkg in packages.split():
2255 for postfix in postfixes:
2256 if pkg.endswith(postfix):
2257 if not postfix in pkgs:
2258 pkgs[postfix] = {}
2259 pkgs[postfix][pkg] = (pkg[:-len(postfix)], post_getname)
2260
2261 for prefix in prefixes:
2262 if pkg.startswith(prefix):
2263 if not prefix in pkgs:
2264 pkgs[prefix] = {}
2265 pkgs[prefix][pkg] = (pkg[:-len(prefix)], pre_getname)
2266
2267 if "-dbg" in pkgs:
2268 pkglibdeps = read_libdep_files(d)
2269 pkglibdeplist = []
2270 for pkg in pkglibdeps:
2271 for k in pkglibdeps[pkg]:
2272 add_dep(pkglibdeplist, k)
Brad Bishop6e60e8b2018-02-01 10:27:11 -05002273 dbgdefaultdeps = ((d.getVar('DEPCHAIN_DBGDEFAULTDEPS') == '1') or (bb.data.inherits_class('packagegroup', d)))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002274
2275 for suffix in pkgs:
2276 for pkg in pkgs[suffix]:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05002277 if d.getVarFlag('RRECOMMENDS_' + pkg, 'nodeprrecs'):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002278 continue
2279 (base, func) = pkgs[suffix][pkg]
2280 if suffix == "-dev":
2281 pkg_adddeprrecs(pkg, base, suffix, func, depends, d)
2282 elif suffix == "-dbg":
2283 if not dbgdefaultdeps:
2284 pkg_addrrecs(pkg, base, suffix, func, pkglibdeplist, d)
2285 continue
2286 if len(pkgs[suffix]) == 1:
2287 pkg_addrrecs(pkg, base, suffix, func, rdepends, d)
2288 else:
2289 rdeps = []
Brad Bishop6e60e8b2018-02-01 10:27:11 -05002290 for dep in bb.utils.explode_deps(d.getVar('RDEPENDS_' + base) or ""):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002291 add_dep(rdeps, dep)
2292 pkg_addrrecs(pkg, base, suffix, func, rdeps, d)
2293}
2294
2295# Since bitbake can't determine which variables are accessed during package
2296# iteration, we need to list them here:
Andrew Geissler1e34c2d2020-05-29 16:02:59 -05002297PACKAGEVARS = "FILES RDEPENDS RRECOMMENDS SUMMARY DESCRIPTION RSUGGESTS RPROVIDES RCONFLICTS PKG ALLOW_EMPTY pkg_postinst pkg_postrm pkg_postinst_ontarget INITSCRIPT_NAME INITSCRIPT_PARAMS DEBIAN_NOAUTONAME ALTERNATIVE PKGE PKGV PKGR USERADD_PARAM GROUPADD_PARAM CONFFILES SYSTEMD_SERVICE LICENSE SECTION pkg_preinst pkg_prerm RREPLACES GROUPMEMS_PARAM SYSTEMD_AUTO_ENABLE SKIP_FILEDEPS PRIVATE_LIBS PACKAGE_ADD_METADATA"
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002298
Andrew Geissler82c905d2020-04-13 13:39:40 -05002299def gen_packagevar(d, pkgvars="PACKAGEVARS"):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002300 ret = []
Brad Bishop6e60e8b2018-02-01 10:27:11 -05002301 pkgs = (d.getVar("PACKAGES") or "").split()
Andrew Geissler82c905d2020-04-13 13:39:40 -05002302 vars = (d.getVar(pkgvars) or "").split()
2303 for v in vars:
2304 ret.append(v)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002305 for p in pkgs:
2306 for v in vars:
2307 ret.append(v + "_" + p)
2308
2309 # Ensure that changes to INCOMPATIBLE_LICENSE re-run do_package for
2310 # affected recipes.
2311 ret.append('LICENSE_EXCLUSION-%s' % p)
2312 return " ".join(ret)
2313
2314PACKAGE_PREPROCESS_FUNCS ?= ""
2315# Functions for setting up PKGD
2316PACKAGEBUILDPKGD ?= " \
Brad Bishop96ff1982019-08-19 13:50:42 -04002317 package_prepare_pkgdata \
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002318 perform_packagecopy \
2319 ${PACKAGE_PREPROCESS_FUNCS} \
2320 split_and_strip_files \
2321 fixup_perms \
2322 "
2323# Functions which split PKGD up into separate packages
2324PACKAGESPLITFUNCS ?= " \
2325 package_do_split_locales \
2326 populate_packages"
2327# Functions which process metadata based on split packages
2328PACKAGEFUNCS += " \
2329 package_fixsymlinks \
2330 package_name_hook \
2331 package_do_filedeps \
2332 package_do_shlibs \
2333 package_do_pkgconfig \
2334 read_shlibdeps \
2335 package_depchains \
2336 emit_pkgdata"
2337
2338python do_package () {
2339 # Change the following version to cause sstate to invalidate the package
2340 # cache. This is useful if an item this class depends on changes in a
2341 # way that the output of this class changes. rpmdeps is a good example
2342 # as any change to rpmdeps requires this to be rerun.
Andrew Geissler6ce62a22020-11-30 19:58:47 -06002343 # PACKAGE_BBCLASS_VERSION = "4"
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002344
2345 # Init cachedpath
2346 global cpath
2347 cpath = oe.cachedpath.CachedPath()
2348
2349 ###########################################################################
2350 # Sanity test the setup
2351 ###########################################################################
2352
Brad Bishop6e60e8b2018-02-01 10:27:11 -05002353 packages = (d.getVar('PACKAGES') or "").split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002354 if len(packages) < 1:
2355 bb.debug(1, "No packages to build, skipping do_package")
2356 return
2357
Brad Bishop6e60e8b2018-02-01 10:27:11 -05002358 workdir = d.getVar('WORKDIR')
2359 outdir = d.getVar('DEPLOY_DIR')
2360 dest = d.getVar('D')
2361 dvar = d.getVar('PKGD')
2362 pn = d.getVar('PN')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002363
2364 if not workdir or not outdir or not dest or not dvar or not pn:
2365 msg = "WORKDIR, DEPLOY_DIR, D, PN and PKGD all must be defined, unable to package"
2366 package_qa_handle_error("var-undefined", msg, d)
2367 return
2368
Andrew Geisslerc9f78652020-09-18 14:11:35 -05002369 bb.build.exec_func("package_convert_pr_autoinc", d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002370
2371 ###########################################################################
2372 # Optimisations
2373 ###########################################################################
2374
2375 # Continually expanding complex expressions is inefficient, particularly
2376 # when we write to the datastore and invalidate the expansion cache. This
2377 # code pre-expands some frequently used variables
2378
2379 def expandVar(x, d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -05002380 d.setVar(x, d.getVar(x))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002381
2382 for x in 'PN', 'PV', 'BPN', 'TARGET_SYS', 'EXTENDPRAUTO':
2383 expandVar(x, d)
2384
2385 ###########################################################################
2386 # Setup PKGD (from D)
2387 ###########################################################################
2388
Brad Bishop6e60e8b2018-02-01 10:27:11 -05002389 for f in (d.getVar('PACKAGEBUILDPKGD') or '').split():
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002390 bb.build.exec_func(f, d)
2391
2392 ###########################################################################
2393 # Split up PKGD into PKGDEST
2394 ###########################################################################
2395
2396 cpath = oe.cachedpath.CachedPath()
2397
Brad Bishop6e60e8b2018-02-01 10:27:11 -05002398 for f in (d.getVar('PACKAGESPLITFUNCS') or '').split():
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002399 bb.build.exec_func(f, d)
2400
2401 ###########################################################################
2402 # Process PKGDEST
2403 ###########################################################################
2404
2405 # Build global list of files in each split package
2406 global pkgfiles
2407 pkgfiles = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -05002408 packages = d.getVar('PACKAGES').split()
2409 pkgdest = d.getVar('PKGDEST')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002410 for pkg in packages:
2411 pkgfiles[pkg] = []
2412 for walkroot, dirs, files in cpath.walk(pkgdest + "/" + pkg):
2413 for file in files:
2414 pkgfiles[pkg].append(walkroot + os.sep + file)
2415
Brad Bishop6e60e8b2018-02-01 10:27:11 -05002416 for f in (d.getVar('PACKAGEFUNCS') or '').split():
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002417 bb.build.exec_func(f, d)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05002418
Brad Bishop6e60e8b2018-02-01 10:27:11 -05002419 qa_sane = d.getVar("QA_SANE")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05002420 if not qa_sane:
2421 bb.fatal("Fatal QA errors found, failing task.")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002422}
2423
2424do_package[dirs] = "${SHLIBSWORKDIR} ${PKGDESTWORK} ${D}"
2425do_package[vardeps] += "${PACKAGEBUILDPKGD} ${PACKAGESPLITFUNCS} ${PACKAGEFUNCS} ${@gen_packagevar(d)}"
2426addtask package after do_install
2427
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002428SSTATETASKS += "do_package"
2429do_package[cleandirs] = "${PKGDEST} ${PKGDESTWORK}"
2430do_package[sstate-plaindirs] = "${PKGD} ${PKGDEST} ${PKGDESTWORK}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002431do_package_setscene[dirs] = "${STAGING_DIR}"
2432
2433python do_package_setscene () {
2434 sstate_setscene(d)
2435}
2436addtask do_package_setscene
2437
Brad Bishopc68388fc2019-08-26 01:33:31 -04002438# Copy from PKGDESTWORK to tempdirectory as tempdirectory can be cleaned at both
2439# do_package_setscene and do_packagedata_setscene leading to races
2440python do_packagedata () {
Andrew Geisslerc9f78652020-09-18 14:11:35 -05002441 bb.build.exec_func("package_get_auto_pr", d)
2442
Brad Bishopc68388fc2019-08-26 01:33:31 -04002443 src = d.expand("${PKGDESTWORK}")
2444 dest = d.expand("${WORKDIR}/pkgdata-pdata-input")
2445 oe.path.copyhardlinktree(src, dest)
Andrew Geisslerc9f78652020-09-18 14:11:35 -05002446
2447 bb.build.exec_func("packagedata_translate_pr_autoinc", d)
2448}
2449
2450# Translate the EXTENDPRAUTO and AUTOINC to the final values
2451packagedata_translate_pr_autoinc() {
2452 find ${WORKDIR}/pkgdata-pdata-input -type f | xargs --no-run-if-empty \
2453 sed -e 's,@PRSERV_PV_AUTOINC@,${PRSERV_PV_AUTOINC},g' \
2454 -e 's,@EXTENDPRAUTO@,${EXTENDPRAUTO},g' -i
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002455}
2456
2457addtask packagedata before do_build after do_package
2458
2459SSTATETASKS += "do_packagedata"
Brad Bishopc68388fc2019-08-26 01:33:31 -04002460do_packagedata[sstate-inputdirs] = "${WORKDIR}/pkgdata-pdata-input"
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002461do_packagedata[sstate-outputdirs] = "${PKGDATA_DIR}"
Brad Bishop316dfdd2018-06-25 12:45:53 -04002462do_packagedata[stamp-extra-info] = "${MACHINE_ARCH}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002463
2464python do_packagedata_setscene () {
2465 sstate_setscene(d)
2466}
2467addtask do_packagedata_setscene
2468
2469#
2470# Helper functions for the package writing classes
2471#
2472
2473def mapping_rename_hook(d):
2474 """
2475 Rewrite variables to account for package renaming in things
2476 like debian.bbclass or manual PKG variable name changes
2477 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -05002478 pkg = d.getVar("PKG")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002479 runtime_mapping_rename("RDEPENDS", pkg, d)
2480 runtime_mapping_rename("RRECOMMENDS", pkg, d)
2481 runtime_mapping_rename("RSUGGESTS", pkg, d)