blob: df2c40c3c5df6b727a846adac2f118137f65d268 [file] [log] [blame]
Patrick Williams92b42cb2022-09-03 06:53:57 -05001#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: MIT
5#
6
7# BB Class inspired by ebuild.sh
8#
9# This class will test files after installation for certain
10# security issues and other kind of issues.
11#
12# Checks we do:
13# -Check the ownership and permissions
14# -Check the RUNTIME path for the $TMPDIR
15# -Check if .la files wrongly point to workdir
16# -Check if .pc files wrongly point to workdir
17# -Check if packages contains .debug directories or .so files
18# where they should be in -dev or -dbg
19# -Check if config.log contains traces to broken autoconf tests
20# -Check invalid characters (non-utf8) on some package metadata
21# -Ensure that binaries in base_[bindir|sbindir|libdir] do not link
22# into exec_prefix
23# -Check that scripts in base_[bindir|sbindir|libdir] do not reference
24# files under exec_prefix
25# -Check if the package name is upper case
26
27# Elect whether a given type of error is a warning or error, they may
28# have been set by other files.
29WARN_QA ?= " libdir xorg-driver-abi buildpaths \
30 textrel incompatible-license files-invalid \
31 infodir build-deps src-uri-bad symlink-to-sysroot multilib \
32 invalid-packageconfig host-user-contaminated uppercase-pn patch-fuzz \
33 mime mime-xdg unlisted-pkg-lics unhandled-features-check \
34 missing-update-alternatives native-last missing-ptest \
35 license-exists license-no-generic license-syntax license-format \
36 license-incompatible license-file-missing obsolete-license \
37 "
38ERROR_QA ?= "dev-so debug-deps dev-deps debug-files arch pkgconfig la \
39 perms dep-cmp pkgvarcheck perm-config perm-line perm-link \
40 split-strip packages-list pkgv-undefined var-undefined \
41 version-going-backwards expanded-d invalid-chars \
42 license-checksum dev-elf file-rdeps configure-unsafe \
43 configure-gettext perllocalpod shebang-size \
44 already-stripped installed-vs-shipped ldflags compile-host-path \
45 install-host-path pn-overrides unknown-configure-option \
46 useless-rpaths rpaths staticdev empty-dirs \
47 "
48# Add usrmerge QA check based on distro feature
49ERROR_QA:append = "${@bb.utils.contains('DISTRO_FEATURES', 'usrmerge', ' usrmerge', '', d)}"
50
51FAKEROOT_QA = "host-user-contaminated"
52FAKEROOT_QA[doc] = "QA tests which need to run under fakeroot. If any \
53enabled tests are listed here, the do_package_qa task will run under fakeroot."
54
55ALL_QA = "${WARN_QA} ${ERROR_QA}"
56
57UNKNOWN_CONFIGURE_OPT_IGNORE ?= "--enable-nls --disable-nls --disable-silent-rules --disable-dependency-tracking --with-libtool-sysroot --disable-static"
58
59# This is a list of directories that are expected to be empty.
60QA_EMPTY_DIRS ?= " \
61 /dev/pts \
62 /media \
63 /proc \
64 /run \
65 /tmp \
66 ${localstatedir}/run \
67 ${localstatedir}/volatile \
68"
69# It is possible to specify why a directory is expected to be empty by defining
70# QA_EMPTY_DIRS_RECOMMENDATION:<path>, which will then be included in the error
71# message if the directory is not empty. If it is not specified for a directory,
72# then "but it is expected to be empty" will be used.
73
74def package_qa_clean_path(path, d, pkg=None):
75 """
76 Remove redundant paths from the path for display. If pkg isn't set then
77 TMPDIR is stripped, otherwise PKGDEST/pkg is stripped.
78 """
79 if pkg:
80 path = path.replace(os.path.join(d.getVar("PKGDEST"), pkg), "/")
81 return path.replace(d.getVar("TMPDIR"), "/").replace("//", "/")
82
83QAPATHTEST[shebang-size] = "package_qa_check_shebang_size"
84def package_qa_check_shebang_size(path, name, d, elf, messages):
85 import stat
86 if os.path.islink(path) or stat.S_ISFIFO(os.stat(path).st_mode) or elf:
87 return
88
89 try:
90 with open(path, 'rb') as f:
91 stanza = f.readline(130)
92 except IOError:
93 return
94
95 if stanza.startswith(b'#!'):
96 #Shebang not found
97 try:
98 stanza = stanza.decode("utf-8")
99 except UnicodeDecodeError:
100 #If it is not a text file, it is not a script
101 return
102
103 if len(stanza) > 129:
104 oe.qa.add_message(messages, "shebang-size", "%s: %s maximum shebang size exceeded, the maximum size is 128." % (name, package_qa_clean_path(path, d)))
105 return
106
107QAPATHTEST[libexec] = "package_qa_check_libexec"
108def package_qa_check_libexec(path,name, d, elf, messages):
109
110 # Skip the case where the default is explicitly /usr/libexec
111 libexec = d.getVar('libexecdir')
112 if libexec == "/usr/libexec":
113 return True
114
115 if 'libexec' in path.split(os.path.sep):
116 oe.qa.add_message(messages, "libexec", "%s: %s is using libexec please relocate to %s" % (name, package_qa_clean_path(path, d), libexec))
117 return False
118
119 return True
120
121QAPATHTEST[rpaths] = "package_qa_check_rpath"
122def package_qa_check_rpath(file,name, d, elf, messages):
123 """
124 Check for dangerous RPATHs
125 """
126 if not elf:
127 return
128
129 if os.path.islink(file):
130 return
131
132 bad_dirs = [d.getVar('BASE_WORKDIR'), d.getVar('STAGING_DIR_TARGET')]
133
134 phdrs = elf.run_objdump("-p", d)
135
136 import re
137 rpath_re = re.compile(r"\s+RPATH\s+(.*)")
138 for line in phdrs.split("\n"):
139 m = rpath_re.match(line)
140 if m:
141 rpath = m.group(1)
142 for dir in bad_dirs:
143 if dir in rpath:
144 oe.qa.add_message(messages, "rpaths", "package %s contains bad RPATH %s in file %s" % (name, rpath, file))
145
146QAPATHTEST[useless-rpaths] = "package_qa_check_useless_rpaths"
147def package_qa_check_useless_rpaths(file, name, d, elf, messages):
148 """
149 Check for RPATHs that are useless but not dangerous
150 """
151 def rpath_eq(a, b):
152 return os.path.normpath(a) == os.path.normpath(b)
153
154 if not elf:
155 return
156
157 if os.path.islink(file):
158 return
159
160 libdir = d.getVar("libdir")
161 base_libdir = d.getVar("base_libdir")
162
163 phdrs = elf.run_objdump("-p", d)
164
165 import re
166 rpath_re = re.compile(r"\s+RPATH\s+(.*)")
167 for line in phdrs.split("\n"):
168 m = rpath_re.match(line)
169 if m:
170 rpath = m.group(1)
171 if rpath_eq(rpath, libdir) or rpath_eq(rpath, base_libdir):
172 # The dynamic linker searches both these places anyway. There is no point in
173 # looking there again.
174 oe.qa.add_message(messages, "useless-rpaths", "%s: %s contains probably-redundant RPATH %s" % (name, package_qa_clean_path(file, d, name), rpath))
175
176QAPATHTEST[dev-so] = "package_qa_check_dev"
177def package_qa_check_dev(path, name, d, elf, messages):
178 """
179 Check for ".so" library symlinks in non-dev packages
180 """
181
182 if not name.endswith("-dev") and not name.endswith("-dbg") and not name.endswith("-ptest") and not name.startswith("nativesdk-") and path.endswith(".so") and os.path.islink(path):
183 oe.qa.add_message(messages, "dev-so", "non -dev/-dbg/nativesdk- package %s contains symlink .so '%s'" % \
184 (name, package_qa_clean_path(path, d, name)))
185
186QAPATHTEST[dev-elf] = "package_qa_check_dev_elf"
187def package_qa_check_dev_elf(path, name, d, elf, messages):
188 """
189 Check that -dev doesn't contain real shared libraries. The test has to
190 check that the file is not a link and is an ELF object as some recipes
191 install link-time .so files that are linker scripts.
192 """
193 if name.endswith("-dev") and path.endswith(".so") and not os.path.islink(path) and elf:
194 oe.qa.add_message(messages, "dev-elf", "-dev package %s contains non-symlink .so '%s'" % \
195 (name, package_qa_clean_path(path, d, name)))
196
197QAPATHTEST[staticdev] = "package_qa_check_staticdev"
198def package_qa_check_staticdev(path, name, d, elf, messages):
199 """
200 Check for ".a" library in non-staticdev packages
201 There are a number of exceptions to this rule, -pic packages can contain
202 static libraries, the _nonshared.a belong with their -dev packages and
203 libgcc.a, libgcov.a will be skipped in their packages
204 """
205
206 if not name.endswith("-pic") and not name.endswith("-staticdev") and not name.endswith("-ptest") and path.endswith(".a") and not path.endswith("_nonshared.a") and not '/usr/lib/debug-static/' in path and not '/.debug-static/' in path:
207 oe.qa.add_message(messages, "staticdev", "non -staticdev package contains static .a library: %s path '%s'" % \
208 (name, package_qa_clean_path(path,d, name)))
209
210QAPATHTEST[mime] = "package_qa_check_mime"
211def package_qa_check_mime(path, name, d, elf, messages):
212 """
213 Check if package installs mime types to /usr/share/mime/packages
214 while no inheriting mime.bbclass
215 """
216
217 if d.getVar("datadir") + "/mime/packages" in path and path.endswith('.xml') and not bb.data.inherits_class("mime", d):
218 oe.qa.add_message(messages, "mime", "package contains mime types but does not inherit mime: %s path '%s'" % \
219 (name, package_qa_clean_path(path,d)))
220
221QAPATHTEST[mime-xdg] = "package_qa_check_mime_xdg"
222def package_qa_check_mime_xdg(path, name, d, elf, messages):
223 """
224 Check if package installs desktop file containing MimeType and requires
225 mime-types.bbclass to create /usr/share/applications/mimeinfo.cache
226 """
227
228 if d.getVar("datadir") + "/applications" in path and path.endswith('.desktop') and not bb.data.inherits_class("mime-xdg", d):
229 mime_type_found = False
230 try:
231 with open(path, 'r') as f:
232 for line in f.read().split('\n'):
233 if 'MimeType' in line:
234 mime_type_found = True
235 break;
236 except:
237 # At least libreoffice installs symlinks with absolute paths that are dangling here.
238 # We could implement some magic but for few (one) recipes it is not worth the effort so just warn:
239 wstr = "%s cannot open %s - is it a symlink with absolute path?\n" % (name, package_qa_clean_path(path,d))
240 wstr += "Please check if (linked) file contains key 'MimeType'.\n"
241 pkgname = name
242 if name == d.getVar('PN'):
243 pkgname = '${PN}'
244 wstr += "If yes: add \'inhert mime-xdg\' and \'MIME_XDG_PACKAGES += \"%s\"\' / if no add \'INSANE_SKIP:%s += \"mime-xdg\"\' to recipe." % (pkgname, pkgname)
245 oe.qa.add_message(messages, "mime-xdg", wstr)
246 if mime_type_found:
247 oe.qa.add_message(messages, "mime-xdg", "package contains desktop file with key 'MimeType' but does not inhert mime-xdg: %s path '%s'" % \
248 (name, package_qa_clean_path(path,d)))
249
250def package_qa_check_libdir(d):
251 """
252 Check for wrong library installation paths. For instance, catch
253 recipes installing /lib/bar.so when ${base_libdir}="lib32" or
254 installing in /usr/lib64 when ${libdir}="/usr/lib"
255 """
256 import re
257
258 pkgdest = d.getVar('PKGDEST')
259 base_libdir = d.getVar("base_libdir") + os.sep
260 libdir = d.getVar("libdir") + os.sep
261 libexecdir = d.getVar("libexecdir") + os.sep
262 exec_prefix = d.getVar("exec_prefix") + os.sep
263
264 messages = []
265
266 # The re's are purposely fuzzy, as some there are some .so.x.y.z files
267 # that don't follow the standard naming convention. It checks later
268 # that they are actual ELF files
269 lib_re = re.compile(r"^/lib.+\.so(\..+)?$")
270 exec_re = re.compile(r"^%s.*/lib.+\.so(\..+)?$" % exec_prefix)
271
272 for root, dirs, files in os.walk(pkgdest):
273 if root == pkgdest:
274 # Skip subdirectories for any packages with libdir in INSANE_SKIP
275 skippackages = []
276 for package in dirs:
277 if 'libdir' in (d.getVar('INSANE_SKIP:' + package) or "").split():
278 bb.note("Package %s skipping libdir QA test" % (package))
279 skippackages.append(package)
280 elif d.getVar('PACKAGE_DEBUG_SPLIT_STYLE') == 'debug-file-directory' and package.endswith("-dbg"):
281 bb.note("Package %s skipping libdir QA test for PACKAGE_DEBUG_SPLIT_STYLE equals debug-file-directory" % (package))
282 skippackages.append(package)
283 for package in skippackages:
284 dirs.remove(package)
285 for file in files:
286 full_path = os.path.join(root, file)
287 rel_path = os.path.relpath(full_path, pkgdest)
288 if os.sep in rel_path:
289 package, rel_path = rel_path.split(os.sep, 1)
290 rel_path = os.sep + rel_path
291 if lib_re.match(rel_path):
292 if base_libdir not in rel_path:
293 # make sure it's an actual ELF file
294 elf = oe.qa.ELFFile(full_path)
295 try:
296 elf.open()
297 messages.append("%s: found library in wrong location: %s" % (package, rel_path))
298 except (oe.qa.NotELFFileError):
299 pass
300 if exec_re.match(rel_path):
301 if libdir not in rel_path and libexecdir not in rel_path:
302 # make sure it's an actual ELF file
303 elf = oe.qa.ELFFile(full_path)
304 try:
305 elf.open()
306 messages.append("%s: found library in wrong location: %s" % (package, rel_path))
307 except (oe.qa.NotELFFileError):
308 pass
309
310 if messages:
311 oe.qa.handle_error("libdir", "\n".join(messages), d)
312
313QAPATHTEST[debug-files] = "package_qa_check_dbg"
314def package_qa_check_dbg(path, name, d, elf, messages):
315 """
316 Check for ".debug" files or directories outside of the dbg package
317 """
318
319 if not "-dbg" in name and not "-ptest" in name:
320 if '.debug' in path.split(os.path.sep):
321 oe.qa.add_message(messages, "debug-files", "non debug package contains .debug directory: %s path %s" % \
322 (name, package_qa_clean_path(path,d)))
323
324QAPATHTEST[arch] = "package_qa_check_arch"
325def package_qa_check_arch(path,name,d, elf, messages):
326 """
327 Check if archs are compatible
328 """
329 import re, oe.elf
330
331 if not elf:
332 return
333
334 target_os = d.getVar('HOST_OS')
335 target_arch = d.getVar('HOST_ARCH')
336 provides = d.getVar('PROVIDES')
337 bpn = d.getVar('BPN')
338
339 if target_arch == "allarch":
340 pn = d.getVar('PN')
341 oe.qa.add_message(messages, "arch", pn + ": Recipe inherits the allarch class, but has packaged architecture-specific binaries")
342 return
343
344 # FIXME: Cross package confuse this check, so just skip them
345 for s in ['cross', 'nativesdk', 'cross-canadian']:
346 if bb.data.inherits_class(s, d):
347 return
348
349 # avoid following links to /usr/bin (e.g. on udev builds)
350 # we will check the files pointed to anyway...
351 if os.path.islink(path):
352 return
353
354 #if this will throw an exception, then fix the dict above
355 (machine, osabi, abiversion, littleendian, bits) \
356 = oe.elf.machine_dict(d)[target_os][target_arch]
357
358 # Check the architecture and endiannes of the binary
359 is_32 = (("virtual/kernel" in provides) or bb.data.inherits_class("module", d)) and \
360 (target_os == "linux-gnux32" or target_os == "linux-muslx32" or \
361 target_os == "linux-gnu_ilp32" or re.match(r'mips64.*32', d.getVar('DEFAULTTUNE')))
362 is_bpf = (oe.qa.elf_machine_to_string(elf.machine()) == "BPF")
363 if not ((machine == elf.machine()) or is_32 or is_bpf):
364 oe.qa.add_message(messages, "arch", "Architecture did not match (%s, expected %s) in %s" % \
365 (oe.qa.elf_machine_to_string(elf.machine()), oe.qa.elf_machine_to_string(machine), package_qa_clean_path(path, d, name)))
366 elif not ((bits == elf.abiSize()) or is_32 or is_bpf):
367 oe.qa.add_message(messages, "arch", "Bit size did not match (%d, expected %d) in %s" % \
368 (elf.abiSize(), bits, package_qa_clean_path(path, d, name)))
369 elif not ((littleendian == elf.isLittleEndian()) or is_bpf):
370 oe.qa.add_message(messages, "arch", "Endiannes did not match (%d, expected %d) in %s" % \
371 (elf.isLittleEndian(), littleendian, package_qa_clean_path(path,d, name)))
372
373QAPATHTEST[desktop] = "package_qa_check_desktop"
374def package_qa_check_desktop(path, name, d, elf, messages):
375 """
376 Run all desktop files through desktop-file-validate.
377 """
378 if path.endswith(".desktop"):
379 desktop_file_validate = os.path.join(d.getVar('STAGING_BINDIR_NATIVE'),'desktop-file-validate')
380 output = os.popen("%s %s" % (desktop_file_validate, path))
381 # This only produces output on errors
382 for l in output:
383 oe.qa.add_message(messages, "desktop", "Desktop file issue: " + l.strip())
384
385QAPATHTEST[textrel] = "package_qa_textrel"
386def package_qa_textrel(path, name, d, elf, messages):
387 """
388 Check if the binary contains relocations in .text
389 """
390
391 if not elf:
392 return
393
394 if os.path.islink(path):
395 return
396
397 phdrs = elf.run_objdump("-p", d)
398 sane = True
399
400 import re
401 textrel_re = re.compile(r"\s+TEXTREL\s+")
402 for line in phdrs.split("\n"):
403 if textrel_re.match(line):
404 sane = False
405 break
406
407 if not sane:
408 path = package_qa_clean_path(path, d, name)
409 oe.qa.add_message(messages, "textrel", "%s: ELF binary %s has relocations in .text" % (name, path))
410
411QAPATHTEST[ldflags] = "package_qa_hash_style"
412def package_qa_hash_style(path, name, d, elf, messages):
413 """
414 Check if the binary has the right hash style...
415 """
416
417 if not elf:
418 return
419
420 if os.path.islink(path):
421 return
422
423 gnu_hash = "--hash-style=gnu" in d.getVar('LDFLAGS')
424 if not gnu_hash:
425 gnu_hash = "--hash-style=both" in d.getVar('LDFLAGS')
426 if not gnu_hash:
427 return
428
429 sane = False
430 has_syms = False
431
432 phdrs = elf.run_objdump("-p", d)
433
434 # If this binary has symbols, we expect it to have GNU_HASH too.
435 for line in phdrs.split("\n"):
436 if "SYMTAB" in line:
437 has_syms = True
438 if "GNU_HASH" in line or "MIPS_XHASH" in line:
439 sane = True
440 if ("[mips32]" in line or "[mips64]" in line) and d.getVar('TCLIBC') == "musl":
441 sane = True
442 if has_syms and not sane:
443 path = package_qa_clean_path(path, d, name)
444 oe.qa.add_message(messages, "ldflags", "File %s in package %s doesn't have GNU_HASH (didn't pass LDFLAGS?)" % (path, name))
445
446
447QAPATHTEST[buildpaths] = "package_qa_check_buildpaths"
448def package_qa_check_buildpaths(path, name, d, elf, messages):
449 """
450 Check for build paths inside target files and error if paths are not
451 explicitly ignored.
452 """
453 import stat
454
455 # Ignore symlinks/devs/fifos
456 mode = os.lstat(path).st_mode
457 if stat.S_ISLNK(mode) or stat.S_ISBLK(mode) or stat.S_ISFIFO(mode) or stat.S_ISCHR(mode) or stat.S_ISSOCK(mode):
458 return
459
460 tmpdir = bytes(d.getVar('TMPDIR'), encoding="utf-8")
461 with open(path, 'rb') as f:
462 file_content = f.read()
463 if tmpdir in file_content:
464 trimmed = path.replace(os.path.join (d.getVar("PKGDEST"), name), "")
465 oe.qa.add_message(messages, "buildpaths", "File %s in package %s contains reference to TMPDIR" % (trimmed, name))
466
467
468QAPATHTEST[xorg-driver-abi] = "package_qa_check_xorg_driver_abi"
469def package_qa_check_xorg_driver_abi(path, name, d, elf, messages):
470 """
471 Check that all packages containing Xorg drivers have ABI dependencies
472 """
473
474 # Skip dev, dbg or nativesdk packages
475 if name.endswith("-dev") or name.endswith("-dbg") or name.startswith("nativesdk-"):
476 return
477
478 driverdir = d.expand("${libdir}/xorg/modules/drivers/")
479 if driverdir in path and path.endswith(".so"):
480 mlprefix = d.getVar('MLPREFIX') or ''
481 for rdep in bb.utils.explode_deps(d.getVar('RDEPENDS:' + name) or ""):
482 if rdep.startswith("%sxorg-abi-" % mlprefix):
483 return
484 oe.qa.add_message(messages, "xorg-driver-abi", "Package %s contains Xorg driver (%s) but no xorg-abi- dependencies" % (name, os.path.basename(path)))
485
486QAPATHTEST[infodir] = "package_qa_check_infodir"
487def package_qa_check_infodir(path, name, d, elf, messages):
488 """
489 Check that /usr/share/info/dir isn't shipped in a particular package
490 """
491 infodir = d.expand("${infodir}/dir")
492
493 if infodir in path:
494 oe.qa.add_message(messages, "infodir", "The /usr/share/info/dir file is not meant to be shipped in a particular package.")
495
496QAPATHTEST[symlink-to-sysroot] = "package_qa_check_symlink_to_sysroot"
497def package_qa_check_symlink_to_sysroot(path, name, d, elf, messages):
498 """
499 Check that the package doesn't contain any absolute symlinks to the sysroot.
500 """
501 if os.path.islink(path):
502 target = os.readlink(path)
503 if os.path.isabs(target):
504 tmpdir = d.getVar('TMPDIR')
505 if target.startswith(tmpdir):
506 trimmed = path.replace(os.path.join (d.getVar("PKGDEST"), name), "")
507 oe.qa.add_message(messages, "symlink-to-sysroot", "Symlink %s in %s points to TMPDIR" % (trimmed, name))
508
509# Check license variables
510do_populate_lic[postfuncs] += "populate_lic_qa_checksum"
511python populate_lic_qa_checksum() {
512 """
513 Check for changes in the license files.
514 """
515
516 lic_files = d.getVar('LIC_FILES_CHKSUM') or ''
517 lic = d.getVar('LICENSE')
518 pn = d.getVar('PN')
519
520 if lic == "CLOSED":
521 return
522
523 if not lic_files and d.getVar('SRC_URI'):
524 oe.qa.handle_error("license-checksum", pn + ": Recipe file fetches files and does not have license file information (LIC_FILES_CHKSUM)", d)
525
526 srcdir = d.getVar('S')
527 corebase_licensefile = d.getVar('COREBASE') + "/LICENSE"
528 for url in lic_files.split():
529 try:
530 (type, host, path, user, pswd, parm) = bb.fetch.decodeurl(url)
531 except bb.fetch.MalformedUrl:
532 oe.qa.handle_error("license-checksum", pn + ": LIC_FILES_CHKSUM contains an invalid URL: " + url, d)
533 continue
534 srclicfile = os.path.join(srcdir, path)
535 if not os.path.isfile(srclicfile):
536 oe.qa.handle_error("license-checksum", pn + ": LIC_FILES_CHKSUM points to an invalid file: " + srclicfile, d)
537 continue
538
539 if (srclicfile == corebase_licensefile):
540 bb.warn("${COREBASE}/LICENSE is not a valid license file, please use '${COMMON_LICENSE_DIR}/MIT' for a MIT License file in LIC_FILES_CHKSUM. This will become an error in the future")
541
542 recipemd5 = parm.get('md5', '')
543 beginline, endline = 0, 0
544 if 'beginline' in parm:
545 beginline = int(parm['beginline'])
546 if 'endline' in parm:
547 endline = int(parm['endline'])
548
549 if (not beginline) and (not endline):
550 md5chksum = bb.utils.md5_file(srclicfile)
551 with open(srclicfile, 'r', errors='replace') as f:
552 license = f.read().splitlines()
553 else:
554 with open(srclicfile, 'rb') as f:
555 import hashlib
556 lineno = 0
557 license = []
Patrick Williams2390b1b2022-11-03 13:47:49 -0500558 try:
559 m = hashlib.new('MD5', usedforsecurity=False)
560 except TypeError:
561 m = hashlib.new('MD5')
Patrick Williams92b42cb2022-09-03 06:53:57 -0500562 for line in f:
563 lineno += 1
564 if (lineno >= beginline):
565 if ((lineno <= endline) or not endline):
566 m.update(line)
567 license.append(line.decode('utf-8', errors='replace').rstrip())
568 else:
569 break
570 md5chksum = m.hexdigest()
571 if recipemd5 == md5chksum:
572 bb.note (pn + ": md5 checksum matched for ", url)
573 else:
574 if recipemd5:
575 msg = pn + ": The LIC_FILES_CHKSUM does not match for " + url
576 msg = msg + "\n" + pn + ": The new md5 checksum is " + md5chksum
577 max_lines = int(d.getVar('QA_MAX_LICENSE_LINES') or 20)
578 if not license or license[-1] != '':
579 # Ensure that our license text ends with a line break
580 # (will be added with join() below).
581 license.append('')
582 remove = len(license) - max_lines
583 if remove > 0:
584 start = max_lines // 2
585 end = start + remove - 1
586 del license[start:end]
587 license.insert(start, '...')
588 msg = msg + "\n" + pn + ": Here is the selected license text:" + \
589 "\n" + \
590 "{:v^70}".format(" beginline=%d " % beginline if beginline else "") + \
591 "\n" + "\n".join(license) + \
592 "{:^^70}".format(" endline=%d " % endline if endline else "")
593 if beginline:
594 if endline:
595 srcfiledesc = "%s (lines %d through to %d)" % (srclicfile, beginline, endline)
596 else:
597 srcfiledesc = "%s (beginning on line %d)" % (srclicfile, beginline)
598 elif endline:
599 srcfiledesc = "%s (ending on line %d)" % (srclicfile, endline)
600 else:
601 srcfiledesc = srclicfile
602 msg = msg + "\n" + pn + ": Check if the license information has changed in %s to verify that the LICENSE value \"%s\" remains valid" % (srcfiledesc, lic)
603
604 else:
605 msg = pn + ": LIC_FILES_CHKSUM is not specified for " + url
606 msg = msg + "\n" + pn + ": The md5 checksum is " + md5chksum
607 oe.qa.handle_error("license-checksum", msg, d)
608
609 oe.qa.exit_if_errors(d)
610}
611
612def qa_check_staged(path,d):
613 """
614 Check staged la and pc files for common problems like references to the work
615 directory.
616
617 As this is run after every stage we should be able to find the one
618 responsible for the errors easily even if we look at every .pc and .la file.
619 """
620
621 tmpdir = d.getVar('TMPDIR')
622 workdir = os.path.join(tmpdir, "work")
623 recipesysroot = d.getVar("RECIPE_SYSROOT")
624
625 if bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d):
626 pkgconfigcheck = workdir
627 else:
628 pkgconfigcheck = tmpdir
629
630 skip = (d.getVar('INSANE_SKIP') or "").split()
631 skip_la = False
632 if 'la' in skip:
633 bb.note("Recipe %s skipping qa checking: la" % d.getVar('PN'))
634 skip_la = True
635
636 skip_pkgconfig = False
637 if 'pkgconfig' in skip:
638 bb.note("Recipe %s skipping qa checking: pkgconfig" % d.getVar('PN'))
639 skip_pkgconfig = True
640
641 skip_shebang_size = False
642 if 'shebang-size' in skip:
643 bb.note("Recipe %s skipping qa checkking: shebang-size" % d.getVar('PN'))
644 skip_shebang_size = True
645
646 # find all .la and .pc files
647 # read the content
648 # and check for stuff that looks wrong
649 for root, dirs, files in os.walk(path):
650 for file in files:
651 path = os.path.join(root,file)
652 if file.endswith(".la") and not skip_la:
653 with open(path) as f:
654 file_content = f.read()
655 file_content = file_content.replace(recipesysroot, "")
656 if workdir in file_content:
657 error_msg = "%s failed sanity test (workdir) in path %s" % (file,root)
658 oe.qa.handle_error("la", error_msg, d)
659 elif file.endswith(".pc") and not skip_pkgconfig:
660 with open(path) as f:
661 file_content = f.read()
662 file_content = file_content.replace(recipesysroot, "")
663 if pkgconfigcheck in file_content:
664 error_msg = "%s failed sanity test (tmpdir) in path %s" % (file,root)
665 oe.qa.handle_error("pkgconfig", error_msg, d)
666
667 if not skip_shebang_size:
668 errors = {}
669 package_qa_check_shebang_size(path, "", d, None, errors)
670 for e in errors:
671 oe.qa.handle_error(e, errors[e], d)
672
673
674# Run all package-wide warnfuncs and errorfuncs
675def package_qa_package(warnfuncs, errorfuncs, package, d):
676 warnings = {}
677 errors = {}
678
679 for func in warnfuncs:
680 func(package, d, warnings)
681 for func in errorfuncs:
682 func(package, d, errors)
683
684 for w in warnings:
685 oe.qa.handle_error(w, warnings[w], d)
686 for e in errors:
687 oe.qa.handle_error(e, errors[e], d)
688
689 return len(errors) == 0
690
691# Run all recipe-wide warnfuncs and errorfuncs
692def package_qa_recipe(warnfuncs, errorfuncs, pn, d):
693 warnings = {}
694 errors = {}
695
696 for func in warnfuncs:
697 func(pn, d, warnings)
698 for func in errorfuncs:
699 func(pn, d, errors)
700
701 for w in warnings:
702 oe.qa.handle_error(w, warnings[w], d)
703 for e in errors:
704 oe.qa.handle_error(e, errors[e], d)
705
706 return len(errors) == 0
707
708def prepopulate_objdump_p(elf, d):
709 output = elf.run_objdump("-p", d)
710 return (elf.name, output)
711
712# Walk over all files in a directory and call func
713def package_qa_walk(warnfuncs, errorfuncs, package, d):
714 #if this will throw an exception, then fix the dict above
715 target_os = d.getVar('HOST_OS')
716 target_arch = d.getVar('HOST_ARCH')
717
718 warnings = {}
719 errors = {}
720 elves = {}
721 for path in pkgfiles[package]:
722 elf = None
723 if os.path.isfile(path):
724 elf = oe.qa.ELFFile(path)
725 try:
726 elf.open()
727 elf.close()
728 except oe.qa.NotELFFileError:
729 elf = None
730 if elf:
731 elves[path] = elf
732
733 results = oe.utils.multiprocess_launch(prepopulate_objdump_p, elves.values(), d, extraargs=(d,))
734 for item in results:
735 elves[item[0]].set_objdump("-p", item[1])
736
737 for path in pkgfiles[package]:
738 if path in elves:
739 elves[path].open()
740 for func in warnfuncs:
741 func(path, package, d, elves.get(path), warnings)
742 for func in errorfuncs:
743 func(path, package, d, elves.get(path), errors)
744 if path in elves:
745 elves[path].close()
746
747 for w in warnings:
748 oe.qa.handle_error(w, warnings[w], d)
749 for e in errors:
750 oe.qa.handle_error(e, errors[e], d)
751
752def package_qa_check_rdepends(pkg, pkgdest, skip, taskdeps, packages, d):
753 # Don't do this check for kernel/module recipes, there aren't too many debug/development
754 # packages and you can get false positives e.g. on kernel-module-lirc-dev
755 if bb.data.inherits_class("kernel", d) or bb.data.inherits_class("module-base", d):
756 return
757
758 if not "-dbg" in pkg and not "packagegroup-" in pkg and not "-image" in pkg:
759 localdata = bb.data.createCopy(d)
760 localdata.setVar('OVERRIDES', localdata.getVar('OVERRIDES') + ':' + pkg)
761
762 # Now check the RDEPENDS
763 rdepends = bb.utils.explode_deps(localdata.getVar('RDEPENDS') or "")
764
765 # Now do the sanity check!!!
766 if "build-deps" not in skip:
767 for rdepend in rdepends:
768 if "-dbg" in rdepend and "debug-deps" not in skip:
769 error_msg = "%s rdepends on %s" % (pkg,rdepend)
770 oe.qa.handle_error("debug-deps", error_msg, d)
771 if (not "-dev" in pkg and not "-staticdev" in pkg) and rdepend.endswith("-dev") and "dev-deps" not in skip:
772 error_msg = "%s rdepends on %s" % (pkg, rdepend)
773 oe.qa.handle_error("dev-deps", error_msg, d)
774 if rdepend not in packages:
775 rdep_data = oe.packagedata.read_subpkgdata(rdepend, d)
776 if rdep_data and 'PN' in rdep_data and rdep_data['PN'] in taskdeps:
777 continue
778 if not rdep_data or not 'PN' in rdep_data:
779 pkgdata_dir = d.getVar("PKGDATA_DIR")
780 try:
781 possibles = os.listdir("%s/runtime-rprovides/%s/" % (pkgdata_dir, rdepend))
782 except OSError:
783 possibles = []
784 for p in possibles:
785 rdep_data = oe.packagedata.read_subpkgdata(p, d)
786 if rdep_data and 'PN' in rdep_data and rdep_data['PN'] in taskdeps:
787 break
788 if rdep_data and 'PN' in rdep_data and rdep_data['PN'] in taskdeps:
789 continue
790 if rdep_data and 'PN' in rdep_data:
791 error_msg = "%s rdepends on %s, but it isn't a build dependency, missing %s in DEPENDS or PACKAGECONFIG?" % (pkg, rdepend, rdep_data['PN'])
792 else:
793 error_msg = "%s rdepends on %s, but it isn't a build dependency?" % (pkg, rdepend)
794 oe.qa.handle_error("build-deps", error_msg, d)
795
796 if "file-rdeps" not in skip:
797 ignored_file_rdeps = set(['/bin/sh', '/usr/bin/env', 'rtld(GNU_HASH)'])
798 if bb.data.inherits_class('nativesdk', d):
799 ignored_file_rdeps |= set(['/bin/bash', '/usr/bin/perl', 'perl'])
800 # For Saving the FILERDEPENDS
801 filerdepends = {}
802 rdep_data = oe.packagedata.read_subpkgdata(pkg, d)
803 for key in rdep_data:
804 if key.startswith("FILERDEPENDS:"):
805 for subkey in bb.utils.explode_deps(rdep_data[key]):
806 if subkey not in ignored_file_rdeps and \
807 not subkey.startswith('perl('):
808 # We already know it starts with FILERDEPENDS_
809 filerdepends[subkey] = key[13:]
810
811 if filerdepends:
812 done = rdepends[:]
813 # Add the rprovides of itself
814 if pkg not in done:
815 done.insert(0, pkg)
816
817 # The python is not a package, but python-core provides it, so
818 # skip checking /usr/bin/python if python is in the rdeps, in
819 # case there is a RDEPENDS:pkg = "python" in the recipe.
820 for py in [ d.getVar('MLPREFIX') + "python", "python" ]:
821 if py in done:
822 filerdepends.pop("/usr/bin/python",None)
823 done.remove(py)
824 for rdep in done:
825 # The file dependencies may contain package names, e.g.,
826 # perl
827 filerdepends.pop(rdep,None)
828
829 # For Saving the FILERPROVIDES, RPROVIDES and FILES_INFO
830 rdep_data = oe.packagedata.read_subpkgdata(rdep, d)
831 for key in rdep_data:
832 if key.startswith("FILERPROVIDES:") or key.startswith("RPROVIDES:"):
833 for subkey in bb.utils.explode_deps(rdep_data[key]):
834 filerdepends.pop(subkey,None)
835 # Add the files list to the rprovides
836 if key.startswith("FILES_INFO:"):
837 # Use eval() to make it as a dict
838 for subkey in eval(rdep_data[key]):
839 filerdepends.pop(subkey,None)
840 if not filerdepends:
841 # Break if all the file rdepends are met
842 break
843 if filerdepends:
844 for key in filerdepends:
845 error_msg = "%s contained in package %s requires %s, but no providers found in RDEPENDS:%s?" % \
846 (filerdepends[key].replace(":%s" % pkg, "").replace("@underscore@", "_"), pkg, key, pkg)
847 oe.qa.handle_error("file-rdeps", error_msg, d)
848package_qa_check_rdepends[vardepsexclude] = "OVERRIDES"
849
850def package_qa_check_deps(pkg, pkgdest, d):
851
852 localdata = bb.data.createCopy(d)
853 localdata.setVar('OVERRIDES', pkg)
854
855 def check_valid_deps(var):
856 try:
857 rvar = bb.utils.explode_dep_versions2(localdata.getVar(var) or "")
858 except ValueError as e:
859 bb.fatal("%s:%s: %s" % (var, pkg, e))
860 for dep in rvar:
861 for v in rvar[dep]:
862 if v and not v.startswith(('< ', '= ', '> ', '<= ', '>=')):
863 error_msg = "%s:%s is invalid: %s (%s) only comparisons <, =, >, <=, and >= are allowed" % (var, pkg, dep, v)
864 oe.qa.handle_error("dep-cmp", error_msg, d)
865
866 check_valid_deps('RDEPENDS')
867 check_valid_deps('RRECOMMENDS')
868 check_valid_deps('RSUGGESTS')
869 check_valid_deps('RPROVIDES')
870 check_valid_deps('RREPLACES')
871 check_valid_deps('RCONFLICTS')
872
873QAPKGTEST[usrmerge] = "package_qa_check_usrmerge"
874def package_qa_check_usrmerge(pkg, d, messages):
875
876 pkgdest = d.getVar('PKGDEST')
877 pkg_dir = pkgdest + os.sep + pkg + os.sep
878 merged_dirs = ['bin', 'sbin', 'lib'] + d.getVar('MULTILIB_VARIANTS').split()
879 for f in merged_dirs:
880 if os.path.exists(pkg_dir + f) and not os.path.islink(pkg_dir + f):
881 msg = "%s package is not obeying usrmerge distro feature. /%s should be relocated to /usr." % (pkg, f)
882 oe.qa.add_message(messages, "usrmerge", msg)
883 return False
884 return True
885
886QAPKGTEST[perllocalpod] = "package_qa_check_perllocalpod"
887def package_qa_check_perllocalpod(pkg, d, messages):
888 """
889 Check that the recipe didn't ship a perlocal.pod file, which shouldn't be
890 installed in a distribution package. cpan.bbclass sets NO_PERLLOCAL=1 to
891 handle this for most recipes.
892 """
893 import glob
894 pkgd = oe.path.join(d.getVar('PKGDEST'), pkg)
895 podpath = oe.path.join(pkgd, d.getVar("libdir"), "perl*", "*", "*", "perllocal.pod")
896
897 matches = glob.glob(podpath)
898 if matches:
899 matches = [package_qa_clean_path(path, d, pkg) for path in matches]
900 msg = "%s contains perllocal.pod (%s), should not be installed" % (pkg, " ".join(matches))
901 oe.qa.add_message(messages, "perllocalpod", msg)
902
903QAPKGTEST[expanded-d] = "package_qa_check_expanded_d"
904def package_qa_check_expanded_d(package, d, messages):
905 """
906 Check for the expanded D (${D}) value in pkg_* and FILES
907 variables, warn the user to use it correctly.
908 """
909 sane = True
910 expanded_d = d.getVar('D')
911
912 for var in 'FILES','pkg_preinst', 'pkg_postinst', 'pkg_prerm', 'pkg_postrm':
913 bbvar = d.getVar(var + ":" + package) or ""
914 if expanded_d in bbvar:
915 if var == 'FILES':
916 oe.qa.add_message(messages, "expanded-d", "FILES in %s recipe should not contain the ${D} variable as it references the local build directory not the target filesystem, best solution is to remove the ${D} reference" % package)
917 sane = False
918 else:
919 oe.qa.add_message(messages, "expanded-d", "%s in %s recipe contains ${D}, it should be replaced by $D instead" % (var, package))
920 sane = False
921 return sane
922
923QAPKGTEST[unlisted-pkg-lics] = "package_qa_check_unlisted_pkg_lics"
924def package_qa_check_unlisted_pkg_lics(package, d, messages):
925 """
926 Check that all licenses for a package are among the licenses for the recipe.
927 """
928 pkg_lics = d.getVar('LICENSE:' + package)
929 if not pkg_lics:
930 return True
931
932 recipe_lics_set = oe.license.list_licenses(d.getVar('LICENSE'))
933 package_lics = oe.license.list_licenses(pkg_lics)
934 unlisted = package_lics - recipe_lics_set
935 if unlisted:
936 oe.qa.add_message(messages, "unlisted-pkg-lics",
937 "LICENSE:%s includes licenses (%s) that are not "
938 "listed in LICENSE" % (package, ' '.join(unlisted)))
939 return False
940 obsolete = set(oe.license.obsolete_license_list()) & package_lics - recipe_lics_set
941 if obsolete:
942 oe.qa.add_message(messages, "obsolete-license",
943 "LICENSE:%s includes obsolete licenses %s" % (package, ' '.join(obsolete)))
944 return False
945 return True
946
947QAPKGTEST[empty-dirs] = "package_qa_check_empty_dirs"
948def package_qa_check_empty_dirs(pkg, d, messages):
949 """
950 Check for the existence of files in directories that are expected to be
951 empty.
952 """
953
954 pkgd = oe.path.join(d.getVar('PKGDEST'), pkg)
955 for dir in (d.getVar('QA_EMPTY_DIRS') or "").split():
956 empty_dir = oe.path.join(pkgd, dir)
957 if os.path.exists(empty_dir) and os.listdir(empty_dir):
958 recommendation = (d.getVar('QA_EMPTY_DIRS_RECOMMENDATION:' + dir) or
959 "but it is expected to be empty")
960 msg = "%s installs files in %s, %s" % (pkg, dir, recommendation)
961 oe.qa.add_message(messages, "empty-dirs", msg)
962
963def package_qa_check_encoding(keys, encode, d):
964 def check_encoding(key, enc):
965 sane = True
966 value = d.getVar(key)
967 if value:
968 try:
969 s = value.encode(enc)
970 except UnicodeDecodeError as e:
971 error_msg = "%s has non %s characters" % (key,enc)
972 sane = False
973 oe.qa.handle_error("invalid-chars", error_msg, d)
974 return sane
975
976 for key in keys:
977 sane = check_encoding(key, encode)
978 if not sane:
979 break
980
981HOST_USER_UID := "${@os.getuid()}"
982HOST_USER_GID := "${@os.getgid()}"
983
984QAPATHTEST[host-user-contaminated] = "package_qa_check_host_user"
985def package_qa_check_host_user(path, name, d, elf, messages):
986 """Check for paths outside of /home which are owned by the user running bitbake."""
987
988 if not os.path.lexists(path):
989 return
990
991 dest = d.getVar('PKGDEST')
992 pn = d.getVar('PN')
993 home = os.path.join(dest, name, 'home')
994 if path == home or path.startswith(home + os.sep):
995 return
996
997 try:
998 stat = os.lstat(path)
999 except OSError as exc:
1000 import errno
1001 if exc.errno != errno.ENOENT:
1002 raise
1003 else:
1004 check_uid = int(d.getVar('HOST_USER_UID'))
1005 if stat.st_uid == check_uid:
1006 oe.qa.add_message(messages, "host-user-contaminated", "%s: %s is owned by uid %d, which is the same as the user running bitbake. This may be due to host contamination" % (pn, package_qa_clean_path(path, d, name), check_uid))
1007 return False
1008
1009 check_gid = int(d.getVar('HOST_USER_GID'))
1010 if stat.st_gid == check_gid:
1011 oe.qa.add_message(messages, "host-user-contaminated", "%s: %s is owned by gid %d, which is the same as the user running bitbake. This may be due to host contamination" % (pn, package_qa_clean_path(path, d, name), check_gid))
1012 return False
1013 return True
1014
1015QARECIPETEST[unhandled-features-check] = "package_qa_check_unhandled_features_check"
1016def package_qa_check_unhandled_features_check(pn, d, messages):
1017 if not bb.data.inherits_class('features_check', d):
1018 var_set = False
1019 for kind in ['DISTRO', 'MACHINE', 'COMBINED']:
1020 for var in ['ANY_OF_' + kind + '_FEATURES', 'REQUIRED_' + kind + '_FEATURES', 'CONFLICT_' + kind + '_FEATURES']:
1021 if d.getVar(var) is not None or d.hasOverrides(var):
1022 var_set = True
1023 if var_set:
1024 oe.qa.handle_error("unhandled-features-check", "%s: recipe doesn't inherit features_check" % pn, d)
1025
1026QARECIPETEST[missing-update-alternatives] = "package_qa_check_missing_update_alternatives"
1027def package_qa_check_missing_update_alternatives(pn, d, messages):
1028 # Look at all packages and find out if any of those sets ALTERNATIVE variable
1029 # without inheriting update-alternatives class
1030 for pkg in (d.getVar('PACKAGES') or '').split():
1031 if d.getVar('ALTERNATIVE:%s' % pkg) and not bb.data.inherits_class('update-alternatives', d):
1032 oe.qa.handle_error("missing-update-alternatives", "%s: recipe defines ALTERNATIVE:%s but doesn't inherit update-alternatives. This might fail during do_rootfs later!" % (pn, pkg), d)
1033
1034# The PACKAGE FUNC to scan each package
1035python do_package_qa () {
1036 import subprocess
1037 import oe.packagedata
1038
1039 bb.note("DO PACKAGE QA")
1040
1041 main_lic = d.getVar('LICENSE')
1042
1043 # Check for obsolete license references in main LICENSE (packages are checked below for any changes)
1044 main_licenses = oe.license.list_licenses(d.getVar('LICENSE'))
1045 obsolete = set(oe.license.obsolete_license_list()) & main_licenses
1046 if obsolete:
1047 oe.qa.handle_error("obsolete-license", "Recipe LICENSE includes obsolete licenses %s" % ' '.join(obsolete), d)
1048
1049 bb.build.exec_func("read_subpackage_metadata", d)
1050
1051 # Check non UTF-8 characters on recipe's metadata
1052 package_qa_check_encoding(['DESCRIPTION', 'SUMMARY', 'LICENSE', 'SECTION'], 'utf-8', d)
1053
1054 logdir = d.getVar('T')
1055 pn = d.getVar('PN')
1056
1057 # Scan the packages...
1058 pkgdest = d.getVar('PKGDEST')
1059 packages = set((d.getVar('PACKAGES') or '').split())
1060
1061 global pkgfiles
1062 pkgfiles = {}
1063 for pkg in packages:
1064 pkgfiles[pkg] = []
1065 pkgdir = os.path.join(pkgdest, pkg)
1066 for walkroot, dirs, files in os.walk(pkgdir):
1067 # Don't walk into top-level CONTROL or DEBIAN directories as these
1068 # are temporary directories created by do_package.
1069 if walkroot == pkgdir:
1070 for control in ("CONTROL", "DEBIAN"):
1071 if control in dirs:
1072 dirs.remove(control)
1073 for file in files:
1074 pkgfiles[pkg].append(os.path.join(walkroot, file))
1075
1076 # no packages should be scanned
1077 if not packages:
1078 return
1079
1080 import re
1081 # The package name matches the [a-z0-9.+-]+ regular expression
1082 pkgname_pattern = re.compile(r"^[a-z0-9.+-]+$")
1083
1084 taskdepdata = d.getVar("BB_TASKDEPDATA", False)
1085 taskdeps = set()
1086 for dep in taskdepdata:
1087 taskdeps.add(taskdepdata[dep][0])
1088
1089 def parse_test_matrix(matrix_name):
1090 testmatrix = d.getVarFlags(matrix_name) or {}
1091 g = globals()
1092 warnchecks = []
1093 for w in (d.getVar("WARN_QA") or "").split():
1094 if w in skip:
1095 continue
1096 if w in testmatrix and testmatrix[w] in g:
1097 warnchecks.append(g[testmatrix[w]])
1098
1099 errorchecks = []
1100 for e in (d.getVar("ERROR_QA") or "").split():
1101 if e in skip:
1102 continue
1103 if e in testmatrix and testmatrix[e] in g:
1104 errorchecks.append(g[testmatrix[e]])
1105 return warnchecks, errorchecks
1106
1107 for package in packages:
1108 skip = set((d.getVar('INSANE_SKIP') or "").split() +
1109 (d.getVar('INSANE_SKIP:' + package) or "").split())
1110 if skip:
1111 bb.note("Package %s skipping QA tests: %s" % (package, str(skip)))
1112
1113 bb.note("Checking Package: %s" % package)
1114 # Check package name
1115 if not pkgname_pattern.match(package):
1116 oe.qa.handle_error("pkgname",
1117 "%s doesn't match the [a-z0-9.+-]+ regex" % package, d)
1118
1119 warn_checks, error_checks = parse_test_matrix("QAPATHTEST")
1120 package_qa_walk(warn_checks, error_checks, package, d)
1121
1122 warn_checks, error_checks = parse_test_matrix("QAPKGTEST")
1123 package_qa_package(warn_checks, error_checks, package, d)
1124
1125 package_qa_check_rdepends(package, pkgdest, skip, taskdeps, packages, d)
1126 package_qa_check_deps(package, pkgdest, d)
1127
1128 warn_checks, error_checks = parse_test_matrix("QARECIPETEST")
1129 package_qa_recipe(warn_checks, error_checks, pn, d)
1130
1131 if 'libdir' in d.getVar("ALL_QA").split():
1132 package_qa_check_libdir(d)
1133
1134 oe.qa.exit_if_errors(d)
1135}
1136
1137# binutils is used for most checks, so need to set as dependency
1138# POPULATESYSROOTDEPS is defined in staging class.
1139do_package_qa[depends] += "${POPULATESYSROOTDEPS}"
1140do_package_qa[vardeps] = "${@bb.utils.contains('ERROR_QA', 'empty-dirs', 'QA_EMPTY_DIRS', '', d)}"
1141do_package_qa[vardepsexclude] = "BB_TASKDEPDATA"
1142do_package_qa[rdeptask] = "do_packagedata"
1143addtask do_package_qa after do_packagedata do_package before do_build
1144
1145# Add the package specific INSANE_SKIPs to the sstate dependencies
1146python() {
1147 pkgs = (d.getVar('PACKAGES') or '').split()
1148 for pkg in pkgs:
1149 d.appendVarFlag("do_package_qa", "vardeps", " INSANE_SKIP:{}".format(pkg))
1150}
1151
1152SSTATETASKS += "do_package_qa"
1153do_package_qa[sstate-inputdirs] = ""
1154do_package_qa[sstate-outputdirs] = ""
1155python do_package_qa_setscene () {
1156 sstate_setscene(d)
1157}
1158addtask do_package_qa_setscene
1159
1160python do_qa_sysroot() {
1161 bb.note("QA checking do_populate_sysroot")
1162 sysroot_destdir = d.expand('${SYSROOT_DESTDIR}')
1163 for sysroot_dir in d.expand('${SYSROOT_DIRS}').split():
1164 qa_check_staged(sysroot_destdir + sysroot_dir, d)
1165 oe.qa.exit_with_message_if_errors("do_populate_sysroot for this recipe installed files with QA issues", d)
1166}
1167do_populate_sysroot[postfuncs] += "do_qa_sysroot"
1168
1169python do_qa_patch() {
1170 import subprocess
1171
1172 ###########################################################################
1173 # Check patch.log for fuzz warnings
1174 #
1175 # Further information on why we check for patch fuzz warnings:
1176 # http://lists.openembedded.org/pipermail/openembedded-core/2018-March/148675.html
1177 # https://bugzilla.yoctoproject.org/show_bug.cgi?id=10450
1178 ###########################################################################
1179
1180 logdir = d.getVar('T')
1181 patchlog = os.path.join(logdir,"log.do_patch")
1182
1183 if os.path.exists(patchlog):
1184 fuzzheader = '--- Patch fuzz start ---'
1185 fuzzfooter = '--- Patch fuzz end ---'
1186 statement = "grep -e '%s' %s > /dev/null" % (fuzzheader, patchlog)
1187 if subprocess.call(statement, shell=True) == 0:
1188 msg = "Fuzz detected:\n\n"
1189 fuzzmsg = ""
1190 inFuzzInfo = False
1191 f = open(patchlog, "r")
1192 for line in f:
1193 if fuzzheader in line:
1194 inFuzzInfo = True
1195 fuzzmsg = ""
1196 elif fuzzfooter in line:
1197 fuzzmsg = fuzzmsg.replace('\n\n', '\n')
1198 msg += fuzzmsg
1199 msg += "\n"
1200 inFuzzInfo = False
1201 elif inFuzzInfo and not 'Now at patch' in line:
1202 fuzzmsg += line
1203 f.close()
1204 msg += "The context lines in the patches can be updated with devtool:\n"
1205 msg += "\n"
1206 msg += " devtool modify %s\n" % d.getVar('PN')
1207 msg += " devtool finish --force-patch-refresh %s <layer_path>\n\n" % d.getVar('PN')
1208 msg += "Don't forget to review changes done by devtool!\n"
1209 if bb.utils.filter('ERROR_QA', 'patch-fuzz', d):
1210 bb.error(msg)
1211 elif bb.utils.filter('WARN_QA', 'patch-fuzz', d):
1212 bb.warn(msg)
1213 msg = "Patch log indicates that patches do not apply cleanly."
1214 oe.qa.handle_error("patch-fuzz", msg, d)
1215
1216 # Check if the patch contains a correctly formatted and spelled Upstream-Status
1217 import re
1218 from oe import patch
1219
1220 coremeta_path = os.path.join(d.getVar('COREBASE'), 'meta', '')
1221 for url in patch.src_patches(d):
1222 (_, _, fullpath, _, _, _) = bb.fetch.decodeurl(url)
1223
1224 # skip patches not in oe-core
1225 if not os.path.abspath(fullpath).startswith(coremeta_path):
1226 continue
1227
1228 kinda_status_re = re.compile(r"^.*upstream.*status.*$", re.IGNORECASE | re.MULTILINE)
1229 strict_status_re = re.compile(r"^Upstream-Status: (Pending|Submitted|Denied|Accepted|Inappropriate|Backport|Inactive-Upstream)( .+)?$", re.MULTILINE)
1230 guidelines = "https://www.openembedded.org/wiki/Commit_Patch_Message_Guidelines#Patch_Header_Recommendations:_Upstream-Status"
1231
1232 with open(fullpath, encoding='utf-8', errors='ignore') as f:
1233 file_content = f.read()
1234 match_kinda = kinda_status_re.search(file_content)
1235 match_strict = strict_status_re.search(file_content)
1236
1237 if not match_strict:
1238 if match_kinda:
1239 bb.error("Malformed Upstream-Status in patch\n%s\nPlease correct according to %s :\n%s" % (fullpath, guidelines, match_kinda.group(0)))
1240 else:
1241 bb.error("Missing Upstream-Status in patch\n%s\nPlease add according to %s ." % (fullpath, guidelines))
1242}
1243
1244python do_qa_configure() {
1245 import subprocess
1246
1247 ###########################################################################
1248 # Check config.log for cross compile issues
1249 ###########################################################################
1250
1251 configs = []
1252 workdir = d.getVar('WORKDIR')
1253
1254 skip = (d.getVar('INSANE_SKIP') or "").split()
1255 skip_configure_unsafe = False
1256 if 'configure-unsafe' in skip:
1257 bb.note("Recipe %s skipping qa checking: configure-unsafe" % d.getVar('PN'))
1258 skip_configure_unsafe = True
1259
1260 if bb.data.inherits_class('autotools', d) and not skip_configure_unsafe:
1261 bb.note("Checking autotools environment for common misconfiguration")
1262 for root, dirs, files in os.walk(workdir):
1263 statement = "grep -q -F -e 'is unsafe for cross-compilation' %s" % \
1264 os.path.join(root,"config.log")
1265 if "config.log" in files:
1266 if subprocess.call(statement, shell=True) == 0:
1267 error_msg = """This autoconf log indicates errors, it looked at host include and/or library paths while determining system capabilities.
1268Rerun configure task after fixing this."""
1269 oe.qa.handle_error("configure-unsafe", error_msg, d)
1270
1271 if "configure.ac" in files:
1272 configs.append(os.path.join(root,"configure.ac"))
1273 if "configure.in" in files:
1274 configs.append(os.path.join(root, "configure.in"))
1275
1276 ###########################################################################
1277 # Check gettext configuration and dependencies are correct
1278 ###########################################################################
1279
1280 skip_configure_gettext = False
1281 if 'configure-gettext' in skip:
1282 bb.note("Recipe %s skipping qa checking: configure-gettext" % d.getVar('PN'))
1283 skip_configure_gettext = True
1284
1285 cnf = d.getVar('EXTRA_OECONF') or ""
1286 if not ("gettext" in d.getVar('P') or "gcc-runtime" in d.getVar('P') or \
1287 "--disable-nls" in cnf or skip_configure_gettext):
1288 ml = d.getVar("MLPREFIX") or ""
1289 if bb.data.inherits_class('cross-canadian', d):
1290 gt = "nativesdk-gettext"
1291 else:
1292 gt = "gettext-native"
1293 deps = bb.utils.explode_deps(d.getVar('DEPENDS') or "")
1294 if gt not in deps:
1295 for config in configs:
1296 gnu = "grep \"^[[:space:]]*AM_GNU_GETTEXT\" %s >/dev/null" % config
1297 if subprocess.call(gnu, shell=True) == 0:
1298 error_msg = "AM_GNU_GETTEXT used but no inherit gettext"
1299 oe.qa.handle_error("configure-gettext", error_msg, d)
1300
1301 ###########################################################################
1302 # Check unrecognised configure options (with a white list)
1303 ###########################################################################
1304 if bb.data.inherits_class("autotools", d):
1305 bb.note("Checking configure output for unrecognised options")
1306 try:
1307 if bb.data.inherits_class("autotools", d):
1308 flag = "WARNING: unrecognized options:"
1309 log = os.path.join(d.getVar('B'), 'config.log')
1310 output = subprocess.check_output(['grep', '-F', flag, log]).decode("utf-8").replace(', ', ' ').replace('"', '')
1311 options = set()
1312 for line in output.splitlines():
1313 options |= set(line.partition(flag)[2].split())
1314 ignore_opts = set(d.getVar("UNKNOWN_CONFIGURE_OPT_IGNORE").split())
1315 options -= ignore_opts
1316 if options:
1317 pn = d.getVar('PN')
1318 error_msg = pn + ": configure was passed unrecognised options: " + " ".join(options)
1319 oe.qa.handle_error("unknown-configure-option", error_msg, d)
1320 except subprocess.CalledProcessError:
1321 pass
1322
1323 # Check invalid PACKAGECONFIG
1324 pkgconfig = (d.getVar("PACKAGECONFIG") or "").split()
1325 if pkgconfig:
1326 pkgconfigflags = d.getVarFlags("PACKAGECONFIG") or {}
1327 for pconfig in pkgconfig:
1328 if pconfig not in pkgconfigflags:
1329 pn = d.getVar('PN')
1330 error_msg = "%s: invalid PACKAGECONFIG: %s" % (pn, pconfig)
1331 oe.qa.handle_error("invalid-packageconfig", error_msg, d)
1332
1333 oe.qa.exit_if_errors(d)
1334}
1335
1336def unpack_check_src_uri(pn, d):
1337 import re
1338
1339 skip = (d.getVar('INSANE_SKIP') or "").split()
1340 if 'src-uri-bad' in skip:
1341 bb.note("Recipe %s skipping qa checking: src-uri-bad" % d.getVar('PN'))
1342 return
1343
1344 if "${PN}" in d.getVar("SRC_URI", False):
1345 oe.qa.handle_error("src-uri-bad", "%s: SRC_URI uses PN not BPN" % pn, d)
1346
1347 for url in d.getVar("SRC_URI").split():
1348 # Search for github and gitlab URLs that pull unstable archives (comment for future greppers)
Patrick Williams7784c422022-11-17 07:29:11 -06001349 if re.search(r"git(hu|la)b\.com/.+/.+/archive/.+", url) or "//codeload.github.com/" in url:
Patrick Williams92b42cb2022-09-03 06:53:57 -05001350 oe.qa.handle_error("src-uri-bad", "%s: SRC_URI uses unstable GitHub/GitLab archives, convert recipe to use git protocol" % pn, d)
1351
1352python do_qa_unpack() {
1353 src_uri = d.getVar('SRC_URI')
1354 s_dir = d.getVar('S')
1355 if src_uri and not os.path.exists(s_dir):
1356 bb.warn('%s: the directory %s (%s) pointed to by the S variable doesn\'t exist - please set S within the recipe to point to where the source has been unpacked to' % (d.getVar('PN'), d.getVar('S', False), s_dir))
1357
1358 unpack_check_src_uri(d.getVar('PN'), d)
1359}
1360
1361# Check for patch fuzz
1362do_patch[postfuncs] += "do_qa_patch "
1363
1364# Check broken config.log files, for packages requiring Gettext which
1365# don't have it in DEPENDS.
1366#addtask qa_configure after do_configure before do_compile
1367do_configure[postfuncs] += "do_qa_configure "
1368
1369# Check does S exist.
1370do_unpack[postfuncs] += "do_qa_unpack"
1371
1372python () {
1373 import re
1374
1375 tests = d.getVar('ALL_QA').split()
1376 if "desktop" in tests:
1377 d.appendVar("PACKAGE_DEPENDS", " desktop-file-utils-native")
1378
1379 ###########################################################################
1380 # Check various variables
1381 ###########################################################################
1382
1383 # Checking ${FILESEXTRAPATHS}
1384 extrapaths = (d.getVar("FILESEXTRAPATHS") or "")
1385 if '__default' not in extrapaths.split(":"):
1386 msg = "FILESEXTRAPATHS-variable, must always use :prepend (or :append)\n"
1387 msg += "type of assignment, and don't forget the colon.\n"
1388 msg += "Please assign it with the format of:\n"
1389 msg += " FILESEXTRAPATHS:append := \":${THISDIR}/Your_Files_Path\" or\n"
1390 msg += " FILESEXTRAPATHS:prepend := \"${THISDIR}/Your_Files_Path:\"\n"
1391 msg += "in your bbappend file\n\n"
1392 msg += "Your incorrect assignment is:\n"
1393 msg += "%s\n" % extrapaths
1394 bb.warn(msg)
1395
1396 overrides = d.getVar('OVERRIDES').split(':')
1397 pn = d.getVar('PN')
1398 if pn in overrides:
1399 msg = 'Recipe %s has PN of "%s" which is in OVERRIDES, this can result in unexpected behaviour.' % (d.getVar("FILE"), pn)
1400 oe.qa.handle_error("pn-overrides", msg, d)
1401 prog = re.compile(r'[A-Z]')
1402 if prog.search(pn):
1403 oe.qa.handle_error("uppercase-pn", 'PN: %s is upper case, this can result in unexpected behavior.' % pn, d)
1404
1405 # Some people mistakenly use DEPENDS:${PN} instead of DEPENDS and wonder
1406 # why it doesn't work.
1407 if (d.getVar(d.expand('DEPENDS:${PN}'))):
1408 oe.qa.handle_error("pkgvarcheck", "recipe uses DEPENDS:${PN}, should use DEPENDS", d)
1409
1410 issues = []
1411 if (d.getVar('PACKAGES') or "").split():
1412 for dep in (d.getVar('QADEPENDS') or "").split():
1413 d.appendVarFlag('do_package_qa', 'depends', " %s:do_populate_sysroot" % dep)
1414 for var in 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RCONFLICTS', 'RPROVIDES', 'RREPLACES', 'FILES', 'pkg_preinst', 'pkg_postinst', 'pkg_prerm', 'pkg_postrm', 'ALLOW_EMPTY':
1415 if d.getVar(var, False):
1416 issues.append(var)
1417
1418 fakeroot_tests = d.getVar('FAKEROOT_QA').split()
1419 if set(tests) & set(fakeroot_tests):
1420 d.setVarFlag('do_package_qa', 'fakeroot', '1')
1421 d.appendVarFlag('do_package_qa', 'depends', ' virtual/fakeroot-native:do_populate_sysroot')
1422 else:
1423 d.setVarFlag('do_package_qa', 'rdeptask', '')
1424 for i in issues:
1425 oe.qa.handle_error("pkgvarcheck", "%s: Variable %s is set as not being package specific, please fix this." % (d.getVar("FILE"), i), d)
1426
1427 if 'native-last' not in (d.getVar('INSANE_SKIP') or "").split():
1428 for native_class in ['native', 'nativesdk']:
1429 if bb.data.inherits_class(native_class, d):
1430
1431 inherited_classes = d.getVar('__inherit_cache', False) or []
1432 needle = "/" + native_class
1433
1434 bbclassextend = (d.getVar('BBCLASSEXTEND') or '').split()
1435 # BBCLASSEXTEND items are always added in the end
1436 skip_classes = bbclassextend
1437 if bb.data.inherits_class('native', d) or 'native' in bbclassextend:
1438 # native also inherits nopackages and relocatable bbclasses
1439 skip_classes.extend(['nopackages', 'relocatable'])
1440
1441 broken_order = []
1442 for class_item in reversed(inherited_classes):
1443 if needle not in class_item:
1444 for extend_item in skip_classes:
1445 if '/%s.bbclass' % extend_item in class_item:
1446 break
1447 else:
1448 pn = d.getVar('PN')
1449 broken_order.append(os.path.basename(class_item))
1450 else:
1451 break
1452 if broken_order:
1453 oe.qa.handle_error("native-last", "%s: native/nativesdk class is not inherited last, this can result in unexpected behaviour. "
1454 "Classes inherited after native/nativesdk: %s" % (pn, " ".join(broken_order)), d)
1455
1456 oe.qa.exit_if_errors(d)
1457}