blob: 5cec7a07fa253bfc5e10886b8d8bd6a438d90248 [file] [log] [blame]
Andrew Geisslerc926e172021-05-07 16:11:35 -05001ROOTFS_LICENSE_DIR = "${IMAGE_ROOTFS}/usr/share/common-licenses"
2
Andrew Geissler5f350902021-07-23 13:09:54 -04003# This requires LICENSE_CREATE_PACKAGE=1 to work too
4COMPLEMENTARY_GLOB[lic-pkgs] = "*-lic"
5
6python() {
7 if not oe.data.typed_value('LICENSE_CREATE_PACKAGE', d):
8 features = set(oe.data.typed_value('IMAGE_FEATURES', d))
9 if 'lic-pkgs' in features:
10 bb.error("'lic-pkgs' in IMAGE_FEATURES but LICENSE_CREATE_PACKAGE not enabled to generate -lic packages")
11}
12
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080013python write_package_manifest() {
14 # Get list of installed packages
15 license_image_dir = d.expand('${LICENSE_DIRECTORY}/${IMAGE_NAME}')
16 bb.utils.mkdirhier(license_image_dir)
17 from oe.rootfs import image_list_installed_packages
18 from oe.utils import format_pkg_list
19
20 pkgs = image_list_installed_packages(d)
21 output = format_pkg_list(pkgs)
22 open(os.path.join(license_image_dir, 'package.manifest'),
23 'w+').write(output)
24}
25
26python license_create_manifest() {
27 import oe.packagedata
28 from oe.rootfs import image_list_installed_packages
29
30 build_images_from_feeds = d.getVar('BUILD_IMAGES_FROM_FEEDS')
31 if build_images_from_feeds == "1":
32 return 0
33
34 pkg_dic = {}
35 for pkg in sorted(image_list_installed_packages(d)):
36 pkg_info = os.path.join(d.getVar('PKGDATA_DIR'),
37 'runtime-reverse', pkg)
38 pkg_name = os.path.basename(os.readlink(pkg_info))
39
40 pkg_dic[pkg_name] = oe.packagedata.read_pkgdatafile(pkg_info)
41 if not "LICENSE" in pkg_dic[pkg_name].keys():
Patrick Williams213cb262021-08-07 19:21:33 -050042 pkg_lic_name = "LICENSE:" + pkg_name
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080043 pkg_dic[pkg_name]["LICENSE"] = pkg_dic[pkg_name][pkg_lic_name]
44
45 rootfs_license_manifest = os.path.join(d.getVar('LICENSE_DIRECTORY'),
46 d.getVar('IMAGE_NAME'), 'license.manifest')
Brad Bishop19323692019-04-05 15:28:33 -040047 write_license_files(d, rootfs_license_manifest, pkg_dic, rootfs=True)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080048}
49
Brad Bishop19323692019-04-05 15:28:33 -040050def write_license_files(d, license_manifest, pkg_dic, rootfs=True):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080051 import re
Brad Bishop19323692019-04-05 15:28:33 -040052 import stat
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080053
54 bad_licenses = (d.getVar("INCOMPATIBLE_LICENSE") or "").split()
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080055 bad_licenses = expand_wildcard_licenses(d, bad_licenses)
56
Brad Bishopf3f93bb2019-10-16 14:33:32 -040057 whitelist = []
58 for lic in bad_licenses:
59 whitelist.extend((d.getVar("WHITELIST_" + lic) or "").split())
60
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080061 with open(license_manifest, "w") as license_file:
62 for pkg in sorted(pkg_dic):
Brad Bishopf3f93bb2019-10-16 14:33:32 -040063 if bad_licenses and pkg not in whitelist:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080064 try:
Andrew Geissler82c905d2020-04-13 13:39:40 -050065 licenses = incompatible_pkg_license(d, bad_licenses, pkg_dic[pkg]["LICENSE"])
66 if licenses:
67 bb.fatal("Package %s cannot be installed into the image because it has incompatible license(s): %s" %(pkg, ' '.join(licenses)))
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080068 (pkg_dic[pkg]["LICENSE"], pkg_dic[pkg]["LICENSES"]) = \
69 oe.license.manifest_licenses(pkg_dic[pkg]["LICENSE"],
70 bad_licenses, canonical_license, d)
71 except oe.license.LicenseError as exc:
72 bb.fatal('%s: %s' % (d.getVar('P'), exc))
73 else:
Brad Bishop977dc1a2019-02-06 16:01:43 -050074 pkg_dic[pkg]["LICENSES"] = re.sub(r'[|&()*]', ' ', pkg_dic[pkg]["LICENSE"])
75 pkg_dic[pkg]["LICENSES"] = re.sub(r' *', ' ', pkg_dic[pkg]["LICENSES"])
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080076 pkg_dic[pkg]["LICENSES"] = pkg_dic[pkg]["LICENSES"].split()
Brad Bishopf3f93bb2019-10-16 14:33:32 -040077 if pkg in whitelist:
78 bb.warn("Including %s with an incompatible license %s into the image, because it has been whitelisted." %(pkg, pkg_dic[pkg]["LICENSE"]))
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080079
80 if not "IMAGE_MANIFEST" in pkg_dic[pkg]:
81 # Rootfs manifest
82 license_file.write("PACKAGE NAME: %s\n" % pkg)
83 license_file.write("PACKAGE VERSION: %s\n" % pkg_dic[pkg]["PV"])
84 license_file.write("RECIPE NAME: %s\n" % pkg_dic[pkg]["PN"])
85 license_file.write("LICENSE: %s\n\n" % pkg_dic[pkg]["LICENSE"])
86
87 # If the package doesn't contain any file, that is, its size is 0, the license
88 # isn't relevant as far as the final image is concerned. So doing license check
89 # doesn't make much sense, skip it.
90 if pkg_dic[pkg]["PKGSIZE_%s" % pkg] == "0":
91 continue
92 else:
93 # Image manifest
94 license_file.write("RECIPE NAME: %s\n" % pkg_dic[pkg]["PN"])
95 license_file.write("VERSION: %s\n" % pkg_dic[pkg]["PV"])
96 license_file.write("LICENSE: %s\n" % pkg_dic[pkg]["LICENSE"])
97 license_file.write("FILES: %s\n\n" % pkg_dic[pkg]["FILES"])
98
99 for lic in pkg_dic[pkg]["LICENSES"]:
100 lic_file = os.path.join(d.getVar('LICENSE_DIRECTORY'),
101 pkg_dic[pkg]["PN"], "generic_%s" %
Brad Bishop977dc1a2019-02-06 16:01:43 -0500102 re.sub(r'\+', '', lic))
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800103 # add explicity avoid of CLOSED license because isn't generic
104 if lic == "CLOSED":
105 continue
106
107 if not os.path.exists(lic_file):
108 bb.warn("The license listed %s was not in the "\
109 "licenses collected for recipe %s"
110 % (lic, pkg_dic[pkg]["PN"]))
111
112 # Two options here:
113 # - Just copy the manifest
114 # - Copy the manifest and the license directories
115 # With both options set we see a .5 M increase in core-image-minimal
116 copy_lic_manifest = d.getVar('COPY_LIC_MANIFEST')
117 copy_lic_dirs = d.getVar('COPY_LIC_DIRS')
Brad Bishop19323692019-04-05 15:28:33 -0400118 if rootfs and copy_lic_manifest == "1":
Andrew Geisslerc926e172021-05-07 16:11:35 -0500119 rootfs_license_dir = d.getVar('ROOTFS_LICENSE_DIR')
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800120 bb.utils.mkdirhier(rootfs_license_dir)
121 rootfs_license_manifest = os.path.join(rootfs_license_dir,
122 os.path.split(license_manifest)[1])
123 if not os.path.exists(rootfs_license_manifest):
Brad Bishopc342db32019-05-15 21:57:59 -0400124 oe.path.copyhardlink(license_manifest, rootfs_license_manifest)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800125
126 if copy_lic_dirs == "1":
127 for pkg in sorted(pkg_dic):
128 pkg_rootfs_license_dir = os.path.join(rootfs_license_dir, pkg)
129 bb.utils.mkdirhier(pkg_rootfs_license_dir)
130 pkg_license_dir = os.path.join(d.getVar('LICENSE_DIRECTORY'),
131 pkg_dic[pkg]["PN"])
132
133 pkg_manifest_licenses = [canonical_license(d, lic) \
134 for lic in pkg_dic[pkg]["LICENSES"]]
135
136 licenses = os.listdir(pkg_license_dir)
137 for lic in licenses:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800138 pkg_license = os.path.join(pkg_license_dir, lic)
139 pkg_rootfs_license = os.path.join(pkg_rootfs_license_dir, lic)
140
Brad Bishop977dc1a2019-02-06 16:01:43 -0500141 if re.match(r"^generic_.*$", lic):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800142 generic_lic = canonical_license(d,
Brad Bishop977dc1a2019-02-06 16:01:43 -0500143 re.search(r"^generic_(.*)$", lic).group(1))
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800144
145 # Do not copy generic license into package if isn't
146 # declared into LICENSES of the package.
Brad Bishop977dc1a2019-02-06 16:01:43 -0500147 if not re.sub(r'\+$', '', generic_lic) in \
148 [re.sub(r'\+', '', lic) for lic in \
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800149 pkg_manifest_licenses]:
150 continue
151
152 if oe.license.license_ok(generic_lic,
153 bad_licenses) == False:
154 continue
155
Andrew Geissler6ce62a22020-11-30 19:58:47 -0600156 # Make sure we use only canonical name for the license file
Andrew Geisslerc926e172021-05-07 16:11:35 -0500157 generic_lic_file = "generic_%s" % generic_lic
158 rootfs_license = os.path.join(rootfs_license_dir, generic_lic_file)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800159 if not os.path.exists(rootfs_license):
Brad Bishopc342db32019-05-15 21:57:59 -0400160 oe.path.copyhardlink(pkg_license, rootfs_license)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800161
162 if not os.path.exists(pkg_rootfs_license):
Andrew Geisslerc926e172021-05-07 16:11:35 -0500163 os.symlink(os.path.join('..', generic_lic_file), pkg_rootfs_license)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800164 else:
165 if (oe.license.license_ok(canonical_license(d,
166 lic), bad_licenses) == False or
167 os.path.exists(pkg_rootfs_license)):
168 continue
169
Brad Bishopc342db32019-05-15 21:57:59 -0400170 oe.path.copyhardlink(pkg_license, pkg_rootfs_license)
Brad Bishop19323692019-04-05 15:28:33 -0400171 # Fixup file ownership and permissions
172 for walkroot, dirs, files in os.walk(rootfs_license_dir):
173 for f in files:
174 p = os.path.join(walkroot, f)
175 os.lchown(p, 0, 0)
176 if not os.path.islink(p):
177 os.chmod(p, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
178 for dir in dirs:
179 p = os.path.join(walkroot, dir)
180 os.lchown(p, 0, 0)
181 os.chmod(p, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
182
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800183
184
185def license_deployed_manifest(d):
186 """
187 Write the license manifest for the deployed recipes.
188 The deployed recipes usually includes the bootloader
189 and extra files to boot the target.
190 """
191
192 dep_dic = {}
193 man_dic = {}
194 lic_dir = d.getVar("LICENSE_DIRECTORY")
195
196 dep_dic = get_deployed_dependencies(d)
197 for dep in dep_dic.keys():
198 man_dic[dep] = {}
199 # It is necessary to mark this will be used for image manifest
200 man_dic[dep]["IMAGE_MANIFEST"] = True
201 man_dic[dep]["PN"] = dep
202 man_dic[dep]["FILES"] = \
203 " ".join(get_deployed_files(dep_dic[dep]))
204 with open(os.path.join(lic_dir, dep, "recipeinfo"), "r") as f:
205 for line in f.readlines():
206 key,val = line.split(": ", 1)
207 man_dic[dep][key] = val[:-1]
208
209 lic_manifest_dir = os.path.join(d.getVar('LICENSE_DIRECTORY'),
210 d.getVar('IMAGE_NAME'))
211 bb.utils.mkdirhier(lic_manifest_dir)
212 image_license_manifest = os.path.join(lic_manifest_dir, 'image_license.manifest')
Brad Bishop19323692019-04-05 15:28:33 -0400213 write_license_files(d, image_license_manifest, man_dic, rootfs=False)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800214
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500215 link_name = d.getVar('IMAGE_LINK_NAME')
216 if link_name:
217 lic_manifest_symlink_dir = os.path.join(d.getVar('LICENSE_DIRECTORY'),
218 link_name)
219 # remove old symlink
220 if os.path.islink(lic_manifest_symlink_dir):
221 os.unlink(lic_manifest_symlink_dir)
222
223 # create the image dir symlink
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600224 if lic_manifest_dir != lic_manifest_symlink_dir:
225 os.symlink(lic_manifest_dir, lic_manifest_symlink_dir)
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500226
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800227def get_deployed_dependencies(d):
228 """
229 Get all the deployed dependencies of an image
230 """
231
232 deploy = {}
233 # Get all the dependencies for the current task (rootfs).
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800234 taskdata = d.getVar("BB_TASKDEPDATA", False)
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600235 pn = d.getVar("PN", True)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800236 depends = list(set([dep[0] for dep
237 in list(taskdata.values())
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600238 if not dep[0].endswith("-native") and not dep[0] == pn]))
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800239
240 # To verify what was deployed it checks the rootfs dependencies against
241 # the SSTATE_MANIFESTS for "deploy" task.
242 # The manifest file name contains the arch. Because we are not running
243 # in the recipe context it is necessary to check every arch used.
244 sstate_manifest_dir = d.getVar("SSTATE_MANIFESTS")
245 archs = list(set(d.getVar("SSTATE_ARCHS").split()))
246 for dep in depends:
247 for arch in archs:
248 sstate_manifest_file = os.path.join(sstate_manifest_dir,
249 "manifest-%s-%s.deploy" % (arch, dep))
250 if os.path.exists(sstate_manifest_file):
251 deploy[dep] = sstate_manifest_file
252 break
253
254 return deploy
255get_deployed_dependencies[vardepsexclude] = "BB_TASKDEPDATA"
256
257def get_deployed_files(man_file):
258 """
259 Get the files deployed from the sstate manifest
260 """
261
262 dep_files = []
263 excluded_files = []
264 with open(man_file, "r") as manifest:
265 all_files = manifest.read()
266 for f in all_files.splitlines():
267 if ((not (os.path.islink(f) or os.path.isdir(f))) and
268 not os.path.basename(f) in excluded_files):
269 dep_files.append(os.path.basename(f))
270 return dep_files
271
Patrick Williams213cb262021-08-07 19:21:33 -0500272ROOTFS_POSTPROCESS_COMMAND:prepend = "write_package_manifest; license_create_manifest; "
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800273do_rootfs[recrdeptask] += "do_populate_lic"
274
275python do_populate_lic_deploy() {
276 license_deployed_manifest(d)
277}
278
279addtask populate_lic_deploy before do_build after do_image_complete
280do_populate_lic_deploy[recrdeptask] += "do_populate_lic do_deploy"
281
Andrew Geisslerc926e172021-05-07 16:11:35 -0500282python license_qa_dead_symlink() {
283 import os
284
285 for root, dirs, files in os.walk(d.getVar('ROOTFS_LICENSE_DIR')):
286 for file in files:
287 full_path = root + "/" + file
288 if os.path.islink(full_path) and not os.path.exists(full_path):
289 bb.error("broken symlink: " + full_path)
290}
291IMAGE_QA_COMMANDS += "license_qa_dead_symlink"