blob: 06dd4a8c1171c39bfc6b683978a8d66d5888134d [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# Populates LICENSE_DIRECTORY as set in distro config with the license files as set by
2# LIC_FILES_CHKSUM.
3# TODO:
4# - There is a real issue revolving around license naming standards.
5
6LICENSE_DIRECTORY ??= "${DEPLOY_DIR}/licenses"
7LICSSTATEDIR = "${WORKDIR}/license-destdir/"
8
9# Create extra package with license texts and add it to RRECOMMENDS_${PN}
10LICENSE_CREATE_PACKAGE[type] = "boolean"
11LICENSE_CREATE_PACKAGE ??= "0"
12LICENSE_PACKAGE_SUFFIX ??= "-lic"
13LICENSE_FILES_DIRECTORY ??= "${datadir}/licenses/"
14
15addtask populate_lic after do_patch before do_build
16do_populate_lic[dirs] = "${LICSSTATEDIR}/${PN}"
17do_populate_lic[cleandirs] = "${LICSSTATEDIR}"
18
19python write_package_manifest() {
20 # Get list of installed packages
21 license_image_dir = d.expand('${LICENSE_DIRECTORY}/${IMAGE_NAME}')
22 bb.utils.mkdirhier(license_image_dir)
23 from oe.rootfs import image_list_installed_packages
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050024 from oe.utils import format_pkg_list
25
26 pkgs = image_list_installed_packages(d)
27 output = format_pkg_list(pkgs)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050028 open(os.path.join(license_image_dir, 'package.manifest'),
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050029 'w+').write(output)
30}
31
32python write_deploy_manifest() {
33 license_deployed_manifest(d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050034}
35
36python license_create_manifest() {
Patrick Williamsc124f4f2015-09-15 14:41:29 -050037 import oe.packagedata
38 from oe.rootfs import image_list_installed_packages
39
Brad Bishop6e60e8b2018-02-01 10:27:11 -050040 build_images_from_feeds = d.getVar('BUILD_IMAGES_FROM_FEEDS')
Patrick Williamsc124f4f2015-09-15 14:41:29 -050041 if build_images_from_feeds == "1":
42 return 0
43
44 pkg_dic = {}
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050045 for pkg in sorted(image_list_installed_packages(d)):
Brad Bishop6e60e8b2018-02-01 10:27:11 -050046 pkg_info = os.path.join(d.getVar('PKGDATA_DIR'),
Patrick Williamsc124f4f2015-09-15 14:41:29 -050047 'runtime-reverse', pkg)
48 pkg_name = os.path.basename(os.readlink(pkg_info))
49
50 pkg_dic[pkg_name] = oe.packagedata.read_pkgdatafile(pkg_info)
51 if not "LICENSE" in pkg_dic[pkg_name].keys():
52 pkg_lic_name = "LICENSE_" + pkg_name
53 pkg_dic[pkg_name]["LICENSE"] = pkg_dic[pkg_name][pkg_lic_name]
54
Brad Bishop6e60e8b2018-02-01 10:27:11 -050055 rootfs_license_manifest = os.path.join(d.getVar('LICENSE_DIRECTORY'),
56 d.getVar('IMAGE_NAME'), 'license.manifest')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050057 write_license_files(d, rootfs_license_manifest, pkg_dic)
58}
59
60def write_license_files(d, license_manifest, pkg_dic):
61 import re
62
Brad Bishop6e60e8b2018-02-01 10:27:11 -050063 bad_licenses = (d.getVar("INCOMPATIBLE_LICENSE") or "").split()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050064 bad_licenses = map(lambda l: canonical_license(d, l), bad_licenses)
65 bad_licenses = expand_wildcard_licenses(d, bad_licenses)
66
Patrick Williamsc124f4f2015-09-15 14:41:29 -050067 with open(license_manifest, "w") as license_file:
68 for pkg in sorted(pkg_dic):
69 if bad_licenses:
70 try:
71 (pkg_dic[pkg]["LICENSE"], pkg_dic[pkg]["LICENSES"]) = \
72 oe.license.manifest_licenses(pkg_dic[pkg]["LICENSE"],
73 bad_licenses, canonical_license, d)
74 except oe.license.LicenseError as exc:
Brad Bishop6e60e8b2018-02-01 10:27:11 -050075 bb.fatal('%s: %s' % (d.getVar('P'), exc))
Patrick Williamsc124f4f2015-09-15 14:41:29 -050076 else:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050077 pkg_dic[pkg]["LICENSES"] = re.sub('[|&()*]', ' ', pkg_dic[pkg]["LICENSE"])
Patrick Williamsc124f4f2015-09-15 14:41:29 -050078 pkg_dic[pkg]["LICENSES"] = re.sub(' *', ' ', pkg_dic[pkg]["LICENSES"])
79 pkg_dic[pkg]["LICENSES"] = pkg_dic[pkg]["LICENSES"].split()
80
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050081 if not "IMAGE_MANIFEST" in pkg_dic[pkg]:
82 # Rootfs manifest
83 license_file.write("PACKAGE NAME: %s\n" % pkg)
84 license_file.write("PACKAGE VERSION: %s\n" % pkg_dic[pkg]["PV"])
85 license_file.write("RECIPE NAME: %s\n" % pkg_dic[pkg]["PN"])
86 license_file.write("LICENSE: %s\n\n" % pkg_dic[pkg]["LICENSE"])
Patrick Williamsc124f4f2015-09-15 14:41:29 -050087
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050088 # If the package doesn't contain any file, that is, its size is 0, the license
89 # isn't relevant as far as the final image is concerned. So doing license check
90 # doesn't make much sense, skip it.
91 if pkg_dic[pkg]["PKGSIZE_%s" % pkg] == "0":
92 continue
93 else:
94 # Image manifest
95 license_file.write("RECIPE NAME: %s\n" % pkg_dic[pkg]["PN"])
96 license_file.write("VERSION: %s\n" % pkg_dic[pkg]["PV"])
97 license_file.write("LICENSE: %s\n" % pkg_dic[pkg]["LICENSE"])
98 license_file.write("FILES: %s\n\n" % pkg_dic[pkg]["FILES"])
Patrick Williamsc124f4f2015-09-15 14:41:29 -050099
100 for lic in pkg_dic[pkg]["LICENSES"]:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500101 lic_file = os.path.join(d.getVar('LICENSE_DIRECTORY'),
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500102 pkg_dic[pkg]["PN"], "generic_%s" %
103 re.sub('\+', '', lic))
104 # add explicity avoid of CLOSED license because isn't generic
105 if lic == "CLOSED":
106 continue
107
108 if not os.path.exists(lic_file):
109 bb.warn("The license listed %s was not in the "\
110 "licenses collected for recipe %s"
111 % (lic, pkg_dic[pkg]["PN"]))
112
113 # Two options here:
114 # - Just copy the manifest
115 # - Copy the manifest and the license directories
116 # With both options set we see a .5 M increase in core-image-minimal
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500117 copy_lic_manifest = d.getVar('COPY_LIC_MANIFEST')
118 copy_lic_dirs = d.getVar('COPY_LIC_DIRS')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500119 if copy_lic_manifest == "1":
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500120 rootfs_license_dir = os.path.join(d.getVar('IMAGE_ROOTFS'),
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500121 'usr', 'share', 'common-licenses')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500122 bb.utils.mkdirhier(rootfs_license_dir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500123 rootfs_license_manifest = os.path.join(rootfs_license_dir,
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500124 os.path.split(license_manifest)[1])
125 if not os.path.exists(rootfs_license_manifest):
126 os.link(license_manifest, rootfs_license_manifest)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500127
128 if copy_lic_dirs == "1":
129 for pkg in sorted(pkg_dic):
130 pkg_rootfs_license_dir = os.path.join(rootfs_license_dir, pkg)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500131 bb.utils.mkdirhier(pkg_rootfs_license_dir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500132 pkg_license_dir = os.path.join(d.getVar('LICENSE_DIRECTORY'),
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500133 pkg_dic[pkg]["PN"])
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500134
135 pkg_manifest_licenses = [canonical_license(d, lic) \
136 for lic in pkg_dic[pkg]["LICENSES"]]
137
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500138 licenses = os.listdir(pkg_license_dir)
139 for lic in licenses:
140 rootfs_license = os.path.join(rootfs_license_dir, lic)
141 pkg_license = os.path.join(pkg_license_dir, lic)
142 pkg_rootfs_license = os.path.join(pkg_rootfs_license_dir, lic)
143
144 if re.match("^generic_.*$", lic):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500145 generic_lic = canonical_license(d,
146 re.search("^generic_(.*)$", lic).group(1))
147
148 # Do not copy generic license into package if isn't
149 # declared into LICENSES of the package.
150 if not re.sub('\+$', '', generic_lic) in \
151 [re.sub('\+', '', lic) for lic in \
152 pkg_manifest_licenses]:
153 continue
154
155 if oe.license.license_ok(generic_lic,
156 bad_licenses) == False:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500157 continue
158
159 if not os.path.exists(rootfs_license):
160 os.link(pkg_license, rootfs_license)
161
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500162 if not os.path.exists(pkg_rootfs_license):
163 os.symlink(os.path.join('..', lic), pkg_rootfs_license)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500164 else:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500165 if (oe.license.license_ok(canonical_license(d,
166 lic), bad_licenses) == False or
167 os.path.exists(pkg_rootfs_license)):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500168 continue
169
170 os.link(pkg_license, pkg_rootfs_license)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500171
172
173def license_deployed_manifest(d):
174 """
175 Write the license manifest for the deployed recipes.
176 The deployed recipes usually includes the bootloader
177 and extra files to boot the target.
178 """
179
180 dep_dic = {}
181 man_dic = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500182 lic_dir = d.getVar("LICENSE_DIRECTORY")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500183
184 dep_dic = get_deployed_dependencies(d)
185 for dep in dep_dic.keys():
186 man_dic[dep] = {}
187 # It is necessary to mark this will be used for image manifest
188 man_dic[dep]["IMAGE_MANIFEST"] = True
189 man_dic[dep]["PN"] = dep
190 man_dic[dep]["FILES"] = \
191 " ".join(get_deployed_files(dep_dic[dep]))
192 with open(os.path.join(lic_dir, dep, "recipeinfo"), "r") as f:
193 for line in f.readlines():
194 key,val = line.split(": ", 1)
195 man_dic[dep][key] = val[:-1]
196
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500197 lic_manifest_dir = os.path.join(d.getVar('LICENSE_DIRECTORY'),
198 d.getVar('IMAGE_NAME'))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600199 bb.utils.mkdirhier(lic_manifest_dir)
200 image_license_manifest = os.path.join(lic_manifest_dir, 'image_license.manifest')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500201 write_license_files(d, image_license_manifest, man_dic)
202
203def get_deployed_dependencies(d):
204 """
205 Get all the deployed dependencies of an image
206 """
207
208 deploy = {}
209 # Get all the dependencies for the current task (rootfs).
210 # Also get EXTRA_IMAGEDEPENDS because the bootloader is
211 # usually in this var and not listed in rootfs.
212 # At last, get the dependencies from boot classes because
213 # it might contain the bootloader.
214 taskdata = d.getVar("BB_TASKDEPDATA", False)
215 depends = list(set([dep[0] for dep
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600216 in list(taskdata.values())
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500217 if not dep[0].endswith("-native")]))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500218 extra_depends = d.getVar("EXTRA_IMAGEDEPENDS")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500219 boot_depends = get_boot_dependencies(d)
220 depends.extend(extra_depends.split())
221 depends.extend(boot_depends)
222 depends = list(set(depends))
223
224 # To verify what was deployed it checks the rootfs dependencies against
225 # the SSTATE_MANIFESTS for "deploy" task.
226 # The manifest file name contains the arch. Because we are not running
227 # in the recipe context it is necessary to check every arch used.
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500228 sstate_manifest_dir = d.getVar("SSTATE_MANIFESTS")
Brad Bishop316dfdd2018-06-25 12:45:53 -0400229 archs = list(set(d.getVar("SSTATE_ARCHS").split()))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500230 for dep in depends:
231 # Some recipes have an arch on their own, so we try that first.
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500232 special_arch = d.getVar("PACKAGE_ARCH_pn-%s" % dep)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500233 if special_arch:
234 sstate_manifest_file = os.path.join(sstate_manifest_dir,
235 "manifest-%s-%s.deploy" % (special_arch, dep))
236 if os.path.exists(sstate_manifest_file):
237 deploy[dep] = sstate_manifest_file
238 continue
239
240 for arch in archs:
241 sstate_manifest_file = os.path.join(sstate_manifest_dir,
242 "manifest-%s-%s.deploy" % (arch, dep))
243 if os.path.exists(sstate_manifest_file):
244 deploy[dep] = sstate_manifest_file
245 break
246
247 return deploy
248get_deployed_dependencies[vardepsexclude] = "BB_TASKDEPDATA"
249
250def get_boot_dependencies(d):
251 """
252 Return the dependencies from boot tasks
253 """
254
255 depends = []
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500256 taskdepdata = d.getVar("BB_TASKDEPDATA", False)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500257 # Only bootimg includes the depends flag
258 boot_depends_string = d.getVarFlag("do_bootimg", "depends") or ""
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500259 boot_depends = [dep.split(":")[0] for dep
260 in boot_depends_string.split()
261 if not dep.split(":")[0].endswith("-native")]
262 for dep in boot_depends:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500263 info_file = os.path.join(d.getVar("LICENSE_DIRECTORY"),
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500264 dep, "recipeinfo")
265 # If the recipe and dependency name is the same
266 if os.path.exists(info_file):
267 depends.append(dep)
268 # We need to search for the provider of the dependency
269 else:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600270 for taskdep in taskdepdata.values():
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500271 # The fifth field contains what the task provides
272 if dep in taskdep[4]:
273 info_file = os.path.join(
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500274 d.getVar("LICENSE_DIRECTORY"),
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500275 taskdep[0], "recipeinfo")
276 if os.path.exists(info_file):
277 depends.append(taskdep[0])
278 break
279 return depends
280get_boot_dependencies[vardepsexclude] = "BB_TASKDEPDATA"
281
282def get_deployed_files(man_file):
283 """
284 Get the files deployed from the sstate manifest
285 """
286
287 dep_files = []
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500288 excluded_files = []
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500289 with open(man_file, "r") as manifest:
290 all_files = manifest.read()
291 for f in all_files.splitlines():
292 if ((not (os.path.islink(f) or os.path.isdir(f))) and
293 not os.path.basename(f) in excluded_files):
294 dep_files.append(os.path.basename(f))
295 return dep_files
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500296
297python do_populate_lic() {
298 """
299 Populate LICENSE_DIRECTORY with licenses.
300 """
301 lic_files_paths = find_license_files(d)
302
303 # The base directory we wrangle licenses to
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500304 destdir = os.path.join(d.getVar('LICSSTATEDIR'), d.getVar('PN'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500305 copy_license_files(lic_files_paths, destdir)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500306 info = get_recipe_info(d)
307 with open(os.path.join(destdir, "recipeinfo"), "w") as f:
308 for key in sorted(info.keys()):
309 f.write("%s: %s\n" % (key, info[key]))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500310}
311
312# it would be better to copy them in do_install_append, but find_license_filesa is python
313python perform_packagecopy_prepend () {
314 enabled = oe.data.typed_value('LICENSE_CREATE_PACKAGE', d)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500315 if d.getVar('CLASSOVERRIDE') == 'class-target' and enabled:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500316 lic_files_paths = find_license_files(d)
317
318 # LICENSE_FILES_DIRECTORY starts with '/' so os.path.join cannot be used to join D and LICENSE_FILES_DIRECTORY
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500319 destdir = d.getVar('D') + os.path.join(d.getVar('LICENSE_FILES_DIRECTORY'), d.getVar('PN'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500320 copy_license_files(lic_files_paths, destdir)
321 add_package_and_files(d)
322}
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500323perform_packagecopy[vardeps] += "LICENSE_CREATE_PACKAGE"
324
325def get_recipe_info(d):
326 info = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500327 info["PV"] = d.getVar("PV")
328 info["PR"] = d.getVar("PR")
329 info["LICENSE"] = d.getVar("LICENSE")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500330 return info
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500331
332def add_package_and_files(d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500333 packages = d.getVar('PACKAGES')
334 files = d.getVar('LICENSE_FILES_DIRECTORY')
335 pn = d.getVar('PN')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500336 pn_lic = "%s%s" % (pn, d.getVar('LICENSE_PACKAGE_SUFFIX', False))
Brad Bishop316dfdd2018-06-25 12:45:53 -0400337 if pn_lic in packages.split():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500338 bb.warn("%s package already existed in %s." % (pn_lic, pn))
339 else:
340 # first in PACKAGES to be sure that nothing else gets LICENSE_FILES_DIRECTORY
341 d.setVar('PACKAGES', "%s %s" % (pn_lic, packages))
342 d.setVar('FILES_' + pn_lic, files)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500343 rrecommends_pn = d.getVar('RRECOMMENDS_' + pn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500344 if rrecommends_pn:
345 d.setVar('RRECOMMENDS_' + pn, "%s %s" % (pn_lic, rrecommends_pn))
346 else:
347 d.setVar('RRECOMMENDS_' + pn, "%s" % (pn_lic))
348
349def copy_license_files(lic_files_paths, destdir):
350 import shutil
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600351 import errno
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500352
353 bb.utils.mkdirhier(destdir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500354 for (basename, path, beginline, endline) in lic_files_paths:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500355 try:
356 src = path
357 dst = os.path.join(destdir, basename)
358 if os.path.exists(dst):
359 os.remove(dst)
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500360 if os.path.islink(src):
361 src = os.path.realpath(src)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500362 canlink = os.access(src, os.W_OK) and (os.stat(src).st_dev == os.stat(destdir).st_dev) and beginline is None and endline is None
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600363 if canlink:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500364 try:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600365 os.link(src, dst)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500366 except OSError as err:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600367 if err.errno == errno.EXDEV:
368 # Copy license files if hard-link is not possible even if st_dev is the
369 # same on source and destination (docker container with device-mapper?)
370 canlink = False
371 else:
372 raise
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500373 # Only chown if we did hardling, and, we're running under pseudo
374 if canlink and os.environ.get('PSEUDO_DISABLED') == '0':
375 os.chown(dst,0,0)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600376 if not canlink:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500377 begin_idx = int(beginline)-1 if beginline is not None else None
378 end_idx = int(endline) if endline is not None else None
379 if begin_idx is None and end_idx is None:
380 shutil.copyfile(src, dst)
381 else:
382 with open(src, 'rb') as src_f:
383 with open(dst, 'wb') as dst_f:
384 dst_f.write(b''.join(src_f.readlines()[begin_idx:end_idx]))
385
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500386 except Exception as e:
387 bb.warn("Could not copy license file %s to %s: %s" % (src, dst, e))
388
389def find_license_files(d):
390 """
391 Creates list of files used in LIC_FILES_CHKSUM and generic LICENSE files.
392 """
393 import shutil
394 import oe.license
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600395 from collections import defaultdict, OrderedDict
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500396
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500397 # All the license files for the package
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500398 lic_files = d.getVar('LIC_FILES_CHKSUM') or ""
399 pn = d.getVar('PN')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500400 # The license files are located in S/LIC_FILE_CHECKSUM.
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500401 srcdir = d.getVar('S')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500402 # Directory we store the generic licenses as set in the distro configuration
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500403 generic_directory = d.getVar('COMMON_LICENSE_DIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500404 # List of basename, path tuples
405 lic_files_paths = []
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500406 # hash for keep track generic lics mappings
407 non_generic_lics = {}
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600408 # Entries from LIC_FILES_CHKSUM
409 lic_chksums = {}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500410 license_source_dirs = []
411 license_source_dirs.append(generic_directory)
412 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500413 additional_lic_dirs = d.getVar('LICENSE_PATH').split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500414 for lic_dir in additional_lic_dirs:
415 license_source_dirs.append(lic_dir)
416 except:
417 pass
418
419 class FindVisitor(oe.license.LicenseVisitor):
420 def visit_Str(self, node):
421 #
422 # Until I figure out what to do with
423 # the two modifiers I support (or greater = +
424 # and "with exceptions" being *
425 # we'll just strip out the modifier and put
426 # the base license.
427 find_license(node.s.replace("+", "").replace("*", ""))
428 self.generic_visit(node)
429
430 def find_license(license_type):
431 try:
432 bb.utils.mkdirhier(gen_lic_dest)
433 except:
434 pass
435 spdx_generic = None
436 license_source = None
437 # If the generic does not exist we need to check to see if there is an SPDX mapping to it,
438 # unless NO_GENERIC_LICENSE is set.
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500439 for lic_dir in license_source_dirs:
440 if not os.path.isfile(os.path.join(lic_dir, license_type)):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500441 if d.getVarFlag('SPDXLICENSEMAP', license_type) != None:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500442 # Great, there is an SPDXLICENSEMAP. We can copy!
443 bb.debug(1, "We need to use a SPDXLICENSEMAP for %s" % (license_type))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500444 spdx_generic = d.getVarFlag('SPDXLICENSEMAP', license_type)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500445 license_source = lic_dir
446 break
447 elif os.path.isfile(os.path.join(lic_dir, license_type)):
448 spdx_generic = license_type
449 license_source = lic_dir
450 break
451
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500452 non_generic_lic = d.getVarFlag('NO_GENERIC_LICENSE', license_type)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500453 if spdx_generic and license_source:
454 # we really should copy to generic_ + spdx_generic, however, that ends up messing the manifest
455 # audit up. This should be fixed in emit_pkgdata (or, we actually got and fix all the recipes)
456
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500457 lic_files_paths.append(("generic_" + license_type, os.path.join(license_source, spdx_generic),
458 None, None))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500459
460 # The user may attempt to use NO_GENERIC_LICENSE for a generic license which doesn't make sense
461 # and should not be allowed, warn the user in this case.
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500462 if d.getVarFlag('NO_GENERIC_LICENSE', license_type):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500463 bb.warn("%s: %s is a generic license, please don't use NO_GENERIC_LICENSE for it." % (pn, license_type))
464
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600465 elif non_generic_lic and non_generic_lic in lic_chksums:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500466 # if NO_GENERIC_LICENSE is set, we copy the license files from the fetched source
467 # of the package rather than the license_source_dirs.
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600468 lic_files_paths.append(("generic_" + license_type,
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500469 os.path.join(srcdir, non_generic_lic), None, None))
470 non_generic_lics[non_generic_lic] = license_type
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500471 else:
472 # Add explicity avoid of CLOSED license because this isn't generic
473 if license_type != 'CLOSED':
474 # And here is where we warn people that their licenses are lousy
475 bb.warn("%s: No generic license file exists for: %s in any provider" % (pn, license_type))
476 pass
477
478 if not generic_directory:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600479 bb.fatal("COMMON_LICENSE_DIR is unset. Please set this in your distro config")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500480
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500481 for url in lic_files.split():
482 try:
483 (type, host, path, user, pswd, parm) = bb.fetch.decodeurl(url)
484 except bb.fetch.MalformedUrl:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500485 bb.fatal("%s: LIC_FILES_CHKSUM contains an invalid URL: %s" % (d.getVar('PF'), url))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500486 # We want the license filename and path
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500487 chksum = parm.get('md5', None)
488 beginline = parm.get('beginline')
489 endline = parm.get('endline')
490 lic_chksums[path] = (chksum, beginline, endline)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500491
492 v = FindVisitor()
493 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500494 v.visit_string(d.getVar('LICENSE'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500495 except oe.license.InvalidLicense as exc:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500496 bb.fatal('%s: %s' % (d.getVar('PF'), exc))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500497 except SyntaxError:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500498 bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF')))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600499 # Add files from LIC_FILES_CHKSUM to list of license files
500 lic_chksum_paths = defaultdict(OrderedDict)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500501 for path, data in sorted(lic_chksums.items()):
502 lic_chksum_paths[os.path.basename(path)][data] = (os.path.join(srcdir, path), data[1], data[2])
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600503 for basename, files in lic_chksum_paths.items():
504 if len(files) == 1:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500505 # Don't copy again a LICENSE already handled as non-generic
506 if basename in non_generic_lics:
507 continue
508 data = list(files.values())[0]
509 lic_files_paths.append(tuple([basename] + list(data)))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600510 else:
511 # If there are multiple different license files with identical
512 # basenames we rename them to <file>.0, <file>.1, ...
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500513 for i, data in enumerate(files.values()):
514 lic_files_paths.append(tuple(["%s.%d" % (basename, i)] + list(data)))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600515
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500516 return lic_files_paths
517
518def return_spdx(d, license):
519 """
520 This function returns the spdx mapping of a license if it exists.
521 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500522 return d.getVarFlag('SPDXLICENSEMAP', license)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500523
524def canonical_license(d, license):
525 """
526 Return the canonical (SPDX) form of the license if available (so GPLv3
527 becomes GPL-3.0), for the license named 'X+', return canonical form of
528 'X' if availabel and the tailing '+' (so GPLv3+ becomes GPL-3.0+),
529 or the passed license if there is no canonical form.
530 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500531 lic = d.getVarFlag('SPDXLICENSEMAP', license) or ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500532 if not lic and license.endswith('+'):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500533 lic = d.getVarFlag('SPDXLICENSEMAP', license.rstrip('+'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500534 if lic:
535 lic += '+'
536 return lic or license
537
538def expand_wildcard_licenses(d, wildcard_licenses):
539 """
540 Return actual spdx format license names if wildcard used. We expand
541 wildcards from SPDXLICENSEMAP flags and SRC_DISTRIBUTE_LICENSES values.
542 """
543 import fnmatch
544 licenses = []
545 spdxmapkeys = d.getVarFlags('SPDXLICENSEMAP').keys()
546 for wld_lic in wildcard_licenses:
547 spdxflags = fnmatch.filter(spdxmapkeys, wld_lic)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500548 licenses += [d.getVarFlag('SPDXLICENSEMAP', flag) for flag in spdxflags]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500549
550 spdx_lics = (d.getVar('SRC_DISTRIBUTE_LICENSES', False) or '').split()
551 for wld_lic in wildcard_licenses:
552 licenses += fnmatch.filter(spdx_lics, wld_lic)
553
554 licenses = list(set(licenses))
555 return licenses
556
557def incompatible_license_contains(license, truevalue, falsevalue, d):
558 license = canonical_license(d, license)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500559 bad_licenses = (d.getVar('INCOMPATIBLE_LICENSE') or "").split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500560 bad_licenses = expand_wildcard_licenses(d, bad_licenses)
561 return truevalue if license in bad_licenses else falsevalue
562
563def incompatible_license(d, dont_want_licenses, package=None):
564 """
565 This function checks if a recipe has only incompatible licenses. It also
566 take into consideration 'or' operand. dont_want_licenses should be passed
567 as canonical (SPDX) names.
568 """
569 import oe.license
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500570 license = d.getVar("LICENSE_%s" % package) if package else None
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500571 if not license:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500572 license = d.getVar('LICENSE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500573
574 # Handles an "or" or two license sets provided by
575 # flattened_licenses(), pick one that works if possible.
576 def choose_lic_set(a, b):
577 return a if all(oe.license.license_ok(canonical_license(d, lic),
578 dont_want_licenses) for lic in a) else b
579
580 try:
581 licenses = oe.license.flattened_licenses(license, choose_lic_set)
582 except oe.license.LicenseError as exc:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500583 bb.fatal('%s: %s' % (d.getVar('P'), exc))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500584 return any(not oe.license.license_ok(canonical_license(d, l), \
585 dont_want_licenses) for l in licenses)
586
587def check_license_flags(d):
588 """
589 This function checks if a recipe has any LICENSE_FLAGS that
590 aren't whitelisted.
591
592 If it does, it returns the first LICENSE_FLAGS item missing from the
593 whitelist, or all of the LICENSE_FLAGS if there is no whitelist.
594
595 If everything is is properly whitelisted, it returns None.
596 """
597
598 def license_flag_matches(flag, whitelist, pn):
599 """
600 Return True if flag matches something in whitelist, None if not.
601
602 Before we test a flag against the whitelist, we append _${PN}
603 to it. We then try to match that string against the
604 whitelist. This covers the normal case, where we expect
605 LICENSE_FLAGS to be a simple string like 'commercial', which
606 the user typically matches exactly in the whitelist by
607 explicitly appending the package name e.g 'commercial_foo'.
608 If we fail the match however, we then split the flag across
609 '_' and append each fragment and test until we either match or
610 run out of fragments.
611 """
612 flag_pn = ("%s_%s" % (flag, pn))
613 for candidate in whitelist:
614 if flag_pn == candidate:
615 return True
616
617 flag_cur = ""
618 flagments = flag_pn.split("_")
619 flagments.pop() # we've already tested the full string
620 for flagment in flagments:
621 if flag_cur:
622 flag_cur += "_"
623 flag_cur += flagment
624 for candidate in whitelist:
625 if flag_cur == candidate:
626 return True
627 return False
628
629 def all_license_flags_match(license_flags, whitelist):
630 """ Return first unmatched flag, None if all flags match """
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500631 pn = d.getVar('PN')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500632 split_whitelist = whitelist.split()
633 for flag in license_flags.split():
634 if not license_flag_matches(flag, split_whitelist, pn):
635 return flag
636 return None
637
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500638 license_flags = d.getVar('LICENSE_FLAGS')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500639 if license_flags:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500640 whitelist = d.getVar('LICENSE_FLAGS_WHITELIST')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500641 if not whitelist:
642 return license_flags
643 unmatched_flag = all_license_flags_match(license_flags, whitelist)
644 if unmatched_flag:
645 return unmatched_flag
646 return None
647
648def check_license_format(d):
649 """
650 This function checks if LICENSE is well defined,
651 Validate operators in LICENSES.
652 No spaces are allowed between LICENSES.
653 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500654 pn = d.getVar('PN')
655 licenses = d.getVar('LICENSE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500656 from oe.license import license_operator, license_operator_chars, license_pattern
657
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600658 elements = list(filter(lambda x: x.strip(), license_operator.split(licenses)))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500659 for pos, element in enumerate(elements):
660 if license_pattern.match(element):
661 if pos > 0 and license_pattern.match(elements[pos - 1]):
662 bb.warn('%s: LICENSE value "%s" has an invalid format - license names ' \
663 'must be separated by the following characters to indicate ' \
664 'the license selection: %s' %
665 (pn, licenses, license_operator_chars))
666 elif not license_operator.match(element):
667 bb.warn('%s: LICENSE value "%s" has an invalid separator "%s" that is not ' \
668 'in the valid list of separators (%s)' %
669 (pn, licenses, element, license_operator_chars))
670
671SSTATETASKS += "do_populate_lic"
672do_populate_lic[sstate-inputdirs] = "${LICSSTATEDIR}"
673do_populate_lic[sstate-outputdirs] = "${LICENSE_DIRECTORY}/"
674
675ROOTFS_POSTPROCESS_COMMAND_prepend = "write_package_manifest; license_create_manifest; "
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500676do_rootfs[recrdeptask] += "do_populate_lic"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500677
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500678IMAGE_POSTPROCESS_COMMAND_prepend = "write_deploy_manifest; "
679do_image[recrdeptask] += "do_populate_lic"
680
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500681python do_populate_lic_setscene () {
682 sstate_setscene(d)
683}
684addtask do_populate_lic_setscene