blob: 5103ed8533d6d8348be78784796f303bea156566 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# Populates LICENSE_DIRECTORY as set in distro config with the license files as set by
2# LIC_FILES_CHKSUM.
3# TODO:
4# - There is a real issue revolving around license naming standards.
5
6LICENSE_DIRECTORY ??= "${DEPLOY_DIR}/licenses"
7LICSSTATEDIR = "${WORKDIR}/license-destdir/"
8
9# Create extra package with license texts and add it to RRECOMMENDS_${PN}
10LICENSE_CREATE_PACKAGE[type] = "boolean"
11LICENSE_CREATE_PACKAGE ??= "0"
12LICENSE_PACKAGE_SUFFIX ??= "-lic"
13LICENSE_FILES_DIRECTORY ??= "${datadir}/licenses/"
14
15addtask populate_lic after do_patch before do_build
16do_populate_lic[dirs] = "${LICSSTATEDIR}/${PN}"
17do_populate_lic[cleandirs] = "${LICSSTATEDIR}"
18
19python write_package_manifest() {
20 # Get list of installed packages
21 license_image_dir = d.expand('${LICENSE_DIRECTORY}/${IMAGE_NAME}')
22 bb.utils.mkdirhier(license_image_dir)
23 from oe.rootfs import image_list_installed_packages
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050024 from oe.utils import format_pkg_list
25
26 pkgs = image_list_installed_packages(d)
27 output = format_pkg_list(pkgs)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050028 open(os.path.join(license_image_dir, 'package.manifest'),
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050029 'w+').write(output)
30}
31
32python write_deploy_manifest() {
33 license_deployed_manifest(d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050034}
35
36python license_create_manifest() {
Patrick Williamsc124f4f2015-09-15 14:41:29 -050037 import oe.packagedata
38 from oe.rootfs import image_list_installed_packages
39
Brad Bishop6e60e8b2018-02-01 10:27:11 -050040 build_images_from_feeds = d.getVar('BUILD_IMAGES_FROM_FEEDS')
Patrick Williamsc124f4f2015-09-15 14:41:29 -050041 if build_images_from_feeds == "1":
42 return 0
43
44 pkg_dic = {}
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050045 for pkg in sorted(image_list_installed_packages(d)):
Brad Bishop6e60e8b2018-02-01 10:27:11 -050046 pkg_info = os.path.join(d.getVar('PKGDATA_DIR'),
Patrick Williamsc124f4f2015-09-15 14:41:29 -050047 'runtime-reverse', pkg)
48 pkg_name = os.path.basename(os.readlink(pkg_info))
49
50 pkg_dic[pkg_name] = oe.packagedata.read_pkgdatafile(pkg_info)
51 if not "LICENSE" in pkg_dic[pkg_name].keys():
52 pkg_lic_name = "LICENSE_" + pkg_name
53 pkg_dic[pkg_name]["LICENSE"] = pkg_dic[pkg_name][pkg_lic_name]
54
Brad Bishop6e60e8b2018-02-01 10:27:11 -050055 rootfs_license_manifest = os.path.join(d.getVar('LICENSE_DIRECTORY'),
56 d.getVar('IMAGE_NAME'), 'license.manifest')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050057 write_license_files(d, rootfs_license_manifest, pkg_dic)
58}
59
60def write_license_files(d, license_manifest, pkg_dic):
61 import re
62
Brad Bishop6e60e8b2018-02-01 10:27:11 -050063 bad_licenses = (d.getVar("INCOMPATIBLE_LICENSE") or "").split()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050064 bad_licenses = map(lambda l: canonical_license(d, l), bad_licenses)
65 bad_licenses = expand_wildcard_licenses(d, bad_licenses)
66
Patrick Williamsc124f4f2015-09-15 14:41:29 -050067 with open(license_manifest, "w") as license_file:
68 for pkg in sorted(pkg_dic):
69 if bad_licenses:
70 try:
71 (pkg_dic[pkg]["LICENSE"], pkg_dic[pkg]["LICENSES"]) = \
72 oe.license.manifest_licenses(pkg_dic[pkg]["LICENSE"],
73 bad_licenses, canonical_license, d)
74 except oe.license.LicenseError as exc:
Brad Bishop6e60e8b2018-02-01 10:27:11 -050075 bb.fatal('%s: %s' % (d.getVar('P'), exc))
Patrick Williamsc124f4f2015-09-15 14:41:29 -050076 else:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050077 pkg_dic[pkg]["LICENSES"] = re.sub('[|&()*]', ' ', pkg_dic[pkg]["LICENSE"])
Patrick Williamsc124f4f2015-09-15 14:41:29 -050078 pkg_dic[pkg]["LICENSES"] = re.sub(' *', ' ', pkg_dic[pkg]["LICENSES"])
79 pkg_dic[pkg]["LICENSES"] = pkg_dic[pkg]["LICENSES"].split()
80
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050081 if not "IMAGE_MANIFEST" in pkg_dic[pkg]:
82 # Rootfs manifest
83 license_file.write("PACKAGE NAME: %s\n" % pkg)
84 license_file.write("PACKAGE VERSION: %s\n" % pkg_dic[pkg]["PV"])
85 license_file.write("RECIPE NAME: %s\n" % pkg_dic[pkg]["PN"])
86 license_file.write("LICENSE: %s\n\n" % pkg_dic[pkg]["LICENSE"])
Patrick Williamsc124f4f2015-09-15 14:41:29 -050087
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050088 # If the package doesn't contain any file, that is, its size is 0, the license
89 # isn't relevant as far as the final image is concerned. So doing license check
90 # doesn't make much sense, skip it.
91 if pkg_dic[pkg]["PKGSIZE_%s" % pkg] == "0":
92 continue
93 else:
94 # Image manifest
95 license_file.write("RECIPE NAME: %s\n" % pkg_dic[pkg]["PN"])
96 license_file.write("VERSION: %s\n" % pkg_dic[pkg]["PV"])
97 license_file.write("LICENSE: %s\n" % pkg_dic[pkg]["LICENSE"])
98 license_file.write("FILES: %s\n\n" % pkg_dic[pkg]["FILES"])
Patrick Williamsc124f4f2015-09-15 14:41:29 -050099
100 for lic in pkg_dic[pkg]["LICENSES"]:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500101 lic_file = os.path.join(d.getVar('LICENSE_DIRECTORY'),
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500102 pkg_dic[pkg]["PN"], "generic_%s" %
103 re.sub('\+', '', lic))
104 # add explicity avoid of CLOSED license because isn't generic
105 if lic == "CLOSED":
106 continue
107
108 if not os.path.exists(lic_file):
109 bb.warn("The license listed %s was not in the "\
110 "licenses collected for recipe %s"
111 % (lic, pkg_dic[pkg]["PN"]))
112
113 # Two options here:
114 # - Just copy the manifest
115 # - Copy the manifest and the license directories
116 # With both options set we see a .5 M increase in core-image-minimal
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500117 copy_lic_manifest = d.getVar('COPY_LIC_MANIFEST')
118 copy_lic_dirs = d.getVar('COPY_LIC_DIRS')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500119 if copy_lic_manifest == "1":
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500120 rootfs_license_dir = os.path.join(d.getVar('IMAGE_ROOTFS'),
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500121 'usr', 'share', 'common-licenses')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500122 bb.utils.mkdirhier(rootfs_license_dir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500123 rootfs_license_manifest = os.path.join(rootfs_license_dir,
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500124 os.path.split(license_manifest)[1])
125 if not os.path.exists(rootfs_license_manifest):
126 os.link(license_manifest, rootfs_license_manifest)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500127
128 if copy_lic_dirs == "1":
129 for pkg in sorted(pkg_dic):
130 pkg_rootfs_license_dir = os.path.join(rootfs_license_dir, pkg)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500131 bb.utils.mkdirhier(pkg_rootfs_license_dir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500132 pkg_license_dir = os.path.join(d.getVar('LICENSE_DIRECTORY'),
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500133 pkg_dic[pkg]["PN"])
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500134
135 pkg_manifest_licenses = [canonical_license(d, lic) \
136 for lic in pkg_dic[pkg]["LICENSES"]]
137
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500138 licenses = os.listdir(pkg_license_dir)
139 for lic in licenses:
140 rootfs_license = os.path.join(rootfs_license_dir, lic)
141 pkg_license = os.path.join(pkg_license_dir, lic)
142 pkg_rootfs_license = os.path.join(pkg_rootfs_license_dir, lic)
143
144 if re.match("^generic_.*$", lic):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500145 generic_lic = canonical_license(d,
146 re.search("^generic_(.*)$", lic).group(1))
147
148 # Do not copy generic license into package if isn't
149 # declared into LICENSES of the package.
150 if not re.sub('\+$', '', generic_lic) in \
151 [re.sub('\+', '', lic) for lic in \
152 pkg_manifest_licenses]:
153 continue
154
155 if oe.license.license_ok(generic_lic,
156 bad_licenses) == False:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500157 continue
158
159 if not os.path.exists(rootfs_license):
160 os.link(pkg_license, rootfs_license)
161
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500162 if not os.path.exists(pkg_rootfs_license):
163 os.symlink(os.path.join('..', lic), pkg_rootfs_license)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500164 else:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500165 if (oe.license.license_ok(canonical_license(d,
166 lic), bad_licenses) == False or
167 os.path.exists(pkg_rootfs_license)):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500168 continue
169
170 os.link(pkg_license, pkg_rootfs_license)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500171
172
173def license_deployed_manifest(d):
174 """
175 Write the license manifest for the deployed recipes.
176 The deployed recipes usually includes the bootloader
177 and extra files to boot the target.
178 """
179
180 dep_dic = {}
181 man_dic = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500182 lic_dir = d.getVar("LICENSE_DIRECTORY")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500183
184 dep_dic = get_deployed_dependencies(d)
185 for dep in dep_dic.keys():
186 man_dic[dep] = {}
187 # It is necessary to mark this will be used for image manifest
188 man_dic[dep]["IMAGE_MANIFEST"] = True
189 man_dic[dep]["PN"] = dep
190 man_dic[dep]["FILES"] = \
191 " ".join(get_deployed_files(dep_dic[dep]))
192 with open(os.path.join(lic_dir, dep, "recipeinfo"), "r") as f:
193 for line in f.readlines():
194 key,val = line.split(": ", 1)
195 man_dic[dep][key] = val[:-1]
196
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500197 lic_manifest_dir = os.path.join(d.getVar('LICENSE_DIRECTORY'),
198 d.getVar('IMAGE_NAME'))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600199 bb.utils.mkdirhier(lic_manifest_dir)
200 image_license_manifest = os.path.join(lic_manifest_dir, 'image_license.manifest')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500201 write_license_files(d, image_license_manifest, man_dic)
202
203def get_deployed_dependencies(d):
204 """
205 Get all the deployed dependencies of an image
206 """
207
208 deploy = {}
209 # Get all the dependencies for the current task (rootfs).
210 # Also get EXTRA_IMAGEDEPENDS because the bootloader is
211 # usually in this var and not listed in rootfs.
212 # At last, get the dependencies from boot classes because
213 # it might contain the bootloader.
214 taskdata = d.getVar("BB_TASKDEPDATA", False)
215 depends = list(set([dep[0] for dep
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600216 in list(taskdata.values())
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500217 if not dep[0].endswith("-native")]))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500218 extra_depends = d.getVar("EXTRA_IMAGEDEPENDS")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500219 boot_depends = get_boot_dependencies(d)
220 depends.extend(extra_depends.split())
221 depends.extend(boot_depends)
222 depends = list(set(depends))
223
224 # To verify what was deployed it checks the rootfs dependencies against
225 # the SSTATE_MANIFESTS for "deploy" task.
226 # The manifest file name contains the arch. Because we are not running
227 # in the recipe context it is necessary to check every arch used.
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500228 sstate_manifest_dir = d.getVar("SSTATE_MANIFESTS")
Brad Bishop316dfdd2018-06-25 12:45:53 -0400229 archs = list(set(d.getVar("SSTATE_ARCHS").split()))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500230 for dep in depends:
231 # Some recipes have an arch on their own, so we try that first.
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500232 special_arch = d.getVar("PACKAGE_ARCH_pn-%s" % dep)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500233 if special_arch:
234 sstate_manifest_file = os.path.join(sstate_manifest_dir,
235 "manifest-%s-%s.deploy" % (special_arch, dep))
236 if os.path.exists(sstate_manifest_file):
237 deploy[dep] = sstate_manifest_file
238 continue
239
240 for arch in archs:
241 sstate_manifest_file = os.path.join(sstate_manifest_dir,
242 "manifest-%s-%s.deploy" % (arch, dep))
243 if os.path.exists(sstate_manifest_file):
244 deploy[dep] = sstate_manifest_file
245 break
246
247 return deploy
248get_deployed_dependencies[vardepsexclude] = "BB_TASKDEPDATA"
249
250def get_boot_dependencies(d):
251 """
252 Return the dependencies from boot tasks
253 """
254
255 depends = []
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500256 taskdepdata = d.getVar("BB_TASKDEPDATA", False)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500257 # Only bootimg includes the depends flag
258 boot_depends_string = d.getVarFlag("do_bootimg", "depends") or ""
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500259 boot_depends = [dep.split(":")[0] for dep
260 in boot_depends_string.split()
261 if not dep.split(":")[0].endswith("-native")]
262 for dep in boot_depends:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500263 info_file = os.path.join(d.getVar("LICENSE_DIRECTORY"),
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500264 dep, "recipeinfo")
265 # If the recipe and dependency name is the same
266 if os.path.exists(info_file):
267 depends.append(dep)
268 # We need to search for the provider of the dependency
269 else:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600270 for taskdep in taskdepdata.values():
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500271 # The fifth field contains what the task provides
272 if dep in taskdep[4]:
273 info_file = os.path.join(
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500274 d.getVar("LICENSE_DIRECTORY"),
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500275 taskdep[0], "recipeinfo")
276 if os.path.exists(info_file):
277 depends.append(taskdep[0])
278 break
279 return depends
280get_boot_dependencies[vardepsexclude] = "BB_TASKDEPDATA"
281
282def get_deployed_files(man_file):
283 """
284 Get the files deployed from the sstate manifest
285 """
286
287 dep_files = []
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500288 excluded_files = []
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500289 with open(man_file, "r") as manifest:
290 all_files = manifest.read()
291 for f in all_files.splitlines():
292 if ((not (os.path.islink(f) or os.path.isdir(f))) and
293 not os.path.basename(f) in excluded_files):
294 dep_files.append(os.path.basename(f))
295 return dep_files
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500296
297python do_populate_lic() {
298 """
299 Populate LICENSE_DIRECTORY with licenses.
300 """
301 lic_files_paths = find_license_files(d)
302
303 # The base directory we wrangle licenses to
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500304 destdir = os.path.join(d.getVar('LICSSTATEDIR'), d.getVar('PN'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500305 copy_license_files(lic_files_paths, destdir)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500306 info = get_recipe_info(d)
307 with open(os.path.join(destdir, "recipeinfo"), "w") as f:
308 for key in sorted(info.keys()):
309 f.write("%s: %s\n" % (key, info[key]))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500310}
311
312# it would be better to copy them in do_install_append, but find_license_filesa is python
313python perform_packagecopy_prepend () {
314 enabled = oe.data.typed_value('LICENSE_CREATE_PACKAGE', d)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500315 if d.getVar('CLASSOVERRIDE') == 'class-target' and enabled:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500316 lic_files_paths = find_license_files(d)
317
318 # LICENSE_FILES_DIRECTORY starts with '/' so os.path.join cannot be used to join D and LICENSE_FILES_DIRECTORY
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500319 destdir = d.getVar('D') + os.path.join(d.getVar('LICENSE_FILES_DIRECTORY'), d.getVar('PN'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500320 copy_license_files(lic_files_paths, destdir)
321 add_package_and_files(d)
322}
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500323perform_packagecopy[vardeps] += "LICENSE_CREATE_PACKAGE"
324
325def get_recipe_info(d):
326 info = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500327 info["PV"] = d.getVar("PV")
328 info["PR"] = d.getVar("PR")
329 info["LICENSE"] = d.getVar("LICENSE")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500330 return info
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500331
332def add_package_and_files(d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500333 packages = d.getVar('PACKAGES')
334 files = d.getVar('LICENSE_FILES_DIRECTORY')
335 pn = d.getVar('PN')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500336 pn_lic = "%s%s" % (pn, d.getVar('LICENSE_PACKAGE_SUFFIX', False))
Brad Bishop316dfdd2018-06-25 12:45:53 -0400337 if pn_lic in packages.split():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500338 bb.warn("%s package already existed in %s." % (pn_lic, pn))
339 else:
340 # first in PACKAGES to be sure that nothing else gets LICENSE_FILES_DIRECTORY
341 d.setVar('PACKAGES', "%s %s" % (pn_lic, packages))
342 d.setVar('FILES_' + pn_lic, files)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500343 rrecommends_pn = d.getVar('RRECOMMENDS_' + pn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500344 if rrecommends_pn:
345 d.setVar('RRECOMMENDS_' + pn, "%s %s" % (pn_lic, rrecommends_pn))
346 else:
347 d.setVar('RRECOMMENDS_' + pn, "%s" % (pn_lic))
348
349def copy_license_files(lic_files_paths, destdir):
350 import shutil
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600351 import errno
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500352
353 bb.utils.mkdirhier(destdir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500354 for (basename, path, beginline, endline) in lic_files_paths:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500355 try:
356 src = path
357 dst = os.path.join(destdir, basename)
358 if os.path.exists(dst):
359 os.remove(dst)
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500360 if os.path.islink(src):
361 src = os.path.realpath(src)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500362 canlink = os.access(src, os.W_OK) and (os.stat(src).st_dev == os.stat(destdir).st_dev) and beginline is None and endline is None
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600363 if canlink:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500364 try:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600365 os.link(src, dst)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500366 except OSError as err:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600367 if err.errno == errno.EXDEV:
368 # Copy license files if hard-link is not possible even if st_dev is the
369 # same on source and destination (docker container with device-mapper?)
370 canlink = False
371 else:
372 raise
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500373 # Only chown if we did hardling, and, we're running under pseudo
374 if canlink and os.environ.get('PSEUDO_DISABLED') == '0':
375 os.chown(dst,0,0)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600376 if not canlink:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500377 begin_idx = int(beginline)-1 if beginline is not None else None
378 end_idx = int(endline) if endline is not None else None
379 if begin_idx is None and end_idx is None:
380 shutil.copyfile(src, dst)
381 else:
382 with open(src, 'rb') as src_f:
383 with open(dst, 'wb') as dst_f:
384 dst_f.write(b''.join(src_f.readlines()[begin_idx:end_idx]))
385
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500386 except Exception as e:
387 bb.warn("Could not copy license file %s to %s: %s" % (src, dst, e))
388
389def find_license_files(d):
390 """
391 Creates list of files used in LIC_FILES_CHKSUM and generic LICENSE files.
392 """
393 import shutil
394 import oe.license
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600395 from collections import defaultdict, OrderedDict
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500396
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500397 # All the license files for the package
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500398 lic_files = d.getVar('LIC_FILES_CHKSUM') or ""
399 pn = d.getVar('PN')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500400 # The license files are located in S/LIC_FILE_CHECKSUM.
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500401 srcdir = d.getVar('S')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500402 # Directory we store the generic licenses as set in the distro configuration
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500403 generic_directory = d.getVar('COMMON_LICENSE_DIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500404 # List of basename, path tuples
405 lic_files_paths = []
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500406 # hash for keep track generic lics mappings
407 non_generic_lics = {}
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600408 # Entries from LIC_FILES_CHKSUM
409 lic_chksums = {}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500410 license_source_dirs = []
411 license_source_dirs.append(generic_directory)
412 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500413 additional_lic_dirs = d.getVar('LICENSE_PATH').split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500414 for lic_dir in additional_lic_dirs:
415 license_source_dirs.append(lic_dir)
416 except:
417 pass
418
419 class FindVisitor(oe.license.LicenseVisitor):
420 def visit_Str(self, node):
421 #
422 # Until I figure out what to do with
423 # the two modifiers I support (or greater = +
424 # and "with exceptions" being *
425 # we'll just strip out the modifier and put
426 # the base license.
427 find_license(node.s.replace("+", "").replace("*", ""))
428 self.generic_visit(node)
429
430 def find_license(license_type):
431 try:
432 bb.utils.mkdirhier(gen_lic_dest)
433 except:
434 pass
435 spdx_generic = None
436 license_source = None
437 # If the generic does not exist we need to check to see if there is an SPDX mapping to it,
438 # unless NO_GENERIC_LICENSE is set.
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500439 for lic_dir in license_source_dirs:
440 if not os.path.isfile(os.path.join(lic_dir, license_type)):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500441 if d.getVarFlag('SPDXLICENSEMAP', license_type) != None:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500442 # Great, there is an SPDXLICENSEMAP. We can copy!
443 bb.debug(1, "We need to use a SPDXLICENSEMAP for %s" % (license_type))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500444 spdx_generic = d.getVarFlag('SPDXLICENSEMAP', license_type)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500445 license_source = lic_dir
446 break
447 elif os.path.isfile(os.path.join(lic_dir, license_type)):
448 spdx_generic = license_type
449 license_source = lic_dir
450 break
451
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500452 non_generic_lic = d.getVarFlag('NO_GENERIC_LICENSE', license_type)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500453 if spdx_generic and license_source:
454 # we really should copy to generic_ + spdx_generic, however, that ends up messing the manifest
455 # audit up. This should be fixed in emit_pkgdata (or, we actually got and fix all the recipes)
456
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500457 lic_files_paths.append(("generic_" + license_type, os.path.join(license_source, spdx_generic),
458 None, None))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500459
460 # The user may attempt to use NO_GENERIC_LICENSE for a generic license which doesn't make sense
461 # and should not be allowed, warn the user in this case.
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500462 if d.getVarFlag('NO_GENERIC_LICENSE', license_type):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500463 bb.warn("%s: %s is a generic license, please don't use NO_GENERIC_LICENSE for it." % (pn, license_type))
464
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600465 elif non_generic_lic and non_generic_lic in lic_chksums:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500466 # if NO_GENERIC_LICENSE is set, we copy the license files from the fetched source
467 # of the package rather than the license_source_dirs.
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600468 lic_files_paths.append(("generic_" + license_type,
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500469 os.path.join(srcdir, non_generic_lic), None, None))
470 non_generic_lics[non_generic_lic] = license_type
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500471 else:
472 # Add explicity avoid of CLOSED license because this isn't generic
473 if license_type != 'CLOSED':
474 # And here is where we warn people that their licenses are lousy
475 bb.warn("%s: No generic license file exists for: %s in any provider" % (pn, license_type))
476 pass
477
478 if not generic_directory:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600479 bb.fatal("COMMON_LICENSE_DIR is unset. Please set this in your distro config")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500480
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500481 for url in lic_files.split():
482 try:
Brad Bishop220d5532018-08-14 00:59:39 +0100483 (method, host, path, user, pswd, parm) = bb.fetch.decodeurl(url)
484 if method != "file" or not path:
485 raise bb.fetch.MalformedUrl()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500486 except bb.fetch.MalformedUrl:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500487 bb.fatal("%s: LIC_FILES_CHKSUM contains an invalid URL: %s" % (d.getVar('PF'), url))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500488 # We want the license filename and path
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500489 chksum = parm.get('md5', None)
490 beginline = parm.get('beginline')
491 endline = parm.get('endline')
492 lic_chksums[path] = (chksum, beginline, endline)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500493
494 v = FindVisitor()
495 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500496 v.visit_string(d.getVar('LICENSE'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500497 except oe.license.InvalidLicense as exc:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500498 bb.fatal('%s: %s' % (d.getVar('PF'), exc))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500499 except SyntaxError:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500500 bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF')))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600501 # Add files from LIC_FILES_CHKSUM to list of license files
502 lic_chksum_paths = defaultdict(OrderedDict)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500503 for path, data in sorted(lic_chksums.items()):
504 lic_chksum_paths[os.path.basename(path)][data] = (os.path.join(srcdir, path), data[1], data[2])
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600505 for basename, files in lic_chksum_paths.items():
506 if len(files) == 1:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500507 # Don't copy again a LICENSE already handled as non-generic
508 if basename in non_generic_lics:
509 continue
510 data = list(files.values())[0]
511 lic_files_paths.append(tuple([basename] + list(data)))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600512 else:
513 # If there are multiple different license files with identical
514 # basenames we rename them to <file>.0, <file>.1, ...
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500515 for i, data in enumerate(files.values()):
516 lic_files_paths.append(tuple(["%s.%d" % (basename, i)] + list(data)))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600517
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500518 return lic_files_paths
519
520def return_spdx(d, license):
521 """
522 This function returns the spdx mapping of a license if it exists.
523 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500524 return d.getVarFlag('SPDXLICENSEMAP', license)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500525
526def canonical_license(d, license):
527 """
528 Return the canonical (SPDX) form of the license if available (so GPLv3
529 becomes GPL-3.0), for the license named 'X+', return canonical form of
530 'X' if availabel and the tailing '+' (so GPLv3+ becomes GPL-3.0+),
531 or the passed license if there is no canonical form.
532 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500533 lic = d.getVarFlag('SPDXLICENSEMAP', license) or ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500534 if not lic and license.endswith('+'):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500535 lic = d.getVarFlag('SPDXLICENSEMAP', license.rstrip('+'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500536 if lic:
537 lic += '+'
538 return lic or license
539
540def expand_wildcard_licenses(d, wildcard_licenses):
541 """
542 Return actual spdx format license names if wildcard used. We expand
543 wildcards from SPDXLICENSEMAP flags and SRC_DISTRIBUTE_LICENSES values.
544 """
545 import fnmatch
546 licenses = []
547 spdxmapkeys = d.getVarFlags('SPDXLICENSEMAP').keys()
548 for wld_lic in wildcard_licenses:
549 spdxflags = fnmatch.filter(spdxmapkeys, wld_lic)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500550 licenses += [d.getVarFlag('SPDXLICENSEMAP', flag) for flag in spdxflags]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500551
552 spdx_lics = (d.getVar('SRC_DISTRIBUTE_LICENSES', False) or '').split()
553 for wld_lic in wildcard_licenses:
554 licenses += fnmatch.filter(spdx_lics, wld_lic)
555
556 licenses = list(set(licenses))
557 return licenses
558
559def incompatible_license_contains(license, truevalue, falsevalue, d):
560 license = canonical_license(d, license)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500561 bad_licenses = (d.getVar('INCOMPATIBLE_LICENSE') or "").split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500562 bad_licenses = expand_wildcard_licenses(d, bad_licenses)
563 return truevalue if license in bad_licenses else falsevalue
564
565def incompatible_license(d, dont_want_licenses, package=None):
566 """
567 This function checks if a recipe has only incompatible licenses. It also
568 take into consideration 'or' operand. dont_want_licenses should be passed
569 as canonical (SPDX) names.
570 """
571 import oe.license
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500572 license = d.getVar("LICENSE_%s" % package) if package else None
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500573 if not license:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500574 license = d.getVar('LICENSE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500575
576 # Handles an "or" or two license sets provided by
577 # flattened_licenses(), pick one that works if possible.
578 def choose_lic_set(a, b):
579 return a if all(oe.license.license_ok(canonical_license(d, lic),
580 dont_want_licenses) for lic in a) else b
581
582 try:
583 licenses = oe.license.flattened_licenses(license, choose_lic_set)
584 except oe.license.LicenseError as exc:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500585 bb.fatal('%s: %s' % (d.getVar('P'), exc))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500586 return any(not oe.license.license_ok(canonical_license(d, l), \
587 dont_want_licenses) for l in licenses)
588
589def check_license_flags(d):
590 """
591 This function checks if a recipe has any LICENSE_FLAGS that
592 aren't whitelisted.
593
594 If it does, it returns the first LICENSE_FLAGS item missing from the
595 whitelist, or all of the LICENSE_FLAGS if there is no whitelist.
596
597 If everything is is properly whitelisted, it returns None.
598 """
599
600 def license_flag_matches(flag, whitelist, pn):
601 """
602 Return True if flag matches something in whitelist, None if not.
603
604 Before we test a flag against the whitelist, we append _${PN}
605 to it. We then try to match that string against the
606 whitelist. This covers the normal case, where we expect
607 LICENSE_FLAGS to be a simple string like 'commercial', which
608 the user typically matches exactly in the whitelist by
609 explicitly appending the package name e.g 'commercial_foo'.
610 If we fail the match however, we then split the flag across
611 '_' and append each fragment and test until we either match or
612 run out of fragments.
613 """
614 flag_pn = ("%s_%s" % (flag, pn))
615 for candidate in whitelist:
616 if flag_pn == candidate:
617 return True
618
619 flag_cur = ""
620 flagments = flag_pn.split("_")
621 flagments.pop() # we've already tested the full string
622 for flagment in flagments:
623 if flag_cur:
624 flag_cur += "_"
625 flag_cur += flagment
626 for candidate in whitelist:
627 if flag_cur == candidate:
628 return True
629 return False
630
631 def all_license_flags_match(license_flags, whitelist):
632 """ Return first unmatched flag, None if all flags match """
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500633 pn = d.getVar('PN')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500634 split_whitelist = whitelist.split()
635 for flag in license_flags.split():
636 if not license_flag_matches(flag, split_whitelist, pn):
637 return flag
638 return None
639
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500640 license_flags = d.getVar('LICENSE_FLAGS')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500641 if license_flags:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500642 whitelist = d.getVar('LICENSE_FLAGS_WHITELIST')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500643 if not whitelist:
644 return license_flags
645 unmatched_flag = all_license_flags_match(license_flags, whitelist)
646 if unmatched_flag:
647 return unmatched_flag
648 return None
649
650def check_license_format(d):
651 """
652 This function checks if LICENSE is well defined,
653 Validate operators in LICENSES.
654 No spaces are allowed between LICENSES.
655 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500656 pn = d.getVar('PN')
657 licenses = d.getVar('LICENSE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500658 from oe.license import license_operator, license_operator_chars, license_pattern
659
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600660 elements = list(filter(lambda x: x.strip(), license_operator.split(licenses)))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500661 for pos, element in enumerate(elements):
662 if license_pattern.match(element):
663 if pos > 0 and license_pattern.match(elements[pos - 1]):
664 bb.warn('%s: LICENSE value "%s" has an invalid format - license names ' \
665 'must be separated by the following characters to indicate ' \
666 'the license selection: %s' %
667 (pn, licenses, license_operator_chars))
668 elif not license_operator.match(element):
669 bb.warn('%s: LICENSE value "%s" has an invalid separator "%s" that is not ' \
670 'in the valid list of separators (%s)' %
671 (pn, licenses, element, license_operator_chars))
672
673SSTATETASKS += "do_populate_lic"
674do_populate_lic[sstate-inputdirs] = "${LICSSTATEDIR}"
675do_populate_lic[sstate-outputdirs] = "${LICENSE_DIRECTORY}/"
676
677ROOTFS_POSTPROCESS_COMMAND_prepend = "write_package_manifest; license_create_manifest; "
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500678do_rootfs[recrdeptask] += "do_populate_lic"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500679
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500680IMAGE_POSTPROCESS_COMMAND_prepend = "write_deploy_manifest; "
681do_image[recrdeptask] += "do_populate_lic"
682
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500683python do_populate_lic_setscene () {
684 sstate_setscene(d)
685}
686addtask do_populate_lic_setscene