blob: 8ad4614d61b6a39e33c194ba2ef6f500a6a4e1b4 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# Populates LICENSE_DIRECTORY as set in distro config with the license files as set by
2# LIC_FILES_CHKSUM.
3# TODO:
4# - There is a real issue revolving around license naming standards.
5
6LICENSE_DIRECTORY ??= "${DEPLOY_DIR}/licenses"
7LICSSTATEDIR = "${WORKDIR}/license-destdir/"
8
9# Create extra package with license texts and add it to RRECOMMENDS_${PN}
10LICENSE_CREATE_PACKAGE[type] = "boolean"
11LICENSE_CREATE_PACKAGE ??= "0"
12LICENSE_PACKAGE_SUFFIX ??= "-lic"
13LICENSE_FILES_DIRECTORY ??= "${datadir}/licenses/"
14
15addtask populate_lic after do_patch before do_build
16do_populate_lic[dirs] = "${LICSSTATEDIR}/${PN}"
17do_populate_lic[cleandirs] = "${LICSSTATEDIR}"
18
19python write_package_manifest() {
20 # Get list of installed packages
21 license_image_dir = d.expand('${LICENSE_DIRECTORY}/${IMAGE_NAME}')
22 bb.utils.mkdirhier(license_image_dir)
23 from oe.rootfs import image_list_installed_packages
24 open(os.path.join(license_image_dir, 'package.manifest'),
25 'w+').write(image_list_installed_packages(d))
26}
27
28python license_create_manifest() {
29 import re
30 import oe.packagedata
31 from oe.rootfs import image_list_installed_packages
32
33 bad_licenses = (d.getVar("INCOMPATIBLE_LICENSE", True) or "").split()
34 bad_licenses = map(lambda l: canonical_license(d, l), bad_licenses)
35 bad_licenses = expand_wildcard_licenses(d, bad_licenses)
36
37 build_images_from_feeds = d.getVar('BUILD_IMAGES_FROM_FEEDS', True)
38 if build_images_from_feeds == "1":
39 return 0
40
41 pkg_dic = {}
42 for pkg in image_list_installed_packages(d).splitlines():
43 pkg_info = os.path.join(d.getVar('PKGDATA_DIR', True),
44 'runtime-reverse', pkg)
45 pkg_name = os.path.basename(os.readlink(pkg_info))
46
47 pkg_dic[pkg_name] = oe.packagedata.read_pkgdatafile(pkg_info)
48 if not "LICENSE" in pkg_dic[pkg_name].keys():
49 pkg_lic_name = "LICENSE_" + pkg_name
50 pkg_dic[pkg_name]["LICENSE"] = pkg_dic[pkg_name][pkg_lic_name]
51
52 license_manifest = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
53 d.getVar('IMAGE_NAME', True), 'license.manifest')
54 with open(license_manifest, "w") as license_file:
55 for pkg in sorted(pkg_dic):
56 if bad_licenses:
57 try:
58 (pkg_dic[pkg]["LICENSE"], pkg_dic[pkg]["LICENSES"]) = \
59 oe.license.manifest_licenses(pkg_dic[pkg]["LICENSE"],
60 bad_licenses, canonical_license, d)
61 except oe.license.LicenseError as exc:
62 bb.fatal('%s: %s' % (d.getVar('P', True), exc))
63 else:
64 pkg_dic[pkg]["LICENSES"] = re.sub('[|&()*]', '', pkg_dic[pkg]["LICENSE"])
65 pkg_dic[pkg]["LICENSES"] = re.sub(' *', ' ', pkg_dic[pkg]["LICENSES"])
66 pkg_dic[pkg]["LICENSES"] = pkg_dic[pkg]["LICENSES"].split()
67
68 license_file.write("PACKAGE NAME: %s\n" % pkg)
69 license_file.write("PACKAGE VERSION: %s\n" % pkg_dic[pkg]["PV"])
70 license_file.write("RECIPE NAME: %s\n" % pkg_dic[pkg]["PN"])
71 license_file.write("LICENSE: %s\n\n" % pkg_dic[pkg]["LICENSE"])
72
73 # If the package doesn't contain any file, that is, its size is 0, the license
74 # isn't relevant as far as the final image is concerned. So doing license check
75 # doesn't make much sense, skip it.
76 if pkg_dic[pkg]["PKGSIZE_%s" % pkg] == "0":
77 continue
78
79 for lic in pkg_dic[pkg]["LICENSES"]:
80 lic_file = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
81 pkg_dic[pkg]["PN"], "generic_%s" %
82 re.sub('\+', '', lic))
83 # add explicity avoid of CLOSED license because isn't generic
84 if lic == "CLOSED":
85 continue
86
87 if not os.path.exists(lic_file):
88 bb.warn("The license listed %s was not in the "\
89 "licenses collected for recipe %s"
90 % (lic, pkg_dic[pkg]["PN"]))
91
92 # Two options here:
93 # - Just copy the manifest
94 # - Copy the manifest and the license directories
95 # With both options set we see a .5 M increase in core-image-minimal
96 copy_lic_manifest = d.getVar('COPY_LIC_MANIFEST', True)
97 copy_lic_dirs = d.getVar('COPY_LIC_DIRS', True)
98 if copy_lic_manifest == "1":
99 rootfs_license_dir = os.path.join(d.getVar('IMAGE_ROOTFS', 'True'),
100 'usr', 'share', 'common-licenses')
101 os.makedirs(rootfs_license_dir)
102 rootfs_license_manifest = os.path.join(rootfs_license_dir,
103 'license.manifest')
104 os.link(license_manifest, rootfs_license_manifest)
105
106 if copy_lic_dirs == "1":
107 for pkg in sorted(pkg_dic):
108 pkg_rootfs_license_dir = os.path.join(rootfs_license_dir, pkg)
109 os.makedirs(pkg_rootfs_license_dir)
110 pkg_license_dir = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
111 pkg_dic[pkg]["PN"])
112 licenses = os.listdir(pkg_license_dir)
113 for lic in licenses:
114 rootfs_license = os.path.join(rootfs_license_dir, lic)
115 pkg_license = os.path.join(pkg_license_dir, lic)
116 pkg_rootfs_license = os.path.join(pkg_rootfs_license_dir, lic)
117
118 if re.match("^generic_.*$", lic):
119 generic_lic = re.search("^generic_(.*)$", lic).group(1)
120 if oe.license.license_ok(canonical_license(d,
121 generic_lic), bad_licenses) == False:
122 continue
123
124 if not os.path.exists(rootfs_license):
125 os.link(pkg_license, rootfs_license)
126
127 os.symlink(os.path.join('..', lic), pkg_rootfs_license)
128 else:
129 if oe.license.license_ok(canonical_license(d,
130 lic), bad_licenses) == False:
131 continue
132
133 os.link(pkg_license, pkg_rootfs_license)
134}
135
136python do_populate_lic() {
137 """
138 Populate LICENSE_DIRECTORY with licenses.
139 """
140 lic_files_paths = find_license_files(d)
141
142 # The base directory we wrangle licenses to
143 destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
144 copy_license_files(lic_files_paths, destdir)
145}
146
147# it would be better to copy them in do_install_append, but find_license_filesa is python
148python perform_packagecopy_prepend () {
149 enabled = oe.data.typed_value('LICENSE_CREATE_PACKAGE', d)
150 if d.getVar('CLASSOVERRIDE', True) == 'class-target' and enabled:
151 lic_files_paths = find_license_files(d)
152
153 # LICENSE_FILES_DIRECTORY starts with '/' so os.path.join cannot be used to join D and LICENSE_FILES_DIRECTORY
154 destdir = d.getVar('D', True) + os.path.join(d.getVar('LICENSE_FILES_DIRECTORY', True), d.getVar('PN', True))
155 copy_license_files(lic_files_paths, destdir)
156 add_package_and_files(d)
157}
158
159def add_package_and_files(d):
160 packages = d.getVar('PACKAGES', True)
161 files = d.getVar('LICENSE_FILES_DIRECTORY', True)
162 pn = d.getVar('PN', True)
163 pn_lic = "%s%s" % (pn, d.getVar('LICENSE_PACKAGE_SUFFIX', False))
164 if pn_lic in packages:
165 bb.warn("%s package already existed in %s." % (pn_lic, pn))
166 else:
167 # first in PACKAGES to be sure that nothing else gets LICENSE_FILES_DIRECTORY
168 d.setVar('PACKAGES', "%s %s" % (pn_lic, packages))
169 d.setVar('FILES_' + pn_lic, files)
170 rrecommends_pn = d.getVar('RRECOMMENDS_' + pn, True)
171 if rrecommends_pn:
172 d.setVar('RRECOMMENDS_' + pn, "%s %s" % (pn_lic, rrecommends_pn))
173 else:
174 d.setVar('RRECOMMENDS_' + pn, "%s" % (pn_lic))
175
176def copy_license_files(lic_files_paths, destdir):
177 import shutil
178
179 bb.utils.mkdirhier(destdir)
180 for (basename, path) in lic_files_paths:
181 try:
182 src = path
183 dst = os.path.join(destdir, basename)
184 if os.path.exists(dst):
185 os.remove(dst)
186 if os.access(src, os.W_OK) and (os.stat(src).st_dev == os.stat(destdir).st_dev):
187 os.link(src, dst)
188 else:
189 shutil.copyfile(src, dst)
190 except Exception as e:
191 bb.warn("Could not copy license file %s to %s: %s" % (src, dst, e))
192
193def find_license_files(d):
194 """
195 Creates list of files used in LIC_FILES_CHKSUM and generic LICENSE files.
196 """
197 import shutil
198 import oe.license
199
200 pn = d.getVar('PN', True)
201 for package in d.getVar('PACKAGES', True):
202 if d.getVar('LICENSE_' + package, True):
203 license_types = license_types + ' & ' + \
204 d.getVar('LICENSE_' + package, True)
205
206 #If we get here with no license types, then that means we have a recipe
207 #level license. If so, we grab only those.
208 try:
209 license_types
210 except NameError:
211 # All the license types at the recipe level
212 license_types = d.getVar('LICENSE', True)
213
214 # All the license files for the package
215 lic_files = d.getVar('LIC_FILES_CHKSUM', True)
216 pn = d.getVar('PN', True)
217 # The license files are located in S/LIC_FILE_CHECKSUM.
218 srcdir = d.getVar('S', True)
219 # Directory we store the generic licenses as set in the distro configuration
220 generic_directory = d.getVar('COMMON_LICENSE_DIR', True)
221 # List of basename, path tuples
222 lic_files_paths = []
223 license_source_dirs = []
224 license_source_dirs.append(generic_directory)
225 try:
226 additional_lic_dirs = d.getVar('LICENSE_PATH', True).split()
227 for lic_dir in additional_lic_dirs:
228 license_source_dirs.append(lic_dir)
229 except:
230 pass
231
232 class FindVisitor(oe.license.LicenseVisitor):
233 def visit_Str(self, node):
234 #
235 # Until I figure out what to do with
236 # the two modifiers I support (or greater = +
237 # and "with exceptions" being *
238 # we'll just strip out the modifier and put
239 # the base license.
240 find_license(node.s.replace("+", "").replace("*", ""))
241 self.generic_visit(node)
242
243 def find_license(license_type):
244 try:
245 bb.utils.mkdirhier(gen_lic_dest)
246 except:
247 pass
248 spdx_generic = None
249 license_source = None
250 # If the generic does not exist we need to check to see if there is an SPDX mapping to it,
251 # unless NO_GENERIC_LICENSE is set.
252
253 for lic_dir in license_source_dirs:
254 if not os.path.isfile(os.path.join(lic_dir, license_type)):
255 if d.getVarFlag('SPDXLICENSEMAP', license_type) != None:
256 # Great, there is an SPDXLICENSEMAP. We can copy!
257 bb.debug(1, "We need to use a SPDXLICENSEMAP for %s" % (license_type))
258 spdx_generic = d.getVarFlag('SPDXLICENSEMAP', license_type)
259 license_source = lic_dir
260 break
261 elif os.path.isfile(os.path.join(lic_dir, license_type)):
262 spdx_generic = license_type
263 license_source = lic_dir
264 break
265
266 if spdx_generic and license_source:
267 # we really should copy to generic_ + spdx_generic, however, that ends up messing the manifest
268 # audit up. This should be fixed in emit_pkgdata (or, we actually got and fix all the recipes)
269
270 lic_files_paths.append(("generic_" + license_type, os.path.join(license_source, spdx_generic)))
271
272 # The user may attempt to use NO_GENERIC_LICENSE for a generic license which doesn't make sense
273 # and should not be allowed, warn the user in this case.
274 if d.getVarFlag('NO_GENERIC_LICENSE', license_type):
275 bb.warn("%s: %s is a generic license, please don't use NO_GENERIC_LICENSE for it." % (pn, license_type))
276
277 elif d.getVarFlag('NO_GENERIC_LICENSE', license_type):
278 # if NO_GENERIC_LICENSE is set, we copy the license files from the fetched source
279 # of the package rather than the license_source_dirs.
280 for (basename, path) in lic_files_paths:
281 if d.getVarFlag('NO_GENERIC_LICENSE', license_type) == basename:
282 lic_files_paths.append(("generic_" + license_type, path))
283 break
284 else:
285 # Add explicity avoid of CLOSED license because this isn't generic
286 if license_type != 'CLOSED':
287 # And here is where we warn people that their licenses are lousy
288 bb.warn("%s: No generic license file exists for: %s in any provider" % (pn, license_type))
289 pass
290
291 if not generic_directory:
292 raise bb.build.FuncFailed("COMMON_LICENSE_DIR is unset. Please set this in your distro config")
293
294 if not lic_files:
295 # No recipe should have an invalid license file. This is checked else
296 # where, but let's be pedantic
297 bb.note(pn + ": Recipe file does not have license file information.")
298 return lic_files_paths
299
300 for url in lic_files.split():
301 try:
302 (type, host, path, user, pswd, parm) = bb.fetch.decodeurl(url)
303 except bb.fetch.MalformedUrl:
304 raise bb.build.FuncFailed("%s: LIC_FILES_CHKSUM contains an invalid URL: %s" % (d.getVar('PF', True), url))
305 # We want the license filename and path
306 srclicfile = os.path.join(srcdir, path)
307 lic_files_paths.append((os.path.basename(path), srclicfile))
308
309 v = FindVisitor()
310 try:
311 v.visit_string(license_types)
312 except oe.license.InvalidLicense as exc:
313 bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
314 except SyntaxError:
315 bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
316
317 return lic_files_paths
318
319def return_spdx(d, license):
320 """
321 This function returns the spdx mapping of a license if it exists.
322 """
323 return d.getVarFlag('SPDXLICENSEMAP', license, True)
324
325def canonical_license(d, license):
326 """
327 Return the canonical (SPDX) form of the license if available (so GPLv3
328 becomes GPL-3.0), for the license named 'X+', return canonical form of
329 'X' if availabel and the tailing '+' (so GPLv3+ becomes GPL-3.0+),
330 or the passed license if there is no canonical form.
331 """
332 lic = d.getVarFlag('SPDXLICENSEMAP', license, True) or ""
333 if not lic and license.endswith('+'):
334 lic = d.getVarFlag('SPDXLICENSEMAP', license.rstrip('+'), True)
335 if lic:
336 lic += '+'
337 return lic or license
338
339def expand_wildcard_licenses(d, wildcard_licenses):
340 """
341 Return actual spdx format license names if wildcard used. We expand
342 wildcards from SPDXLICENSEMAP flags and SRC_DISTRIBUTE_LICENSES values.
343 """
344 import fnmatch
345 licenses = []
346 spdxmapkeys = d.getVarFlags('SPDXLICENSEMAP').keys()
347 for wld_lic in wildcard_licenses:
348 spdxflags = fnmatch.filter(spdxmapkeys, wld_lic)
349 licenses += [d.getVarFlag('SPDXLICENSEMAP', flag) for flag in spdxflags]
350
351 spdx_lics = (d.getVar('SRC_DISTRIBUTE_LICENSES', False) or '').split()
352 for wld_lic in wildcard_licenses:
353 licenses += fnmatch.filter(spdx_lics, wld_lic)
354
355 licenses = list(set(licenses))
356 return licenses
357
358def incompatible_license_contains(license, truevalue, falsevalue, d):
359 license = canonical_license(d, license)
360 bad_licenses = (d.getVar('INCOMPATIBLE_LICENSE', True) or "").split()
361 bad_licenses = expand_wildcard_licenses(d, bad_licenses)
362 return truevalue if license in bad_licenses else falsevalue
363
364def incompatible_license(d, dont_want_licenses, package=None):
365 """
366 This function checks if a recipe has only incompatible licenses. It also
367 take into consideration 'or' operand. dont_want_licenses should be passed
368 as canonical (SPDX) names.
369 """
370 import oe.license
371 license = d.getVar("LICENSE_%s" % package, True) if package else None
372 if not license:
373 license = d.getVar('LICENSE', True)
374
375 # Handles an "or" or two license sets provided by
376 # flattened_licenses(), pick one that works if possible.
377 def choose_lic_set(a, b):
378 return a if all(oe.license.license_ok(canonical_license(d, lic),
379 dont_want_licenses) for lic in a) else b
380
381 try:
382 licenses = oe.license.flattened_licenses(license, choose_lic_set)
383 except oe.license.LicenseError as exc:
384 bb.fatal('%s: %s' % (d.getVar('P', True), exc))
385 return any(not oe.license.license_ok(canonical_license(d, l), \
386 dont_want_licenses) for l in licenses)
387
388def check_license_flags(d):
389 """
390 This function checks if a recipe has any LICENSE_FLAGS that
391 aren't whitelisted.
392
393 If it does, it returns the first LICENSE_FLAGS item missing from the
394 whitelist, or all of the LICENSE_FLAGS if there is no whitelist.
395
396 If everything is is properly whitelisted, it returns None.
397 """
398
399 def license_flag_matches(flag, whitelist, pn):
400 """
401 Return True if flag matches something in whitelist, None if not.
402
403 Before we test a flag against the whitelist, we append _${PN}
404 to it. We then try to match that string against the
405 whitelist. This covers the normal case, where we expect
406 LICENSE_FLAGS to be a simple string like 'commercial', which
407 the user typically matches exactly in the whitelist by
408 explicitly appending the package name e.g 'commercial_foo'.
409 If we fail the match however, we then split the flag across
410 '_' and append each fragment and test until we either match or
411 run out of fragments.
412 """
413 flag_pn = ("%s_%s" % (flag, pn))
414 for candidate in whitelist:
415 if flag_pn == candidate:
416 return True
417
418 flag_cur = ""
419 flagments = flag_pn.split("_")
420 flagments.pop() # we've already tested the full string
421 for flagment in flagments:
422 if flag_cur:
423 flag_cur += "_"
424 flag_cur += flagment
425 for candidate in whitelist:
426 if flag_cur == candidate:
427 return True
428 return False
429
430 def all_license_flags_match(license_flags, whitelist):
431 """ Return first unmatched flag, None if all flags match """
432 pn = d.getVar('PN', True)
433 split_whitelist = whitelist.split()
434 for flag in license_flags.split():
435 if not license_flag_matches(flag, split_whitelist, pn):
436 return flag
437 return None
438
439 license_flags = d.getVar('LICENSE_FLAGS', True)
440 if license_flags:
441 whitelist = d.getVar('LICENSE_FLAGS_WHITELIST', True)
442 if not whitelist:
443 return license_flags
444 unmatched_flag = all_license_flags_match(license_flags, whitelist)
445 if unmatched_flag:
446 return unmatched_flag
447 return None
448
449def check_license_format(d):
450 """
451 This function checks if LICENSE is well defined,
452 Validate operators in LICENSES.
453 No spaces are allowed between LICENSES.
454 """
455 pn = d.getVar('PN', True)
456 licenses = d.getVar('LICENSE', True)
457 from oe.license import license_operator, license_operator_chars, license_pattern
458
459 elements = filter(lambda x: x.strip(), license_operator.split(licenses))
460 for pos, element in enumerate(elements):
461 if license_pattern.match(element):
462 if pos > 0 and license_pattern.match(elements[pos - 1]):
463 bb.warn('%s: LICENSE value "%s" has an invalid format - license names ' \
464 'must be separated by the following characters to indicate ' \
465 'the license selection: %s' %
466 (pn, licenses, license_operator_chars))
467 elif not license_operator.match(element):
468 bb.warn('%s: LICENSE value "%s" has an invalid separator "%s" that is not ' \
469 'in the valid list of separators (%s)' %
470 (pn, licenses, element, license_operator_chars))
471
472SSTATETASKS += "do_populate_lic"
473do_populate_lic[sstate-inputdirs] = "${LICSSTATEDIR}"
474do_populate_lic[sstate-outputdirs] = "${LICENSE_DIRECTORY}/"
475
476ROOTFS_POSTPROCESS_COMMAND_prepend = "write_package_manifest; license_create_manifest; "
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500477do_rootfs[recrdeptask] += "do_populate_lic"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500478
479do_populate_lic_setscene[dirs] = "${LICSSTATEDIR}/${PN}"
480do_populate_lic_setscene[cleandirs] = "${LICSSTATEDIR}"
481python do_populate_lic_setscene () {
482 sstate_setscene(d)
483}
484addtask do_populate_lic_setscene