Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 1 | # |
| 2 | # SPDX-License-Identifier: GPL-2.0-only |
| 3 | # |
| 4 | |
| 5 | DEPLOY_DIR_SPDX ??= "${DEPLOY_DIR}/spdx/${MACHINE}" |
| 6 | |
| 7 | # The product name that the CVE database uses. Defaults to BPN, but may need to |
| 8 | # be overriden per recipe (for example tiff.bb sets CVE_PRODUCT=libtiff). |
| 9 | CVE_PRODUCT ??= "${BPN}" |
| 10 | CVE_VERSION ??= "${PV}" |
| 11 | |
| 12 | SPDXDIR ??= "${WORKDIR}/spdx" |
| 13 | SPDXDEPLOY = "${SPDXDIR}/deploy" |
| 14 | SPDXWORK = "${SPDXDIR}/work" |
| 15 | |
Patrick Williams | 93c203f | 2021-10-06 16:15:23 -0500 | [diff] [blame] | 16 | SPDX_TOOL_NAME ??= "oe-spdx-creator" |
| 17 | SPDX_TOOL_VERSION ??= "1.0" |
| 18 | |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 19 | SPDXRUNTIMEDEPLOY = "${SPDXDIR}/runtime-deploy" |
| 20 | |
| 21 | SPDX_INCLUDE_SOURCES ??= "0" |
| 22 | SPDX_INCLUDE_PACKAGED ??= "0" |
| 23 | SPDX_ARCHIVE_SOURCES ??= "0" |
| 24 | SPDX_ARCHIVE_PACKAGED ??= "0" |
| 25 | |
| 26 | SPDX_UUID_NAMESPACE ??= "sbom.openembedded.org" |
| 27 | SPDX_NAMESPACE_PREFIX ??= "http://spdx.org/spdxdoc" |
| 28 | |
| 29 | SPDX_LICENSES ??= "${COREBASE}/meta/files/spdx-licenses.json" |
| 30 | |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 31 | SPDX_ORG ??= "OpenEmbedded ()" |
Andrew Geissler | 7e0e3c0 | 2022-02-25 20:34:39 +0000 | [diff] [blame^] | 32 | SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}" |
| 33 | SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created from \ |
| 34 | this recipe. For SPDX documents create using this class during the build, this \ |
| 35 | is the contact information for the person or organization who is doing the \ |
| 36 | build." |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 37 | |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 38 | do_image_complete[depends] = "virtual/kernel:do_create_spdx" |
| 39 | |
Andrew Geissler | 7e0e3c0 | 2022-02-25 20:34:39 +0000 | [diff] [blame^] | 40 | def extract_licenses(filename): |
| 41 | import re |
| 42 | |
| 43 | lic_regex = re.compile(b'^\W*SPDX-License-Identifier:\s*([ \w\d.()+-]+?)(?:\s+\W*)?$', re.MULTILINE) |
| 44 | |
| 45 | try: |
| 46 | with open(filename, 'rb') as f: |
| 47 | size = min(15000, os.stat(filename).st_size) |
| 48 | txt = f.read(size) |
| 49 | licenses = re.findall(lic_regex, txt) |
| 50 | if licenses: |
| 51 | ascii_licenses = [lic.decode('ascii') for lic in licenses] |
| 52 | return ascii_licenses |
| 53 | except Exception as e: |
| 54 | bb.warn(f"Exception reading {filename}: {e}") |
| 55 | return None |
| 56 | |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 57 | def get_doc_namespace(d, doc): |
| 58 | import uuid |
| 59 | namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE")) |
| 60 | return "%s/%s-%s" % (d.getVar("SPDX_NAMESPACE_PREFIX"), doc.name, str(uuid.uuid5(namespace_uuid, doc.name))) |
| 61 | |
Andrew Geissler | eff2747 | 2021-10-29 15:35:00 -0500 | [diff] [blame] | 62 | def create_annotation(d, comment): |
| 63 | from datetime import datetime, timezone |
| 64 | |
| 65 | creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") |
| 66 | annotation = oe.spdx.SPDXAnnotation() |
| 67 | annotation.annotationDate = creation_time |
| 68 | annotation.annotationType = "OTHER" |
| 69 | annotation.annotator = "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION")) |
| 70 | annotation.comment = comment |
| 71 | return annotation |
| 72 | |
Patrick Williams | 93c203f | 2021-10-06 16:15:23 -0500 | [diff] [blame] | 73 | def recipe_spdx_is_native(d, recipe): |
| 74 | return any(a.annotationType == "OTHER" and |
| 75 | a.annotator == "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION")) and |
| 76 | a.comment == "isNative" for a in recipe.annotations) |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 77 | |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 78 | def is_work_shared_spdx(d): |
| 79 | return bb.data.inherits_class('kernel', d) or ('work-shared' in d.getVar('WORKDIR')) |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 80 | |
| 81 | python() { |
| 82 | import json |
| 83 | if d.getVar("SPDX_LICENSE_DATA"): |
| 84 | return |
| 85 | |
| 86 | with open(d.getVar("SPDX_LICENSES"), "r") as f: |
| 87 | data = json.load(f) |
| 88 | # Transform the license array to a dictionary |
| 89 | data["licenses"] = {l["licenseId"]: l for l in data["licenses"]} |
| 90 | d.setVar("SPDX_LICENSE_DATA", data) |
| 91 | } |
| 92 | |
| 93 | def convert_license_to_spdx(lic, document, d, existing={}): |
| 94 | from pathlib import Path |
| 95 | import oe.spdx |
| 96 | |
| 97 | available_licenses = d.getVar("AVAILABLE_LICENSES").split() |
| 98 | license_data = d.getVar("SPDX_LICENSE_DATA") |
| 99 | extracted = {} |
| 100 | |
| 101 | def add_extracted_license(ident, name): |
| 102 | nonlocal document |
| 103 | |
| 104 | if name in extracted: |
| 105 | return |
| 106 | |
| 107 | extracted_info = oe.spdx.SPDXExtractedLicensingInfo() |
| 108 | extracted_info.name = name |
| 109 | extracted_info.licenseId = ident |
| 110 | extracted_info.extractedText = None |
| 111 | |
| 112 | if name == "PD": |
| 113 | # Special-case this. |
| 114 | extracted_info.extractedText = "Software released to the public domain" |
| 115 | elif name in available_licenses: |
| 116 | # This license can be found in COMMON_LICENSE_DIR or LICENSE_PATH |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 117 | for directory in [d.getVar('COMMON_LICENSE_DIR')] + (d.getVar('LICENSE_PATH') or '').split(): |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 118 | try: |
| 119 | with (Path(directory) / name).open(errors="replace") as f: |
| 120 | extracted_info.extractedText = f.read() |
| 121 | break |
| 122 | except FileNotFoundError: |
| 123 | pass |
| 124 | if extracted_info.extractedText is None: |
| 125 | # Error out, as the license was in available_licenses so should |
| 126 | # be on disk somewhere. |
| 127 | bb.error("Cannot find text for license %s" % name) |
| 128 | else: |
| 129 | # If it's not SPDX, or PD, or in available licenses, then NO_GENERIC_LICENSE must be set |
| 130 | filename = d.getVarFlag('NO_GENERIC_LICENSE', name) |
| 131 | if filename: |
| 132 | filename = d.expand("${S}/" + filename) |
| 133 | with open(filename, errors="replace") as f: |
| 134 | extracted_info.extractedText = f.read() |
| 135 | else: |
| 136 | bb.error("Cannot find any text for license %s" % name) |
| 137 | |
| 138 | extracted[name] = extracted_info |
| 139 | document.hasExtractedLicensingInfos.append(extracted_info) |
| 140 | |
| 141 | def convert(l): |
| 142 | if l == "(" or l == ")": |
| 143 | return l |
| 144 | |
| 145 | if l == "&": |
| 146 | return "AND" |
| 147 | |
| 148 | if l == "|": |
| 149 | return "OR" |
| 150 | |
| 151 | if l == "CLOSED": |
| 152 | return "NONE" |
| 153 | |
| 154 | spdx_license = d.getVarFlag("SPDXLICENSEMAP", l) or l |
| 155 | if spdx_license in license_data["licenses"]: |
| 156 | return spdx_license |
| 157 | |
| 158 | try: |
| 159 | spdx_license = existing[l] |
| 160 | except KeyError: |
| 161 | spdx_license = "LicenseRef-" + l |
| 162 | add_extracted_license(spdx_license, l) |
| 163 | |
| 164 | return spdx_license |
| 165 | |
| 166 | lic_split = lic.replace("(", " ( ").replace(")", " ) ").split() |
| 167 | |
| 168 | return ' '.join(convert(l) for l in lic_split) |
| 169 | |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 170 | def process_sources(d): |
| 171 | pn = d.getVar('PN') |
| 172 | assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split() |
| 173 | if pn in assume_provided: |
| 174 | for p in d.getVar("PROVIDES").split(): |
| 175 | if p != pn: |
| 176 | pn = p |
| 177 | break |
| 178 | |
| 179 | # glibc-locale: do_fetch, do_unpack and do_patch tasks have been deleted, |
| 180 | # so avoid archiving source here. |
| 181 | if pn.startswith('glibc-locale'): |
| 182 | return False |
| 183 | if d.getVar('PN') == "libtool-cross": |
| 184 | return False |
| 185 | if d.getVar('PN') == "libgcc-initial": |
| 186 | return False |
| 187 | if d.getVar('PN') == "shadow-sysroot": |
| 188 | return False |
| 189 | |
| 190 | # We just archive gcc-source for all the gcc related recipes |
| 191 | if d.getVar('BPN') in ['gcc', 'libgcc']: |
| 192 | bb.debug(1, 'spdx: There is bug in scan of %s is, do nothing' % pn) |
| 193 | return False |
| 194 | |
| 195 | return True |
| 196 | |
| 197 | |
| 198 | def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archive=None, ignore_dirs=[], ignore_top_level_dirs=[]): |
| 199 | from pathlib import Path |
| 200 | import oe.spdx |
| 201 | import hashlib |
| 202 | |
| 203 | source_date_epoch = d.getVar("SOURCE_DATE_EPOCH") |
| 204 | if source_date_epoch: |
| 205 | source_date_epoch = int(source_date_epoch) |
| 206 | |
| 207 | sha1s = [] |
| 208 | spdx_files = [] |
| 209 | |
| 210 | file_counter = 1 |
| 211 | for subdir, dirs, files in os.walk(topdir): |
| 212 | dirs[:] = [d for d in dirs if d not in ignore_dirs] |
| 213 | if subdir == str(topdir): |
| 214 | dirs[:] = [d for d in dirs if d not in ignore_top_level_dirs] |
| 215 | |
| 216 | for file in files: |
| 217 | filepath = Path(subdir) / file |
| 218 | filename = str(filepath.relative_to(topdir)) |
| 219 | |
| 220 | if filepath.is_file() and not filepath.is_symlink(): |
| 221 | spdx_file = oe.spdx.SPDXFile() |
| 222 | spdx_file.SPDXID = get_spdxid(file_counter) |
| 223 | for t in get_types(filepath): |
| 224 | spdx_file.fileTypes.append(t) |
| 225 | spdx_file.fileName = filename |
| 226 | |
| 227 | if archive is not None: |
| 228 | with filepath.open("rb") as f: |
| 229 | info = archive.gettarinfo(fileobj=f) |
| 230 | info.name = filename |
| 231 | info.uid = 0 |
| 232 | info.gid = 0 |
| 233 | info.uname = "root" |
| 234 | info.gname = "root" |
| 235 | |
| 236 | if source_date_epoch is not None and info.mtime > source_date_epoch: |
| 237 | info.mtime = source_date_epoch |
| 238 | |
| 239 | archive.addfile(info, f) |
| 240 | |
| 241 | sha1 = bb.utils.sha1_file(filepath) |
| 242 | sha1s.append(sha1) |
| 243 | spdx_file.checksums.append(oe.spdx.SPDXChecksum( |
| 244 | algorithm="SHA1", |
| 245 | checksumValue=sha1, |
| 246 | )) |
| 247 | spdx_file.checksums.append(oe.spdx.SPDXChecksum( |
| 248 | algorithm="SHA256", |
| 249 | checksumValue=bb.utils.sha256_file(filepath), |
| 250 | )) |
| 251 | |
Andrew Geissler | 7e0e3c0 | 2022-02-25 20:34:39 +0000 | [diff] [blame^] | 252 | if "SOURCE" in spdx_file.fileTypes: |
| 253 | extracted_lics = extract_licenses(filepath) |
| 254 | if extracted_lics: |
| 255 | spdx_file.licenseInfoInFiles = extracted_lics |
| 256 | |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 257 | doc.files.append(spdx_file) |
| 258 | doc.add_relationship(spdx_pkg, "CONTAINS", spdx_file) |
| 259 | spdx_pkg.hasFiles.append(spdx_file.SPDXID) |
| 260 | |
| 261 | spdx_files.append(spdx_file) |
| 262 | |
| 263 | file_counter += 1 |
| 264 | |
| 265 | sha1s.sort() |
| 266 | verifier = hashlib.sha1() |
| 267 | for v in sha1s: |
| 268 | verifier.update(v.encode("utf-8")) |
| 269 | spdx_pkg.packageVerificationCode.packageVerificationCodeValue = verifier.hexdigest() |
| 270 | |
| 271 | return spdx_files |
| 272 | |
| 273 | |
| 274 | def add_package_sources_from_debug(d, package_doc, spdx_package, package, package_files, sources): |
| 275 | from pathlib import Path |
| 276 | import hashlib |
| 277 | import oe.packagedata |
| 278 | import oe.spdx |
| 279 | |
| 280 | debug_search_paths = [ |
| 281 | Path(d.getVar('PKGD')), |
| 282 | Path(d.getVar('STAGING_DIR_TARGET')), |
| 283 | Path(d.getVar('STAGING_DIR_NATIVE')), |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 284 | Path(d.getVar('STAGING_KERNEL_DIR')), |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 285 | ] |
| 286 | |
| 287 | pkg_data = oe.packagedata.read_subpkgdata_extended(package, d) |
| 288 | |
| 289 | if pkg_data is None: |
| 290 | return |
| 291 | |
| 292 | for file_path, file_data in pkg_data["files_info"].items(): |
| 293 | if not "debugsrc" in file_data: |
| 294 | continue |
| 295 | |
| 296 | for pkg_file in package_files: |
| 297 | if file_path.lstrip("/") == pkg_file.fileName.lstrip("/"): |
| 298 | break |
| 299 | else: |
| 300 | bb.fatal("No package file found for %s" % str(file_path)) |
| 301 | continue |
| 302 | |
| 303 | for debugsrc in file_data["debugsrc"]: |
| 304 | ref_id = "NOASSERTION" |
| 305 | for search in debug_search_paths: |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 306 | if debugsrc.startswith("/usr/src/kernel"): |
| 307 | debugsrc_path = search / debugsrc.replace('/usr/src/kernel/', '') |
| 308 | else: |
| 309 | debugsrc_path = search / debugsrc.lstrip("/") |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 310 | if not debugsrc_path.exists(): |
| 311 | continue |
| 312 | |
| 313 | file_sha256 = bb.utils.sha256_file(debugsrc_path) |
| 314 | |
| 315 | if file_sha256 in sources: |
| 316 | source_file = sources[file_sha256] |
| 317 | |
| 318 | doc_ref = package_doc.find_external_document_ref(source_file.doc.documentNamespace) |
| 319 | if doc_ref is None: |
| 320 | doc_ref = oe.spdx.SPDXExternalDocumentRef() |
| 321 | doc_ref.externalDocumentId = "DocumentRef-dependency-" + source_file.doc.name |
| 322 | doc_ref.spdxDocument = source_file.doc.documentNamespace |
| 323 | doc_ref.checksum.algorithm = "SHA1" |
| 324 | doc_ref.checksum.checksumValue = source_file.doc_sha1 |
| 325 | package_doc.externalDocumentRefs.append(doc_ref) |
| 326 | |
| 327 | ref_id = "%s:%s" % (doc_ref.externalDocumentId, source_file.file.SPDXID) |
| 328 | else: |
| 329 | bb.debug(1, "Debug source %s with SHA256 %s not found in any dependency" % (str(debugsrc_path), file_sha256)) |
| 330 | break |
| 331 | else: |
| 332 | bb.debug(1, "Debug source %s not found" % debugsrc) |
| 333 | |
| 334 | package_doc.add_relationship(pkg_file, "GENERATED_FROM", ref_id, comment=debugsrc) |
| 335 | |
| 336 | def collect_dep_recipes(d, doc, spdx_recipe): |
| 337 | from pathlib import Path |
| 338 | import oe.sbom |
| 339 | import oe.spdx |
| 340 | |
| 341 | deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) |
| 342 | |
| 343 | dep_recipes = [] |
| 344 | taskdepdata = d.getVar("BB_TASKDEPDATA", False) |
| 345 | deps = sorted(set( |
| 346 | dep[0] for dep in taskdepdata.values() if |
| 347 | dep[1] == "do_create_spdx" and dep[0] != d.getVar("PN") |
| 348 | )) |
| 349 | for dep_pn in deps: |
| 350 | dep_recipe_path = deploy_dir_spdx / "recipes" / ("recipe-%s.spdx.json" % dep_pn) |
| 351 | |
| 352 | spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_recipe_path) |
| 353 | |
| 354 | for pkg in spdx_dep_doc.packages: |
| 355 | if pkg.name == dep_pn: |
| 356 | spdx_dep_recipe = pkg |
| 357 | break |
| 358 | else: |
| 359 | continue |
| 360 | |
| 361 | dep_recipes.append(oe.sbom.DepRecipe(spdx_dep_doc, spdx_dep_sha1, spdx_dep_recipe)) |
| 362 | |
| 363 | dep_recipe_ref = oe.spdx.SPDXExternalDocumentRef() |
| 364 | dep_recipe_ref.externalDocumentId = "DocumentRef-dependency-" + spdx_dep_doc.name |
| 365 | dep_recipe_ref.spdxDocument = spdx_dep_doc.documentNamespace |
| 366 | dep_recipe_ref.checksum.algorithm = "SHA1" |
| 367 | dep_recipe_ref.checksum.checksumValue = spdx_dep_sha1 |
| 368 | |
| 369 | doc.externalDocumentRefs.append(dep_recipe_ref) |
| 370 | |
| 371 | doc.add_relationship( |
| 372 | "%s:%s" % (dep_recipe_ref.externalDocumentId, spdx_dep_recipe.SPDXID), |
| 373 | "BUILD_DEPENDENCY_OF", |
| 374 | spdx_recipe |
| 375 | ) |
| 376 | |
| 377 | return dep_recipes |
| 378 | |
| 379 | collect_dep_recipes[vardepsexclude] += "BB_TASKDEPDATA" |
| 380 | |
| 381 | |
| 382 | def collect_dep_sources(d, dep_recipes): |
| 383 | import oe.sbom |
| 384 | |
| 385 | sources = {} |
| 386 | for dep in dep_recipes: |
Patrick Williams | 93c203f | 2021-10-06 16:15:23 -0500 | [diff] [blame] | 387 | # Don't collect sources from native recipes as they |
| 388 | # match non-native sources also. |
| 389 | if recipe_spdx_is_native(d, dep.recipe): |
| 390 | continue |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 391 | recipe_files = set(dep.recipe.hasFiles) |
| 392 | |
| 393 | for spdx_file in dep.doc.files: |
| 394 | if spdx_file.SPDXID not in recipe_files: |
| 395 | continue |
| 396 | |
| 397 | if "SOURCE" in spdx_file.fileTypes: |
| 398 | for checksum in spdx_file.checksums: |
| 399 | if checksum.algorithm == "SHA256": |
| 400 | sources[checksum.checksumValue] = oe.sbom.DepSource(dep.doc, dep.doc_sha1, dep.recipe, spdx_file) |
| 401 | break |
| 402 | |
| 403 | return sources |
| 404 | |
| 405 | |
| 406 | python do_create_spdx() { |
| 407 | from datetime import datetime, timezone |
| 408 | import oe.sbom |
| 409 | import oe.spdx |
| 410 | import uuid |
| 411 | from pathlib import Path |
| 412 | from contextlib import contextmanager |
| 413 | import oe.cve_check |
| 414 | |
| 415 | @contextmanager |
| 416 | def optional_tarfile(name, guard, mode="w"): |
| 417 | import tarfile |
| 418 | import bb.compress.zstd |
| 419 | |
| 420 | num_threads = int(d.getVar("BB_NUMBER_THREADS")) |
| 421 | |
| 422 | if guard: |
| 423 | name.parent.mkdir(parents=True, exist_ok=True) |
| 424 | with bb.compress.zstd.open(name, mode=mode + "b", num_threads=num_threads) as f: |
| 425 | with tarfile.open(fileobj=f, mode=mode + "|") as tf: |
| 426 | yield tf |
| 427 | else: |
| 428 | yield None |
| 429 | |
| 430 | |
| 431 | deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) |
| 432 | spdx_workdir = Path(d.getVar("SPDXWORK")) |
| 433 | include_packaged = d.getVar("SPDX_INCLUDE_PACKAGED") == "1" |
| 434 | include_sources = d.getVar("SPDX_INCLUDE_SOURCES") == "1" |
| 435 | archive_sources = d.getVar("SPDX_ARCHIVE_SOURCES") == "1" |
| 436 | archive_packaged = d.getVar("SPDX_ARCHIVE_PACKAGED") == "1" |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 437 | |
| 438 | creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") |
| 439 | |
| 440 | doc = oe.spdx.SPDXDocument() |
| 441 | |
| 442 | doc.name = "recipe-" + d.getVar("PN") |
| 443 | doc.documentNamespace = get_doc_namespace(d, doc) |
| 444 | doc.creationInfo.created = creation_time |
| 445 | doc.creationInfo.comment = "This document was created by analyzing recipe files during the build." |
| 446 | doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] |
| 447 | doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 448 | doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 449 | doc.creationInfo.creators.append("Person: N/A ()") |
| 450 | |
| 451 | recipe = oe.spdx.SPDXPackage() |
| 452 | recipe.name = d.getVar("PN") |
| 453 | recipe.versionInfo = d.getVar("PV") |
| 454 | recipe.SPDXID = oe.sbom.get_recipe_spdxid(d) |
Andrew Geissler | 7e0e3c0 | 2022-02-25 20:34:39 +0000 | [diff] [blame^] | 455 | recipe.packageSupplier = d.getVar("SPDX_SUPPLIER") |
Andrew Geissler | eff2747 | 2021-10-29 15:35:00 -0500 | [diff] [blame] | 456 | if bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d): |
| 457 | recipe.annotations.append(create_annotation(d, "isNative")) |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 458 | |
| 459 | for s in d.getVar('SRC_URI').split(): |
| 460 | if not s.startswith("file://"): |
| 461 | recipe.downloadLocation = s |
| 462 | break |
| 463 | else: |
| 464 | recipe.downloadLocation = "NOASSERTION" |
| 465 | |
| 466 | homepage = d.getVar("HOMEPAGE") |
| 467 | if homepage: |
| 468 | recipe.homepage = homepage |
| 469 | |
| 470 | license = d.getVar("LICENSE") |
| 471 | if license: |
| 472 | recipe.licenseDeclared = convert_license_to_spdx(license, doc, d) |
| 473 | |
| 474 | summary = d.getVar("SUMMARY") |
| 475 | if summary: |
| 476 | recipe.summary = summary |
| 477 | |
| 478 | description = d.getVar("DESCRIPTION") |
| 479 | if description: |
| 480 | recipe.description = description |
| 481 | |
| 482 | # Some CVEs may be patched during the build process without incrementing the version number, |
| 483 | # so querying for CVEs based on the CPE id can lead to false positives. To account for this, |
| 484 | # save the CVEs fixed by patches to source information field in the SPDX. |
| 485 | patched_cves = oe.cve_check.get_patched_cves(d) |
| 486 | patched_cves = list(patched_cves) |
| 487 | patched_cves = ' '.join(patched_cves) |
| 488 | if patched_cves: |
| 489 | recipe.sourceInfo = "CVEs fixed: " + patched_cves |
| 490 | |
| 491 | cpe_ids = oe.cve_check.get_cpe_ids(d.getVar("CVE_PRODUCT"), d.getVar("CVE_VERSION")) |
| 492 | if cpe_ids: |
| 493 | for cpe_id in cpe_ids: |
| 494 | cpe = oe.spdx.SPDXExternalReference() |
| 495 | cpe.referenceCategory = "SECURITY" |
| 496 | cpe.referenceType = "http://spdx.org/rdf/references/cpe23Type" |
| 497 | cpe.referenceLocator = cpe_id |
| 498 | recipe.externalRefs.append(cpe) |
| 499 | |
| 500 | doc.packages.append(recipe) |
| 501 | doc.add_relationship(doc, "DESCRIBES", recipe) |
| 502 | |
| 503 | if process_sources(d) and include_sources: |
| 504 | recipe_archive = deploy_dir_spdx / "recipes" / (doc.name + ".tar.zst") |
| 505 | with optional_tarfile(recipe_archive, archive_sources) as archive: |
| 506 | spdx_get_src(d) |
| 507 | |
| 508 | add_package_files( |
| 509 | d, |
| 510 | doc, |
| 511 | recipe, |
| 512 | spdx_workdir, |
| 513 | lambda file_counter: "SPDXRef-SourceFile-%s-%d" % (d.getVar("PN"), file_counter), |
| 514 | lambda filepath: ["SOURCE"], |
| 515 | ignore_dirs=[".git"], |
| 516 | ignore_top_level_dirs=["temp"], |
| 517 | archive=archive, |
| 518 | ) |
| 519 | |
| 520 | if archive is not None: |
| 521 | recipe.packageFileName = str(recipe_archive.name) |
| 522 | |
| 523 | dep_recipes = collect_dep_recipes(d, doc, recipe) |
| 524 | |
| 525 | doc_sha1 = oe.sbom.write_doc(d, doc, "recipes") |
| 526 | dep_recipes.append(oe.sbom.DepRecipe(doc, doc_sha1, recipe)) |
| 527 | |
| 528 | recipe_ref = oe.spdx.SPDXExternalDocumentRef() |
| 529 | recipe_ref.externalDocumentId = "DocumentRef-recipe-" + recipe.name |
| 530 | recipe_ref.spdxDocument = doc.documentNamespace |
| 531 | recipe_ref.checksum.algorithm = "SHA1" |
| 532 | recipe_ref.checksum.checksumValue = doc_sha1 |
| 533 | |
| 534 | sources = collect_dep_sources(d, dep_recipes) |
| 535 | found_licenses = {license.name:recipe_ref.externalDocumentId + ":" + license.licenseId for license in doc.hasExtractedLicensingInfos} |
| 536 | |
Patrick Williams | 93c203f | 2021-10-06 16:15:23 -0500 | [diff] [blame] | 537 | if not recipe_spdx_is_native(d, recipe): |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 538 | bb.build.exec_func("read_subpackage_metadata", d) |
| 539 | |
| 540 | pkgdest = Path(d.getVar("PKGDEST")) |
| 541 | for package in d.getVar("PACKAGES").split(): |
| 542 | if not oe.packagedata.packaged(package, d): |
| 543 | continue |
| 544 | |
| 545 | package_doc = oe.spdx.SPDXDocument() |
| 546 | pkg_name = d.getVar("PKG:%s" % package) or package |
| 547 | package_doc.name = pkg_name |
| 548 | package_doc.documentNamespace = get_doc_namespace(d, package_doc) |
| 549 | package_doc.creationInfo.created = creation_time |
| 550 | package_doc.creationInfo.comment = "This document was created by analyzing packages created during the build." |
| 551 | package_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] |
| 552 | package_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 553 | package_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 554 | package_doc.creationInfo.creators.append("Person: N/A ()") |
| 555 | package_doc.externalDocumentRefs.append(recipe_ref) |
| 556 | |
| 557 | package_license = d.getVar("LICENSE:%s" % package) or d.getVar("LICENSE") |
| 558 | |
| 559 | spdx_package = oe.spdx.SPDXPackage() |
| 560 | |
| 561 | spdx_package.SPDXID = oe.sbom.get_package_spdxid(pkg_name) |
| 562 | spdx_package.name = pkg_name |
| 563 | spdx_package.versionInfo = d.getVar("PV") |
| 564 | spdx_package.licenseDeclared = convert_license_to_spdx(package_license, package_doc, d, found_licenses) |
Andrew Geissler | 7e0e3c0 | 2022-02-25 20:34:39 +0000 | [diff] [blame^] | 565 | spdx_package.packageSupplier = d.getVar("SPDX_SUPPLIER") |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 566 | |
| 567 | package_doc.packages.append(spdx_package) |
| 568 | |
| 569 | package_doc.add_relationship(spdx_package, "GENERATED_FROM", "%s:%s" % (recipe_ref.externalDocumentId, recipe.SPDXID)) |
| 570 | package_doc.add_relationship(package_doc, "DESCRIBES", spdx_package) |
| 571 | |
| 572 | package_archive = deploy_dir_spdx / "packages" / (package_doc.name + ".tar.zst") |
| 573 | with optional_tarfile(package_archive, archive_packaged) as archive: |
| 574 | package_files = add_package_files( |
| 575 | d, |
| 576 | package_doc, |
| 577 | spdx_package, |
| 578 | pkgdest / package, |
| 579 | lambda file_counter: oe.sbom.get_packaged_file_spdxid(pkg_name, file_counter), |
| 580 | lambda filepath: ["BINARY"], |
| 581 | archive=archive, |
| 582 | ) |
| 583 | |
| 584 | if archive is not None: |
| 585 | spdx_package.packageFileName = str(package_archive.name) |
| 586 | |
| 587 | add_package_sources_from_debug(d, package_doc, spdx_package, package, package_files, sources) |
| 588 | |
| 589 | oe.sbom.write_doc(d, package_doc, "packages") |
| 590 | } |
| 591 | # NOTE: depending on do_unpack is a hack that is necessary to get it's dependencies for archive the source |
Andrew Geissler | 7e0e3c0 | 2022-02-25 20:34:39 +0000 | [diff] [blame^] | 592 | addtask do_create_spdx after do_package do_packagedata do_unpack before do_populate_sdk do_build do_rm_work |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 593 | |
| 594 | SSTATETASKS += "do_create_spdx" |
| 595 | do_create_spdx[sstate-inputdirs] = "${SPDXDEPLOY}" |
| 596 | do_create_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}" |
| 597 | |
| 598 | python do_create_spdx_setscene () { |
| 599 | sstate_setscene(d) |
| 600 | } |
| 601 | addtask do_create_spdx_setscene |
| 602 | |
| 603 | do_create_spdx[dirs] = "${SPDXDEPLOY} ${SPDXWORK}" |
| 604 | do_create_spdx[cleandirs] = "${SPDXDEPLOY} ${SPDXWORK}" |
| 605 | do_create_spdx[depends] += "${PATCHDEPENDENCY}" |
| 606 | do_create_spdx[deptask] = "do_create_spdx" |
| 607 | |
| 608 | def collect_package_providers(d): |
| 609 | from pathlib import Path |
| 610 | import oe.sbom |
| 611 | import oe.spdx |
| 612 | import json |
| 613 | |
| 614 | deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) |
| 615 | |
| 616 | providers = {} |
| 617 | |
| 618 | taskdepdata = d.getVar("BB_TASKDEPDATA", False) |
| 619 | deps = sorted(set( |
| 620 | dep[0] for dep in taskdepdata.values() if dep[0] != d.getVar("PN") |
| 621 | )) |
| 622 | deps.append(d.getVar("PN")) |
| 623 | |
| 624 | for dep_pn in deps: |
| 625 | recipe_data = oe.packagedata.read_pkgdata(dep_pn, d) |
| 626 | |
| 627 | for pkg in recipe_data.get("PACKAGES", "").split(): |
| 628 | |
| 629 | pkg_data = oe.packagedata.read_subpkgdata_dict(pkg, d) |
| 630 | rprovides = set(n for n, _ in bb.utils.explode_dep_versions2(pkg_data.get("RPROVIDES", "")).items()) |
| 631 | rprovides.add(pkg) |
| 632 | |
| 633 | for r in rprovides: |
| 634 | providers[r] = pkg |
| 635 | |
| 636 | return providers |
| 637 | |
| 638 | collect_package_providers[vardepsexclude] += "BB_TASKDEPDATA" |
| 639 | |
| 640 | python do_create_runtime_spdx() { |
| 641 | from datetime import datetime, timezone |
| 642 | import oe.sbom |
| 643 | import oe.spdx |
| 644 | import oe.packagedata |
| 645 | from pathlib import Path |
| 646 | |
| 647 | deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) |
| 648 | spdx_deploy = Path(d.getVar("SPDXRUNTIMEDEPLOY")) |
Andrew Geissler | eff2747 | 2021-10-29 15:35:00 -0500 | [diff] [blame] | 649 | is_native = bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d) |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 650 | |
| 651 | creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") |
| 652 | |
| 653 | providers = collect_package_providers(d) |
| 654 | |
| 655 | if not is_native: |
| 656 | bb.build.exec_func("read_subpackage_metadata", d) |
| 657 | |
| 658 | dep_package_cache = {} |
| 659 | |
| 660 | pkgdest = Path(d.getVar("PKGDEST")) |
| 661 | for package in d.getVar("PACKAGES").split(): |
| 662 | localdata = bb.data.createCopy(d) |
| 663 | pkg_name = d.getVar("PKG:%s" % package) or package |
| 664 | localdata.setVar("PKG", pkg_name) |
| 665 | localdata.setVar('OVERRIDES', d.getVar("OVERRIDES", False) + ":" + package) |
| 666 | |
| 667 | if not oe.packagedata.packaged(package, localdata): |
| 668 | continue |
| 669 | |
| 670 | pkg_spdx_path = deploy_dir_spdx / "packages" / (pkg_name + ".spdx.json") |
| 671 | |
| 672 | package_doc, package_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path) |
| 673 | |
| 674 | for p in package_doc.packages: |
| 675 | if p.name == pkg_name: |
| 676 | spdx_package = p |
| 677 | break |
| 678 | else: |
| 679 | bb.fatal("Package '%s' not found in %s" % (pkg_name, pkg_spdx_path)) |
| 680 | |
| 681 | runtime_doc = oe.spdx.SPDXDocument() |
| 682 | runtime_doc.name = "runtime-" + pkg_name |
| 683 | runtime_doc.documentNamespace = get_doc_namespace(localdata, runtime_doc) |
| 684 | runtime_doc.creationInfo.created = creation_time |
| 685 | runtime_doc.creationInfo.comment = "This document was created by analyzing package runtime dependencies." |
| 686 | runtime_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] |
| 687 | runtime_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 688 | runtime_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 689 | runtime_doc.creationInfo.creators.append("Person: N/A ()") |
| 690 | |
| 691 | package_ref = oe.spdx.SPDXExternalDocumentRef() |
| 692 | package_ref.externalDocumentId = "DocumentRef-package-" + package |
| 693 | package_ref.spdxDocument = package_doc.documentNamespace |
| 694 | package_ref.checksum.algorithm = "SHA1" |
| 695 | package_ref.checksum.checksumValue = package_doc_sha1 |
| 696 | |
| 697 | runtime_doc.externalDocumentRefs.append(package_ref) |
| 698 | |
| 699 | runtime_doc.add_relationship( |
| 700 | runtime_doc.SPDXID, |
| 701 | "AMENDS", |
| 702 | "%s:%s" % (package_ref.externalDocumentId, package_doc.SPDXID) |
| 703 | ) |
| 704 | |
| 705 | deps = bb.utils.explode_dep_versions2(localdata.getVar("RDEPENDS") or "") |
| 706 | seen_deps = set() |
| 707 | for dep, _ in deps.items(): |
| 708 | if dep in seen_deps: |
| 709 | continue |
| 710 | |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 711 | if dep not in providers: |
| 712 | continue |
| 713 | |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 714 | dep = providers[dep] |
| 715 | |
| 716 | if not oe.packagedata.packaged(dep, localdata): |
| 717 | continue |
| 718 | |
| 719 | dep_pkg_data = oe.packagedata.read_subpkgdata_dict(dep, d) |
| 720 | dep_pkg = dep_pkg_data["PKG"] |
| 721 | |
| 722 | if dep in dep_package_cache: |
| 723 | (dep_spdx_package, dep_package_ref) = dep_package_cache[dep] |
| 724 | else: |
| 725 | dep_path = deploy_dir_spdx / "packages" / ("%s.spdx.json" % dep_pkg) |
| 726 | |
| 727 | spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_path) |
| 728 | |
| 729 | for pkg in spdx_dep_doc.packages: |
| 730 | if pkg.name == dep_pkg: |
| 731 | dep_spdx_package = pkg |
| 732 | break |
| 733 | else: |
| 734 | bb.fatal("Package '%s' not found in %s" % (dep_pkg, dep_path)) |
| 735 | |
| 736 | dep_package_ref = oe.spdx.SPDXExternalDocumentRef() |
| 737 | dep_package_ref.externalDocumentId = "DocumentRef-runtime-dependency-" + spdx_dep_doc.name |
| 738 | dep_package_ref.spdxDocument = spdx_dep_doc.documentNamespace |
| 739 | dep_package_ref.checksum.algorithm = "SHA1" |
| 740 | dep_package_ref.checksum.checksumValue = spdx_dep_sha1 |
| 741 | |
| 742 | dep_package_cache[dep] = (dep_spdx_package, dep_package_ref) |
| 743 | |
| 744 | runtime_doc.externalDocumentRefs.append(dep_package_ref) |
| 745 | |
| 746 | runtime_doc.add_relationship( |
| 747 | "%s:%s" % (dep_package_ref.externalDocumentId, dep_spdx_package.SPDXID), |
| 748 | "RUNTIME_DEPENDENCY_OF", |
| 749 | "%s:%s" % (package_ref.externalDocumentId, spdx_package.SPDXID) |
| 750 | ) |
| 751 | seen_deps.add(dep) |
| 752 | |
| 753 | oe.sbom.write_doc(d, runtime_doc, "runtime", spdx_deploy) |
| 754 | } |
| 755 | |
| 756 | addtask do_create_runtime_spdx after do_create_spdx before do_build do_rm_work |
| 757 | SSTATETASKS += "do_create_runtime_spdx" |
| 758 | do_create_runtime_spdx[sstate-inputdirs] = "${SPDXRUNTIMEDEPLOY}" |
| 759 | do_create_runtime_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}" |
| 760 | |
| 761 | python do_create_runtime_spdx_setscene () { |
| 762 | sstate_setscene(d) |
| 763 | } |
| 764 | addtask do_create_runtime_spdx_setscene |
| 765 | |
| 766 | do_create_runtime_spdx[dirs] = "${SPDXRUNTIMEDEPLOY}" |
| 767 | do_create_runtime_spdx[cleandirs] = "${SPDXRUNTIMEDEPLOY}" |
| 768 | do_create_runtime_spdx[rdeptask] = "do_create_spdx" |
| 769 | |
| 770 | def spdx_get_src(d): |
| 771 | """ |
| 772 | save patched source of the recipe in SPDX_WORKDIR. |
| 773 | """ |
| 774 | import shutil |
| 775 | spdx_workdir = d.getVar('SPDXWORK') |
| 776 | spdx_sysroot_native = d.getVar('STAGING_DIR_NATIVE') |
| 777 | pn = d.getVar('PN') |
| 778 | |
| 779 | workdir = d.getVar("WORKDIR") |
| 780 | |
| 781 | try: |
| 782 | # The kernel class functions require it to be on work-shared, so we dont change WORKDIR |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 783 | if not is_work_shared_spdx(d): |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 784 | # Change the WORKDIR to make do_unpack do_patch run in another dir. |
| 785 | d.setVar('WORKDIR', spdx_workdir) |
| 786 | # Restore the original path to recipe's native sysroot (it's relative to WORKDIR). |
| 787 | d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native) |
| 788 | |
| 789 | # The changed 'WORKDIR' also caused 'B' changed, create dir 'B' for the |
| 790 | # possibly requiring of the following tasks (such as some recipes's |
| 791 | # do_patch required 'B' existed). |
| 792 | bb.utils.mkdirhier(d.getVar('B')) |
| 793 | |
| 794 | bb.build.exec_func('do_unpack', d) |
| 795 | # Copy source of kernel to spdx_workdir |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 796 | if is_work_shared_spdx(d): |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 797 | d.setVar('WORKDIR', spdx_workdir) |
| 798 | d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native) |
| 799 | src_dir = spdx_workdir + "/" + d.getVar('PN')+ "-" + d.getVar('PV') + "-" + d.getVar('PR') |
| 800 | bb.utils.mkdirhier(src_dir) |
| 801 | if bb.data.inherits_class('kernel',d): |
| 802 | share_src = d.getVar('STAGING_KERNEL_DIR') |
| 803 | cmd_copy_share = "cp -rf " + share_src + "/* " + src_dir + "/" |
| 804 | cmd_copy_kernel_result = os.popen(cmd_copy_share).read() |
| 805 | bb.note("cmd_copy_kernel_result = " + cmd_copy_kernel_result) |
| 806 | |
| 807 | git_path = src_dir + "/.git" |
| 808 | if os.path.exists(git_path): |
| 809 | shutils.rmtree(git_path) |
| 810 | |
| 811 | # Make sure gcc and kernel sources are patched only once |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 812 | if not (d.getVar('SRC_URI') == "" or is_work_shared_spdx(d)): |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 813 | bb.build.exec_func('do_patch', d) |
| 814 | |
| 815 | # Some userland has no source. |
| 816 | if not os.path.exists( spdx_workdir ): |
| 817 | bb.utils.mkdirhier(spdx_workdir) |
| 818 | finally: |
| 819 | d.setVar("WORKDIR", workdir) |
| 820 | |
| 821 | do_rootfs[recrdeptask] += "do_create_spdx do_create_runtime_spdx" |
| 822 | |
| 823 | ROOTFS_POSTUNINSTALL_COMMAND =+ "image_combine_spdx ; " |
Andrew Geissler | 7e0e3c0 | 2022-02-25 20:34:39 +0000 | [diff] [blame^] | 824 | |
| 825 | do_populate_sdk[recrdeptask] += "do_create_spdx do_create_runtime_spdx" |
| 826 | POPULATE_SDK_POST_HOST_COMMAND:append:task-populate-sdk = " sdk_host_combine_spdx; " |
| 827 | POPULATE_SDK_POST_TARGET_COMMAND:append:task-populate-sdk = " sdk_target_combine_spdx; " |
| 828 | |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 829 | python image_combine_spdx() { |
| 830 | import os |
Andrew Geissler | 7e0e3c0 | 2022-02-25 20:34:39 +0000 | [diff] [blame^] | 831 | import oe.sbom |
| 832 | from pathlib import Path |
| 833 | from oe.rootfs import image_list_installed_packages |
| 834 | |
| 835 | image_name = d.getVar("IMAGE_NAME") |
| 836 | image_link_name = d.getVar("IMAGE_LINK_NAME") |
| 837 | imgdeploydir = Path(d.getVar("IMGDEPLOYDIR")) |
| 838 | img_spdxid = oe.sbom.get_image_spdxid(image_name) |
| 839 | packages = image_list_installed_packages(d) |
| 840 | |
| 841 | combine_spdx(d, image_name, imgdeploydir, img_spdxid, packages) |
| 842 | |
| 843 | if image_link_name: |
| 844 | image_spdx_path = imgdeploydir / (image_name + ".spdx.json") |
| 845 | image_spdx_link = imgdeploydir / (image_link_name + ".spdx.json") |
| 846 | image_spdx_link.symlink_to(os.path.relpath(image_spdx_path, image_spdx_link.parent)) |
| 847 | |
| 848 | def make_image_link(target_path, suffix): |
| 849 | if image_link_name: |
| 850 | link = imgdeploydir / (image_link_name + suffix) |
| 851 | link.symlink_to(os.path.relpath(target_path, link.parent)) |
| 852 | |
| 853 | spdx_tar_path = imgdeploydir / (image_name + ".spdx.tar.zst") |
| 854 | make_image_link(spdx_tar_path, ".spdx.tar.zst") |
| 855 | spdx_index_path = imgdeploydir / (image_name + ".spdx.index.json") |
| 856 | make_image_link(spdx_index_path, ".spdx.index.json") |
| 857 | } |
| 858 | |
| 859 | python sdk_host_combine_spdx() { |
| 860 | sdk_combine_spdx(d, "host") |
| 861 | } |
| 862 | |
| 863 | python sdk_target_combine_spdx() { |
| 864 | sdk_combine_spdx(d, "target") |
| 865 | } |
| 866 | |
| 867 | def sdk_combine_spdx(d, sdk_type): |
| 868 | import oe.sbom |
| 869 | from pathlib import Path |
| 870 | from oe.sdk import sdk_list_installed_packages |
| 871 | |
| 872 | sdk_name = d.getVar("SDK_NAME") + "-" + sdk_type |
| 873 | sdk_deploydir = Path(d.getVar("SDKDEPLOYDIR")) |
| 874 | sdk_spdxid = oe.sbom.get_sdk_spdxid(sdk_name) |
| 875 | sdk_packages = sdk_list_installed_packages(d, sdk_type == "target") |
| 876 | combine_spdx(d, sdk_name, sdk_deploydir, sdk_spdxid, sdk_packages) |
| 877 | |
| 878 | def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages): |
| 879 | import os |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 880 | import oe.spdx |
| 881 | import oe.sbom |
| 882 | import io |
| 883 | import json |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 884 | from datetime import timezone, datetime |
| 885 | from pathlib import Path |
| 886 | import tarfile |
| 887 | import bb.compress.zstd |
| 888 | |
| 889 | creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 890 | deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 891 | source_date_epoch = d.getVar("SOURCE_DATE_EPOCH") |
| 892 | |
| 893 | doc = oe.spdx.SPDXDocument() |
Andrew Geissler | 7e0e3c0 | 2022-02-25 20:34:39 +0000 | [diff] [blame^] | 894 | doc.name = rootfs_name |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 895 | doc.documentNamespace = get_doc_namespace(d, doc) |
| 896 | doc.creationInfo.created = creation_time |
| 897 | doc.creationInfo.comment = "This document was created by analyzing the source of the Yocto recipe during the build." |
| 898 | doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] |
| 899 | doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") |
Andrew Geissler | 595f630 | 2022-01-24 19:11:47 +0000 | [diff] [blame] | 900 | doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 901 | doc.creationInfo.creators.append("Person: N/A ()") |
| 902 | |
| 903 | image = oe.spdx.SPDXPackage() |
| 904 | image.name = d.getVar("PN") |
| 905 | image.versionInfo = d.getVar("PV") |
Andrew Geissler | 7e0e3c0 | 2022-02-25 20:34:39 +0000 | [diff] [blame^] | 906 | image.SPDXID = rootfs_spdxid |
| 907 | image.packageSupplier = d.getVar("SPDX_SUPPLIER") |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 908 | |
| 909 | doc.packages.append(image) |
| 910 | |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 911 | for name in sorted(packages.keys()): |
| 912 | pkg_spdx_path = deploy_dir_spdx / "packages" / (name + ".spdx.json") |
| 913 | pkg_doc, pkg_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path) |
| 914 | |
| 915 | for p in pkg_doc.packages: |
| 916 | if p.name == name: |
| 917 | pkg_ref = oe.spdx.SPDXExternalDocumentRef() |
| 918 | pkg_ref.externalDocumentId = "DocumentRef-%s" % pkg_doc.name |
| 919 | pkg_ref.spdxDocument = pkg_doc.documentNamespace |
| 920 | pkg_ref.checksum.algorithm = "SHA1" |
| 921 | pkg_ref.checksum.checksumValue = pkg_doc_sha1 |
| 922 | |
| 923 | doc.externalDocumentRefs.append(pkg_ref) |
| 924 | doc.add_relationship(image, "CONTAINS", "%s:%s" % (pkg_ref.externalDocumentId, p.SPDXID)) |
| 925 | break |
| 926 | else: |
| 927 | bb.fatal("Unable to find package with name '%s' in SPDX file %s" % (name, pkg_spdx_path)) |
| 928 | |
| 929 | runtime_spdx_path = deploy_dir_spdx / "runtime" / ("runtime-" + name + ".spdx.json") |
| 930 | runtime_doc, runtime_doc_sha1 = oe.sbom.read_doc(runtime_spdx_path) |
| 931 | |
| 932 | runtime_ref = oe.spdx.SPDXExternalDocumentRef() |
| 933 | runtime_ref.externalDocumentId = "DocumentRef-%s" % runtime_doc.name |
| 934 | runtime_ref.spdxDocument = runtime_doc.documentNamespace |
| 935 | runtime_ref.checksum.algorithm = "SHA1" |
| 936 | runtime_ref.checksum.checksumValue = runtime_doc_sha1 |
| 937 | |
| 938 | # "OTHER" isn't ideal here, but I can't find a relationship that makes sense |
| 939 | doc.externalDocumentRefs.append(runtime_ref) |
| 940 | doc.add_relationship( |
| 941 | image, |
| 942 | "OTHER", |
| 943 | "%s:%s" % (runtime_ref.externalDocumentId, runtime_doc.SPDXID), |
| 944 | comment="Runtime dependencies for %s" % name |
| 945 | ) |
| 946 | |
Andrew Geissler | 7e0e3c0 | 2022-02-25 20:34:39 +0000 | [diff] [blame^] | 947 | image_spdx_path = rootfs_deploydir / (rootfs_name + ".spdx.json") |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 948 | |
| 949 | with image_spdx_path.open("wb") as f: |
| 950 | doc.to_json(f, sort_keys=True) |
| 951 | |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 952 | num_threads = int(d.getVar("BB_NUMBER_THREADS")) |
| 953 | |
| 954 | visited_docs = set() |
| 955 | |
| 956 | index = {"documents": []} |
| 957 | |
Andrew Geissler | 7e0e3c0 | 2022-02-25 20:34:39 +0000 | [diff] [blame^] | 958 | spdx_tar_path = rootfs_deploydir / (rootfs_name + ".spdx.tar.zst") |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 959 | with bb.compress.zstd.open(spdx_tar_path, "w", num_threads=num_threads) as f: |
| 960 | with tarfile.open(fileobj=f, mode="w|") as tar: |
| 961 | def collect_spdx_document(path): |
| 962 | nonlocal tar |
| 963 | nonlocal deploy_dir_spdx |
| 964 | nonlocal source_date_epoch |
| 965 | nonlocal index |
| 966 | |
| 967 | if path in visited_docs: |
| 968 | return |
| 969 | |
| 970 | visited_docs.add(path) |
| 971 | |
| 972 | with path.open("rb") as f: |
| 973 | doc, sha1 = oe.sbom.read_doc(f) |
| 974 | f.seek(0) |
| 975 | |
| 976 | if doc.documentNamespace in visited_docs: |
| 977 | return |
| 978 | |
| 979 | bb.note("Adding SPDX document %s" % path) |
| 980 | visited_docs.add(doc.documentNamespace) |
| 981 | info = tar.gettarinfo(fileobj=f) |
| 982 | |
| 983 | info.name = doc.name + ".spdx.json" |
| 984 | info.uid = 0 |
| 985 | info.gid = 0 |
| 986 | info.uname = "root" |
| 987 | info.gname = "root" |
| 988 | |
| 989 | if source_date_epoch is not None and info.mtime > int(source_date_epoch): |
| 990 | info.mtime = int(source_date_epoch) |
| 991 | |
| 992 | tar.addfile(info, f) |
| 993 | |
| 994 | index["documents"].append({ |
| 995 | "filename": info.name, |
| 996 | "documentNamespace": doc.documentNamespace, |
| 997 | "sha1": sha1, |
| 998 | }) |
| 999 | |
| 1000 | for ref in doc.externalDocumentRefs: |
| 1001 | ref_path = deploy_dir_spdx / "by-namespace" / ref.spdxDocument.replace("/", "_") |
| 1002 | collect_spdx_document(ref_path) |
| 1003 | |
| 1004 | collect_spdx_document(image_spdx_path) |
| 1005 | |
| 1006 | index["documents"].sort(key=lambda x: x["filename"]) |
| 1007 | |
| 1008 | index_str = io.BytesIO(json.dumps(index, sort_keys=True).encode("utf-8")) |
| 1009 | |
| 1010 | info = tarfile.TarInfo() |
| 1011 | info.name = "index.json" |
| 1012 | info.size = len(index_str.getvalue()) |
| 1013 | info.uid = 0 |
| 1014 | info.gid = 0 |
| 1015 | info.uname = "root" |
| 1016 | info.gname = "root" |
| 1017 | |
| 1018 | tar.addfile(info, fileobj=index_str) |
| 1019 | |
Andrew Geissler | 7e0e3c0 | 2022-02-25 20:34:39 +0000 | [diff] [blame^] | 1020 | spdx_index_path = rootfs_deploydir / (rootfs_name + ".spdx.index.json") |
Andrew Geissler | 5199d83 | 2021-09-24 16:47:35 -0500 | [diff] [blame] | 1021 | with spdx_index_path.open("w") as f: |
| 1022 | json.dump(index, f, sort_keys=True) |