| # |
| # Toaster helper class |
| # |
| # Copyright (C) 2013 Intel Corporation |
| # |
| # Released under the MIT license (see COPYING.MIT) |
| # |
| # This bbclass is designed to extract data used by OE-Core during the build process, |
| # for recording in the Toaster system. |
| # The data access is synchronous, preserving the build data integrity across |
| # different builds. |
| # |
| # The data is transferred through the event system, using the MetadataEvent objects. |
| # |
| # The model is to enable the datadump functions as postfuncs, and have the dump |
| # executed after the real taskfunc has been executed. This prevents task signature changing |
| # is toaster is enabled or not. Build performance is not affected if Toaster is not enabled. |
| # |
| # To enable, use INHERIT in local.conf: |
| # |
| # INHERIT += "toaster" |
| # |
| # |
| # |
| # |
| |
| # Find and dump layer info when we got the layers parsed |
| |
| |
| |
| python toaster_layerinfo_dumpdata() { |
| import subprocess |
| |
| def _get_git_branch(layer_path): |
| branch = subprocess.Popen("git symbolic-ref HEAD 2>/dev/null ", cwd=layer_path, shell=True, stdout=subprocess.PIPE).communicate()[0] |
| branch = branch.replace('refs/heads/', '').rstrip() |
| return branch |
| |
| def _get_git_revision(layer_path): |
| revision = subprocess.Popen("git rev-parse HEAD 2>/dev/null ", cwd=layer_path, shell=True, stdout=subprocess.PIPE).communicate()[0].rstrip() |
| return revision |
| |
| def _get_url_map_name(layer_name): |
| """ Some layers have a different name on openembedded.org site, |
| this method returns the correct name to use in the URL |
| """ |
| |
| url_name = layer_name |
| url_mapping = {'meta': 'openembedded-core'} |
| |
| for key in url_mapping.keys(): |
| if key == layer_name: |
| url_name = url_mapping[key] |
| |
| return url_name |
| |
| def _get_layer_version_information(layer_path): |
| |
| layer_version_info = {} |
| layer_version_info['branch'] = _get_git_branch(layer_path) |
| layer_version_info['commit'] = _get_git_revision(layer_path) |
| layer_version_info['priority'] = 0 |
| |
| return layer_version_info |
| |
| |
| def _get_layer_dict(layer_path): |
| |
| layer_info = {} |
| layer_name = layer_path.split('/')[-1] |
| layer_url = 'http://layers.openembedded.org/layerindex/layer/{layer}/' |
| layer_url_name = _get_url_map_name(layer_name) |
| |
| layer_info['name'] = layer_url_name |
| layer_info['local_path'] = layer_path |
| layer_info['layer_index_url'] = layer_url.format(layer=layer_url_name) |
| layer_info['version'] = _get_layer_version_information(layer_path) |
| |
| return layer_info |
| |
| |
| bblayers = e.data.getVar("BBLAYERS", True) |
| |
| llayerinfo = {} |
| |
| for layer in { l for l in bblayers.strip().split(" ") if len(l) }: |
| llayerinfo[layer] = _get_layer_dict(layer) |
| |
| |
| bb.event.fire(bb.event.MetadataEvent("LayerInfo", llayerinfo), e.data) |
| } |
| |
| # Dump package file info data |
| |
| def _toaster_load_pkgdatafile(dirpath, filepath): |
| import json |
| import re |
| pkgdata = {} |
| with open(os.path.join(dirpath, filepath), "r") as fin: |
| for line in fin: |
| try: |
| kn, kv = line.strip().split(": ", 1) |
| m = re.match(r"^PKG_([^A-Z:]*)", kn) |
| if m: |
| pkgdata['OPKGN'] = m.group(1) |
| kn = "_".join([x for x in kn.split("_") if x.isupper()]) |
| pkgdata[kn] = kv.strip() |
| if kn == 'FILES_INFO': |
| pkgdata[kn] = json.loads(kv) |
| |
| except ValueError: |
| pass # ignore lines without valid key: value pairs |
| return pkgdata |
| |
| python toaster_package_dumpdata() { |
| """ |
| Dumps the data about the packages created by a recipe |
| """ |
| |
| # No need to try and dumpdata if the recipe isn't generating packages |
| if not d.getVar('PACKAGES', True): |
| return |
| |
| pkgdatadir = d.getVar('PKGDESTWORK', True) |
| lpkgdata = {} |
| datadir = os.path.join(pkgdatadir, 'runtime') |
| |
| # scan and send data for each generated package |
| for datafile in os.listdir(datadir): |
| if not datafile.endswith('.packaged'): |
| lpkgdata = _toaster_load_pkgdatafile(datadir, datafile) |
| # Fire an event containing the pkg data |
| bb.event.fire(bb.event.MetadataEvent("SinglePackageInfo", lpkgdata), d) |
| } |
| |
| # 2. Dump output image files information |
| |
| python toaster_image_dumpdata() { |
| """ |
| Image filename for output images is not standardized. |
| image_types.bbclass will spell out IMAGE_CMD_xxx variables that actually |
| have hardcoded ways to create image file names in them. |
| So we look for files starting with the set name. |
| |
| We also look for other files in the images/ directory which don't |
| match IMAGE_NAME, such as the kernel bzImage, modules tarball etc. |
| """ |
| |
| dir_to_walk = d.getVar('DEPLOY_DIR_IMAGE', True); |
| image_name = d.getVar('IMAGE_NAME', True); |
| image_info_data = {} |
| artifact_info_data = {} |
| |
| # collect all images and artifacts in the images directory |
| for dirpath, dirnames, filenames in os.walk(dir_to_walk): |
| for filename in filenames: |
| full_path = os.path.join(dirpath, filename) |
| try: |
| if filename.startswith(image_name): |
| # image |
| image_info_data[full_path] = os.stat(full_path).st_size |
| else: |
| # other non-image artifact |
| if not os.path.islink(full_path): |
| artifact_info_data[full_path] = os.stat(full_path).st_size |
| except OSError as e: |
| bb.event.fire(bb.event.MetadataEvent("OSErrorException", e), d) |
| |
| bb.event.fire(bb.event.MetadataEvent("ImageFileSize", image_info_data), d) |
| bb.event.fire(bb.event.MetadataEvent("ArtifactFileSize", artifact_info_data), d) |
| } |
| |
| python toaster_artifact_dumpdata() { |
| """ |
| Dump data about artifacts in the SDK_DEPLOY directory |
| """ |
| |
| dir_to_walk = d.getVar("SDK_DEPLOY", True) |
| artifact_info_data = {} |
| |
| # collect all artifacts in the sdk directory |
| for dirpath, dirnames, filenames in os.walk(dir_to_walk): |
| for filename in filenames: |
| full_path = os.path.join(dirpath, filename) |
| try: |
| if not os.path.islink(full_path): |
| artifact_info_data[full_path] = os.stat(full_path).st_size |
| except OSError as e: |
| bb.event.fire(bb.event.MetadataEvent("OSErrorException", e), d) |
| |
| bb.event.fire(bb.event.MetadataEvent("ArtifactFileSize", artifact_info_data), d) |
| } |
| |
| # collect list of buildstats files based on fired events; when the build completes, collect all stats and fire an event with collected data |
| |
| python toaster_collect_task_stats() { |
| import bb.build |
| import bb.event |
| import bb.data |
| import bb.utils |
| import os |
| |
| toaster_statlist_file = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist") |
| |
| if not e.data.getVar('BUILDSTATS_BASE', True): |
| return # if we don't have buildstats, we cannot collect stats |
| |
| def stat_to_float(value): |
| return float(value.strip('% \n\r')) |
| |
| def _append_read_list(v): |
| lock = bb.utils.lockfile(e.data.expand("${TOPDIR}/toaster.lock"), False, True) |
| |
| with open(toaster_statlist_file, "a") as fout: |
| taskdir = e.data.expand("${BUILDSTATS_BASE}/${BUILDNAME}/${PF}") |
| fout.write("%s::%s::%s::%s\n" % (e.taskfile, e.taskname, os.path.join(taskdir, e.task), e.data.expand("${PN}"))) |
| |
| bb.utils.unlockfile(lock) |
| |
| def _read_stats(filename): |
| # seconds |
| cpu_time_user = 0 |
| cpu_time_system = 0 |
| |
| # bytes |
| disk_io_read = 0 |
| disk_io_write = 0 |
| |
| started = 0 |
| ended = 0 |
| |
| taskname = '' |
| |
| statinfo = {} |
| |
| with open(filename, 'r') as task_bs: |
| for line in task_bs.readlines(): |
| k,v = line.strip().split(": ", 1) |
| statinfo[k] = v |
| |
| if "Started" in statinfo: |
| started = stat_to_float(statinfo["Started"]) |
| |
| if "Ended" in statinfo: |
| ended = stat_to_float(statinfo["Ended"]) |
| |
| if "Child rusage ru_utime" in statinfo: |
| cpu_time_user = cpu_time_user + stat_to_float(statinfo["Child rusage ru_utime"]) |
| |
| if "Child rusage ru_stime" in statinfo: |
| cpu_time_system = cpu_time_system + stat_to_float(statinfo["Child rusage ru_stime"]) |
| |
| if "IO write_bytes" in statinfo: |
| write_bytes = int(statinfo["IO write_bytes"].strip('% \n\r')) |
| disk_io_write = disk_io_write + write_bytes |
| |
| if "IO read_bytes" in statinfo: |
| read_bytes = int(statinfo["IO read_bytes"].strip('% \n\r')) |
| disk_io_read = disk_io_read + read_bytes |
| |
| return { |
| 'stat_file': filename, |
| 'cpu_time_user': cpu_time_user, |
| 'cpu_time_system': cpu_time_system, |
| 'disk_io_read': disk_io_read, |
| 'disk_io_write': disk_io_write, |
| 'started': started, |
| 'ended': ended |
| } |
| |
| if isinstance(e, (bb.build.TaskSucceeded, bb.build.TaskFailed)): |
| _append_read_list(e) |
| pass |
| |
| if isinstance(e, bb.event.BuildCompleted) and os.path.exists(toaster_statlist_file): |
| events = [] |
| with open(toaster_statlist_file, "r") as fin: |
| for line in fin: |
| (taskfile, taskname, filename, recipename) = line.strip().split("::") |
| stats = _read_stats(filename) |
| events.append((taskfile, taskname, stats, recipename)) |
| bb.event.fire(bb.event.MetadataEvent("BuildStatsList", events), e.data) |
| os.unlink(toaster_statlist_file) |
| } |
| |
| # dump relevant build history data as an event when the build is completed |
| |
| python toaster_buildhistory_dump() { |
| import re |
| BUILDHISTORY_DIR = e.data.expand("${TOPDIR}/buildhistory") |
| BUILDHISTORY_DIR_IMAGE_BASE = e.data.expand("%s/images/${MACHINE_ARCH}/${TCLIBC}/"% BUILDHISTORY_DIR) |
| pkgdata_dir = e.data.getVar("PKGDATA_DIR", True) |
| |
| |
| # scan the build targets for this build |
| images = {} |
| allpkgs = {} |
| files = {} |
| for target in e._pkgs: |
| target = target.split(':')[0] # strip ':<task>' suffix from the target |
| installed_img_path = e.data.expand(os.path.join(BUILDHISTORY_DIR_IMAGE_BASE, target)) |
| if os.path.exists(installed_img_path): |
| images[target] = {} |
| files[target] = {} |
| files[target]['dirs'] = [] |
| files[target]['syms'] = [] |
| files[target]['files'] = [] |
| with open("%s/installed-package-sizes.txt" % installed_img_path, "r") as fin: |
| for line in fin: |
| line = line.rstrip(";") |
| psize, px = line.split("\t") |
| punit, pname = px.split(" ") |
| # this size is "installed-size" as it measures how much space it takes on disk |
| images[target][pname.strip()] = {'size':int(psize)*1024, 'depends' : []} |
| |
| with open("%s/depends.dot" % installed_img_path, "r") as fin: |
| p = re.compile(r' -> ') |
| dot = re.compile(r'.*style=dotted') |
| for line in fin: |
| line = line.rstrip(';') |
| linesplit = p.split(line) |
| if len(linesplit) == 2: |
| pname = linesplit[0].rstrip('"').strip('"') |
| dependsname = linesplit[1].split(" ")[0].strip().strip(";").strip('"').rstrip('"') |
| deptype = "depends" |
| if dot.match(line): |
| deptype = "recommends" |
| if not pname in images[target]: |
| images[target][pname] = {'size': 0, 'depends' : []} |
| if not dependsname in images[target]: |
| images[target][dependsname] = {'size': 0, 'depends' : []} |
| images[target][pname]['depends'].append((dependsname, deptype)) |
| |
| with open("%s/files-in-image.txt" % installed_img_path, "r") as fin: |
| for line in fin: |
| lc = [ x for x in line.strip().split(" ") if len(x) > 0 ] |
| if lc[0].startswith("l"): |
| files[target]['syms'].append(lc) |
| elif lc[0].startswith("d"): |
| files[target]['dirs'].append(lc) |
| else: |
| files[target]['files'].append(lc) |
| |
| for pname in images[target]: |
| if not pname in allpkgs: |
| try: |
| pkgdata = _toaster_load_pkgdatafile("%s/runtime-reverse/" % pkgdata_dir, pname) |
| except IOError as err: |
| if err.errno == 2: |
| # We expect this e.g. for RRECOMMENDS that are unsatisfied at runtime |
| continue |
| else: |
| raise |
| allpkgs[pname] = pkgdata |
| |
| |
| data = { 'pkgdata' : allpkgs, 'imgdata' : images, 'filedata' : files } |
| |
| bb.event.fire(bb.event.MetadataEvent("ImagePkgList", data), e.data) |
| |
| } |
| |
| # dump information related to license manifest path |
| |
| python toaster_licensemanifest_dump() { |
| deploy_dir = d.getVar('DEPLOY_DIR', True); |
| image_name = d.getVar('IMAGE_NAME', True); |
| |
| data = { 'deploy_dir' : deploy_dir, 'image_name' : image_name } |
| |
| bb.event.fire(bb.event.MetadataEvent("LicenseManifestPath", data), d) |
| } |
| |
| # set event handlers |
| addhandler toaster_layerinfo_dumpdata |
| toaster_layerinfo_dumpdata[eventmask] = "bb.event.TreeDataPreparationCompleted" |
| |
| addhandler toaster_collect_task_stats |
| toaster_collect_task_stats[eventmask] = "bb.event.BuildCompleted bb.build.TaskSucceeded bb.build.TaskFailed" |
| |
| addhandler toaster_buildhistory_dump |
| toaster_buildhistory_dump[eventmask] = "bb.event.BuildCompleted" |
| |
| do_packagedata_setscene[postfuncs] += "toaster_package_dumpdata " |
| do_packagedata_setscene[vardepsexclude] += "toaster_package_dumpdata " |
| |
| do_package[postfuncs] += "toaster_package_dumpdata " |
| do_package[vardepsexclude] += "toaster_package_dumpdata " |
| |
| do_image_complete[postfuncs] += "toaster_image_dumpdata " |
| do_image_complete[vardepsexclude] += "toaster_image_dumpdata " |
| |
| do_rootfs[postfuncs] += "toaster_licensemanifest_dump " |
| do_rootfs[vardepsexclude] += "toaster_licensemanifest_dump " |
| |
| do_populate_sdk[postfuncs] += "toaster_artifact_dumpdata " |
| do_populate_sdk[vardepsexclude] += "toaster_artifact_dumpdata " |