blob: f365c091420076ca10328f46df3445c72468822d [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001#
2# Toaster helper class
3#
4# Copyright (C) 2013 Intel Corporation
5#
6# Released under the MIT license (see COPYING.MIT)
7#
8# This bbclass is designed to extract data used by OE-Core during the build process,
9# for recording in the Toaster system.
10# The data access is synchronous, preserving the build data integrity across
11# different builds.
12#
13# The data is transferred through the event system, using the MetadataEvent objects.
14#
15# The model is to enable the datadump functions as postfuncs, and have the dump
16# executed after the real taskfunc has been executed. This prevents task signature changing
17# is toaster is enabled or not. Build performance is not affected if Toaster is not enabled.
18#
19# To enable, use INHERIT in local.conf:
20#
21# INHERIT += "toaster"
22#
23#
24#
25#
26
27# Find and dump layer info when we got the layers parsed
28
29
30
31python toaster_layerinfo_dumpdata() {
32 import subprocess
33
34 def _get_git_branch(layer_path):
35 branch = subprocess.Popen("git symbolic-ref HEAD 2>/dev/null ", cwd=layer_path, shell=True, stdout=subprocess.PIPE).communicate()[0]
Patrick Williamsc0f7c042017-02-23 20:41:17 -060036 branch = branch.decode('utf-8')
Patrick Williamsc124f4f2015-09-15 14:41:29 -050037 branch = branch.replace('refs/heads/', '').rstrip()
38 return branch
39
40 def _get_git_revision(layer_path):
41 revision = subprocess.Popen("git rev-parse HEAD 2>/dev/null ", cwd=layer_path, shell=True, stdout=subprocess.PIPE).communicate()[0].rstrip()
42 return revision
43
44 def _get_url_map_name(layer_name):
45 """ Some layers have a different name on openembedded.org site,
46 this method returns the correct name to use in the URL
47 """
48
49 url_name = layer_name
50 url_mapping = {'meta': 'openembedded-core'}
51
52 for key in url_mapping.keys():
53 if key == layer_name:
54 url_name = url_mapping[key]
55
56 return url_name
57
58 def _get_layer_version_information(layer_path):
59
60 layer_version_info = {}
61 layer_version_info['branch'] = _get_git_branch(layer_path)
62 layer_version_info['commit'] = _get_git_revision(layer_path)
63 layer_version_info['priority'] = 0
64
65 return layer_version_info
66
67
68 def _get_layer_dict(layer_path):
69
70 layer_info = {}
71 layer_name = layer_path.split('/')[-1]
72 layer_url = 'http://layers.openembedded.org/layerindex/layer/{layer}/'
73 layer_url_name = _get_url_map_name(layer_name)
74
75 layer_info['name'] = layer_url_name
76 layer_info['local_path'] = layer_path
77 layer_info['layer_index_url'] = layer_url.format(layer=layer_url_name)
78 layer_info['version'] = _get_layer_version_information(layer_path)
79
80 return layer_info
81
82
Brad Bishop6e60e8b2018-02-01 10:27:11 -050083 bblayers = e.data.getVar("BBLAYERS")
Patrick Williamsc124f4f2015-09-15 14:41:29 -050084
85 llayerinfo = {}
86
87 for layer in { l for l in bblayers.strip().split(" ") if len(l) }:
88 llayerinfo[layer] = _get_layer_dict(layer)
89
90
91 bb.event.fire(bb.event.MetadataEvent("LayerInfo", llayerinfo), e.data)
92}
93
94# Dump package file info data
95
96def _toaster_load_pkgdatafile(dirpath, filepath):
97 import json
98 import re
99 pkgdata = {}
100 with open(os.path.join(dirpath, filepath), "r") as fin:
101 for line in fin:
102 try:
103 kn, kv = line.strip().split(": ", 1)
Andrew Geissler9aee5002022-03-30 16:27:02 +0000104 m = re.match(r"^PKG:([^A-Z:]*)", kn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500105 if m:
106 pkgdata['OPKGN'] = m.group(1)
Andrew Geissler9aee5002022-03-30 16:27:02 +0000107 kn = kn.split(":")[0]
108 pkgdata[kn] = kv
Andrew Geisslerd159c7f2021-09-02 21:05:58 -0500109 if kn.startswith('FILES_INFO'):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500110 pkgdata[kn] = json.loads(kv)
111
112 except ValueError:
113 pass # ignore lines without valid key: value pairs
114 return pkgdata
115
Andrew Geissler82c905d2020-04-13 13:39:40 -0500116def _toaster_dumpdata(pkgdatadir, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500117 """
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500118 Dumps the data about the packages created by a recipe
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500119 """
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500120
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500121 # No need to try and dumpdata if the recipe isn't generating packages
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500122 if not d.getVar('PACKAGES'):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500123 return
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500124
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500125 lpkgdata = {}
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500126 datadir = os.path.join(pkgdatadir, 'runtime')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500127
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500128 # scan and send data for each generated package
Andrew Geissler82c905d2020-04-13 13:39:40 -0500129 if os.path.exists(datadir):
130 for datafile in os.listdir(datadir):
131 if not datafile.endswith('.packaged'):
132 lpkgdata = _toaster_load_pkgdatafile(datadir, datafile)
133 # Fire an event containing the pkg data
134 bb.event.fire(bb.event.MetadataEvent("SinglePackageInfo", lpkgdata), d)
135
136python toaster_package_dumpdata() {
137 _toaster_dumpdata(d.getVar('PKGDESTWORK'), d)
138}
139
140python toaster_packagedata_dumpdata() {
141 # This path needs to match do_packagedata[sstate-inputdirs]
142 _toaster_dumpdata(os.path.join(d.getVar('WORKDIR'), 'pkgdata-pdata-input'), d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500143}
144
145# 2. Dump output image files information
146
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500147python toaster_artifact_dumpdata() {
148 """
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600149 Dump data about SDK variables
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500150 """
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500151
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600152 event_data = {
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500153 "TOOLCHAIN_OUTPUTNAME": d.getVar("TOOLCHAIN_OUTPUTNAME")
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600154 }
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500155
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600156 bb.event.fire(bb.event.MetadataEvent("SDKArtifactInfo", event_data), d)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500157}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500158
159# collect list of buildstats files based on fired events; when the build completes, collect all stats and fire an event with collected data
160
161python toaster_collect_task_stats() {
162 import bb.build
163 import bb.event
164 import bb.data
165 import bb.utils
166 import os
167
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500168 if not e.data.getVar('BUILDSTATS_BASE'):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500169 return # if we don't have buildstats, we cannot collect stats
170
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500171 toaster_statlist_file = os.path.join(e.data.getVar('BUILDSTATS_BASE'), "toasterstatlist")
172
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500173 def stat_to_float(value):
174 return float(value.strip('% \n\r'))
175
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500176 def _append_read_list(v):
177 lock = bb.utils.lockfile(e.data.expand("${TOPDIR}/toaster.lock"), False, True)
178
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500179 with open(toaster_statlist_file, "a") as fout:
180 taskdir = e.data.expand("${BUILDSTATS_BASE}/${BUILDNAME}/${PF}")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500181 fout.write("%s::%s::%s::%s\n" % (e.taskfile, e.taskname, os.path.join(taskdir, e.task), e.data.expand("${PN}")))
182
183 bb.utils.unlockfile(lock)
184
185 def _read_stats(filename):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500186 # seconds
187 cpu_time_user = 0
188 cpu_time_system = 0
189
190 # bytes
191 disk_io_read = 0
192 disk_io_write = 0
193
194 started = 0
195 ended = 0
196
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500197 taskname = ''
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500198
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500199 statinfo = {}
200
201 with open(filename, 'r') as task_bs:
202 for line in task_bs.readlines():
203 k,v = line.strip().split(": ", 1)
204 statinfo[k] = v
205
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500206 if "Started" in statinfo:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500207 started = stat_to_float(statinfo["Started"])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500208
209 if "Ended" in statinfo:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500210 ended = stat_to_float(statinfo["Ended"])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500211
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500212 if "Child rusage ru_utime" in statinfo:
213 cpu_time_user = cpu_time_user + stat_to_float(statinfo["Child rusage ru_utime"])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500214
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500215 if "Child rusage ru_stime" in statinfo:
216 cpu_time_system = cpu_time_system + stat_to_float(statinfo["Child rusage ru_stime"])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500217
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500218 if "IO write_bytes" in statinfo:
219 write_bytes = int(statinfo["IO write_bytes"].strip('% \n\r'))
220 disk_io_write = disk_io_write + write_bytes
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500221
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500222 if "IO read_bytes" in statinfo:
223 read_bytes = int(statinfo["IO read_bytes"].strip('% \n\r'))
224 disk_io_read = disk_io_read + read_bytes
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500225
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500226 return {
227 'stat_file': filename,
228 'cpu_time_user': cpu_time_user,
229 'cpu_time_system': cpu_time_system,
230 'disk_io_read': disk_io_read,
231 'disk_io_write': disk_io_write,
232 'started': started,
233 'ended': ended
234 }
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500235
236 if isinstance(e, (bb.build.TaskSucceeded, bb.build.TaskFailed)):
237 _append_read_list(e)
238 pass
239
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500240 if isinstance(e, bb.event.BuildCompleted) and os.path.exists(toaster_statlist_file):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500241 events = []
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500242 with open(toaster_statlist_file, "r") as fin:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500243 for line in fin:
244 (taskfile, taskname, filename, recipename) = line.strip().split("::")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500245 stats = _read_stats(filename)
246 events.append((taskfile, taskname, stats, recipename))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500247 bb.event.fire(bb.event.MetadataEvent("BuildStatsList", events), e.data)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500248 os.unlink(toaster_statlist_file)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500249}
250
251# dump relevant build history data as an event when the build is completed
252
253python toaster_buildhistory_dump() {
254 import re
255 BUILDHISTORY_DIR = e.data.expand("${TOPDIR}/buildhistory")
256 BUILDHISTORY_DIR_IMAGE_BASE = e.data.expand("%s/images/${MACHINE_ARCH}/${TCLIBC}/"% BUILDHISTORY_DIR)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500257 pkgdata_dir = e.data.getVar("PKGDATA_DIR")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500258
259
260 # scan the build targets for this build
261 images = {}
262 allpkgs = {}
263 files = {}
264 for target in e._pkgs:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500265 target = target.split(':')[0] # strip ':<task>' suffix from the target
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500266 installed_img_path = e.data.expand(os.path.join(BUILDHISTORY_DIR_IMAGE_BASE, target))
267 if os.path.exists(installed_img_path):
268 images[target] = {}
269 files[target] = {}
270 files[target]['dirs'] = []
271 files[target]['syms'] = []
272 files[target]['files'] = []
273 with open("%s/installed-package-sizes.txt" % installed_img_path, "r") as fin:
274 for line in fin:
275 line = line.rstrip(";")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500276 psize, punit, pname = line.split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500277 # this size is "installed-size" as it measures how much space it takes on disk
278 images[target][pname.strip()] = {'size':int(psize)*1024, 'depends' : []}
279
280 with open("%s/depends.dot" % installed_img_path, "r") as fin:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500281 p = re.compile(r'\s*"(?P<name>[^"]+)"\s*->\s*"(?P<dep>[^"]+)"(?P<rec>.*?\[style=dotted\])?')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500282 for line in fin:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500283 m = p.match(line)
284 if not m:
285 continue
286 pname = m.group('name')
287 dependsname = m.group('dep')
288 deptype = 'recommends' if m.group('rec') else 'depends'
289
290 # If RPM is used for packaging, then there may be
291 # dependencies such as "/bin/sh", which will confuse
292 # _toaster_load_pkgdatafile() later on. While at it, ignore
293 # any dependencies that contain parentheses, e.g.,
294 # "libc.so.6(GLIBC_2.7)".
295 if dependsname.startswith('/') or '(' in dependsname:
296 continue
297
298 if not pname in images[target]:
299 images[target][pname] = {'size': 0, 'depends' : []}
300 if not dependsname in images[target]:
301 images[target][dependsname] = {'size': 0, 'depends' : []}
302 images[target][pname]['depends'].append((dependsname, deptype))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500303
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600304 # files-in-image.txt is only generated if an image file is created,
305 # so the file entries ('syms', 'dirs', 'files') for a target will be
306 # empty for rootfs builds and other "image" tasks which don't
307 # produce image files
308 # (e.g. "bitbake core-image-minimal -c populate_sdk")
309 files_in_image_path = "%s/files-in-image.txt" % installed_img_path
310 if os.path.exists(files_in_image_path):
311 with open(files_in_image_path, "r") as fin:
312 for line in fin:
313 lc = [ x for x in line.strip().split(" ") if len(x) > 0 ]
314 if lc[0].startswith("l"):
315 files[target]['syms'].append(lc)
316 elif lc[0].startswith("d"):
317 files[target]['dirs'].append(lc)
318 else:
319 files[target]['files'].append(lc)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500320
321 for pname in images[target]:
322 if not pname in allpkgs:
323 try:
324 pkgdata = _toaster_load_pkgdatafile("%s/runtime-reverse/" % pkgdata_dir, pname)
325 except IOError as err:
326 if err.errno == 2:
327 # We expect this e.g. for RRECOMMENDS that are unsatisfied at runtime
328 continue
329 else:
330 raise
331 allpkgs[pname] = pkgdata
332
333
334 data = { 'pkgdata' : allpkgs, 'imgdata' : images, 'filedata' : files }
335
336 bb.event.fire(bb.event.MetadataEvent("ImagePkgList", data), e.data)
337
338}
339
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600340# get list of artifacts from sstate manifest
341python toaster_artifacts() {
342 if e.taskname in ["do_deploy", "do_image_complete", "do_populate_sdk", "do_populate_sdk_ext"]:
343 d2 = d.createCopy()
344 d2.setVar('FILE', e.taskfile)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500345 # Use 'stamp-extra-info' if present, else use workaround
346 # to determine 'SSTATE_MANMACH'
347 extrainf = d2.getVarFlag(e.taskname, 'stamp-extra-info')
348 if extrainf:
349 d2.setVar('SSTATE_MANMACH', extrainf)
350 else:
351 if "do_populate_sdk" == e.taskname:
352 d2.setVar('SSTATE_MANMACH', d2.expand("${MACHINE}${SDKMACHINE}"))
353 else:
354 d2.setVar('SSTATE_MANMACH', d2.expand("${MACHINE}"))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600355 manifest = oe.sstatesig.sstate_get_manifest_filename(e.taskname[3:], d2)[0]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500356
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600357 if os.access(manifest, os.R_OK):
358 with open(manifest) as fmanifest:
359 artifacts = [fname.strip() for fname in fmanifest]
360 data = {"task": e.taskid, "artifacts": artifacts}
361 bb.event.fire(bb.event.MetadataEvent("TaskArtifacts", data), d2)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500362}
363
364# set event handlers
365addhandler toaster_layerinfo_dumpdata
366toaster_layerinfo_dumpdata[eventmask] = "bb.event.TreeDataPreparationCompleted"
367
368addhandler toaster_collect_task_stats
369toaster_collect_task_stats[eventmask] = "bb.event.BuildCompleted bb.build.TaskSucceeded bb.build.TaskFailed"
370
371addhandler toaster_buildhistory_dump
372toaster_buildhistory_dump[eventmask] = "bb.event.BuildCompleted"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500373
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600374addhandler toaster_artifacts
375toaster_artifacts[eventmask] = "bb.runqueue.runQueueTaskSkipped bb.runqueue.runQueueTaskCompleted"
376
Andrew Geissler82c905d2020-04-13 13:39:40 -0500377do_packagedata_setscene[postfuncs] += "toaster_packagedata_dumpdata "
378do_packagedata_setscene[vardepsexclude] += "toaster_packagedata_dumpdata "
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500379
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500380do_package[postfuncs] += "toaster_package_dumpdata "
381do_package[vardepsexclude] += "toaster_package_dumpdata "
382
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500383#do_populate_sdk[postfuncs] += "toaster_artifact_dumpdata "
384#do_populate_sdk[vardepsexclude] += "toaster_artifact_dumpdata "
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600385
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500386#do_populate_sdk_ext[postfuncs] += "toaster_artifact_dumpdata "
387#do_populate_sdk_ext[vardepsexclude] += "toaster_artifact_dumpdata "
388