blob: 7d99228c739bdfa33518b26b53db728d4bf88bed [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001#!/bin/bash
2#
3# This script runs a series of tests (with and without sstate) and reports build time (and tmp/ size)
4#
5# Build performance test script
6#
7# Copyright 2013 Intel Corporation
8#
9# This program is free software; you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation; either version 2 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program; if not, write to the Free Software
21# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
22#
23#
24# AUTHORS:
25# Stefan Stanacar <stefanx.stanacar@intel.com>
26
27
28ME=$(basename $0)
29
30#
31# usage and setup
32#
33
34usage () {
35cat << EOT
36Usage: $ME [-h]
37 $ME [-c <commit>] [-v] [-m <val>] [-j <val>] [-t <val>] [-i <image-name>] [-d <path>]
38Options:
39 -h
40 Display this help and exit.
41 -c <commit>
42 git checkout <commit> before anything else
43 -v
44 Show bitbake output, don't redirect it to a log.
45 -m <machine>
46 Value for MACHINE. Default is qemux86.
47 -j <val>
48 Value for PARALLEL_MAKE. Default is 8.
49 -t <val>
50 Value for BB_NUMBER_THREADS. Default is 8.
51 -i <image-name>
52 Instead of timing against core-image-sato, use <image-name>
53 -d <path>
54 Use <path> as DL_DIR
55 -p <githash>
56 Cherry pick githash onto the commit
57
58Note: current working directory must be inside a poky git clone.
59
60EOT
61}
62
63
64if clonedir=$(git rev-parse --show-toplevel); then
65 cd $clonedir
66else
67 echo "The current working dir doesn't seem to be a poky git clone. Please cd there before running $ME"
68 exit 1
69fi
70
71IMAGE="core-image-sato"
72verbose=0
73dldir=
74commit=
75pmake=
76cherrypicks=
77while getopts "hvc:m:j:t:i:d:p:" opt; do
78 case $opt in
79 h) usage
80 exit 0
81 ;;
82 v) verbose=1
83 ;;
84 c) commit=$OPTARG
85 ;;
86 m) export MACHINE=$OPTARG
87 ;;
88 j) pmake=$OPTARG
89 ;;
90 t) export BB_NUMBER_THREADS=$OPTARG
91 ;;
92 i) IMAGE=$OPTARG
93 ;;
94 d) dldir=$OPTARG
95 ;;
96 p) cherrypicks="$cherrypicks $OPTARG"
97 ;;
98 *) usage
99 exit 1
100 ;;
101 esac
102done
103
104
105#drop cached credentials and test for sudo access without a password
106sudo -k -n ls > /dev/null 2>&1
107reqpass=$?
108if [ $reqpass -ne 0 ]; then
109 echo "The script requires sudo access to drop caches between builds (echo 3 > /proc/sys/vm/drop_caches)"
110 read -s -p "Please enter your sudo password: " pass
111 echo
112fi
113
114if [ -n "$commit" ]; then
115 echo "git checkout -f $commit"
116 git pull > /dev/null 2>&1
117 git checkout -f $commit || exit 1
118 git pull > /dev/null 2>&1
119fi
120
121if [ -n "$cherrypicks" ]; then
122 for c in $cherrypicks; do
123 git cherry-pick $c
124 done
125fi
126
127rev=$(git rev-parse --short HEAD) || exit 1
128OUTDIR="$clonedir/build-perf-test/results-$rev-`date "+%Y%m%d%H%M%S"`"
129BUILDDIR="$OUTDIR/build"
130resultsfile="$OUTDIR/results.log"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500131cmdoutput="$OUTDIR/commands.log"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500132myoutput="$OUTDIR/output.log"
133globalres="$clonedir/build-perf-test/globalres.log"
134
135mkdir -p $OUTDIR || exit 1
136
137log () {
138 local msg="$1"
139 echo "`date`: $msg" | tee -a $myoutput
140}
141
142
143#
144# Config stuff
145#
146
147branch=`git branch 2>&1 | grep "^* " | tr -d "* "`
148gitcommit=$(git rev-parse HEAD) || exit 1
149log "Running on $branch:$gitcommit"
150
151source ./oe-init-build-env $OUTDIR/build >/dev/null || exit 1
152cd $OUTDIR/build
153
154[ -n "$MACHINE" ] || export MACHINE="qemux86"
155[ -n "$BB_NUMBER_THREADS" ] || export BB_NUMBER_THREADS="8"
156
157if [ -n "$pmake" ]; then
158 export PARALLEL_MAKE="-j $pmake"
159else
160 export PARALLEL_MAKE="-j 8"
161fi
162
163if [ -n "$dldir" ]; then
164 echo "DL_DIR = \"$dldir\"" >> conf/local.conf
165else
166 echo "DL_DIR = \"$clonedir/build-perf-test/downloads\"" >> conf/local.conf
167fi
168
169# Sometimes I've noticed big differences in timings for the same commit, on the same machine
170# Disabling the network sanity check helps a bit (because of my crappy network connection and/or proxy)
171echo "CONNECTIVITY_CHECK_URIS =\"\"" >> conf/local.conf
172
173
174#
175# Functions
176#
177
178declare -a TIMES
179time_count=0
180declare -a SIZES
181size_count=0
182
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500183time_cmd () {
184 log " Timing: $*"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500185
186 if [ $verbose -eq 0 ]; then
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500187 /usr/bin/time -v -o $resultsfile "$@" >> $cmdoutput
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500188 else
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500189 /usr/bin/time -v -o $resultsfile "$@"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500190 fi
191 ret=$?
192 if [ $ret -eq 0 ]; then
193 t=`grep wall $resultsfile | sed 's/.*m:ss): //'`
194 log " TIME: $t"
195 TIMES[(( time_count++ ))]="$t"
196 else
197 log "ERROR: exit status was non-zero, will report time as 0."
198 TIMES[(( time_count++ ))]="0"
199 fi
200
201 #time by default overwrites the output file and we want to keep the results
202 #it has an append option but I don't want to clobber the results in the same file
203 i=`ls $OUTDIR/results.log* |wc -l`
204 mv $resultsfile "${resultsfile}.${i}"
205 log "More stats can be found in ${resultsfile}.${i}"
206}
207
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500208bbtime () {
209 time_cmd bitbake "$@"
210}
211
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500212#we don't time bitbake here
213bbnotime () {
214 local arg="$@"
215 log " Running: bitbake ${arg}"
216 if [ $verbose -eq 0 ]; then
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500217 bitbake ${arg} >> $cmdoutput
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500218 else
219 bitbake ${arg}
220 fi
221 ret=$?
222 if [ $ret -eq 0 ]; then
223 log " Finished bitbake ${arg}"
224 else
225 log "ERROR: exit status was non-zero. Exit.."
226 exit $ret
227 fi
228
229}
230
231do_rmtmp() {
232 log " Removing tmp"
233 rm -rf bitbake.lock pseudodone conf/sanity_info cache tmp
234}
235do_rmsstate () {
236 log " Removing sstate-cache"
237 rm -rf sstate-cache
238}
239do_sync () {
240 log " Syncing and dropping caches"
241 sync; sync
242 if [ $reqpass -eq 0 ]; then
243 sudo sh -c "echo 3 > /proc/sys/vm/drop_caches"
244 else
245 echo "$pass" | sudo -S sh -c "echo 3 > /proc/sys/vm/drop_caches"
246 echo
247 fi
248 sleep 3
249}
250
251write_results() {
252 echo -n "`uname -n`,$branch:$gitcommit,`git describe`," >> $globalres
253 for i in "${TIMES[@]}"; do
254 echo -n "$i," >> $globalres
255 done
256 for i in "${SIZES[@]}"; do
257 echo -n "$i," >> $globalres
258 done
259 echo >> $globalres
260 sed -i '$ s/,$//' $globalres
261}
262
263####
264
265#
266# Test 1
267# Measure: Wall clock of "bitbake core-image-sato" and size of tmp/dir (w/o rm_work and w/ rm_work)
268# Pre: Downloaded sources, no sstate
269# Steps:
270# Part1:
271# - fetchall
272# - clean build dir
273# - time bitbake core-image-sato
274# - collect data
275# Part2:
276# - bitbake virtual/kernel -c cleansstate
277# - time bitbake virtual/kernel
278# Part3:
279# - add INHERIT to local.conf
280# - clean build dir
281# - build
282# - report size, remove INHERIT
283
284test1_p1 () {
285 log "Running Test 1, part 1/3: Measure wall clock of bitbake $IMAGE and size of tmp/ dir"
286 bbnotime $IMAGE -c fetchall
287 do_rmtmp
288 do_rmsstate
289 do_sync
290 bbtime $IMAGE
291 s=`du -s tmp | sed 's/tmp//' | sed 's/[ \t]*$//'`
292 SIZES[(( size_count++ ))]="$s"
293 log "SIZE of tmp dir is: $s"
294 log "Buildstats are saved in $OUTDIR/buildstats-test1"
295 mv tmp/buildstats $OUTDIR/buildstats-test1
296}
297
298
299test1_p2 () {
300 log "Running Test 1, part 2/3: bitbake virtual/kernel -c cleansstate and time bitbake virtual/kernel"
301 bbnotime virtual/kernel -c cleansstate
302 do_sync
303 bbtime virtual/kernel
304}
305
306test1_p3 () {
307 log "Running Test 1, part 3/3: Build $IMAGE w/o sstate and report size of tmp/dir with rm_work enabled"
308 echo "INHERIT += \"rm_work\"" >> conf/local.conf
309 do_rmtmp
310 do_rmsstate
311 do_sync
312 bbtime $IMAGE
313 sed -i 's/INHERIT += \"rm_work\"//' conf/local.conf
314 s=`du -s tmp | sed 's/tmp//' | sed 's/[ \t]*$//'`
315 SIZES[(( size_count++ ))]="$s"
316 log "SIZE of tmp dir is: $s"
317 log "Buildstats are saved in $OUTDIR/buildstats-test13"
318 mv tmp/buildstats $OUTDIR/buildstats-test13
319}
320
321
322#
323# Test 2
324# Measure: Wall clock of "bitbake core-image-sato" and size of tmp/dir
325# Pre: populated sstate cache
326
327test2 () {
328 # Assuming test 1 has run
329 log "Running Test 2: Measure wall clock of bitbake $IMAGE -c rootfs with sstate"
330 do_rmtmp
331 do_sync
332 bbtime $IMAGE -c rootfs
333}
334
335
336# Test 3
337# parsing time metrics
338#
339# Start with
340# i) "rm -rf tmp/cache; time bitbake -p"
341# ii) "rm -rf tmp/cache/default-glibc/; time bitbake -p"
342# iii) "time bitbake -p"
343
344
345test3 () {
346 log "Running Test 3: Parsing time metrics (bitbake -p)"
347 log " Removing tmp/cache && cache"
348 rm -rf tmp/cache cache
349 bbtime -p
350 log " Removing tmp/cache/default-glibc/"
351 rm -rf tmp/cache/default-glibc/
352 bbtime -p
353 bbtime -p
354}
355
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500356#
357# Test 4 - eSDK
358# Measure: eSDK size and installation time
359test4 () {
360 log "Running Test 4: eSDK size and installation time"
361 bbnotime $IMAGE -c do_populate_sdk_ext
362
363 esdk_installer=(tmp/deploy/sdk/*-toolchain-ext-*.sh)
364
365 if [ ${#esdk_installer[*]} -eq 1 ]; then
366 s=$((`stat -c %s "$esdk_installer"` / 1024))
367 SIZES[(( size_count++ ))]="$s"
368 log "Download SIZE of eSDK is: $s kB"
369
370 do_sync
371 time_cmd "$esdk_installer" -y -d "tmp/esdk-deploy"
372
373 s=$((`du -sb "tmp/esdk-deploy" | cut -f1` / 1024))
374 SIZES[(( size_count++ ))]="$s"
375 log "Install SIZE of eSDK is: $s kB"
376 else
377 log "ERROR: other than one sdk found (${esdk_installer[*]}), reporting size and time as 0."
378 SIZES[(( size_count++ ))]="0"
379 TIMES[(( time_count++ ))]="0"
380 fi
381
382}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500383
384
385# RUN!
386
387test1_p1
388test1_p2
389test1_p3
390test2
391test3
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500392test4
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500393
394# if we got til here write to global results
395write_results
396
397log "All done, cleaning up..."
398
399do_rmtmp
400do_rmsstate