poky/scripts/contrib/build-perf-test.sh - stefanberger/openbmc - Gitiles

 #!/bin/bash
 #
 # This script runs a series of tests  (with and without sstate) and reports build time (and tmp/ size)
 #
 # Build performance test script
 #
 # Copyright 2013 Intel Corporation
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 2 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 #
 #
 # AUTHORS:
 # Stefan Stanacar <stefanx.stanacar@intel.com>


 ME=$(basename $0)

 #
 # usage and setup
 #

 usage () {
 cat << EOT
 Usage: $ME [-h]
        $ME [-c <commit>] [-v] [-m <val>] [-j <val>] [-t <val>] [-i <image-name>] [-d <path>]
 Options:
         -h
                 Display this help and exit.
         -c <commit>
                 git checkout <commit> before anything else
         -v
                 Show bitbake output, don't redirect it to a log.
         -m <machine>
                 Value for MACHINE. Default is qemux86.
         -j <val>
                 Value for PARALLEL_MAKE. Default is 8.
         -t <val>
                 Value for BB_NUMBER_THREADS. Default is 8.
         -i <image-name>
                 Instead of timing against core-image-sato, use <image-name>
         -d <path>
                 Use <path> as DL_DIR
         -p <githash>
                 Cherry pick githash onto the commit

 Note: current working directory must be inside a poky git clone.

 EOT
 }


 if clonedir=$(git rev-parse --show-toplevel); then
         cd $clonedir
 else
         echo "The current working dir doesn't seem to be a poky git clone. Please cd there before running $ME"
         exit 1
 fi

 IMAGE="core-image-sato"
 verbose=0
 dldir=
 commit=
 pmake=
 cherrypicks=
 while getopts "hvc:m:j:t:i:d:p:" opt; do
         case $opt in
                 h)      usage
                         exit 0
                         ;;
                 v)      verbose=1
                         ;;
                 c)      commit=$OPTARG
                         ;;
                 m)      export MACHINE=$OPTARG
                         ;;
                 j)      pmake=$OPTARG
                         ;;
                 t)      export BB_NUMBER_THREADS=$OPTARG
                         ;;
                 i)      IMAGE=$OPTARG
                         ;;
                 d)      dldir=$OPTARG
                         ;;
                 p)      cherrypicks="$cherrypicks $OPTARG"
                         ;;
                 *)      usage
                         exit 1
                         ;;
         esac
 done


 #drop cached credentials and test for sudo access without a password
 sudo -k -n ls > /dev/null 2>&1
 reqpass=$?
 if [ $reqpass -ne 0 ]; then
     echo "The script requires sudo access to drop caches between builds (echo 3 > /proc/sys/vm/drop_caches)"
     read -s -p "Please enter your sudo password: " pass
     echo
 fi

 if [ -n "$commit" ]; then
             echo "git checkout -f $commit"
             git pull > /dev/null 2>&1
             git checkout -f $commit || exit 1
             git pull > /dev/null 2>&1
 fi

 if [ -n "$cherrypicks" ]; then
     for c in $cherrypicks; do
         git cherry-pick $c
     done
 fi

 rev=$(git rev-parse --short HEAD)  || exit 1
 OUTDIR="$clonedir/build-perf-test/results-$rev-`date "+%Y%m%d%H%M%S"`"
 BUILDDIR="$OUTDIR/build"
 resultsfile="$OUTDIR/results.log"
 cmdoutput="$OUTDIR/commands.log"
 myoutput="$OUTDIR/output.log"
 globalres="$clonedir/build-perf-test/globalres.log"

 mkdir -p $OUTDIR || exit 1

 log () {
     local msg="$1"
     echo "`date`: $msg" | tee -a $myoutput
 }


 #
 # Config stuff
 #

 branch=`git branch 2>&1 | grep "^* " | tr -d "* "`
 gitcommit=$(git rev-parse HEAD)  || exit 1
 log "Running on $branch:$gitcommit"

 source ./oe-init-build-env $OUTDIR/build >/dev/null || exit 1
 cd $OUTDIR/build

 [ -n "$MACHINE" ] || export MACHINE="qemux86"
 [ -n "$BB_NUMBER_THREADS" ] || export BB_NUMBER_THREADS="8"

 if [ -n "$pmake" ]; then
         export PARALLEL_MAKE="-j $pmake"
 else
         export PARALLEL_MAKE="-j 8"
 fi

 if [ -n "$dldir" ]; then
     echo "DL_DIR = \"$dldir\"" >> conf/local.conf
 else
     echo "DL_DIR = \"$clonedir/build-perf-test/downloads\"" >> conf/local.conf
 fi

 # Sometimes I've noticed big differences in timings for the same commit, on the same machine
 # Disabling the network sanity check helps a bit (because of my crappy network connection and/or proxy)
 echo "CONNECTIVITY_CHECK_URIS =\"\"" >> conf/local.conf


 #
 # Functions
 #

 declare -a TIMES
 time_count=0
 declare -a SIZES
 size_count=0

 time_cmd () {
     log "   Timing: $*"

     if [ $verbose -eq 0 ]; then
         /usr/bin/time -v -o $resultsfile "$@" >> $cmdoutput
     else
         /usr/bin/time -v -o $resultsfile "$@"
     fi
     ret=$?
     if [ $ret -eq 0 ]; then
         t=`grep wall $resultsfile | sed 's/.*m:ss): //'`
         log "   TIME: $t"
         TIMES[(( time_count++ ))]="$t"
     else
         log "ERROR: exit status was non-zero, will report time as 0."
         TIMES[(( time_count++ ))]="0"
     fi

     #time by default overwrites the output file and we  want to keep the results
     #it has an append option but I don't want to clobber the results in the same file
     i=`ls $OUTDIR/results.log* |wc -l`
     mv $resultsfile "${resultsfile}.${i}"
     log "More stats can be found in ${resultsfile}.${i}"
 }

 bbtime () {
     time_cmd bitbake "$@"
 }

 #we don't time bitbake here
 bbnotime () {
     local arg="$@"
     log "   Running: bitbake ${arg}"
     if [ $verbose -eq 0 ]; then
         bitbake ${arg} >> $cmdoutput
     else
         bitbake ${arg}
     fi
     ret=$?
     if [ $ret -eq 0 ]; then
         log "   Finished bitbake ${arg}"
     else
         log "ERROR: exit status was non-zero. Exit.."
         exit $ret
     fi

 }

 do_rmtmp() {
     log "   Removing tmp"
     rm -rf bitbake.lock pseudodone conf/sanity_info cache tmp
 }
 do_rmsstate () {
     log "   Removing sstate-cache"
     rm -rf sstate-cache
 }
 do_sync () {
     log "   Syncing and dropping caches"
     sync; sync
     if [ $reqpass -eq 0 ]; then
         sudo sh -c "echo 3 > /proc/sys/vm/drop_caches"
     else
         echo "$pass" | sudo -S sh -c "echo 3 > /proc/sys/vm/drop_caches"
         echo
     fi
     sleep 3
 }

 write_results() {
     echo -n "`uname -n`,$branch:$gitcommit,`git describe`," >> $globalres
     for i in "${TIMES[@]}"; do
         echo -n "$i," >> $globalres
     done
     for i in "${SIZES[@]}"; do
         echo -n "$i," >> $globalres
     done
     echo >> $globalres
     sed -i '$ s/,$//' $globalres
 }

 ####

 #
 # Test 1
 # Measure: Wall clock of "bitbake core-image-sato" and size of tmp/dir (w/o rm_work and w/ rm_work)
 # Pre: Downloaded sources, no sstate
 # Steps:
 #     Part1:
 #        - fetchall
 #        - clean build dir
 #        - time bitbake core-image-sato
 #        - collect data
 #     Part2:
 #        - bitbake virtual/kernel -c cleansstate
 #        - time bitbake virtual/kernel
 #     Part3:
 #        - add INHERIT to local.conf
 #        - clean build dir
 #        - build
 #        - report size, remove INHERIT

 test1_p1 () {
     log "Running Test 1, part 1/3: Measure wall clock of bitbake $IMAGE and size of tmp/ dir"
     bbnotime $IMAGE --runall=fetch
     do_rmtmp
     do_rmsstate
     do_sync
     bbtime $IMAGE
     s=`du -s tmp | sed 's/tmp//' | sed 's/[ \t]*$//'`
     SIZES[(( size_count++ ))]="$s"
     log "SIZE of tmp dir is: $s"
     log "Buildstats are saved in $OUTDIR/buildstats-test1"
     mv tmp/buildstats $OUTDIR/buildstats-test1
 }


 test1_p2 () {
     log "Running Test 1, part 2/3: bitbake virtual/kernel -c cleansstate and time bitbake virtual/kernel"
     bbnotime virtual/kernel -c cleansstate
     do_sync
     bbtime virtual/kernel
 }

 test1_p3 () {
     log "Running Test 1, part 3/3: Build $IMAGE w/o sstate and report size of tmp/dir with rm_work enabled"
     echo "INHERIT += \"rm_work\"" >> conf/local.conf
     do_rmtmp
     do_rmsstate
     do_sync
     bbtime $IMAGE
     sed -i 's/INHERIT += \"rm_work\"//' conf/local.conf
     s=`du -s tmp | sed 's/tmp//' | sed 's/[ \t]*$//'`
     SIZES[(( size_count++ ))]="$s"
     log "SIZE of tmp dir is: $s"
     log "Buildstats are saved in $OUTDIR/buildstats-test13"
     mv tmp/buildstats $OUTDIR/buildstats-test13
 }


 #
 # Test 2
 # Measure: Wall clock of "bitbake core-image-sato" and size of tmp/dir
 # Pre: populated sstate cache

 test2 () {
     # Assuming test 1 has run
     log "Running Test 2: Measure wall clock of bitbake $IMAGE -c rootfs with sstate"
     do_rmtmp
     do_sync
     bbtime $IMAGE -c rootfs
 }


 # Test 3
 # parsing time metrics
 #
 #  Start with
 #   i) "rm -rf tmp/cache; time bitbake -p"
 #  ii) "rm -rf tmp/cache/default-glibc/; time bitbake -p"
 # iii) "time bitbake -p"


 test3 () {
     log "Running Test 3: Parsing time metrics (bitbake -p)"
     log "   Removing tmp/cache && cache"
     rm -rf tmp/cache cache
     bbtime -p
     log "   Removing tmp/cache/default-glibc/"
     rm -rf tmp/cache/default-glibc/
     bbtime -p
     bbtime -p
 }

 #
 # Test 4 - eSDK
 # Measure: eSDK size and installation time
 test4 () {
     log "Running Test 4: eSDK size and installation time"
     bbnotime $IMAGE -c do_populate_sdk_ext

     esdk_installer=(tmp/deploy/sdk/*-toolchain-ext-*.sh)

     if [ ${#esdk_installer[*]} -eq 1 ]; then
         s=$((`stat -c %s "$esdk_installer"` / 1024))
         SIZES[(( size_count++ ))]="$s"
         log "Download SIZE of eSDK is: $s kB"

         do_sync
         time_cmd "$esdk_installer" -y -d "tmp/esdk-deploy"

         s=$((`du -sb "tmp/esdk-deploy" | cut -f1` / 1024))
         SIZES[(( size_count++ ))]="$s"
         log "Install SIZE of eSDK is: $s kB"
     else
         log "ERROR: other than one sdk found (${esdk_installer[*]}), reporting size and time as 0."
         SIZES[(( size_count++ ))]="0"
         TIMES[(( time_count++ ))]="0"
     fi

 }


 # RUN!

 test1_p1
 test1_p2
 test1_p3
 test2
 test3
 test4

 # if we got til here write to global results
 write_results

 log "All done, cleaning up..."

 do_rmtmp
 do_rmsstate
	#!/bin/bash
	#
	# This script runs a series of tests (with and without sstate) and reports build time (and tmp/ size)
	#
	# Build performance test script
	#
	# Copyright 2013 Intel Corporation
	#
	# This program is free software; you can redistribute it and/or modify
	# it under the terms of the GNU General Public License as published by
	# the Free Software Foundation; either version 2 of the License, or
	# (at your option) any later version.
	#
	# This program is distributed in the hope that it will be useful,
	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	# GNU General Public License for more details.
	#
	# You should have received a copy of the GNU General Public License
	# along with this program; if not, write to the Free Software
	# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	#
	#
	# AUTHORS:
	# Stefan Stanacar <stefanx.stanacar@intel.com>


	ME=$(basename $0)

	#
	# usage and setup
	#

	usage () {
	cat << EOT
	Usage: $ME [-h]
	$ME [-c <commit>] [-v] [-m <val>] [-j <val>] [-t <val>] [-i <image-name>] [-d <path>]
	Options:
	-h
	Display this help and exit.
	-c <commit>
	git checkout <commit> before anything else
	-v
	Show bitbake output, don't redirect it to a log.
	-m <machine>
	Value for MACHINE. Default is qemux86.
	-j <val>
	Value for PARALLEL_MAKE. Default is 8.
	-t <val>
	Value for BB_NUMBER_THREADS. Default is 8.
	-i <image-name>
	Instead of timing against core-image-sato, use <image-name>
	-d <path>
	Use <path> as DL_DIR
	-p <githash>
	Cherry pick githash onto the commit

	Note: current working directory must be inside a poky git clone.

	EOT
	}


	if clonedir=$(git rev-parse --show-toplevel); then
	cd $clonedir
	else
	echo "The current working dir doesn't seem to be a poky git clone. Please cd there before running $ME"
	exit 1
	fi

	IMAGE="core-image-sato"
	verbose=0
	dldir=
	commit=
	pmake=
	cherrypicks=
	while getopts "hvc:m:j:t:i:d:p:" opt; do
	case $opt in
	h) usage
	exit 0
	;;
	v) verbose=1
	;;
	c) commit=$OPTARG
	;;
	m) export MACHINE=$OPTARG
	;;
	j) pmake=$OPTARG
	;;
	t) export BB_NUMBER_THREADS=$OPTARG
	;;
	i) IMAGE=$OPTARG
	;;
	d) dldir=$OPTARG
	;;
	p) cherrypicks="$cherrypicks $OPTARG"
	;;
	*) usage
	exit 1
	;;
	esac
	done


	#drop cached credentials and test for sudo access without a password
	sudo -k -n ls > /dev/null 2>&1
	reqpass=$?
	if [ $reqpass -ne 0 ]; then
	echo "The script requires sudo access to drop caches between builds (echo 3 > /proc/sys/vm/drop_caches)"
	read -s -p "Please enter your sudo password: " pass
	echo
	fi

	if [ -n "$commit" ]; then
	echo "git checkout -f $commit"
	git pull > /dev/null 2>&1
	git checkout -f $commit \|\| exit 1
	git pull > /dev/null 2>&1
	fi

	if [ -n "$cherrypicks" ]; then
	for c in $cherrypicks; do
	git cherry-pick $c
	done
	fi

	rev=$(git rev-parse --short HEAD) \|\| exit 1
	OUTDIR="$clonedir/build-perf-test/results-$rev-`date "+%Y%m%d%H%M%S"`"
	BUILDDIR="$OUTDIR/build"
	resultsfile="$OUTDIR/results.log"
	cmdoutput="$OUTDIR/commands.log"
	myoutput="$OUTDIR/output.log"
	globalres="$clonedir/build-perf-test/globalres.log"

	mkdir -p $OUTDIR \|\| exit 1

	log () {
	local msg="$1"
	echo "`date`: $msg" \| tee -a $myoutput
	}


	#
	# Config stuff
	#

	branch=`git branch 2>&1 \| grep "^* " \| tr -d "* "`
	gitcommit=$(git rev-parse HEAD) \|\| exit 1
	log "Running on $branch:$gitcommit"

	source ./oe-init-build-env $OUTDIR/build >/dev/null \|\| exit 1
	cd $OUTDIR/build

	[ -n "$MACHINE" ] \|\| export MACHINE="qemux86"
	[ -n "$BB_NUMBER_THREADS" ] \|\| export BB_NUMBER_THREADS="8"

	if [ -n "$pmake" ]; then
	export PARALLEL_MAKE="-j $pmake"
	else
	export PARALLEL_MAKE="-j 8"
	fi

	if [ -n "$dldir" ]; then
	echo "DL_DIR = \"$dldir\"" >> conf/local.conf
	else
	echo "DL_DIR = \"$clonedir/build-perf-test/downloads\"" >> conf/local.conf
	fi

	# Sometimes I've noticed big differences in timings for the same commit, on the same machine
	# Disabling the network sanity check helps a bit (because of my crappy network connection and/or proxy)
	echo "CONNECTIVITY_CHECK_URIS =\"\"" >> conf/local.conf


	#
	# Functions
	#

	declare -a TIMES
	time_count=0
	declare -a SIZES
	size_count=0

	time_cmd () {
	log " Timing: $*"

	if [ $verbose -eq 0 ]; then
	/usr/bin/time -v -o $resultsfile "$@" >> $cmdoutput
	else
	/usr/bin/time -v -o $resultsfile "$@"
	fi
	ret=$?
	if [ $ret -eq 0 ]; then
	t=`grep wall $resultsfile \| sed 's/.*m:ss): //'`
	log " TIME: $t"
	TIMES[(( time_count++ ))]="$t"
	else
	log "ERROR: exit status was non-zero, will report time as 0."
	TIMES[(( time_count++ ))]="0"
	fi

	#time by default overwrites the output file and we want to keep the results
	#it has an append option but I don't want to clobber the results in the same file
	i=`ls $OUTDIR/results.log* \|wc -l`
	mv $resultsfile "${resultsfile}.${i}"
	log "More stats can be found in ${resultsfile}.${i}"
	}

	bbtime () {
	time_cmd bitbake "$@"
	}

	#we don't time bitbake here
	bbnotime () {
	local arg="$@"
	log " Running: bitbake ${arg}"
	if [ $verbose -eq 0 ]; then
	bitbake ${arg} >> $cmdoutput
	else
	bitbake ${arg}
	fi
	ret=$?
	if [ $ret -eq 0 ]; then
	log " Finished bitbake ${arg}"
	else
	log "ERROR: exit status was non-zero. Exit.."
	exit $ret
	fi

	}

	do_rmtmp() {
	log " Removing tmp"
	rm -rf bitbake.lock pseudodone conf/sanity_info cache tmp
	}
	do_rmsstate () {
	log " Removing sstate-cache"
	rm -rf sstate-cache
	}
	do_sync () {
	log " Syncing and dropping caches"
	sync; sync
	if [ $reqpass -eq 0 ]; then
	sudo sh -c "echo 3 > /proc/sys/vm/drop_caches"
	else
	echo "$pass" \| sudo -S sh -c "echo 3 > /proc/sys/vm/drop_caches"
	echo
	fi
	sleep 3
	}

	write_results() {
	echo -n "`uname -n`,$branch:$gitcommit,`git describe`," >> $globalres
	for i in "${TIMES[@]}"; do
	echo -n "$i," >> $globalres
	done
	for i in "${SIZES[@]}"; do
	echo -n "$i," >> $globalres
	done
	echo >> $globalres
	sed -i '$ s/,$//' $globalres
	}

	####

	#
	# Test 1
	# Measure: Wall clock of "bitbake core-image-sato" and size of tmp/dir (w/o rm_work and w/ rm_work)
	# Pre: Downloaded sources, no sstate
	# Steps:
	# Part1:
	# - fetchall
	# - clean build dir
	# - time bitbake core-image-sato
	# - collect data
	# Part2:
	# - bitbake virtual/kernel -c cleansstate
	# - time bitbake virtual/kernel
	# Part3:
	# - add INHERIT to local.conf
	# - clean build dir
	# - build
	# - report size, remove INHERIT

	test1_p1 () {
	log "Running Test 1, part 1/3: Measure wall clock of bitbake $IMAGE and size of tmp/ dir"
	bbnotime $IMAGE --runall=fetch
	do_rmtmp
	do_rmsstate
	do_sync
	bbtime $IMAGE
	s=`du -s tmp \| sed 's/tmp//' \| sed 's/[ \t]*$//'`
	SIZES[(( size_count++ ))]="$s"
	log "SIZE of tmp dir is: $s"
	log "Buildstats are saved in $OUTDIR/buildstats-test1"
	mv tmp/buildstats $OUTDIR/buildstats-test1
	}


	test1_p2 () {
	log "Running Test 1, part 2/3: bitbake virtual/kernel -c cleansstate and time bitbake virtual/kernel"
	bbnotime virtual/kernel -c cleansstate
	do_sync
	bbtime virtual/kernel
	}

	test1_p3 () {
	log "Running Test 1, part 3/3: Build $IMAGE w/o sstate and report size of tmp/dir with rm_work enabled"
	echo "INHERIT += \"rm_work\"" >> conf/local.conf
	do_rmtmp
	do_rmsstate
	do_sync
	bbtime $IMAGE
	sed -i 's/INHERIT += \"rm_work\"//' conf/local.conf
	s=`du -s tmp \| sed 's/tmp//' \| sed 's/[ \t]*$//'`
	SIZES[(( size_count++ ))]="$s"
	log "SIZE of tmp dir is: $s"
	log "Buildstats are saved in $OUTDIR/buildstats-test13"
	mv tmp/buildstats $OUTDIR/buildstats-test13
	}


	#
	# Test 2
	# Measure: Wall clock of "bitbake core-image-sato" and size of tmp/dir
	# Pre: populated sstate cache

	test2 () {
	# Assuming test 1 has run
	log "Running Test 2: Measure wall clock of bitbake $IMAGE -c rootfs with sstate"
	do_rmtmp
	do_sync
	bbtime $IMAGE -c rootfs
	}


	# Test 3
	# parsing time metrics
	#
	# Start with
	# i) "rm -rf tmp/cache; time bitbake -p"
	# ii) "rm -rf tmp/cache/default-glibc/; time bitbake -p"
	# iii) "time bitbake -p"


	test3 () {
	log "Running Test 3: Parsing time metrics (bitbake -p)"
	log " Removing tmp/cache && cache"
	rm -rf tmp/cache cache
	bbtime -p
	log " Removing tmp/cache/default-glibc/"
	rm -rf tmp/cache/default-glibc/
	bbtime -p
	bbtime -p
	}

	#
	# Test 4 - eSDK
	# Measure: eSDK size and installation time
	test4 () {
	log "Running Test 4: eSDK size and installation time"
	bbnotime $IMAGE -c do_populate_sdk_ext

	esdk_installer=(tmp/deploy/sdk/-toolchain-ext-.sh)

	if [ ${#esdk_installer[*]} -eq 1 ]; then
	s=$((`stat -c %s "$esdk_installer"` / 1024))
	SIZES[(( size_count++ ))]="$s"
	log "Download SIZE of eSDK is: $s kB"

	do_sync
	time_cmd "$esdk_installer" -y -d "tmp/esdk-deploy"

	s=$((`du -sb "tmp/esdk-deploy" \| cut -f1` / 1024))
	SIZES[(( size_count++ ))]="$s"
	log "Install SIZE of eSDK is: $s kB"
	else
	log "ERROR: other than one sdk found (${esdk_installer[*]}), reporting size and time as 0."
	SIZES[(( size_count++ ))]="0"
	TIMES[(( time_count++ ))]="0"
	fi

	}


	# RUN!

	test1_p1
	test1_p2
	test1_p3
	test2
	test3
	test4

	# if we got til here write to global results
	write_results

	log "All done, cleaning up..."

	do_rmtmp
	do_rmsstate