poky/scripts/contrib/test_build_time.sh - mdmillerii/openbmc - Gitiles

 #!/bin/bash

 # Build performance regression test script
 #
 # Copyright 2011 Intel Corporation
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 #
 # DESCRIPTION
 # This script is intended to be used in conjunction with "git bisect run"
 # in order to find regressions in build time, however it can also be used
 # independently. It cleans out the build output directories, runs a
 # specified worker script (an example is test_build_time_worker.sh) under
 # TIME(1), logs the results to TEST_LOGDIR (default /tmp) and returns a
 # value telling "git bisect run" whether the build time is good (under
 # the specified threshold) or bad (over it). There is also a tolerance
 # option but it is not particularly useful as it only subtracts the
 # tolerance from the given threshold and uses it as the actual threshold.
 #
 # It is also capable of taking a file listing git revision hashes to be
 # test-applied to the repository in order to get past build failures that
 # would otherwise cause certain revisions to have to be skipped; if a
 # revision does not apply cleanly then the script assumes it does not
 # need to be applied and ignores it.
 #
 # Please see the help output (syntax below) for some important setup
 # instructions.
 #
 # AUTHORS
 # Paul Eggleton <paul.eggleton@linux.intel.com>


 syntax() {
     echo "syntax: $0 <script> <time> <tolerance> [patchrevlist]"
     echo ""
     echo "  script       - worker script file (if in current dir, prefix with ./)"
     echo "  time         - time threshold (in seconds, suffix m for minutes)"
     echo "  tolerance    - tolerance (in seconds, suffix m for minutes or % for"
     echo "                 percentage, can be 0)"
     echo "  patchrevlist - optional file listing revisions to apply as patches on top"
     echo ""
     echo "You must set TEST_BUILDDIR to point to a previously created build directory,"
     echo "however please note that this script will wipe out the TMPDIR defined in"
     echo "TEST_BUILDDIR/conf/local.conf as part of its initial setup (as well as your"
     echo "~/.ccache)"
     echo ""
     echo "To get rid of the sudo prompt, please add the following line to /etc/sudoers"
     echo "(use 'visudo' to edit this; also it is assumed that the user you are running"
     echo "as is a member of the 'wheel' group):"
     echo ""
     echo "%wheel ALL=(ALL) NOPASSWD: /sbin/sysctl -w vm.drop_caches=[1-3]"
     echo ""
     echo "Note: it is recommended that you disable crond and any other process that"
     echo "may cause significant CPU or I/O usage during build performance tests."
 }

 # Note - we exit with 250 here because that will tell git bisect run that
 # something bad happened and stop
 if [ "$1" = "" ] ; then
    syntax
    exit 250
 fi

 if [ "$2" = "" ] ; then
    syntax
    exit 250
 fi

 if [ "$3" = "" ] ; then
    syntax
    exit 250
 fi

 if ! [[ "$2" =~ ^[0-9][0-9m.]*$ ]] ; then
    echo "'$2' is not a valid number for threshold"
    exit 250
 fi

 if ! [[ "$3" =~ ^[0-9][0-9m.%]*$ ]] ; then
    echo "'$3' is not a valid number for tolerance"
    exit 250
 fi

 if [ "$TEST_BUILDDIR" = "" ] ; then
    echo "Please set TEST_BUILDDIR to a previously created build directory"
    exit 250
 fi

 if [ ! -d "$TEST_BUILDDIR" ] ; then
    echo "TEST_BUILDDIR $TEST_BUILDDIR not found"
    exit 250
 fi

 git diff --quiet
 if [ $? != 0 ] ; then
     echo "Working tree is dirty, cannot proceed"
     exit 251
 fi

 if [ "$BB_ENV_EXTRAWHITE" != "" ] ; then
    echo "WARNING: you are running after sourcing the build environment script, this is not recommended"
 fi

 runscript=$1
 timethreshold=$2
 tolerance=$3

 if [ "$4" != "" ] ; then
     patchrevlist=`cat $4`
 else
     patchrevlist=""
 fi

 if [[ timethreshold == *m* ]] ; then
     timethreshold=`echo $timethreshold | sed s/m/*60/ | bc`
 fi

 if [[ $tolerance == *m* ]] ; then
     tolerance=`echo $tolerance | sed s/m/*60/ | bc`
 elif [[ $tolerance == *%* ]] ; then
     tolerance=`echo $tolerance | sed s/%//`
     tolerance=`echo "scale = 2; (($tolerance * $timethreshold) / 100)" | bc`
 fi

 tmpdir=`grep "^TMPDIR" $TEST_BUILDDIR/conf/local.conf | sed -e 's/TMPDIR[ \t]*=[ \t\?]*"//' -e 's/"//'`
 if [ "x$tmpdir" = "x" ]; then
     echo "Unable to determine TMPDIR from $TEST_BUILDDIR/conf/local.conf, bailing out"
     exit 250
 fi
 sstatedir=`grep "^SSTATE_DIR" $TEST_BUILDDIR/conf/local.conf | sed -e 's/SSTATE_DIR[ \t\?]*=[ \t]*"//' -e 's/"//'`
 if [ "x$sstatedir" = "x" ]; then
     echo "Unable to determine SSTATE_DIR from $TEST_BUILDDIR/conf/local.conf, bailing out"
     exit 250
 fi

 if [ `expr length $tmpdir` -lt 4 ] ; then
     echo "TMPDIR $tmpdir is less than 4 characters, bailing out"
     exit 250
 fi

 if [ `expr length $sstatedir` -lt 4 ] ; then
     echo "SSTATE_DIR $sstatedir is less than 4 characters, bailing out"
     exit 250
 fi

 echo -n "About to wipe out TMPDIR $tmpdir, press Ctrl+C to break out...  "
 for i in 9 8 7 6 5 4 3 2 1
 do
     echo -ne "\x08$i"
     sleep 1
 done
 echo

 pushd . > /dev/null

 rm -f pseudodone
 echo "Removing TMPDIR $tmpdir..."
 rm -rf $tmpdir
 echo "Removing TMPDIR $tmpdir-*libc..."
 rm -rf $tmpdir-*libc
 echo "Removing SSTATE_DIR $sstatedir..."
 rm -rf $sstatedir
 echo "Removing ~/.ccache..."
 rm -rf ~/.ccache

 echo "Syncing..."
 sync
 sync
 echo "Dropping VM cache..."
 #echo 3 > /proc/sys/vm/drop_caches
 sudo /sbin/sysctl -w vm.drop_caches=3 > /dev/null

 if [ "$TEST_LOGDIR" = "" ] ; then
     logdir="/tmp"
 else
     logdir="$TEST_LOGDIR"
 fi
 rev=`git rev-parse HEAD`
 logfile="$logdir/timelog_$rev.log"
 echo -n > $logfile

 gitroot=`git rev-parse --show-toplevel`
 cd $gitroot
 for patchrev in $patchrevlist ; do
     echo "Applying $patchrev"
     patchfile=`mktemp`
     git show $patchrev > $patchfile
     git apply --check $patchfile &> /dev/null
     if [ $? != 0 ] ; then
         echo " ... patch does not apply without errors, ignoring"
     else
         echo "Applied $patchrev" >> $logfile
         git apply $patchfile &> /dev/null
     fi
     rm $patchfile
 done

 sync
 echo "Quiescing for 5s..."
 sleep 5

 echo "Running $runscript at $rev..."
 timeoutfile=`mktemp`
 /usr/bin/time -o $timeoutfile -f "%e\nreal\t%E\nuser\t%Us\nsys\t%Ss\nmaxm\t%Mk" $runscript 2>&1 | tee -a $logfile
 exitstatus=$PIPESTATUS

 git reset --hard HEAD > /dev/null
 popd > /dev/null

 timeresult=`head -n1 $timeoutfile`
 cat $timeoutfile | tee -a $logfile
 rm $timeoutfile

 if [ $exitstatus != 0 ] ; then
     # Build failed, exit with 125 to tell git bisect run to skip this rev
     echo "*** Build failed (exit code $exitstatus), skipping..." | tee -a $logfile
     exit 125
 fi

 ret=`echo "scale = 2; $timeresult > $timethreshold - $tolerance" | bc`
 echo "Returning $ret" | tee -a $logfile
 exit $ret
	#!/bin/bash

	# Build performance regression test script
	#
	# Copyright 2011 Intel Corporation
	#
	# SPDX-License-Identifier: GPL-2.0-or-later
	#
	# DESCRIPTION
	# This script is intended to be used in conjunction with "git bisect run"
	# in order to find regressions in build time, however it can also be used
	# independently. It cleans out the build output directories, runs a
	# specified worker script (an example is test_build_time_worker.sh) under
	# TIME(1), logs the results to TEST_LOGDIR (default /tmp) and returns a
	# value telling "git bisect run" whether the build time is good (under
	# the specified threshold) or bad (over it). There is also a tolerance
	# option but it is not particularly useful as it only subtracts the
	# tolerance from the given threshold and uses it as the actual threshold.
	#
	# It is also capable of taking a file listing git revision hashes to be
	# test-applied to the repository in order to get past build failures that
	# would otherwise cause certain revisions to have to be skipped; if a
	# revision does not apply cleanly then the script assumes it does not
	# need to be applied and ignores it.
	#
	# Please see the help output (syntax below) for some important setup
	# instructions.
	#
	# AUTHORS
	# Paul Eggleton <paul.eggleton@linux.intel.com>


	syntax() {
	echo "syntax: $0 <script> <time> <tolerance> [patchrevlist]"
	echo ""
	echo " script - worker script file (if in current dir, prefix with ./)"
	echo " time - time threshold (in seconds, suffix m for minutes)"
	echo " tolerance - tolerance (in seconds, suffix m for minutes or % for"
	echo " percentage, can be 0)"
	echo " patchrevlist - optional file listing revisions to apply as patches on top"
	echo ""
	echo "You must set TEST_BUILDDIR to point to a previously created build directory,"
	echo "however please note that this script will wipe out the TMPDIR defined in"
	echo "TEST_BUILDDIR/conf/local.conf as part of its initial setup (as well as your"
	echo "~/.ccache)"
	echo ""
	echo "To get rid of the sudo prompt, please add the following line to /etc/sudoers"
	echo "(use 'visudo' to edit this; also it is assumed that the user you are running"
	echo "as is a member of the 'wheel' group):"
	echo ""
	echo "%wheel ALL=(ALL) NOPASSWD: /sbin/sysctl -w vm.drop_caches=[1-3]"
	echo ""
	echo "Note: it is recommended that you disable crond and any other process that"
	echo "may cause significant CPU or I/O usage during build performance tests."
	}

	# Note - we exit with 250 here because that will tell git bisect run that
	# something bad happened and stop
	if [ "$1" = "" ] ; then
	syntax
	exit 250
	fi

	if [ "$2" = "" ] ; then
	syntax
	exit 250
	fi

	if [ "$3" = "" ] ; then
	syntax
	exit 250
	fi

	if ! [[ "$2" =~ ^[0-9][0-9m.]*$ ]] ; then
	echo "'$2' is not a valid number for threshold"
	exit 250
	fi

	if ! [[ "$3" =~ ^[0-9][0-9m.%]*$ ]] ; then
	echo "'$3' is not a valid number for tolerance"
	exit 250
	fi

	if [ "$TEST_BUILDDIR" = "" ] ; then
	echo "Please set TEST_BUILDDIR to a previously created build directory"
	exit 250
	fi

	if [ ! -d "$TEST_BUILDDIR" ] ; then
	echo "TEST_BUILDDIR $TEST_BUILDDIR not found"
	exit 250
	fi

	git diff --quiet
	if [ $? != 0 ] ; then
	echo "Working tree is dirty, cannot proceed"
	exit 251
	fi

	if [ "$BB_ENV_EXTRAWHITE" != "" ] ; then
	echo "WARNING: you are running after sourcing the build environment script, this is not recommended"
	fi

	runscript=$1
	timethreshold=$2
	tolerance=$3

	if [ "$4" != "" ] ; then
	patchrevlist=`cat $4`
	else
	patchrevlist=""
	fi

	if [[ timethreshold == m ]] ; then
	timethreshold=`echo $timethreshold \| sed s/m/*60/ \| bc`
	fi

	if [[ $tolerance == m ]] ; then
	tolerance=`echo $tolerance \| sed s/m/*60/ \| bc`
	elif [[ $tolerance == % ]] ; then
	tolerance=`echo $tolerance \| sed s/%//`
	tolerance=`echo "scale = 2; (($tolerance * $timethreshold) / 100)" \| bc`
	fi

	tmpdir=`grep "^TMPDIR" $TEST_BUILDDIR/conf/local.conf \| sed -e 's/TMPDIR[ \t]=[ \t\?]"//' -e 's/"//'`
	if [ "x$tmpdir" = "x" ]; then
	echo "Unable to determine TMPDIR from $TEST_BUILDDIR/conf/local.conf, bailing out"
	exit 250
	fi
	sstatedir=`grep "^SSTATE_DIR" $TEST_BUILDDIR/conf/local.conf \| sed -e 's/SSTATE_DIR[ \t\?]=[ \t]"//' -e 's/"//'`
	if [ "x$sstatedir" = "x" ]; then
	echo "Unable to determine SSTATE_DIR from $TEST_BUILDDIR/conf/local.conf, bailing out"
	exit 250
	fi

	if [ `expr length $tmpdir` -lt 4 ] ; then
	echo "TMPDIR $tmpdir is less than 4 characters, bailing out"
	exit 250
	fi

	if [ `expr length $sstatedir` -lt 4 ] ; then
	echo "SSTATE_DIR $sstatedir is less than 4 characters, bailing out"
	exit 250
	fi

	echo -n "About to wipe out TMPDIR $tmpdir, press Ctrl+C to break out... "
	for i in 9 8 7 6 5 4 3 2 1
	do
	echo -ne "\x08$i"
	sleep 1
	done
	echo

	pushd . > /dev/null

	rm -f pseudodone
	echo "Removing TMPDIR $tmpdir..."
	rm -rf $tmpdir
	echo "Removing TMPDIR $tmpdir-*libc..."
	rm -rf $tmpdir-*libc
	echo "Removing SSTATE_DIR $sstatedir..."
	rm -rf $sstatedir
	echo "Removing ~/.ccache..."
	rm -rf ~/.ccache

	echo "Syncing..."
	sync
	sync
	echo "Dropping VM cache..."
	#echo 3 > /proc/sys/vm/drop_caches
	sudo /sbin/sysctl -w vm.drop_caches=3 > /dev/null

	if [ "$TEST_LOGDIR" = "" ] ; then
	logdir="/tmp"
	else
	logdir="$TEST_LOGDIR"
	fi
	rev=`git rev-parse HEAD`
	logfile="$logdir/timelog_$rev.log"
	echo -n > $logfile

	gitroot=`git rev-parse --show-toplevel`
	cd $gitroot
	for patchrev in $patchrevlist ; do
	echo "Applying $patchrev"
	patchfile=`mktemp`
	git show $patchrev > $patchfile
	git apply --check $patchfile &> /dev/null
	if [ $? != 0 ] ; then
	echo " ... patch does not apply without errors, ignoring"
	else
	echo "Applied $patchrev" >> $logfile
	git apply $patchfile &> /dev/null
	fi
	rm $patchfile
	done

	sync
	echo "Quiescing for 5s..."
	sleep 5

	echo "Running $runscript at $rev..."
	timeoutfile=`mktemp`
	/usr/bin/time -o $timeoutfile -f "%e\nreal\t%E\nuser\t%Us\nsys\t%Ss\nmaxm\t%Mk" $runscript 2>&1 \| tee -a $logfile
	exitstatus=$PIPESTATUS

	git reset --hard HEAD > /dev/null
	popd > /dev/null

	timeresult=`head -n1 $timeoutfile`
	cat $timeoutfile \| tee -a $logfile
	rm $timeoutfile

	if [ $exitstatus != 0 ] ; then
	# Build failed, exit with 125 to tell git bisect run to skip this rev
	echo "*** Build failed (exit code $exitstatus), skipping..." \| tee -a $logfile
	exit 125
	fi

	ret=`echo "scale = 2; $timeresult > $timethreshold - $tolerance" \| bc`
	echo "Returning $ret" \| tee -a $logfile
	exit $ret