| #!/bin/bash |
| |
| # Author: Martin Jansa <martin.jansa@gmail.com> |
| # |
| # Copyright (c) 2013 Martin Jansa <Martin.Jansa@gmail.com> |
| |
| # Used to detect missing dependencies or automagically |
| # enabled dependencies which aren't explicitly enabled |
| # or disabled. Using bash to have PIPESTATUS variable. |
| |
| # It does 3 builds of <target> |
| # 1st to populate sstate-cache directory and sysroot |
| # 2nd to rebuild each recipe with every possible |
| # dependency found in sysroot (which stays populated |
| # from 1st build |
| # 3rd to rebuild each recipe only with dependencies defined |
| # in DEPENDS |
| # 4th (optional) repeat build like 3rd to make sure that |
| # minimal versions of dependencies defined in DEPENDS |
| # is also enough |
| |
| # Global vars |
| tmpdir= |
| targets= |
| recipes= |
| buildhistory= |
| buildtype= |
| default_targets="world" |
| default_buildhistory="buildhistory" |
| default_buildtype="1 2 3 c" |
| |
| usage () { |
| cat << EOF |
| Welcome to utility to detect missing or autoenabled dependencies. |
| WARNING: this utility will completely remove your tmpdir (make sure |
| you don't have important buildhistory or persistent dir there). |
| $0 <OPTION> |
| |
| Options: |
| -h, --help |
| Display this help and exit. |
| |
| --tmpdir=<tmpdir> |
| Specify tmpdir, will use the environment variable TMPDIR if it is not specified. |
| Something like /OE/oe-core/tmp-eglibc (no / at the end). |
| |
| --targets=<targets> |
| List of targets separated by space, will use the environment variable TARGETS if it is not specified. |
| It will run "bitbake <targets>" to populate sysroots. |
| Default value is "world". |
| |
| --recipes=<recipes> |
| File with list of recipes we want to rebuild with minimal and maximal sysroot. |
| Will use the environment variable RECIPES if it is not specified. |
| Default value will use all packages ever recorded in buildhistory directory. |
| |
| --buildhistory=<buildhistory> |
| Path to buildhistory directory, it needs to be enabled in your config, |
| because it's used to detect different dependencies and to create list |
| of recipes to rebuild when it's not specified. |
| Will use the environment variable BUILDHISTORY if it is not specified. |
| Default value is "buildhistory" |
| |
| --buildtype=<buildtype> |
| There are 4 types of build: |
| 1: build to populate sstate-cache directory and sysroot |
| 2: build to rebuild each recipe with every possible dep |
| 3: build to rebuild each recipe with minimal dependencies |
| 4: build to rebuild each recipe again with minimal dependencies |
| c: compare buildhistory directories from build 2 and 3 |
| Will use the environment variable BUILDTYPE if it is not specified. |
| Default value is "1 2 3 c", order is important, type 4 is optional. |
| EOF |
| } |
| |
| # Print error information and exit. |
| echo_error () { |
| echo "ERROR: $1" >&2 |
| exit 1 |
| } |
| |
| while [ -n "$1" ]; do |
| case $1 in |
| --tmpdir=*) |
| tmpdir=`echo $1 | sed -e 's#^--tmpdir=##' | xargs readlink -e` |
| [ -d "$tmpdir" ] || echo_error "Invalid argument to --tmpdir" |
| shift |
| ;; |
| --targets=*) |
| targets=`echo $1 | sed -e 's#^--targets="*\([^"]*\)"*#\1#'` |
| shift |
| ;; |
| --recipes=*) |
| recipes=`echo $1 | sed -e 's#^--recipes="*\([^"]*\)"*#\1#'` |
| shift |
| ;; |
| --buildhistory=*) |
| buildhistory=`echo $1 | sed -e 's#^--buildhistory="*\([^"]*\)"*#\1#'` |
| shift |
| ;; |
| --buildtype=*) |
| buildtype=`echo $1 | sed -e 's#^--buildtype="*\([^"]*\)"*#\1#'` |
| shift |
| ;; |
| --help|-h) |
| usage |
| exit 0 |
| ;; |
| *) |
| echo "Invalid arguments $*" |
| echo_error "Try '$0 -h' for more information." |
| ;; |
| esac |
| done |
| |
| # tmpdir directory, use environment variable TMPDIR |
| # if it was not specified, otherwise, error. |
| [ -n "$tmpdir" ] || tmpdir=$TMPDIR |
| [ -n "$tmpdir" ] || echo_error "No tmpdir found!" |
| [ -d "$tmpdir" ] || echo_error "Invalid tmpdir \"$tmpdir\"" |
| [ -n "$targets" ] || targets=$TARGETS |
| [ -n "$targets" ] || targets=$default_targets |
| [ -n "$recipes" ] || recipes=$RECIPES |
| [ -n "$recipes" -a ! -f "$recipes" ] && echo_error "Invalid file with list of recipes to rebuild" |
| [ -n "$recipes" ] || echo "All packages ever recorded in buildhistory directory will be rebuilt" |
| [ -n "$buildhistory" ] || buildhistory=$BUILDHISTORY |
| [ -n "$buildhistory" ] || buildhistory=$default_buildhistory |
| [ -d "$buildhistory" ] || echo_error "Invalid buildhistory directory \"$buildhistory\"" |
| [ -n "$buildtype" ] || buildtype=$BUILDTYPE |
| [ -n "$buildtype" ] || buildtype=$default_buildtype |
| echo "$buildtype" | grep -v '^[1234c ]*$' && echo_error "Invalid buildtype \"$buildtype\", only some combination of 1, 2, 3, 4, c separated by space is allowed" |
| |
| OUTPUT_BASE=test-dependencies/`date "+%s"` |
| declare -i RESULT=0 |
| |
| build_all() { |
| echo "===== 1st build to populate sstate-cache directory and sysroot =====" |
| OUTPUT1=${OUTPUT_BASE}/${TYPE}_all |
| mkdir -p ${OUTPUT1} |
| echo "Logs will be stored in ${OUTPUT1} directory" |
| bitbake -k $targets 2>&1 | tee -a ${OUTPUT1}/complete.log |
| RESULT+=${PIPESTATUS[0]} |
| grep "ERROR: Task.*failed" ${OUTPUT1}/complete.log > ${OUTPUT1}/failed-tasks.log |
| cat ${OUTPUT1}/failed-tasks.log | sed 's@.*/@@g; s@_.*@@g; s@\.bb, .*@@g; s@\.bb:.*@@g' | sort -u > ${OUTPUT1}/failed-recipes.log |
| } |
| |
| build_every_recipe() { |
| if [ "${TYPE}" = "2" ] ; then |
| echo "===== 2nd build to rebuild each recipe with every possible dep =====" |
| OUTPUT_MAX=${OUTPUT_BASE}/${TYPE}_max |
| OUTPUTB=${OUTPUT_MAX} |
| else |
| echo "===== 3rd or 4th build to rebuild each recipe with minimal dependencies =====" |
| OUTPUT_MIN=${OUTPUT_BASE}/${TYPE}_min |
| OUTPUTB=${OUTPUT_MIN} |
| fi |
| |
| mkdir -p ${OUTPUTB} ${OUTPUTB}/failed ${OUTPUTB}/ok |
| echo "Logs will be stored in ${OUTPUTB} directory" |
| if [ -z "$recipes" ]; then |
| ls -d $buildhistory/packages/*/* | xargs -n 1 basename | sort -u > ${OUTPUTB}/recipe.list |
| recipes=${OUTPUTB}/recipe.list |
| fi |
| if [ "${TYPE}" != "2" ] ; then |
| echo "!!!Removing tmpdir \"$tmpdir\"!!!" |
| rm -rf $tmpdir/deploy $tmpdir/pkgdata $tmpdir/sstate-control $tmpdir/stamps $tmpdir/sysroots $tmpdir/work $tmpdir/work-shared 2>/dev/null |
| fi |
| i=1 |
| count=`cat $recipes ${OUTPUT1}/failed-recipes.log | sort -u | wc -l` |
| for recipe in `cat $recipes ${OUTPUT1}/failed-recipes.log | sort -u`; do |
| echo "Building recipe: ${recipe} ($i/$count)" |
| declare -i RECIPE_RESULT=0 |
| bitbake -c cleansstate ${recipe} > ${OUTPUTB}/${recipe}.log 2>&1; |
| RECIPE_RESULT+=$? |
| bitbake ${recipe} >> ${OUTPUTB}/${recipe}.log 2>&1; |
| RECIPE_RESULT+=$? |
| if [ "${RECIPE_RESULT}" != "0" ] ; then |
| RESULT+=${RECIPE_RESULT} |
| mv ${OUTPUTB}/${recipe}.log ${OUTPUTB}/failed/ |
| grep "ERROR: Task.*failed" ${OUTPUTB}/failed/${recipe}.log | tee -a ${OUTPUTB}/failed-tasks.log |
| grep "ERROR: Task.*failed" ${OUTPUTB}/failed/${recipe}.log | sed 's@.*/@@g; s@_.*@@g; s@\.bb, .*@@g; s@\.bb:.*@@g' >> ${OUTPUTB}/failed-recipes.log |
| # and append also ${recipe} in case the failed task was from some dependency |
| echo ${recipe} >> ${OUTPUTB}/failed-recipes.log |
| else |
| mv ${OUTPUTB}/${recipe}.log ${OUTPUTB}/ok/ |
| fi |
| if [ "${TYPE}" != "2" ] ; then |
| rm -rf $tmpdir/deploy $tmpdir/pkgdata $tmpdir/sstate-control $tmpdir/stamps $tmpdir/sysroots $tmpdir/work $tmpdir/work-shared 2>/dev/null |
| fi |
| i=`expr $i + 1` |
| done |
| echo "Copying buildhistory/packages to ${OUTPUTB}" |
| cp -ra $buildhistory/packages ${OUTPUTB} |
| # This will be usefull to see which library is pulling new dependency |
| echo "Copying do_package logs to ${OUTPUTB}/do_package/" |
| mkdir ${OUTPUTB}/do_package |
| find $tmpdir/work/ -name log.do_package 2>/dev/null| while read f; do |
| # pn is 3 levels back, but we don't know if there is just one log per pn (only one arch and version) |
| # dest=`echo $f | sed 's#^.*/\([^/]*\)/\([^/]*\)/\([^/]*\)/log.do_package#\1#g'` |
| dest=`echo $f | sed "s#$tmpdir/work/##g; s#/#_#g"` |
| cp $f ${OUTPUTB}/do_package/$dest |
| done |
| } |
| |
| compare_deps() { |
| # you can run just compare task with command like this |
| # OUTPUT_BASE=test-dependencies/1373140172 \ |
| # OUTPUT_MAX=${OUTPUT_BASE}/2_max \ |
| # OUTPUT_MIN=${OUTPUT_BASE}/3_min \ |
| # openembedded-core/scripts/test-dependencies.sh --tmpdir=tmp-eglibc --targets=glib-2.0 --recipes=recipe_list --buildtype=c |
| echo "===== Compare dependencies recorded in \"${OUTPUT_MAX}\" and \"${OUTPUT_MIN}\" =====" |
| [ -n "${OUTPUTC}" ] || OUTPUTC=${OUTPUT_BASE}/comp |
| mkdir -p ${OUTPUTC} |
| OUTPUT_FILE=${OUTPUTC}/dependency-changes |
| echo "Differences will be stored in ${OUTPUT_FILE}, dot is shown for every 100 of checked packages" |
| echo > ${OUTPUT_FILE} |
| |
| [ -d ${OUTPUT_MAX} ] || echo_error "Directory with output from build 2 \"${OUTPUT_MAX}\" does not exist" |
| [ -d ${OUTPUT_MIN} ] || echo_error "Directory with output from build 3 \"${OUTPUT_MIN}\" does not exist" |
| [ -d ${OUTPUT_MAX}/packages/ ] || echo_error "Directory with packages from build 2 \"${OUTPUT_MAX}/packages/\" does not exist" |
| [ -d ${OUTPUT_MIN}/packages/ ] || echo_error "Directory with packages from build 3 \"${OUTPUT_MIN}/packages/\" does not exist" |
| i=0 |
| find ${OUTPUT_MAX}/packages/ -name latest | sed "s#${OUTPUT_MAX}/##g" | while read pkg; do |
| max_pkg=${OUTPUT_MAX}/${pkg} |
| min_pkg=${OUTPUT_MIN}/${pkg} |
| # pkg=packages/armv5te-oe-linux-gnueabi/libungif/libungif/latest |
| recipe=`echo "${pkg}" | sed 's#packages/[^/]*/\([^/]*\)/\([^/]*\)/latest#\1#g'` |
| package=`echo "${pkg}" | sed 's#packages/[^/]*/\([^/]*\)/\([^/]*\)/latest#\2#g'` |
| if [ ! -f "${min_pkg}" ] ; then |
| echo "ERROR: ${recipe}: ${package} package isn't created when building with minimal dependencies?" | tee -a ${OUTPUT_FILE} |
| echo ${recipe} >> ${OUTPUTC}/failed-recipes.log |
| continue |
| fi |
| # strip version information in parenthesis |
| max_deps=`grep "^RDEPENDS = " ${max_pkg} | sed 's/^RDEPENDS = / /g; s/$/ /g; s/([^(]*)//g'` |
| min_deps=`grep "^RDEPENDS = " ${min_pkg} | sed 's/^RDEPENDS = / /g; s/$/ /g; s/([^(]*)//g'` |
| if [ "$i" = 100 ] ; then |
| echo -n "." # cheap progressbar |
| i=0 |
| fi |
| if [ "${max_deps}" = "${min_deps}" ] ; then |
| # it's annoying long, but at least it's showing some progress, warnings are grepped at the end |
| echo "NOTE: ${recipe}: ${package} rdepends weren't changed" >> ${OUTPUT_FILE} |
| else |
| missing_deps= |
| for dep in ${max_deps}; do |
| if ! echo "${min_deps}" | grep -q " ${dep} " ; then |
| missing_deps="${missing_deps} ${dep}" |
| echo # to get rid of dots on last line |
| echo "WARN: ${recipe}: ${package} rdepends on ${dep}, but it isn't a build dependency?" | tee -a ${OUTPUT_FILE} |
| fi |
| done |
| if [ -n "${missing_deps}" ] ; then |
| echo ${recipe} >> ${OUTPUTC}/failed-recipes.log |
| fi |
| fi |
| i=`expr $i + 1` |
| done |
| echo # to get rid of dots on last line |
| echo "Found differences: " |
| grep "^WARN: " ${OUTPUT_FILE} | tee ${OUTPUT_FILE}.warn.log |
| echo "Found errors: " |
| grep "^ERROR: " ${OUTPUT_FILE} | tee ${OUTPUT_FILE}.error.log |
| RESULT+=`cat ${OUTPUT_FILE}.warn.log | wc -l` |
| RESULT+=`cat ${OUTPUT_FILE}.error.log | wc -l` |
| } |
| |
| for TYPE in $buildtype; do |
| case ${TYPE} in |
| 1) build_all;; |
| 2) build_every_recipe;; |
| 3) build_every_recipe;; |
| 4) build_every_recipe;; |
| c) compare_deps;; |
| *) echo_error "Invalid buildtype \"$TYPE\"" |
| esac |
| done |
| |
| cat ${OUTPUT_BASE}/*/failed-recipes.log | sort -u >> ${OUTPUT_BASE}/failed-recipes.log |
| |
| if [ "${RESULT}" != "0" ] ; then |
| echo "ERROR: ${RESULT} issues were found in these recipes: `cat ${OUTPUT_BASE}/failed-recipes.log | xargs`" |
| fi |
| |
| echo "INFO: Output written in: ${OUTPUT_BASE}" |
| exit ${RESULT} |