blob: 0170947f0ef55c892fab9c9db2e829b725fd5cce [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001#!/bin/bash
2
3# Author: Martin Jansa <martin.jansa@gmail.com>
4#
5# Copyright (c) 2013 Martin Jansa <Martin.Jansa@gmail.com>
6
7# Used to detect missing dependencies or automagically
8# enabled dependencies which aren't explicitly enabled
9# or disabled. Using bash to have PIPESTATUS variable.
10
11# It does 3 builds of <target>
12# 1st to populate sstate-cache directory and sysroot
13# 2nd to rebuild each recipe with every possible
14# dependency found in sysroot (which stays populated
15# from 1st build
16# 3rd to rebuild each recipe only with dependencies defined
17# in DEPENDS
18# 4th (optional) repeat build like 3rd to make sure that
19# minimal versions of dependencies defined in DEPENDS
20# is also enough
21
22# Global vars
23tmpdir=
24targets=
25recipes=
26buildhistory=
27buildtype=
28default_targets="world"
29default_buildhistory="buildhistory"
30default_buildtype="1 2 3 c"
31
32usage () {
33 cat << EOF
34Welcome to utility to detect missing or autoenabled dependencies.
35WARNING: this utility will completely remove your tmpdir (make sure
36 you don't have important buildhistory or persistent dir there).
37$0 <OPTION>
38
39Options:
40 -h, --help
41 Display this help and exit.
42
43 --tmpdir=<tmpdir>
44 Specify tmpdir, will use the environment variable TMPDIR if it is not specified.
45 Something like /OE/oe-core/tmp-eglibc (no / at the end).
46
47 --targets=<targets>
48 List of targets separated by space, will use the environment variable TARGETS if it is not specified.
49 It will run "bitbake <targets>" to populate sysroots.
50 Default value is "world".
51
52 --recipes=<recipes>
53 File with list of recipes we want to rebuild with minimal and maximal sysroot.
54 Will use the environment variable RECIPES if it is not specified.
55 Default value will use all packages ever recorded in buildhistory directory.
56
57 --buildhistory=<buildhistory>
58 Path to buildhistory directory, it needs to be enabled in your config,
59 because it's used to detect different dependencies and to create list
60 of recipes to rebuild when it's not specified.
61 Will use the environment variable BUILDHISTORY if it is not specified.
62 Default value is "buildhistory"
63
64 --buildtype=<buildtype>
65 There are 4 types of build:
66 1: build to populate sstate-cache directory and sysroot
67 2: build to rebuild each recipe with every possible dep
68 3: build to rebuild each recipe with minimal dependencies
69 4: build to rebuild each recipe again with minimal dependencies
70 c: compare buildhistory directories from build 2 and 3
71 Will use the environment variable BUILDTYPE if it is not specified.
72 Default value is "1 2 3 c", order is important, type 4 is optional.
73EOF
74}
75
76# Print error information and exit.
77echo_error () {
78 echo "ERROR: $1" >&2
79 exit 1
80}
81
82while [ -n "$1" ]; do
83 case $1 in
84 --tmpdir=*)
85 tmpdir=`echo $1 | sed -e 's#^--tmpdir=##' | xargs readlink -e`
86 [ -d "$tmpdir" ] || echo_error "Invalid argument to --tmpdir"
87 shift
88 ;;
89 --targets=*)
90 targets=`echo $1 | sed -e 's#^--targets="*\([^"]*\)"*#\1#'`
91 shift
92 ;;
93 --recipes=*)
94 recipes=`echo $1 | sed -e 's#^--recipes="*\([^"]*\)"*#\1#'`
95 shift
96 ;;
97 --buildhistory=*)
98 buildhistory=`echo $1 | sed -e 's#^--buildhistory="*\([^"]*\)"*#\1#'`
99 shift
100 ;;
101 --buildtype=*)
102 buildtype=`echo $1 | sed -e 's#^--buildtype="*\([^"]*\)"*#\1#'`
103 shift
104 ;;
105 --help|-h)
106 usage
107 exit 0
108 ;;
109 *)
110 echo "Invalid arguments $*"
111 echo_error "Try '$0 -h' for more information."
112 ;;
113 esac
114done
115
116# tmpdir directory, use environment variable TMPDIR
117# if it was not specified, otherwise, error.
118[ -n "$tmpdir" ] || tmpdir=$TMPDIR
119[ -n "$tmpdir" ] || echo_error "No tmpdir found!"
120[ -d "$tmpdir" ] || echo_error "Invalid tmpdir \"$tmpdir\""
121[ -n "$targets" ] || targets=$TARGETS
122[ -n "$targets" ] || targets=$default_targets
123[ -n "$recipes" ] || recipes=$RECIPES
124[ -n "$recipes" -a ! -f "$recipes" ] && echo_error "Invalid file with list of recipes to rebuild"
125[ -n "$recipes" ] || echo "All packages ever recorded in buildhistory directory will be rebuilt"
126[ -n "$buildhistory" ] || buildhistory=$BUILDHISTORY
127[ -n "$buildhistory" ] || buildhistory=$default_buildhistory
128[ -d "$buildhistory" ] || echo_error "Invalid buildhistory directory \"$buildhistory\""
129[ -n "$buildtype" ] || buildtype=$BUILDTYPE
130[ -n "$buildtype" ] || buildtype=$default_buildtype
131echo "$buildtype" | grep -v '^[1234c ]*$' && echo_error "Invalid buildtype \"$buildtype\", only some combination of 1, 2, 3, 4, c separated by space is allowed"
132
133OUTPUT_BASE=test-dependencies/`date "+%s"`
134declare -i RESULT=0
135
136build_all() {
137 echo "===== 1st build to populate sstate-cache directory and sysroot ====="
138 OUTPUT1=${OUTPUT_BASE}/${TYPE}_all
139 mkdir -p ${OUTPUT1}
140 echo "Logs will be stored in ${OUTPUT1} directory"
141 bitbake -k $targets 2>&1 | tee -a ${OUTPUT1}/complete.log
142 RESULT+=${PIPESTATUS[0]}
143 grep "ERROR: Task.*failed" ${OUTPUT1}/complete.log > ${OUTPUT1}/failed-tasks.log
144 cat ${OUTPUT1}/failed-tasks.log | sed 's@.*/@@g; s@_.*@@g; s@\.bb, .*@@g' | sort -u > ${OUTPUT1}/failed-recipes.log
145}
146
147build_every_recipe() {
148 if [ "${TYPE}" = "2" ] ; then
149 echo "===== 2nd build to rebuild each recipe with every possible dep ====="
150 OUTPUT_MAX=${OUTPUT_BASE}/${TYPE}_max
151 OUTPUTB=${OUTPUT_MAX}
152 else
153 echo "===== 3rd or 4th build to rebuild each recipe with minimal dependencies ====="
154 OUTPUT_MIN=${OUTPUT_BASE}/${TYPE}_min
155 OUTPUTB=${OUTPUT_MIN}
156 fi
157
158 mkdir -p ${OUTPUTB} ${OUTPUTB}/failed ${OUTPUTB}/ok
159 echo "Logs will be stored in ${OUTPUTB} directory"
160 if [ -z "$recipes" ]; then
161 ls -d $buildhistory/packages/*/* | xargs -n 1 basename | sort -u > ${OUTPUTB}/recipe.list
162 recipes=${OUTPUTB}/recipe.list
163 fi
164 if [ "${TYPE}" != "2" ] ; then
165 echo "!!!Removing tmpdir \"$tmpdir\"!!!"
166 rm -rf $tmpdir/deploy $tmpdir/pkgdata $tmpdir/sstate-control $tmpdir/stamps $tmpdir/sysroots $tmpdir/work $tmpdir/work-shared 2>/dev/null
167 fi
168 i=1
169 count=`cat $recipes ${OUTPUT1}/failed-recipes.log | sort -u | wc -l`
170 for recipe in `cat $recipes ${OUTPUT1}/failed-recipes.log | sort -u`; do
171 echo "Building recipe: ${recipe} ($i/$count)"
172 declare -i RECIPE_RESULT=0
173 bitbake -c cleansstate ${recipe} > ${OUTPUTB}/${recipe}.log 2>&1;
174 RECIPE_RESULT+=$?
175 bitbake ${recipe} >> ${OUTPUTB}/${recipe}.log 2>&1;
176 RECIPE_RESULT+=$?
177 if [ "${RECIPE_RESULT}" != "0" ] ; then
178 RESULT+=${RECIPE_RESULT}
179 mv ${OUTPUTB}/${recipe}.log ${OUTPUTB}/failed/
180 grep "ERROR: Task.*failed" ${OUTPUTB}/failed/${recipe}.log | tee -a ${OUTPUTB}/failed-tasks.log
181 grep "ERROR: Task.*failed" ${OUTPUTB}/failed/${recipe}.log | sed 's@.*/@@g; s@_.*@@g; s@\.bb, .*@@g' >> ${OUTPUTB}/failed-recipes.log
182 # and append also ${recipe} in case the failed task was from some dependency
183 echo ${recipe} >> ${OUTPUTB}/failed-recipes.log
184 else
185 mv ${OUTPUTB}/${recipe}.log ${OUTPUTB}/ok/
186 fi
187 if [ "${TYPE}" != "2" ] ; then
188 rm -rf $tmpdir/deploy $tmpdir/pkgdata $tmpdir/sstate-control $tmpdir/stamps $tmpdir/sysroots $tmpdir/work $tmpdir/work-shared 2>/dev/null
189 fi
190 i=`expr $i + 1`
191 done
192 echo "Copying buildhistory/packages to ${OUTPUTB}"
193 cp -ra $buildhistory/packages ${OUTPUTB}
194 # This will be usefull to see which library is pulling new dependency
195 echo "Copying do_package logs to ${OUTPUTB}/do_package/"
196 mkdir ${OUTPUTB}/do_package
197 find $tmpdir/work/ -name log.do_package 2>/dev/null| while read f; do
198 # pn is 3 levels back, but we don't know if there is just one log per pn (only one arch and version)
199 # dest=`echo $f | sed 's#^.*/\([^/]*\)/\([^/]*\)/\([^/]*\)/log.do_package#\1#g'`
200 dest=`echo $f | sed "s#$tmpdir/work/##g; s#/#_#g"`
201 cp $f ${OUTPUTB}/do_package/$dest
202 done
203}
204
205compare_deps() {
206 # you can run just compare task with command like this
207 # OUTPUT_BASE=test-dependencies/1373140172 \
208 # OUTPUT_MAX=${OUTPUT_BASE}/2_max \
209 # OUTPUT_MIN=${OUTPUT_BASE}/3_min \
210 # openembedded-core/scripts/test-dependencies.sh --tmpdir=tmp-eglibc --targets=glib-2.0 --recipes=recipe_list --buildtype=c
211 echo "===== Compare dependencies recorded in \"${OUTPUT_MAX}\" and \"${OUTPUT_MIN}\" ====="
212 [ -n "${OUTPUTC}" ] || OUTPUTC=${OUTPUT_BASE}/comp
213 mkdir -p ${OUTPUTC}
214 OUTPUT_FILE=${OUTPUTC}/dependency-changes
215 echo "Differences will be stored in ${OUTPUT_FILE}, dot is shown for every 100 of checked packages"
216 echo > ${OUTPUT_FILE}
217
218 [ -d ${OUTPUT_MAX} ] || echo_error "Directory with output from build 2 \"${OUTPUT_MAX}\" does not exist"
219 [ -d ${OUTPUT_MIN} ] || echo_error "Directory with output from build 3 \"${OUTPUT_MIN}\" does not exist"
220 [ -d ${OUTPUT_MAX}/packages/ ] || echo_error "Directory with packages from build 2 \"${OUTPUT_MAX}/packages/\" does not exist"
221 [ -d ${OUTPUT_MIN}/packages/ ] || echo_error "Directory with packages from build 3 \"${OUTPUT_MIN}/packages/\" does not exist"
222 i=0
223 find ${OUTPUT_MAX}/packages/ -name latest | sed "s#${OUTPUT_MAX}/##g" | while read pkg; do
224 max_pkg=${OUTPUT_MAX}/${pkg}
225 min_pkg=${OUTPUT_MIN}/${pkg}
226 # pkg=packages/armv5te-oe-linux-gnueabi/libungif/libungif/latest
227 recipe=`echo "${pkg}" | sed 's#packages/[^/]*/\([^/]*\)/\([^/]*\)/latest#\1#g'`
228 package=`echo "${pkg}" | sed 's#packages/[^/]*/\([^/]*\)/\([^/]*\)/latest#\2#g'`
229 if [ ! -f "${min_pkg}" ] ; then
230 echo "ERROR: ${recipe}: ${package} package isn't created when building with minimal dependencies?" | tee -a ${OUTPUT_FILE}
231 echo ${recipe} >> ${OUTPUTC}/failed-recipes.log
232 continue
233 fi
234 # strip version information in parenthesis
235 max_deps=`grep "^RDEPENDS = " ${max_pkg} | sed 's/^RDEPENDS = / /g; s/$/ /g; s/([^(]*)//g'`
236 min_deps=`grep "^RDEPENDS = " ${min_pkg} | sed 's/^RDEPENDS = / /g; s/$/ /g; s/([^(]*)//g'`
237 if [ "$i" = 100 ] ; then
238 echo -n "." # cheap progressbar
239 i=0
240 fi
241 if [ "${max_deps}" = "${min_deps}" ] ; then
242 # it's annoying long, but at least it's showing some progress, warnings are grepped at the end
243 echo "NOTE: ${recipe}: ${package} rdepends weren't changed" >> ${OUTPUT_FILE}
244 else
245 missing_deps=
246 for dep in ${max_deps}; do
247 if ! echo "${min_deps}" | grep -q " ${dep} " ; then
248 missing_deps="${missing_deps} ${dep}"
249 echo # to get rid of dots on last line
250 echo "WARN: ${recipe}: ${package} rdepends on ${dep}, but it isn't a build dependency?" | tee -a ${OUTPUT_FILE}
251 fi
252 done
253 if [ -n "${missing_deps}" ] ; then
254 echo ${recipe} >> ${OUTPUTC}/failed-recipes.log
255 fi
256 fi
257 i=`expr $i + 1`
258 done
259 echo # to get rid of dots on last line
260 echo "Found differences: "
261 grep "^WARN: " ${OUTPUT_FILE} | tee ${OUTPUT_FILE}.warn.log
262 echo "Found errors: "
263 grep "^ERROR: " ${OUTPUT_FILE} | tee ${OUTPUT_FILE}.error.log
264 RESULT+=`cat ${OUTPUT_FILE}.warn.log | wc -l`
265 RESULT+=`cat ${OUTPUT_FILE}.error.log | wc -l`
266}
267
268for TYPE in $buildtype; do
269 case ${TYPE} in
270 1) build_all;;
271 2) build_every_recipe;;
272 3) build_every_recipe;;
273 4) build_every_recipe;;
274 c) compare_deps;;
275 *) echo_error "Invalid buildtype \"$TYPE\""
276 esac
277done
278
279cat ${OUTPUT_BASE}/*/failed-recipes.log | sort -u >> ${OUTPUT_BASE}/failed-recipes.log
280
281if [ "${RESULT}" != "0" ] ; then
282 echo "ERROR: ${RESULT} issues were found in these recipes: `cat ${OUTPUT_BASE}/failed-recipes.log | xargs`"
283fi
284
285echo "INFO: Output written in: ${OUTPUT_BASE}"
286exit ${RESULT}