blob: 2ab450ab59cda89f6441ecf94255939a72b6a88e [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001#!/bin/bash
2
3# Copyright (c) 2012 Wind River Systems, Inc.
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License version 2 as
7# published by the Free Software Foundation.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
12# See the GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License
15# along with this program; if not, write to the Free Software
16# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17#
18
19# Global vars
20cache_dir=
21confirm=
22fsym=
23total_deleted=0
24verbose=
25debug=0
26
27usage () {
28 cat << EOF
29Welcome to sstate cache management utilities.
30sstate-cache-management.sh <OPTION>
31
32Options:
33 -h, --help
34 Display this help and exit.
35
36 --cache-dir=<sstate cache dir>
37 Specify sstate cache directory, will use the environment
38 variable SSTATE_CACHE_DIR if it is not specified.
39
40 --extra-archs=<arch1>,<arch2>...<archn>
41 Specify list of architectures which should be tested, this list
42 will be extended with native arch, allarch and empty arch. The
43 script won't be trying to generate list of available archs from
44 AVAILTUNES in tune files.
45
46 --extra-layer=<layer1>,<layer2>...<layern>
47 Specify the layer which will be used for searching the archs,
48 it will search the meta and meta-* layers in the top dir by
49 default, and will search meta, meta-*, <layer1>, <layer2>,
50 ...<layern> when specified. Use "," as the separator.
51
52 This is useless for --stamps-dir or when --extra-archs is used.
53
54 -d, --remove-duplicated
55 Remove the duplicated sstate cache files of one package, only
56 the newest one will be kept. The duplicated sstate cache files
57 of one package must have the same arch, which means sstate cache
58 files with multiple archs are not considered duplicate.
59
60 Conflicts with --stamps-dir.
61
62 --stamps-dir=<dir1>,<dir2>...<dirn>
63 Specify the build directory's stamps directories, the sstate
64 cache file which IS USED by these build diretories will be KEPT,
65 other sstate cache files in cache-dir will be removed. Use ","
66 as the separator. For example:
67 --stamps-dir=build1/tmp/stamps,build2/tmp/stamps
68
69 Conflicts with --remove-duplicated.
70
71 -L, --follow-symlink
72 Remove both the symbol link and the destination file, default: no.
73
74 -y, --yes
75 Automatic yes to prompts; assume "yes" as answer to all prompts
76 and run non-interactively.
77
78 -v, --verbose
79 Explain what is being done.
80
81 -D, --debug
82 Show debug info, repeat for more debug info.
83
84EOF
85}
86
87if [ $# -lt 1 ]; then
88 usage
89 exit 0
90fi
91
92# Echo no files to remove
93no_files () {
94 echo No files to remove
95}
96
97# Echo nothing to do
98do_nothing () {
99 echo Nothing to do
100}
101
102# Read the input "y"
103read_confirm () {
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500104 echo "$total_deleted out of $total_files files will be removed! "
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500105 if [ "$confirm" != "y" ]; then
106 echo "Do you want to continue (y/n)? "
107 while read confirm; do
108 [ "$confirm" = "Y" -o "$confirm" = "y" -o "$confirm" = "n" \
109 -o "$confirm" = "N" ] && break
110 echo "Invalid input \"$confirm\", please input 'y' or 'n': "
111 done
112 else
113 echo
114 fi
115}
116
117# Print error information and exit.
118echo_error () {
119 echo "ERROR: $1" >&2
120 exit 1
121}
122
123# Generate the remove list:
124#
125# * Add .done/.siginfo to the remove list
126# * Add destination of symlink to the remove list
127#
128# $1: output file, others: sstate cache file (.tgz)
129gen_rmlist (){
130 local rmlist_file="$1"
131 shift
132 local files="$@"
133 for i in $files; do
134 echo $i >> $rmlist_file
135 # Add the ".siginfo"
136 if [ -e $i.siginfo ]; then
137 echo $i.siginfo >> $rmlist_file
138 fi
139 # Add the destination of symlink
140 if [ -L "$i" ]; then
141 if [ "$fsym" = "y" ]; then
142 dest="`readlink -e $i`"
143 if [ -n "$dest" ]; then
144 echo $dest >> $rmlist_file
145 # Remove the .siginfo when .tgz is removed
146 if [ -f "$dest.siginfo" ]; then
147 echo $dest.siginfo >> $rmlist_file
148 fi
149 fi
150 fi
151 # Add the ".tgz.done" and ".siginfo.done" (may exist in the future)
152 base_fn="${i##/*/}"
153 t_fn="$base_fn.done"
154 s_fn="$base_fn.siginfo.done"
155 for d in $t_fn $s_fn; do
156 if [ -f $cache_dir/$d ]; then
157 echo $cache_dir/$d >> $rmlist_file
158 fi
159 done
160 fi
161 done
162}
163
164# Remove the duplicated cache files for the pkg, keep the newest one
165remove_duplicated () {
166
167 local topdir
168 local oe_core_dir
169 local tunedirs
170 local all_archs
171 local all_machines
172 local ava_archs
173 local arch
174 local file_names
175 local sstate_files_list
176 local fn_tmp
177 local list_suffix=`mktemp` || exit 1
178
179 if [ -z "$extra_archs" ] ; then
180 # Find out the archs in all the layers
181 echo "Figuring out the archs in the layers ... "
182 oe_core_dir=$(dirname $(dirname $(readlink -e $0)))
183 topdir=$(dirname $oe_core_dir)
184 tunedirs="`find $topdir/meta* ${oe_core_dir}/meta* $layers -path '*/meta*/conf/machine/include' 2>/dev/null`"
185 [ -n "$tunedirs" ] || echo_error "Can't find the tune directory"
186 all_machines="`find $topdir/meta* ${oe_core_dir}/meta* $layers -path '*/meta*/conf/machine/*' -name '*.conf' 2>/dev/null | sed -e 's/.*\///' -e 's/.conf$//'`"
187 all_archs=`grep -r -h "^AVAILTUNES .*=" $tunedirs | sed -e 's/.*=//' -e 's/\"//g'`
188 fi
189
190 # Use the "_" to substitute "-", e.g., x86-64 to x86_64, but not for extra_archs which can be something like cortexa9t2-vfp-neon
191 # Sort to remove the duplicated ones
192 # Add allarch and builder arch (native)
193 builder_arch=$(uname -m)
194 all_archs="$(echo allarch $all_archs $all_machines $builder_arch \
195 | sed -e 's/-/_/g' -e 's/ /\n/g' | sort -u) $extra_archs"
196 echo "Done"
197
198 # Total number of files including sstate-, .siginfo and .done files
199 total_files=`find $cache_dir -name 'sstate*' | wc -l`
200 # Save all the sstate files in a file
201 sstate_files_list=`mktemp` || exit 1
202 find $cache_dir -name 'sstate:*:*:*:*:*:*:*.tgz*' >$sstate_files_list
203
204 echo "Figuring out the suffixes in the sstate cache dir ... "
205 sstate_suffixes="`sed 's%.*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^_]*_\([^:]*\)\.tgz.*%\1%g' $sstate_files_list | sort -u`"
206 echo "Done"
207 echo "The following suffixes have been found in the cache dir:"
208 echo $sstate_suffixes
209
210 echo "Figuring out the archs in the sstate cache dir ... "
211 # Using this SSTATE_PKGSPEC definition it's 6th colon separated field
212 # SSTATE_PKGSPEC = "sstate:${PN}:${PACKAGE_ARCH}${TARGET_VENDOR}-${TARGET_OS}:${PV}:${PR}:${SSTATE_PKGARCH}:${SSTATE_VERSION}:"
213 for arch in $all_archs; do
214 grep -q ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:$arch:[^:]*:[^:]*\.tgz$" $sstate_files_list
215 [ $? -eq 0 ] && ava_archs="$ava_archs $arch"
216 # ${builder_arch}_$arch used by toolchain sstate
217 grep -q ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:${builder_arch}_$arch:[^:]*:[^:]*\.tgz$" $sstate_files_list
218 [ $? -eq 0 ] && ava_archs="$ava_archs ${builder_arch}_$arch"
219 done
220 echo "Done"
221 echo "The following archs have been found in the cache dir:"
222 echo $ava_archs
223 echo ""
224
225 # Save the file list which needs to be removed
226 local remove_listdir=`mktemp -d` || exit 1
227 for suffix in $sstate_suffixes; do
228 if [ "$suffix" = "populate_lic" ] ; then
229 echo "Skipping populate_lic, because removing duplicates doesn't work correctly for them (use --stamps-dir instead)"
230 continue
231 fi
232 # Total number of files including .siginfo and .done files
233 total_files_suffix=`grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix\.tgz.*" $sstate_files_list | wc -l 2>/dev/null`
234 total_tgz_suffix=`grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix\.tgz$" $sstate_files_list | wc -l 2>/dev/null`
235 # Save the file list to a file, some suffix's file may not exist
236 grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix\.tgz.*" $sstate_files_list >$list_suffix 2>/dev/null
237 local deleted_tgz=0
238 local deleted_files=0
239 for ext in tgz tgz.siginfo tgz.done; do
240 echo "Figuring out the sstate:xxx_$suffix.$ext ... "
241 # Uniq BPNs
242 file_names=`for arch in $ava_archs ""; do
243 sed -ne "s%.*/sstate:\([^:]*\):[^:]*:[^:]*:[^:]*:$arch:[^:]*:[^:]*\.${ext}$%\1%p" $list_suffix
244 done | sort -u`
245
246 fn_tmp=`mktemp` || exit 1
247 rm_list="$remove_listdir/sstate:xxx_$suffix"
248 for fn in $file_names; do
249 [ -z "$verbose" ] || echo "Analyzing sstate:$fn-xxx_$suffix.${ext}"
250 for arch in $ava_archs ""; do
251 grep -h ".*/sstate:$fn:[^:]*:[^:]*:[^:]*:$arch:[^:]*:[^:]*\.${ext}$" $list_suffix >$fn_tmp
252 if [ -s $fn_tmp ] ; then
253 [ $debug -gt 1 ] && echo "Available files for $fn-$arch- with suffix $suffix.${ext}:" && cat $fn_tmp
254 # Use the modification time
255 to_del=$(ls -t $(cat $fn_tmp) | sed -n '1!p')
256 [ $debug -gt 2 ] && echo "Considering to delete: $to_del"
257 # The sstate file which is downloaded from the SSTATE_MIRROR is
258 # put in SSTATE_DIR, and there is a symlink in SSTATE_DIR/??/ to
259 # it, so filter it out from the remove list if it should not be
260 # removed.
261 to_keep=$(ls -t $(cat $fn_tmp) | sed -n '1p')
262 [ $debug -gt 2 ] && echo "Considering to keep: $to_keep"
263 for k in $to_keep; do
264 if [ -L "$k" ]; then
265 # The symlink's destination
266 k_dest="`readlink -e $k`"
267 # Maybe it is the one in cache_dir
268 k_maybe="$cache_dir/${k##/*/}"
269 # Remove it from the remove list if they are the same.
270 if [ "$k_dest" = "$k_maybe" ]; then
271 to_del="`echo $to_del | sed 's#'\"$k_maybe\"'##g'`"
272 fi
273 fi
274 done
275 rm -f $fn_tmp
276 [ $debug -gt 2 ] && echo "Decided to delete: $to_del"
277 gen_rmlist $rm_list.$ext "$to_del"
278 fi
279 done
280 done
281 done
282 deleted_tgz=`cat $rm_list.* 2>/dev/null | grep ".tgz$" | wc -l`
283 deleted_files=`cat $rm_list.* 2>/dev/null | wc -l`
284 [ "$deleted_files" -gt 0 -a $debug -gt 0 ] && cat $rm_list.*
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500285 echo "($deleted_tgz out of $total_tgz_suffix .tgz files for $suffix suffix will be removed or $deleted_files out of $total_files_suffix when counting also .siginfo and .done files)"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500286 let total_deleted=$total_deleted+$deleted_files
287 done
288 deleted_tgz=0
289 rm_old_list=$remove_listdir/sstate-old-filenames
290 find $cache_dir -name 'sstate-*.tgz' >$rm_old_list
291 [ -s "$rm_old_list" ] && deleted_tgz=`cat $rm_old_list | grep ".tgz$" | wc -l`
292 [ -s "$rm_old_list" ] && deleted_files=`cat $rm_old_list | wc -l`
293 [ -s "$rm_old_list" -a $debug -gt 0 ] && cat $rm_old_list
294 echo "($deleted_tgz .tgz files with old sstate-* filenames will be removed or $deleted_files when counting also .siginfo and .done files)"
295 let total_deleted=$total_deleted+$deleted_files
296
297 rm -f $list_suffix
298 rm -f $sstate_files_list
299 if [ $total_deleted -gt 0 ]; then
300 read_confirm
301 if [ "$confirm" = "y" -o "$confirm" = "Y" ]; then
302 for list in `ls $remove_listdir/`; do
303 echo "Removing $list.tgz (`cat $remove_listdir/$list | wc -w` files) ... "
304 # Remove them one by one to avoid the argument list too long error
305 for i in `cat $remove_listdir/$list`; do
306 rm -f $verbose $i
307 done
308 echo "Done"
309 done
310 echo "$total_deleted files have been removed!"
311 else
312 do_nothing
313 fi
314 else
315 no_files
316 fi
317 [ -d $remove_listdir ] && rm -fr $remove_listdir
318}
319
320# Remove the sstate file by stamps dir, the file not used by the stamps dir
321# will be removed.
322rm_by_stamps (){
323
324 local cache_list=`mktemp` || exit 1
325 local keep_list=`mktemp` || exit 1
326 local rm_list=`mktemp` || exit 1
327 local sums
328 local all_sums
329
330 # Total number of files including sstate-, .siginfo and .done files
331 total_files=`find $cache_dir -type f -name 'sstate*' | wc -l`
332 # Save all the state file list to a file
333 find $cache_dir -type f -name 'sstate*' | sort -u -o $cache_list
334
335 echo "Figuring out the suffixes in the sstate cache dir ... "
336 local sstate_suffixes="`sed 's%.*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^_]*_\([^:]*\)\.tgz.*%\1%g' $cache_list | sort -u`"
337 echo "Done"
338 echo "The following suffixes have been found in the cache dir:"
339 echo $sstate_suffixes
340
341 # Figure out all the md5sums in the stamps dir.
342 echo "Figuring out all the md5sums in stamps dir ... "
343 for i in $sstate_suffixes; do
344 # There is no "\.sigdata" but "_setcene" when it is mirrored
345 # from the SSTATE_MIRRORS, use them to figure out the sum.
346 sums=`find $stamps -maxdepth 3 -name "*.do_$i.*" \
347 -o -name "*.do_${i}_setscene.*" | \
348 sed -ne 's#.*_setscene\.##p' -e 's#.*\.sigdata\.##p' | \
349 sed -e 's#\..*##' | sort -u`
350 all_sums="$all_sums $sums"
351 done
352 echo "Done"
353
354 echo "Figuring out the files which will be removed ... "
355 for i in $all_sums; do
356 grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:${i}_.*" $cache_list >>$keep_list
357 done
358 echo "Done"
359
360 if [ -s $keep_list ]; then
361 sort -u $keep_list -o $keep_list
362 to_del=`comm -1 -3 $keep_list $cache_list`
363 gen_rmlist $rm_list "$to_del"
364 let total_deleted=`cat $rm_list | sort -u | wc -w`
365 if [ $total_deleted -gt 0 ]; then
366 [ $debug -gt 0 ] && cat $rm_list | sort -u
367 read_confirm
368 if [ "$confirm" = "y" -o "$confirm" = "Y" ]; then
369 echo "Removing sstate cache files ... ($total_deleted files)"
370 # Remove them one by one to avoid the argument list too long error
371 for i in `cat $rm_list | sort -u`; do
372 rm -f $verbose $i
373 done
374 echo "$total_deleted files have been removed"
375 else
376 do_nothing
377 fi
378 else
379 no_files
380 fi
381 else
382 echo_error "All files in cache dir will be removed! Abort!"
383 fi
384
385 rm -f $cache_list
386 rm -f $keep_list
387 rm -f $rm_list
388}
389
390# Parse arguments
391while [ -n "$1" ]; do
392 case $1 in
393 --cache-dir=*)
394 cache_dir=`echo $1 | sed -e 's#^--cache-dir=##' | xargs readlink -e`
395 [ -d "$cache_dir" ] || echo_error "Invalid argument to --cache-dir"
396 shift
397 ;;
398 --remove-duplicated|-d)
399 rm_duplicated="y"
400 shift
401 ;;
402 --yes|-y)
403 confirm="y"
404 shift
405 ;;
406 --follow-symlink|-L)
407 fsym="y"
408 shift
409 ;;
410 --extra-archs=*)
411 extra_archs=`echo $1 | sed -e 's#^--extra-archs=##' -e 's#,# #g'`
412 [ -n "$extra_archs" ] || echo_error "Invalid extra arch parameter"
413 shift
414 ;;
415 --extra-layer=*)
416 extra_layers=`echo $1 | sed -e 's#^--extra-layer=##' -e 's#,# #g'`
417 [ -n "$extra_layers" ] || echo_error "Invalid extra layer parameter"
418 for i in $extra_layers; do
419 l=`readlink -e $i`
420 if [ -d "$l" ]; then
421 layers="$layers $l"
422 else
423 echo_error "Can't find layer $i"
424 fi
425 done
426 shift
427 ;;
428 --stamps-dir=*)
429 stamps=`echo $1 | sed -e 's#^--stamps-dir=##' -e 's#,# #g'`
430 [ -n "$stamps" ] || echo_error "Invalid stamps dir $i"
431 for i in $stamps; do
432 [ -d "$i" ] || echo_error "Invalid stamps dir $i"
433 done
434 shift
435 ;;
436 --verbose|-v)
437 verbose="-v"
438 shift
439 ;;
440 --debug|-D)
441 debug=`expr $debug + 1`
442 echo "Debug level $debug"
443 shift
444 ;;
445 --help|-h)
446 usage
447 exit 0
448 ;;
449 *)
450 echo "Invalid arguments $*"
451 echo_error "Try 'sstate-cache-management.sh -h' for more information."
452 ;;
453 esac
454done
455
456# sstate cache directory, use environment variable SSTATE_CACHE_DIR
457# if it was not specified, otherwise, error.
458[ -n "$cache_dir" ] || cache_dir=$SSTATE_CACHE_DIR
459[ -n "$cache_dir" ] || echo_error "No cache dir found!"
460[ -d "$cache_dir" ] || echo_error "Invalid cache directory \"$cache_dir\""
461
462[ -n "$rm_duplicated" -a -n "$stamps" ] && \
463 echo_error "Can not use both --remove-duplicated and --stamps-dir"
464
465[ "$rm_duplicated" = "y" ] && remove_duplicated
466[ -n "$stamps" ] && rm_by_stamps
467[ -z "$rm_duplicated" -a -z "$stamps" ] && \
468 echo "What do you want to do?"
469exit 0