blob: f1706a2229c7a5f340208b494d6f7d6a2f78cf93 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001#!/bin/bash
2
3# Copyright (c) 2012 Wind River Systems, Inc.
4#
Brad Bishopc342db32019-05-15 21:57:59 -04005# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -05006#
7
8# Global vars
9cache_dir=
10confirm=
11fsym=
12total_deleted=0
13verbose=
14debug=0
15
16usage () {
17 cat << EOF
18Welcome to sstate cache management utilities.
19sstate-cache-management.sh <OPTION>
20
21Options:
22 -h, --help
23 Display this help and exit.
24
25 --cache-dir=<sstate cache dir>
26 Specify sstate cache directory, will use the environment
27 variable SSTATE_CACHE_DIR if it is not specified.
28
29 --extra-archs=<arch1>,<arch2>...<archn>
30 Specify list of architectures which should be tested, this list
31 will be extended with native arch, allarch and empty arch. The
32 script won't be trying to generate list of available archs from
33 AVAILTUNES in tune files.
34
35 --extra-layer=<layer1>,<layer2>...<layern>
36 Specify the layer which will be used for searching the archs,
37 it will search the meta and meta-* layers in the top dir by
38 default, and will search meta, meta-*, <layer1>, <layer2>,
39 ...<layern> when specified. Use "," as the separator.
40
41 This is useless for --stamps-dir or when --extra-archs is used.
42
43 -d, --remove-duplicated
44 Remove the duplicated sstate cache files of one package, only
45 the newest one will be kept. The duplicated sstate cache files
46 of one package must have the same arch, which means sstate cache
47 files with multiple archs are not considered duplicate.
48
49 Conflicts with --stamps-dir.
50
51 --stamps-dir=<dir1>,<dir2>...<dirn>
52 Specify the build directory's stamps directories, the sstate
53 cache file which IS USED by these build diretories will be KEPT,
54 other sstate cache files in cache-dir will be removed. Use ","
55 as the separator. For example:
56 --stamps-dir=build1/tmp/stamps,build2/tmp/stamps
57
58 Conflicts with --remove-duplicated.
59
60 -L, --follow-symlink
61 Remove both the symbol link and the destination file, default: no.
62
63 -y, --yes
64 Automatic yes to prompts; assume "yes" as answer to all prompts
65 and run non-interactively.
66
67 -v, --verbose
68 Explain what is being done.
69
70 -D, --debug
71 Show debug info, repeat for more debug info.
72
73EOF
74}
75
76if [ $# -lt 1 ]; then
77 usage
78 exit 0
79fi
80
81# Echo no files to remove
82no_files () {
83 echo No files to remove
84}
85
86# Echo nothing to do
87do_nothing () {
88 echo Nothing to do
89}
90
91# Read the input "y"
92read_confirm () {
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050093 echo "$total_deleted out of $total_files files will be removed! "
Patrick Williamsc124f4f2015-09-15 14:41:29 -050094 if [ "$confirm" != "y" ]; then
95 echo "Do you want to continue (y/n)? "
96 while read confirm; do
97 [ "$confirm" = "Y" -o "$confirm" = "y" -o "$confirm" = "n" \
98 -o "$confirm" = "N" ] && break
99 echo "Invalid input \"$confirm\", please input 'y' or 'n': "
100 done
101 else
102 echo
103 fi
104}
105
106# Print error information and exit.
107echo_error () {
108 echo "ERROR: $1" >&2
109 exit 1
110}
111
112# Generate the remove list:
113#
114# * Add .done/.siginfo to the remove list
115# * Add destination of symlink to the remove list
116#
117# $1: output file, others: sstate cache file (.tgz)
118gen_rmlist (){
119 local rmlist_file="$1"
120 shift
121 local files="$@"
122 for i in $files; do
123 echo $i >> $rmlist_file
124 # Add the ".siginfo"
125 if [ -e $i.siginfo ]; then
126 echo $i.siginfo >> $rmlist_file
127 fi
128 # Add the destination of symlink
129 if [ -L "$i" ]; then
130 if [ "$fsym" = "y" ]; then
131 dest="`readlink -e $i`"
132 if [ -n "$dest" ]; then
133 echo $dest >> $rmlist_file
134 # Remove the .siginfo when .tgz is removed
135 if [ -f "$dest.siginfo" ]; then
136 echo $dest.siginfo >> $rmlist_file
137 fi
138 fi
139 fi
140 # Add the ".tgz.done" and ".siginfo.done" (may exist in the future)
141 base_fn="${i##/*/}"
142 t_fn="$base_fn.done"
143 s_fn="$base_fn.siginfo.done"
144 for d in $t_fn $s_fn; do
145 if [ -f $cache_dir/$d ]; then
146 echo $cache_dir/$d >> $rmlist_file
147 fi
148 done
149 fi
150 done
151}
152
153# Remove the duplicated cache files for the pkg, keep the newest one
154remove_duplicated () {
155
156 local topdir
157 local oe_core_dir
158 local tunedirs
159 local all_archs
160 local all_machines
161 local ava_archs
162 local arch
163 local file_names
164 local sstate_files_list
165 local fn_tmp
166 local list_suffix=`mktemp` || exit 1
167
168 if [ -z "$extra_archs" ] ; then
169 # Find out the archs in all the layers
170 echo "Figuring out the archs in the layers ... "
171 oe_core_dir=$(dirname $(dirname $(readlink -e $0)))
172 topdir=$(dirname $oe_core_dir)
173 tunedirs="`find $topdir/meta* ${oe_core_dir}/meta* $layers -path '*/meta*/conf/machine/include' 2>/dev/null`"
174 [ -n "$tunedirs" ] || echo_error "Can't find the tune directory"
175 all_machines="`find $topdir/meta* ${oe_core_dir}/meta* $layers -path '*/meta*/conf/machine/*' -name '*.conf' 2>/dev/null | sed -e 's/.*\///' -e 's/.conf$//'`"
176 all_archs=`grep -r -h "^AVAILTUNES .*=" $tunedirs | sed -e 's/.*=//' -e 's/\"//g'`
177 fi
178
179 # Use the "_" to substitute "-", e.g., x86-64 to x86_64, but not for extra_archs which can be something like cortexa9t2-vfp-neon
180 # Sort to remove the duplicated ones
181 # Add allarch and builder arch (native)
182 builder_arch=$(uname -m)
183 all_archs="$(echo allarch $all_archs $all_machines $builder_arch \
184 | sed -e 's/-/_/g' -e 's/ /\n/g' | sort -u) $extra_archs"
185 echo "Done"
186
187 # Total number of files including sstate-, .siginfo and .done files
188 total_files=`find $cache_dir -name 'sstate*' | wc -l`
189 # Save all the sstate files in a file
190 sstate_files_list=`mktemp` || exit 1
191 find $cache_dir -name 'sstate:*:*:*:*:*:*:*.tgz*' >$sstate_files_list
192
193 echo "Figuring out the suffixes in the sstate cache dir ... "
194 sstate_suffixes="`sed 's%.*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^_]*_\([^:]*\)\.tgz.*%\1%g' $sstate_files_list | sort -u`"
195 echo "Done"
196 echo "The following suffixes have been found in the cache dir:"
197 echo $sstate_suffixes
198
199 echo "Figuring out the archs in the sstate cache dir ... "
200 # Using this SSTATE_PKGSPEC definition it's 6th colon separated field
201 # SSTATE_PKGSPEC = "sstate:${PN}:${PACKAGE_ARCH}${TARGET_VENDOR}-${TARGET_OS}:${PV}:${PR}:${SSTATE_PKGARCH}:${SSTATE_VERSION}:"
202 for arch in $all_archs; do
203 grep -q ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:$arch:[^:]*:[^:]*\.tgz$" $sstate_files_list
204 [ $? -eq 0 ] && ava_archs="$ava_archs $arch"
205 # ${builder_arch}_$arch used by toolchain sstate
206 grep -q ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:${builder_arch}_$arch:[^:]*:[^:]*\.tgz$" $sstate_files_list
207 [ $? -eq 0 ] && ava_archs="$ava_archs ${builder_arch}_$arch"
208 done
209 echo "Done"
210 echo "The following archs have been found in the cache dir:"
211 echo $ava_archs
212 echo ""
213
214 # Save the file list which needs to be removed
215 local remove_listdir=`mktemp -d` || exit 1
216 for suffix in $sstate_suffixes; do
217 if [ "$suffix" = "populate_lic" ] ; then
218 echo "Skipping populate_lic, because removing duplicates doesn't work correctly for them (use --stamps-dir instead)"
219 continue
220 fi
221 # Total number of files including .siginfo and .done files
222 total_files_suffix=`grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix\.tgz.*" $sstate_files_list | wc -l 2>/dev/null`
223 total_tgz_suffix=`grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix\.tgz$" $sstate_files_list | wc -l 2>/dev/null`
224 # Save the file list to a file, some suffix's file may not exist
225 grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix\.tgz.*" $sstate_files_list >$list_suffix 2>/dev/null
226 local deleted_tgz=0
227 local deleted_files=0
228 for ext in tgz tgz.siginfo tgz.done; do
229 echo "Figuring out the sstate:xxx_$suffix.$ext ... "
230 # Uniq BPNs
231 file_names=`for arch in $ava_archs ""; do
232 sed -ne "s%.*/sstate:\([^:]*\):[^:]*:[^:]*:[^:]*:$arch:[^:]*:[^:]*\.${ext}$%\1%p" $list_suffix
233 done | sort -u`
234
235 fn_tmp=`mktemp` || exit 1
236 rm_list="$remove_listdir/sstate:xxx_$suffix"
237 for fn in $file_names; do
238 [ -z "$verbose" ] || echo "Analyzing sstate:$fn-xxx_$suffix.${ext}"
239 for arch in $ava_archs ""; do
240 grep -h ".*/sstate:$fn:[^:]*:[^:]*:[^:]*:$arch:[^:]*:[^:]*\.${ext}$" $list_suffix >$fn_tmp
241 if [ -s $fn_tmp ] ; then
242 [ $debug -gt 1 ] && echo "Available files for $fn-$arch- with suffix $suffix.${ext}:" && cat $fn_tmp
243 # Use the modification time
244 to_del=$(ls -t $(cat $fn_tmp) | sed -n '1!p')
245 [ $debug -gt 2 ] && echo "Considering to delete: $to_del"
246 # The sstate file which is downloaded from the SSTATE_MIRROR is
247 # put in SSTATE_DIR, and there is a symlink in SSTATE_DIR/??/ to
248 # it, so filter it out from the remove list if it should not be
249 # removed.
250 to_keep=$(ls -t $(cat $fn_tmp) | sed -n '1p')
251 [ $debug -gt 2 ] && echo "Considering to keep: $to_keep"
252 for k in $to_keep; do
253 if [ -L "$k" ]; then
254 # The symlink's destination
255 k_dest="`readlink -e $k`"
256 # Maybe it is the one in cache_dir
257 k_maybe="$cache_dir/${k##/*/}"
258 # Remove it from the remove list if they are the same.
259 if [ "$k_dest" = "$k_maybe" ]; then
260 to_del="`echo $to_del | sed 's#'\"$k_maybe\"'##g'`"
261 fi
262 fi
263 done
264 rm -f $fn_tmp
265 [ $debug -gt 2 ] && echo "Decided to delete: $to_del"
266 gen_rmlist $rm_list.$ext "$to_del"
267 fi
268 done
269 done
270 done
271 deleted_tgz=`cat $rm_list.* 2>/dev/null | grep ".tgz$" | wc -l`
272 deleted_files=`cat $rm_list.* 2>/dev/null | wc -l`
273 [ "$deleted_files" -gt 0 -a $debug -gt 0 ] && cat $rm_list.*
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500274 echo "($deleted_tgz out of $total_tgz_suffix .tgz files for $suffix suffix will be removed or $deleted_files out of $total_files_suffix when counting also .siginfo and .done files)"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500275 let total_deleted=$total_deleted+$deleted_files
276 done
277 deleted_tgz=0
278 rm_old_list=$remove_listdir/sstate-old-filenames
279 find $cache_dir -name 'sstate-*.tgz' >$rm_old_list
280 [ -s "$rm_old_list" ] && deleted_tgz=`cat $rm_old_list | grep ".tgz$" | wc -l`
281 [ -s "$rm_old_list" ] && deleted_files=`cat $rm_old_list | wc -l`
282 [ -s "$rm_old_list" -a $debug -gt 0 ] && cat $rm_old_list
283 echo "($deleted_tgz .tgz files with old sstate-* filenames will be removed or $deleted_files when counting also .siginfo and .done files)"
284 let total_deleted=$total_deleted+$deleted_files
285
286 rm -f $list_suffix
287 rm -f $sstate_files_list
288 if [ $total_deleted -gt 0 ]; then
289 read_confirm
290 if [ "$confirm" = "y" -o "$confirm" = "Y" ]; then
291 for list in `ls $remove_listdir/`; do
292 echo "Removing $list.tgz (`cat $remove_listdir/$list | wc -w` files) ... "
293 # Remove them one by one to avoid the argument list too long error
294 for i in `cat $remove_listdir/$list`; do
295 rm -f $verbose $i
296 done
297 echo "Done"
298 done
299 echo "$total_deleted files have been removed!"
300 else
301 do_nothing
302 fi
303 else
304 no_files
305 fi
306 [ -d $remove_listdir ] && rm -fr $remove_listdir
307}
308
309# Remove the sstate file by stamps dir, the file not used by the stamps dir
310# will be removed.
311rm_by_stamps (){
312
313 local cache_list=`mktemp` || exit 1
314 local keep_list=`mktemp` || exit 1
315 local rm_list=`mktemp` || exit 1
316 local sums
317 local all_sums
318
319 # Total number of files including sstate-, .siginfo and .done files
320 total_files=`find $cache_dir -type f -name 'sstate*' | wc -l`
321 # Save all the state file list to a file
322 find $cache_dir -type f -name 'sstate*' | sort -u -o $cache_list
323
324 echo "Figuring out the suffixes in the sstate cache dir ... "
325 local sstate_suffixes="`sed 's%.*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^_]*_\([^:]*\)\.tgz.*%\1%g' $cache_list | sort -u`"
326 echo "Done"
327 echo "The following suffixes have been found in the cache dir:"
328 echo $sstate_suffixes
329
330 # Figure out all the md5sums in the stamps dir.
331 echo "Figuring out all the md5sums in stamps dir ... "
332 for i in $sstate_suffixes; do
333 # There is no "\.sigdata" but "_setcene" when it is mirrored
334 # from the SSTATE_MIRRORS, use them to figure out the sum.
335 sums=`find $stamps -maxdepth 3 -name "*.do_$i.*" \
336 -o -name "*.do_${i}_setscene.*" | \
337 sed -ne 's#.*_setscene\.##p' -e 's#.*\.sigdata\.##p' | \
338 sed -e 's#\..*##' | sort -u`
339 all_sums="$all_sums $sums"
340 done
341 echo "Done"
342
343 echo "Figuring out the files which will be removed ... "
344 for i in $all_sums; do
345 grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:${i}_.*" $cache_list >>$keep_list
346 done
347 echo "Done"
348
349 if [ -s $keep_list ]; then
350 sort -u $keep_list -o $keep_list
351 to_del=`comm -1 -3 $keep_list $cache_list`
352 gen_rmlist $rm_list "$to_del"
353 let total_deleted=`cat $rm_list | sort -u | wc -w`
354 if [ $total_deleted -gt 0 ]; then
355 [ $debug -gt 0 ] && cat $rm_list | sort -u
356 read_confirm
357 if [ "$confirm" = "y" -o "$confirm" = "Y" ]; then
358 echo "Removing sstate cache files ... ($total_deleted files)"
359 # Remove them one by one to avoid the argument list too long error
360 for i in `cat $rm_list | sort -u`; do
361 rm -f $verbose $i
362 done
363 echo "$total_deleted files have been removed"
364 else
365 do_nothing
366 fi
367 else
368 no_files
369 fi
370 else
371 echo_error "All files in cache dir will be removed! Abort!"
372 fi
373
374 rm -f $cache_list
375 rm -f $keep_list
376 rm -f $rm_list
377}
378
379# Parse arguments
380while [ -n "$1" ]; do
381 case $1 in
382 --cache-dir=*)
383 cache_dir=`echo $1 | sed -e 's#^--cache-dir=##' | xargs readlink -e`
384 [ -d "$cache_dir" ] || echo_error "Invalid argument to --cache-dir"
385 shift
386 ;;
387 --remove-duplicated|-d)
388 rm_duplicated="y"
389 shift
390 ;;
391 --yes|-y)
392 confirm="y"
393 shift
394 ;;
395 --follow-symlink|-L)
396 fsym="y"
397 shift
398 ;;
399 --extra-archs=*)
400 extra_archs=`echo $1 | sed -e 's#^--extra-archs=##' -e 's#,# #g'`
401 [ -n "$extra_archs" ] || echo_error "Invalid extra arch parameter"
402 shift
403 ;;
404 --extra-layer=*)
405 extra_layers=`echo $1 | sed -e 's#^--extra-layer=##' -e 's#,# #g'`
406 [ -n "$extra_layers" ] || echo_error "Invalid extra layer parameter"
407 for i in $extra_layers; do
408 l=`readlink -e $i`
409 if [ -d "$l" ]; then
410 layers="$layers $l"
411 else
412 echo_error "Can't find layer $i"
413 fi
414 done
415 shift
416 ;;
417 --stamps-dir=*)
418 stamps=`echo $1 | sed -e 's#^--stamps-dir=##' -e 's#,# #g'`
419 [ -n "$stamps" ] || echo_error "Invalid stamps dir $i"
420 for i in $stamps; do
421 [ -d "$i" ] || echo_error "Invalid stamps dir $i"
422 done
423 shift
424 ;;
425 --verbose|-v)
426 verbose="-v"
427 shift
428 ;;
429 --debug|-D)
430 debug=`expr $debug + 1`
431 echo "Debug level $debug"
432 shift
433 ;;
434 --help|-h)
435 usage
436 exit 0
437 ;;
438 *)
439 echo "Invalid arguments $*"
440 echo_error "Try 'sstate-cache-management.sh -h' for more information."
441 ;;
442 esac
443done
444
445# sstate cache directory, use environment variable SSTATE_CACHE_DIR
446# if it was not specified, otherwise, error.
447[ -n "$cache_dir" ] || cache_dir=$SSTATE_CACHE_DIR
448[ -n "$cache_dir" ] || echo_error "No cache dir found!"
449[ -d "$cache_dir" ] || echo_error "Invalid cache directory \"$cache_dir\""
450
451[ -n "$rm_duplicated" -a -n "$stamps" ] && \
452 echo_error "Can not use both --remove-duplicated and --stamps-dir"
453
454[ "$rm_duplicated" = "y" ] && remove_duplicated
455[ -n "$stamps" ] && rm_by_stamps
456[ -z "$rm_duplicated" -a -z "$stamps" ] && \
457 echo "What do you want to do?"
458exit 0