* [PATCH 1/5] sstate-cache-management.sh: Fix remove_duplicated() to work after SSTATE_SWSPEC change
2014-01-29 21:03 [PATCH 0/5] sstate-cache-management fixes Martin Jansa
@ 2014-01-29 21:03 ` Martin Jansa
2014-01-29 21:03 ` [PATCH 2/5] sstate-cache-management.sh: Fix rm_by_stamps() " Martin Jansa
` (3 subsequent siblings)
4 siblings, 0 replies; 6+ messages in thread
From: Martin Jansa @ 2014-01-29 21:03 UTC (permalink / raw)
To: openembedded-core
* format of filenames for sstate archives was changed in:
commit 6f823a23c5f1d0ffa0a27db1c1bc1907de788505
Author: Richard Purdie <richard.purdie@linuxfoundation.org>
Date: Fri Dec 20 12:06:02 2013 +0000
Subject: sstate: Convert to use ':' as a filename sperator and use SSTATE_SWSPEC globally
* remove_duplicated() wasn't able to find available architectures and
duplicate files since this change
* add extra step to remove old sstate archives starting with sstate-
(instead of sstate:)
Signed-off-by: Martin Jansa <Martin.Jansa@gmail.com>
---
scripts/sstate-cache-management.sh | 36 +++++++++++++++++++++++-------------
1 file changed, 23 insertions(+), 13 deletions(-)
diff --git a/scripts/sstate-cache-management.sh b/scripts/sstate-cache-management.sh
index e2baf17..21ca470 100755
--- a/scripts/sstate-cache-management.sh
+++ b/scripts/sstate-cache-management.sh
@@ -181,23 +181,26 @@ remove_duplicated () {
# Add the qemu and native archs
# Use the "_" to substitute "-", e.g., x86-64 to x86_64
# Sort to remove the duplicated ones
- all_archs=$(echo $all_archs $all_machines $(uname -m) \
+ # Add allarch
+ all_archs=$(echo allarch $all_archs $all_machines $(uname -m) \
| sed -e 's/-/_/g' -e 's/ /\n/g' | sort -u)
echo "Done"
# Save all the sstate files in a file
sstate_list=`mktemp` || exit 1
- find $cache_dir -name 'sstate-*.tgz' >$sstate_list
+ find $cache_dir -name 'sstate:*:*:*:*:*:*:*.tgz' >$sstate_list
echo -n "Figuring out the suffixes in the sstate cache dir ... "
- sstate_suffixes="`sed 's/.*_\([^_]*\)\.tgz$/\1/g' $sstate_list | sort -u`"
+ sstate_suffixes="`sed 's%.*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^_]*_\([^:]*\)\.tgz$%\1%g' $sstate_list | sort -u`"
echo "Done"
echo "The following suffixes have been found in the cache dir:"
echo $sstate_suffixes
echo -n "Figuring out the archs in the sstate cache dir ... "
+ # Using this SSTATE_PKGSPEC definition it's 6th colon separated field
+ # SSTATE_PKGSPEC = "sstate:${PN}:${PACKAGE_ARCH}${TARGET_VENDOR}-${TARGET_OS}:${PV}:${PR}:${SSTATE_PKGARCH}:${SSTATE_VERSION}:"
for arch in $all_archs; do
- grep -q "\-$arch-" $sstate_list
+ grep -q ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:$arch:[^:]*:[^:]*\.tgz$" $sstate_list
[ $? -eq 0 ] && ava_archs="$ava_archs $arch"
done
echo "Done"
@@ -210,21 +213,20 @@ remove_duplicated () {
for suffix in $sstate_suffixes; do
# Save the file list to a file, some suffix's file may not exist
- grep "sstate-.*_$suffix.tgz" $sstate_list >$list_suffix 2>/dev/null
+ grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix.tgz" $sstate_list >$list_suffix 2>/dev/null
local deleted=0
- echo -n "Figuring out the sstate-xxx_$suffix.tgz ... "
- # There are at list 6 dashes (-) after arch, use this to avoid the
- # greedy match of sed.
- file_names=`for arch in $ava_archs; do
- sed -ne 's#.*/\(sstate-.*\)-'"$arch"'-.*-.*-.*-.*-.*-.*#\1#p' $list_suffix
+ echo -n "Figuring out the sstate:xxx_$suffix.tgz ... "
+ # Uniq BPNs
+ file_names=`for arch in $ava_archs ""; do
+ sed -ne "s%.*/sstate:\([^:]*\):[^:]*:[^:]*:[^:]*:$arch:[^:]*:[^:]*\.tgz$%\1%p" $list_suffix
done | sort -u`
fn_tmp=`mktemp` || exit 1
- rm_list="$remove_listdir/sstate-xxx_$suffix"
+ rm_list="$remove_listdir/sstate:xxx_$suffix"
for fn in $file_names; do
- [ -z "$verbose" ] || echo "Analyzing $fn-xxx_$suffix.tgz"
+ [ -z "$verbose" ] || echo "Analyzing sstate:$fn-xxx_$suffix.tgz"
for arch in $ava_archs; do
- grep -h "/$fn-$arch-" $list_suffix >$fn_tmp
+ grep -h ".*/sstate:$fn:[^:]*:[^:]*:[^:]*:$arch:[^:]*:[^:]*\.tgz$" $list_suffix >$fn_tmp
if [ -s $fn_tmp ] ; then
[ $debug -gt 1 ] && echo "Available files for $fn-$arch- with suffix $suffix:" && cat $fn_tmp
# Use the modification time
@@ -259,6 +261,14 @@ remove_duplicated () {
echo "($deleted files will be removed)"
let total_deleted=$total_deleted+$deleted
done
+ deleted=0
+ rm_old_list=$remove_listdir/sstate-old-filenames
+ find $cache_dir -name 'sstate-*.tgz' >$rm_old_list
+ [ ! -s "$rm_old_list" ] || deleted=`cat $rm_old_list | wc -l`
+ [ -s "$rm_old_list" -a $debug -gt 0 ] && cat $rm_old_list
+ echo "($deleted files with old sstate-* filenames will be removed)"
+ let total_deleted=$total_deleted+$deleted
+
rm -f $list_suffix
rm -f $sstate_list
if [ $total_deleted -gt 0 ]; then
--
1.8.5.3
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH 2/5] sstate-cache-management.sh: Fix rm_by_stamps() to work after SSTATE_SWSPEC change
2014-01-29 21:03 [PATCH 0/5] sstate-cache-management fixes Martin Jansa
2014-01-29 21:03 ` [PATCH 1/5] sstate-cache-management.sh: Fix remove_duplicated() to work after SSTATE_SWSPEC change Martin Jansa
@ 2014-01-29 21:03 ` Martin Jansa
2014-01-29 21:03 ` [PATCH 3/5] sstate-cache-management.sh: Show total number of files when showing how many will be deleted Martin Jansa
` (2 subsequent siblings)
4 siblings, 0 replies; 6+ messages in thread
From: Martin Jansa @ 2014-01-29 21:03 UTC (permalink / raw)
To: openembedded-core
* format of filenames for sstate archives was changed in:
commit 6f823a23c5f1d0ffa0a27db1c1bc1907de788505
Author: Richard Purdie <richard.purdie@linuxfoundation.org>
Date: Fri Dec 20 12:06:02 2013 +0000
Subject: sstate: Convert to use ':' as a filename sperator and use SSTATE_SWSPEC globally
* this one doesn't need special care for old sstate- names
they will be removed automatically as they don't match with
any checksum in rigth format from stamps directory
Signed-off-by: Martin Jansa <Martin.Jansa@gmail.com>
---
scripts/sstate-cache-management.sh | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/scripts/sstate-cache-management.sh b/scripts/sstate-cache-management.sh
index 21ca470..30ba8c6 100755
--- a/scripts/sstate-cache-management.sh
+++ b/scripts/sstate-cache-management.sh
@@ -320,11 +320,11 @@ rm_by_stamps (){
echo "Done"
# Save all the state file list to a file
- find $cache_dir -name 'sstate-*.tgz' | sort -u -o $cache_list
+ find $cache_dir -name 'sstate*.tgz' | sort -u -o $cache_list
echo -n "Figuring out the files which will be removed ... "
for i in $all_sums; do
- grep ".*-${i}_.*" $cache_list >>$keep_list
+ grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:${i}_.*" $cache_list >>$keep_list
done
echo "Done"
@@ -332,7 +332,7 @@ rm_by_stamps (){
sort -u $keep_list -o $keep_list
to_del=`comm -1 -3 $keep_list $cache_list`
gen_rmlist $rm_list "$to_del"
- let total_deleted=(`cat $rm_list | wc -w`)
+ let total_deleted=`cat $rm_list | wc -w`
if [ $total_deleted -gt 0 ]; then
[ $debug -gt 0 ] && cat $rm_list
read_confirm
--
1.8.5.3
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH 3/5] sstate-cache-management.sh: Show total number of files when showing how many will be deleted
2014-01-29 21:03 [PATCH 0/5] sstate-cache-management fixes Martin Jansa
2014-01-29 21:03 ` [PATCH 1/5] sstate-cache-management.sh: Fix remove_duplicated() to work after SSTATE_SWSPEC change Martin Jansa
2014-01-29 21:03 ` [PATCH 2/5] sstate-cache-management.sh: Fix rm_by_stamps() " Martin Jansa
@ 2014-01-29 21:03 ` Martin Jansa
2014-01-29 21:03 ` [PATCH 4/5] sstate-cache-management.sh: Fix available architectures Martin Jansa
2014-01-29 21:03 ` [PATCH 5/5] sstate-cache-management.sh: don't remove all packagedata sstate archives Martin Jansa
4 siblings, 0 replies; 6+ messages in thread
From: Martin Jansa @ 2014-01-29 21:03 UTC (permalink / raw)
To: openembedded-core
* it's good to see some the ratio of delted files until now it was
showing only when all or none files were to be removed
Signed-off-by: Martin Jansa <Martin.Jansa@gmail.com>
---
scripts/sstate-cache-management.sh | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/scripts/sstate-cache-management.sh b/scripts/sstate-cache-management.sh
index 30ba8c6..035bb25 100755
--- a/scripts/sstate-cache-management.sh
+++ b/scripts/sstate-cache-management.sh
@@ -95,7 +95,7 @@ do_nothing () {
# Read the input "y"
read_confirm () {
- echo -n "$total_deleted files will be removed! "
+ echo -n "$total_deleted from $total_files files will be removed! "
if [ "$confirm" != "y" ]; then
echo -n "Do you want to continue (y/n)? "
while read confirm; do
@@ -186,6 +186,8 @@ remove_duplicated () {
| sed -e 's/-/_/g' -e 's/ /\n/g' | sort -u)
echo "Done"
+ # Total number of files including sstate-, sigdata and .done files
+ total_files=`find $cache_dir -name 'sstate*' | wc -l`
# Save all the sstate files in a file
sstate_list=`mktemp` || exit 1
find $cache_dir -name 'sstate:*:*:*:*:*:*:*.tgz' >$sstate_list
@@ -212,6 +214,8 @@ remove_duplicated () {
local remove_listdir=`mktemp -d` || exit 1
for suffix in $sstate_suffixes; do
+ # Total number of files including sigdata and .done files
+ total_files_suffix=`grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix.*" $sstate_list | wc -l 2>/dev/null`
# Save the file list to a file, some suffix's file may not exist
grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix.tgz" $sstate_list >$list_suffix 2>/dev/null
local deleted=0
@@ -258,7 +262,7 @@ remove_duplicated () {
done
[ ! -s "$rm_list" ] || deleted=`cat $rm_list | wc -l`
[ -s "$rm_list" -a $debug -gt 0 ] && cat $rm_list
- echo "($deleted files will be removed)"
+ echo "($deleted from $total_files_suffix files for $suffix suffix will be removed)"
let total_deleted=$total_deleted+$deleted
done
deleted=0
@@ -319,6 +323,8 @@ rm_by_stamps (){
done
echo "Done"
+ # Total number of files including sstate-, sigdata and .done files
+ total_files=`find $cache_dir -name 'sstate*' | wc -l`
# Save all the state file list to a file
find $cache_dir -name 'sstate*.tgz' | sort -u -o $cache_list
--
1.8.5.3
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH 4/5] sstate-cache-management.sh: Fix available architectures
2014-01-29 21:03 [PATCH 0/5] sstate-cache-management fixes Martin Jansa
` (2 preceding siblings ...)
2014-01-29 21:03 ` [PATCH 3/5] sstate-cache-management.sh: Show total number of files when showing how many will be deleted Martin Jansa
@ 2014-01-29 21:03 ` Martin Jansa
2014-01-29 21:03 ` [PATCH 5/5] sstate-cache-management.sh: don't remove all packagedata sstate archives Martin Jansa
4 siblings, 0 replies; 6+ messages in thread
From: Martin Jansa @ 2014-01-29 21:03 UTC (permalink / raw)
To: openembedded-core
* grep for AVAILTUNES isn't enough in cases where AVAILTUNE doesn't
match exactly with TUNE_PKGARCH, e.g. AVAILTUNE "cortexa8thf-neon"
and TUNE_PKGARCH "cortexa8t2hf-vfp-neon", instead of trying to find
dynamically every available TUNE_PKGARCH (we have _a lot_ of them
even with oe-core only), add parameter --extra-archs where user can
define extra architectures he supports in given build
* Don't replace '-' with '_' for extra-archs, it does apply to MACHINE
names and some AVAILTUNES, but e.g. cortexa8thf-neon shouldn't be
converted to cortexa8thf_neon
* Add empty architecture for populate_lic sstate archives
* Add ${build_arch}_${arch} combinations for toolchain recipes (e.g.
gcc-cross is using x86_64_i586
Signed-off-by: Martin Jansa <Martin.Jansa@gmail.com>
---
scripts/sstate-cache-management.sh | 49 +++++++++++++++++++++++++-------------
1 file changed, 33 insertions(+), 16 deletions(-)
diff --git a/scripts/sstate-cache-management.sh b/scripts/sstate-cache-management.sh
index 035bb25..0d71bfe 100755
--- a/scripts/sstate-cache-management.sh
+++ b/scripts/sstate-cache-management.sh
@@ -37,13 +37,19 @@ Options:
Specify sstate cache directory, will use the environment
variable SSTATE_CACHE_DIR if it is not specified.
+ --extra-archs=<arch1>,<arch2>...<archn>
+ Specify list of architectures which should be tested, this list
+ will be extended with native arch, allarch and empty arch. The
+ script won't be trying to generate list of available archs from
+ AVAILTUNES in tune files.
+
--extra-layer=<layer1>,<layer2>...<layern>
Specify the layer which will be used for searching the archs,
it will search the meta and meta-* layers in the top dir by
default, and will search meta, meta-*, <layer1>, <layer2>,
...<layern> when specified. Use "," as the separator.
- This is useless for --stamps-dir.
+ This is useless for --stamps-dir or when --extra-archs is used.
-d, --remove-duplicated
Remove the duplicated sstate cache files of one package, only
@@ -170,20 +176,23 @@ remove_duplicated () {
local fn_tmp
local list_suffix=`mktemp` || exit 1
- # Find out the archs in all the layers
- echo -n "Figuring out the archs in the layers ... "
- oe_core_dir=$(dirname $(dirname $(readlink -e $0)))
- topdir=$(dirname $oe_core_dir)
- tunedirs="`find $topdir/meta* ${oe_core_dir}/meta* $layers -path '*/meta*/conf/machine/include' 2>/dev/null`"
- [ -n "$tunedirs" ] || echo_error "Can't find the tune directory"
- all_machines="`find $topdir/meta* ${oe_core_dir}/meta* $layers -path '*/meta*/conf/machine/*' -name '*.conf' 2>/dev/null | sed -e 's/.*\///' -e 's/.conf$//'`"
- all_archs=`grep -r -h "^AVAILTUNES .*=" $tunedirs | sed -e 's/.*=//' -e 's/\"//g'`
- # Add the qemu and native archs
- # Use the "_" to substitute "-", e.g., x86-64 to x86_64
+ if [ -z "$extra_archs" ] ; then
+ # Find out the archs in all the layers
+ echo -n "Figuring out the archs in the layers ... "
+ oe_core_dir=$(dirname $(dirname $(readlink -e $0)))
+ topdir=$(dirname $oe_core_dir)
+ tunedirs="`find $topdir/meta* ${oe_core_dir}/meta* $layers -path '*/meta*/conf/machine/include' 2>/dev/null`"
+ [ -n "$tunedirs" ] || echo_error "Can't find the tune directory"
+ all_machines="`find $topdir/meta* ${oe_core_dir}/meta* $layers -path '*/meta*/conf/machine/*' -name '*.conf' 2>/dev/null | sed -e 's/.*\///' -e 's/.conf$//'`"
+ all_archs=`grep -r -h "^AVAILTUNES .*=" $tunedirs | sed -e 's/.*=//' -e 's/\"//g'`
+ fi
+
+ # Use the "_" to substitute "-", e.g., x86-64 to x86_64, but not for extra_archs which can be something like cortexa9t2-vfp-neon
# Sort to remove the duplicated ones
- # Add allarch
- all_archs=$(echo allarch $all_archs $all_machines $(uname -m) \
- | sed -e 's/-/_/g' -e 's/ /\n/g' | sort -u)
+ # Add allarch and builder arch (native)
+ builder_arch=$(uname -m)
+ all_archs="$(echo allarch $all_archs $all_machines $builder_arch \
+ | sed -e 's/-/_/g' -e 's/ /\n/g' | sort -u) $extra_archs"
echo "Done"
# Total number of files including sstate-, sigdata and .done files
@@ -204,6 +213,9 @@ remove_duplicated () {
for arch in $all_archs; do
grep -q ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:$arch:[^:]*:[^:]*\.tgz$" $sstate_list
[ $? -eq 0 ] && ava_archs="$ava_archs $arch"
+ # ${builder_arch}_$arch used by toolchain sstate
+ grep -q ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:${builder_arch}_$arch:[^:]*:[^:]*\.tgz$" $sstate_list
+ [ $? -eq 0 ] && ava_archs="$ava_archs ${builder_arch}_$arch"
done
echo "Done"
echo "The following archs have been found in the cache dir:"
@@ -229,7 +241,7 @@ remove_duplicated () {
rm_list="$remove_listdir/sstate:xxx_$suffix"
for fn in $file_names; do
[ -z "$verbose" ] || echo "Analyzing sstate:$fn-xxx_$suffix.tgz"
- for arch in $ava_archs; do
+ for arch in $ava_archs ""; do
grep -h ".*/sstate:$fn:[^:]*:[^:]*:[^:]*:$arch:[^:]*:[^:]*\.tgz$" $list_suffix >$fn_tmp
if [ -s $fn_tmp ] ; then
[ $debug -gt 1 ] && echo "Available files for $fn-$arch- with suffix $suffix:" && cat $fn_tmp
@@ -384,9 +396,14 @@ while [ -n "$1" ]; do
fsym="y"
shift
;;
+ --extra-archs=*)
+ extra_archs=`echo $1 | sed -e 's#^--extra-archs=##' -e 's#,# #g'`
+ [ -n "$extra_archs" ] || echo_error "Invalid extra arch parameter"
+ shift
+ ;;
--extra-layer=*)
extra_layers=`echo $1 | sed -e 's#^--extra-layer=##' -e 's#,# #g'`
- [ -n "$extra_layers" ] || echo_error "Invalid extra layer $i"
+ [ -n "$extra_layers" ] || echo_error "Invalid extra layer parameter"
for i in $extra_layers; do
l=`readlink -e $i`
if [ -d "$l" ]; then
--
1.8.5.3
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH 5/5] sstate-cache-management.sh: don't remove all packagedata sstate archives
2014-01-29 21:03 [PATCH 0/5] sstate-cache-management fixes Martin Jansa
` (3 preceding siblings ...)
2014-01-29 21:03 ` [PATCH 4/5] sstate-cache-management.sh: Fix available architectures Martin Jansa
@ 2014-01-29 21:03 ` Martin Jansa
4 siblings, 0 replies; 6+ messages in thread
From: Martin Jansa @ 2014-01-29 21:03 UTC (permalink / raw)
To: openembedded-core
* packagedata task was introduced in:
commit 6107ee294afde395e39d084c33e8e94013c625a9
Author: Richard Purdie <richard.purdie@linuxfoundation.org>
Date: Wed Jan 23 14:27:33 2013 +0000
Subject: Split do_packagedata task from do_package
* rm_by_stamps wasn't using do_packagedata or do_packagedata_setscene
stamp files to find which sstate archives to keep, so it was removing
all of them
Signed-off-by: Martin Jansa <Martin.Jansa@gmail.com>
---
scripts/sstate-cache-management.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/scripts/sstate-cache-management.sh b/scripts/sstate-cache-management.sh
index 0d71bfe..34483d0 100755
--- a/scripts/sstate-cache-management.sh
+++ b/scripts/sstate-cache-management.sh
@@ -320,7 +320,7 @@ rm_by_stamps (){
local all_sums
suffixes="populate_sysroot populate_lic package_write_ipk \
- package_write_rpm package_write_deb package deploy"
+ package_write_rpm package_write_deb package packagedata deploy"
# Figure out all the md5sums in the stamps dir.
echo -n "Figuring out all the md5sums in stamps dir ... "
--
1.8.5.3
^ permalink raw reply related [flat|nested] 6+ messages in thread