[PATCH 0/5] sstate-cache-management fixes

[PATCH 0/5] sstate-cache-management fixes

[Martin Jansa]

The following changes since commit c3f3fed1b397b6ee6d44e7d39e63d084e4a88e30:

  toaster.bbclass: save original package name (2014-01-29 17:40:29 +0000)

are available in the git repository at:

  git://git.openembedded.org/openembedded-core-contrib jansa/sstate-management
  http://cgit.openembedded.org/cgit.cgi/openembedded-core-contrib/log/?h=jansa/sstate-management

Martin Jansa (5):
  sstate-cache-management.sh: Fix remove_duplicated() to work after
    SSTATE_SWSPEC change
  sstate-cache-management.sh: Fix rm_by_stamps() to work after
    SSTATE_SWSPEC change
  sstate-cache-management.sh: Show total number of files when showing
    how many will be deleted
  sstate-cache-management.sh: Fix available architectures
  sstate-cache-management.sh: don't remove all packagedata sstate
    archives

 scripts/sstate-cache-management.sh | 99 +++++++++++++++++++++++++-------------
 1 file changed, 66 insertions(+), 33 deletions(-)

-- 
1.8.5.3

[PATCH 1/5] sstate-cache-management.sh: Fix remove_duplicated() to work after SSTATE_SWSPEC change

[Martin Jansa]

* format of filenames for sstate archives was changed in:
  commit 6f823a23c5f1d0ffa0a27db1c1bc1907de788505
  Author: Richard Purdie <richard.purdie@linuxfoundation.org>
  Date:   Fri Dec 20 12:06:02 2013 +0000
  Subject: sstate: Convert to use ':' as a filename sperator and use SSTATE_SWSPEC globally
* remove_duplicated() wasn't able to find available architectures and
  duplicate files since this change
* add extra step to remove old sstate archives starting with sstate-
  (instead of sstate:)

Signed-off-by: Martin Jansa <Martin.Jansa@gmail.com>
---
 scripts/sstate-cache-management.sh | 36 +++++++++++++++++++++++-------------
 1 file changed, 23 insertions(+), 13 deletions(-)

diff --git a/scripts/sstate-cache-management.sh b/scripts/sstate-cache-management.sh
index e2baf17..21ca470 100755
--- a/scripts/sstate-cache-management.sh
+++ b/scripts/sstate-cache-management.sh
@@ -181,23 +181,26 @@ remove_duplicated () {
   # Add the qemu and native archs
   # Use the "_" to substitute "-", e.g., x86-64 to x86_64
   # Sort to remove the duplicated ones
-  all_archs=$(echo $all_archs $all_machines $(uname -m) \
+  # Add allarch
+  all_archs=$(echo allarch $all_archs $all_machines $(uname -m) \
           | sed -e 's/-/_/g' -e 's/ /\n/g' | sort -u)
   echo "Done"
 
   # Save all the sstate files in a file
   sstate_list=`mktemp` || exit 1
-  find $cache_dir -name 'sstate-*.tgz' >$sstate_list
+  find $cache_dir -name 'sstate:*:*:*:*:*:*:*.tgz' >$sstate_list
 
   echo -n "Figuring out the suffixes in the sstate cache dir ... "
-  sstate_suffixes="`sed 's/.*_\([^_]*\)\.tgz$/\1/g' $sstate_list | sort -u`"
+  sstate_suffixes="`sed 's%.*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^_]*_\([^:]*\)\.tgz$%\1%g' $sstate_list | sort -u`"
   echo "Done"
   echo "The following suffixes have been found in the cache dir:"
   echo $sstate_suffixes
 
   echo -n "Figuring out the archs in the sstate cache dir ... "
+  # Using this SSTATE_PKGSPEC definition it's 6th colon separated field
+  # SSTATE_PKGSPEC    = "sstate:${PN}:${PACKAGE_ARCH}${TARGET_VENDOR}-${TARGET_OS}:${PV}:${PR}:${SSTATE_PKGARCH}:${SSTATE_VERSION}:"
   for arch in $all_archs; do
-      grep -q "\-$arch-" $sstate_list
+      grep -q ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:$arch:[^:]*:[^:]*\.tgz$" $sstate_list
       [ $? -eq 0 ] && ava_archs="$ava_archs $arch"
   done
   echo "Done"
@@ -210,21 +213,20 @@ remove_duplicated () {
 
   for suffix in $sstate_suffixes; do
       # Save the file list to a file, some suffix's file may not exist
-      grep "sstate-.*_$suffix.tgz" $sstate_list >$list_suffix 2>/dev/null
+      grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix.tgz" $sstate_list >$list_suffix 2>/dev/null
       local deleted=0
-      echo -n "Figuring out the sstate-xxx_$suffix.tgz ... "
-      # There are at list 6 dashes (-) after arch, use this to avoid the
-      # greedy match of sed.
-      file_names=`for arch in $ava_archs; do
-          sed -ne 's#.*/\(sstate-.*\)-'"$arch"'-.*-.*-.*-.*-.*-.*#\1#p' $list_suffix
+      echo -n "Figuring out the sstate:xxx_$suffix.tgz ... "
+      # Uniq BPNs
+      file_names=`for arch in $ava_archs ""; do
+          sed -ne "s%.*/sstate:\([^:]*\):[^:]*:[^:]*:[^:]*:$arch:[^:]*:[^:]*\.tgz$%\1%p" $list_suffix
       done | sort -u`
 
       fn_tmp=`mktemp` || exit 1
-      rm_list="$remove_listdir/sstate-xxx_$suffix"
+      rm_list="$remove_listdir/sstate:xxx_$suffix"
       for fn in $file_names; do
-          [ -z "$verbose" ] || echo "Analyzing $fn-xxx_$suffix.tgz"
+          [ -z "$verbose" ] || echo "Analyzing sstate:$fn-xxx_$suffix.tgz"
           for arch in $ava_archs; do
-              grep -h "/$fn-$arch-" $list_suffix >$fn_tmp
+              grep -h ".*/sstate:$fn:[^:]*:[^:]*:[^:]*:$arch:[^:]*:[^:]*\.tgz$" $list_suffix >$fn_tmp
               if [ -s $fn_tmp ] ; then
                   [ $debug -gt 1 ] && echo "Available files for $fn-$arch- with suffix $suffix:" && cat $fn_tmp
                   # Use the modification time
@@ -259,6 +261,14 @@ remove_duplicated () {
       echo "($deleted files will be removed)"
       let total_deleted=$total_deleted+$deleted
   done
+  deleted=0
+  rm_old_list=$remove_listdir/sstate-old-filenames
+  find $cache_dir -name 'sstate-*.tgz' >$rm_old_list
+  [ ! -s "$rm_old_list" ] || deleted=`cat $rm_old_list | wc -l`
+  [ -s "$rm_old_list" -a $debug -gt 0 ] && cat $rm_old_list
+  echo "($deleted files with old sstate-* filenames will be removed)"
+  let total_deleted=$total_deleted+$deleted
+
   rm -f $list_suffix
   rm -f $sstate_list
   if [ $total_deleted -gt 0 ]; then
-- 
1.8.5.3

[PATCH 2/5] sstate-cache-management.sh: Fix rm_by_stamps() to work after SSTATE_SWSPEC change

[Martin Jansa]

* format of filenames for sstate archives was changed in:
  commit 6f823a23c5f1d0ffa0a27db1c1bc1907de788505
  Author: Richard Purdie <richard.purdie@linuxfoundation.org>
  Date:   Fri Dec 20 12:06:02 2013 +0000
  Subject: sstate: Convert to use ':' as a filename sperator and use SSTATE_SWSPEC globally
* this one doesn't need special care for old sstate- names
  they will be removed automatically as they don't match with
  any checksum in rigth format from stamps directory

Signed-off-by: Martin Jansa <Martin.Jansa@gmail.com>
---
 scripts/sstate-cache-management.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scripts/sstate-cache-management.sh b/scripts/sstate-cache-management.sh
index 21ca470..30ba8c6 100755
--- a/scripts/sstate-cache-management.sh
+++ b/scripts/sstate-cache-management.sh
@@ -320,11 +320,11 @@ rm_by_stamps (){
   echo "Done"
 
   # Save all the state file list to a file
-  find $cache_dir -name 'sstate-*.tgz' | sort -u -o $cache_list
+  find $cache_dir -name 'sstate*.tgz' | sort -u -o $cache_list
 
   echo -n "Figuring out the files which will be removed ... "
   for i in $all_sums; do
-      grep ".*-${i}_.*" $cache_list >>$keep_list
+      grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:${i}_.*" $cache_list >>$keep_list
   done
   echo "Done"
 
@@ -332,7 +332,7 @@ rm_by_stamps (){
       sort -u $keep_list -o $keep_list
       to_del=`comm -1 -3 $keep_list $cache_list`
       gen_rmlist $rm_list "$to_del"
-      let total_deleted=(`cat $rm_list | wc -w`)
+      let total_deleted=`cat $rm_list | wc -w`
       if [ $total_deleted -gt 0 ]; then
           [ $debug -gt 0 ] && cat $rm_list
           read_confirm
-- 
1.8.5.3

[PATCH 3/5] sstate-cache-management.sh: Show total number of files when showing how many will be deleted

[Martin Jansa]

* it's good to see some the ratio of delted files until now it was
  showing only when all or none files were to be removed

Signed-off-by: Martin Jansa <Martin.Jansa@gmail.com>
---
 scripts/sstate-cache-management.sh | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/scripts/sstate-cache-management.sh b/scripts/sstate-cache-management.sh
index 30ba8c6..035bb25 100755
--- a/scripts/sstate-cache-management.sh
+++ b/scripts/sstate-cache-management.sh
@@ -95,7 +95,7 @@ do_nothing () {
 
 # Read the input "y"
 read_confirm () {
-  echo -n "$total_deleted files will be removed! "
+  echo -n "$total_deleted from $total_files files will be removed! "
   if [ "$confirm" != "y" ]; then
       echo -n "Do you want to continue (y/n)? "
       while read confirm; do
@@ -186,6 +186,8 @@ remove_duplicated () {
           | sed -e 's/-/_/g' -e 's/ /\n/g' | sort -u)
   echo "Done"
 
+  # Total number of files including sstate-, sigdata and .done files
+  total_files=`find $cache_dir -name 'sstate*' | wc -l`
   # Save all the sstate files in a file
   sstate_list=`mktemp` || exit 1
   find $cache_dir -name 'sstate:*:*:*:*:*:*:*.tgz' >$sstate_list
@@ -212,6 +214,8 @@ remove_duplicated () {
   local remove_listdir=`mktemp -d` || exit 1
 
   for suffix in $sstate_suffixes; do
+      # Total number of files including sigdata and .done files
+      total_files_suffix=`grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix.*" $sstate_list | wc -l 2>/dev/null`
       # Save the file list to a file, some suffix's file may not exist
       grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix.tgz" $sstate_list >$list_suffix 2>/dev/null
       local deleted=0
@@ -258,7 +262,7 @@ remove_duplicated () {
       done
       [ ! -s "$rm_list" ] || deleted=`cat $rm_list | wc -l`
       [ -s "$rm_list" -a $debug -gt 0 ] && cat $rm_list
-      echo "($deleted files will be removed)"
+      echo "($deleted from $total_files_suffix files for $suffix suffix will be removed)"
       let total_deleted=$total_deleted+$deleted
   done
   deleted=0
@@ -319,6 +323,8 @@ rm_by_stamps (){
   done
   echo "Done"
 
+  # Total number of files including sstate-, sigdata and .done files
+  total_files=`find $cache_dir -name 'sstate*' | wc -l`
   # Save all the state file list to a file
   find $cache_dir -name 'sstate*.tgz' | sort -u -o $cache_list
 
-- 
1.8.5.3

[PATCH 4/5] sstate-cache-management.sh: Fix available architectures

[Martin Jansa]

* grep for AVAILTUNES isn't enough in cases where AVAILTUNE doesn't
  match exactly with TUNE_PKGARCH, e.g. AVAILTUNE "cortexa8thf-neon"
  and TUNE_PKGARCH "cortexa8t2hf-vfp-neon", instead of trying to find
  dynamically every available TUNE_PKGARCH (we have _a lot_ of them
  even with oe-core only), add parameter --extra-archs where user can
  define extra architectures he supports in given build
* Don't replace '-' with '_' for extra-archs, it does apply to MACHINE
  names and some AVAILTUNES, but e.g. cortexa8thf-neon shouldn't be
  converted to cortexa8thf_neon
* Add empty architecture for populate_lic sstate archives
* Add ${build_arch}_${arch} combinations for toolchain recipes (e.g.
  gcc-cross is using x86_64_i586

Signed-off-by: Martin Jansa <Martin.Jansa@gmail.com>
---
 scripts/sstate-cache-management.sh | 49 +++++++++++++++++++++++++-------------
 1 file changed, 33 insertions(+), 16 deletions(-)

diff --git a/scripts/sstate-cache-management.sh b/scripts/sstate-cache-management.sh
index 035bb25..0d71bfe 100755
--- a/scripts/sstate-cache-management.sh
+++ b/scripts/sstate-cache-management.sh
@@ -37,13 +37,19 @@ Options:
         Specify sstate cache directory, will use the environment
         variable SSTATE_CACHE_DIR if it is not specified.
 
+  --extra-archs=<arch1>,<arch2>...<archn>
+        Specify list of architectures which should be tested, this list
+        will be extended with native arch, allarch and empty arch. The
+        script won't be trying to generate list of available archs from
+        AVAILTUNES in tune files.
+
   --extra-layer=<layer1>,<layer2>...<layern>
         Specify the layer which will be used for searching the archs,
         it will search the meta and meta-* layers in the top dir by
         default, and will search meta, meta-*, <layer1>, <layer2>,
         ...<layern> when specified. Use "," as the separator.
 
-        This is useless for --stamps-dir.
+        This is useless for --stamps-dir or when --extra-archs is used.
 
   -d, --remove-duplicated
         Remove the duplicated sstate cache files of one package, only
@@ -170,20 +176,23 @@ remove_duplicated () {
   local fn_tmp
   local list_suffix=`mktemp` || exit 1
 
-  # Find out the archs in all the layers
-  echo -n "Figuring out the archs in the layers ... "
-  oe_core_dir=$(dirname $(dirname $(readlink -e $0)))
-  topdir=$(dirname $oe_core_dir)
-  tunedirs="`find $topdir/meta* ${oe_core_dir}/meta* $layers -path '*/meta*/conf/machine/include' 2>/dev/null`"
-  [ -n "$tunedirs" ] || echo_error "Can't find the tune directory"
-  all_machines="`find $topdir/meta* ${oe_core_dir}/meta* $layers -path '*/meta*/conf/machine/*' -name '*.conf' 2>/dev/null | sed -e 's/.*\///' -e 's/.conf$//'`"
-  all_archs=`grep -r -h "^AVAILTUNES .*=" $tunedirs | sed -e 's/.*=//' -e 's/\"//g'`
-  # Add the qemu and native archs
-  # Use the "_" to substitute "-", e.g., x86-64 to x86_64
+  if [ -z "$extra_archs" ] ; then
+    # Find out the archs in all the layers
+    echo -n "Figuring out the archs in the layers ... "
+    oe_core_dir=$(dirname $(dirname $(readlink -e $0)))
+    topdir=$(dirname $oe_core_dir)
+    tunedirs="`find $topdir/meta* ${oe_core_dir}/meta* $layers -path '*/meta*/conf/machine/include' 2>/dev/null`"
+    [ -n "$tunedirs" ] || echo_error "Can't find the tune directory"
+    all_machines="`find $topdir/meta* ${oe_core_dir}/meta* $layers -path '*/meta*/conf/machine/*' -name '*.conf' 2>/dev/null | sed -e 's/.*\///' -e 's/.conf$//'`"
+    all_archs=`grep -r -h "^AVAILTUNES .*=" $tunedirs | sed -e 's/.*=//' -e 's/\"//g'`
+  fi
+
+  # Use the "_" to substitute "-", e.g., x86-64 to x86_64, but not for extra_archs which can be something like cortexa9t2-vfp-neon
   # Sort to remove the duplicated ones
-  # Add allarch
-  all_archs=$(echo allarch $all_archs $all_machines $(uname -m) \
-          | sed -e 's/-/_/g' -e 's/ /\n/g' | sort -u)
+  # Add allarch and builder arch (native)
+  builder_arch=$(uname -m)
+  all_archs="$(echo allarch $all_archs $all_machines $builder_arch \
+          | sed -e 's/-/_/g' -e 's/ /\n/g' | sort -u) $extra_archs"
   echo "Done"
 
   # Total number of files including sstate-, sigdata and .done files
@@ -204,6 +213,9 @@ remove_duplicated () {
   for arch in $all_archs; do
       grep -q ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:$arch:[^:]*:[^:]*\.tgz$" $sstate_list
       [ $? -eq 0 ] && ava_archs="$ava_archs $arch"
+      # ${builder_arch}_$arch used by toolchain sstate
+      grep -q ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:${builder_arch}_$arch:[^:]*:[^:]*\.tgz$" $sstate_list
+      [ $? -eq 0 ] && ava_archs="$ava_archs ${builder_arch}_$arch"
   done
   echo "Done"
   echo "The following archs have been found in the cache dir:"
@@ -229,7 +241,7 @@ remove_duplicated () {
       rm_list="$remove_listdir/sstate:xxx_$suffix"
       for fn in $file_names; do
           [ -z "$verbose" ] || echo "Analyzing sstate:$fn-xxx_$suffix.tgz"
-          for arch in $ava_archs; do
+          for arch in $ava_archs ""; do
               grep -h ".*/sstate:$fn:[^:]*:[^:]*:[^:]*:$arch:[^:]*:[^:]*\.tgz$" $list_suffix >$fn_tmp
               if [ -s $fn_tmp ] ; then
                   [ $debug -gt 1 ] && echo "Available files for $fn-$arch- with suffix $suffix:" && cat $fn_tmp
@@ -384,9 +396,14 @@ while [ -n "$1" ]; do
       fsym="y"
       shift
         ;;
+    --extra-archs=*)
+      extra_archs=`echo $1 | sed -e 's#^--extra-archs=##' -e 's#,# #g'`
+      [ -n "$extra_archs" ] || echo_error "Invalid extra arch parameter"
+      shift
+        ;;
     --extra-layer=*)
       extra_layers=`echo $1 | sed -e 's#^--extra-layer=##' -e 's#,# #g'`
-      [ -n "$extra_layers" ] || echo_error "Invalid extra layer $i"
+      [ -n "$extra_layers" ] || echo_error "Invalid extra layer parameter"
       for i in $extra_layers; do
           l=`readlink -e $i`
           if [ -d "$l" ]; then
-- 
1.8.5.3

[PATCH 5/5] sstate-cache-management.sh: don't remove all packagedata sstate archives

[Martin Jansa]

* packagedata task was introduced in:
  commit 6107ee294afde395e39d084c33e8e94013c625a9
  Author: Richard Purdie <richard.purdie@linuxfoundation.org>
  Date:   Wed Jan 23 14:27:33 2013 +0000
  Subject: Split do_packagedata task from do_package
* rm_by_stamps wasn't using do_packagedata or do_packagedata_setscene
  stamp files to find which sstate archives to keep, so it was removing
  all of them

Signed-off-by: Martin Jansa <Martin.Jansa@gmail.com>
---
 scripts/sstate-cache-management.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/sstate-cache-management.sh b/scripts/sstate-cache-management.sh
index 0d71bfe..34483d0 100755
--- a/scripts/sstate-cache-management.sh
+++ b/scripts/sstate-cache-management.sh
@@ -320,7 +320,7 @@ rm_by_stamps (){
   local all_sums
 
   suffixes="populate_sysroot populate_lic package_write_ipk \
-            package_write_rpm package_write_deb package deploy"
+            package_write_rpm package_write_deb package packagedata deploy"
 
   # Figure out all the md5sums in the stamps dir.
   echo -n "Figuring out all the md5sums in stamps dir ... "
-- 
1.8.5.3