[Buildroot] [PATCH 1/2] package/ccache: bump to version 3.2.1

[Buildroot] [PATCH 1/2] package/ccache: bump to version 3.2.1

[Karoly Kasza]

Signed-off-by: Karoly Kasza <kaszak@gmail.com>
---
 package/ccache/ccache.hash |    4 ++--
 package/ccache/ccache.mk   |   10 +++++-----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/package/ccache/ccache.hash b/package/ccache/ccache.hash
index 10caac8..49012af 100644
--- a/package/ccache/ccache.hash
+++ b/package/ccache/ccache.hash
@@ -1,2 +1,2 @@
-# Verified key http://samba.org/ftp/ccache/ccache-3.1.11.tar.xz.asc - sha256 computed locally
-sha256	60e9f3f1e60f528cfce910f3224bcc698029320f6ab77ec09a7b46bda2bc678e	ccache-3.1.11.tar.xz
+# Verified key http://samba.org/ftp/ccache/ccache-3.2.1.tar.xz.asc - sha256 computed locally
+sha256	872af34e1d2cef18694001d29db7f167d644e7602ca32cd6c74b28d81458bd9d	ccache-3.2.1.tar.xz
diff --git a/package/ccache/ccache.mk b/package/ccache/ccache.mk
index 893b6ae..52b5c67 100644
--- a/package/ccache/ccache.mk
+++ b/package/ccache/ccache.mk
@@ -4,7 +4,7 @@
 #
 ################################################################################
 
-CCACHE_VERSION = 3.1.11
+CCACHE_VERSION = 3.2.1
 CCACHE_SITE = http://samba.org/ftp/ccache
 CCACHE_SOURCE = ccache-$(CCACHE_VERSION).tar.xz
 CCACHE_LICENSE = GPLv3+, others
@@ -19,7 +19,7 @@ CCACHE_LICENSE_FILES = LICENSE.txt GPL-3.0.txt
 # to use HOSTCC_NOCCACHE as the compiler. Instead, we take the easy
 # path: tell ccache to use its internal copy of zlib, so that ccache
 # has zero dependency besides the C library.
-HOST_CCACHE_CONF_OPTS += ccache_cv_zlib_1_2_3=no
+HOST_CCACHE_CONF_OPTS += --with-bundled-zlib
 
 # Patch host-ccache as follows:
 #  - Use BR_CACHE_DIR instead of CCACHE_DIR, because CCACHE_DIR
@@ -34,11 +34,11 @@ HOST_CCACHE_CONF_OPTS += ccache_cv_zlib_1_2_3=no
 #    the need to specify BR_CACHE_DIR when invoking ccache directly.
 define HOST_CCACHE_PATCH_CONFIGURATION
 	sed -i 's,getenv("CCACHE_DIR"),getenv("BR_CACHE_DIR"),' $(@D)/ccache.c
-	sed -i 's,getenv("CCACHE_COMPILERCHECK"),"none",' $(@D)/ccache.c
-	sed -i 's,"%s/.ccache","$(BR_CACHE_DIR)",' $(@D)/ccache.c
+	sed -i 's,conf->compiler_check = x_strdup("mtime"),conf->compiler_check = x_strdup("none"),' $(@D)/conf.c
+	sed -i 's,"%s/.ccache","$(BR_CACHE_DIR)",' $(@D)/conf.c
 endef
 
-HOST_CCACHE_POST_CONFIGURE_HOOKS += HOST_CCACHE_PATCH_CONFIGURATION
+HOST_CCACHE_POST_PATCH_HOOKS += HOST_CCACHE_PATCH_CONFIGURATION
 
 define HOST_CCACHE_MAKE_CACHE_DIR
 	mkdir -p $(BR_CACHE_DIR)
-- 
1.7.10.4

[Buildroot] [PATCH 2/2] package/ccache: add wrapper for internal toolchain

[Karoly Kasza]

This patch will make ccache to use mtime for compiler checks except for
BR's internal toolchain, where using mtime would make ccache useless,
because of the recompilations of the binaries. In this case we use
a wrapper script and an md5 hash file with the BR .config options as
suggested previously by Arnout.

Signed-off-by: Karoly Kasza <kaszak@gmail.com>
---
 Config.in                |    7 -------
 package/ccache/ccache.mk |   25 ++++++++++++++++++++-----
 2 files changed, 20 insertions(+), 12 deletions(-)

diff --git a/Config.in b/Config.in
index 2b39d6a..6e7f722 100644
--- a/Config.in
+++ b/Config.in
@@ -257,13 +257,6 @@ config BR2_CCACHE
 	  up future builds. By default, the cache is stored in
 	  $HOME/.buildroot-ccache.
 
-	  Note that Buildroot does not try to invalidate the cache
-	  contents when the compiler changes in an incompatible
-	  way. Therefore, if you make a change to the compiler version
-	  and/or configuration, you are responsible for purging the
-	  ccache cache by removing the $HOME/.buildroot-ccache
-	  directory.
-
 if BR2_CCACHE
 
 config BR2_CCACHE_DIR
diff --git a/package/ccache/ccache.mk b/package/ccache/ccache.mk
index 52b5c67..585e809 100644
--- a/package/ccache/ccache.mk
+++ b/package/ccache/ccache.mk
@@ -26,15 +26,10 @@ HOST_CCACHE_CONF_OPTS += --with-bundled-zlib
 #    is already used by autotargets for the ccache package.
 #    BR_CACHE_DIR is exported by Makefile based on config option
 #    BR2_CCACHE_DIR.
-#  - ccache shouldn't use the compiler binary mtime to detect a change in
-#    the compiler, because in the context of Buildroot, that completely
-#    defeats the purpose of ccache. Of course, that leaves the user
-#    responsible for purging its cache when the compiler changes.
 #  - Change hard-coded last-ditch default to match path in .config, to avoid
 #    the need to specify BR_CACHE_DIR when invoking ccache directly.
 define HOST_CCACHE_PATCH_CONFIGURATION
 	sed -i 's,getenv("CCACHE_DIR"),getenv("BR_CACHE_DIR"),' $(@D)/ccache.c
-	sed -i 's,conf->compiler_check = x_strdup("mtime"),conf->compiler_check = x_strdup("none"),' $(@D)/conf.c
 	sed -i 's,"%s/.ccache","$(BR_CACHE_DIR)",' $(@D)/conf.c
 endef
 
@@ -58,6 +53,26 @@ endef
 HOST_CCACHE_POST_INSTALL_HOOKS += HOST_CCACHE_DO_INITIAL_SETUP
 endif
 
+# When using an internal toolchain, disable mtime and use the hash of the toolchain options instead
+define HOST_CCACHE_INSTALL_WRAPPER
+	mkdir -p $(HOST_DIR)/usr/bin/ccache.bin
+	mv $(HOST_DIR)/usr/bin/ccache $(HOST_DIR)/usr/bin/ccache.bin/ccache
+	echo "#!/bin/sh" > $(HOST_DIR)/usr/bin/ccache
+	echo "if [ \"$$"'1'"\" = \"$(TARGET_CC_NOCCACHE)\" ] || \\" >> $(HOST_DIR)/usr/bin/ccache
+	echo "   [ \"$$"'1'"\" = \"$(TARGET_CXX_NOCCACHE)\" ]; then" >> $(HOST_DIR)/usr/bin/ccache
+	echo "	export CCACHE_COMPILERCHECK=none CCACHE_EXTRAFILES=$(HOST_DIR)/ccache_toolchain.hash" >> $(HOST_DIR)/usr/bin/ccache
+	echo "fi" >> $(HOST_DIR)/usr/bin/ccache
+	echo "$(HOST_DIR)/usr/bin/ccache.bin/ccache \"$$"'@'"\"" >> $(HOST_DIR)/usr/bin/ccache
+	grep "BR2_ARCH\|BR2_TOOLCHAIN\|BR2_GCC\|BR2_BINUTILS\|BR2_UCLIBC\|BR2_GLIBC\|BR2_EGLIBC" $(BR2_CONFIG) | \
+	grep -v "#" | md5sum > $(HOST_DIR)/ccache_toolchain.hash
+	chmod 755 $(HOST_DIR)/usr/bin/ccache
+endef
+
+# Install wrapper script only if internal toolchain is used
+ifeq ($(BR2_TOOLCHAIN_BUILDROOT),y)
+HOST_CCACHE_POST_INSTALL_HOOKS += HOST_CCACHE_INSTALL_WRAPPER
+endif
+
 $(eval $(host-autotools-package))
 
 ifeq ($(BR2_CCACHE),y)
-- 
1.7.10.4

[Buildroot] [PATCH 2/2] package/ccache: add wrapper for internal toolchain

[Thomas Petazzoni]

Dear Karoly Kasza,

On Sun, 26 Apr 2015 11:21:02 +0200, Karoly Kasza wrote:
> This patch will make ccache to use mtime for compiler checks except for
> BR's internal toolchain, where using mtime would make ccache useless,
> because of the recompilations of the binaries. In this case we use
> a wrapper script and an md5 hash file with the BR .config options as
> suggested previously by Arnout.

Well, even for external toolchains using the mtime on the compiler in
$(HOST_DIR)/usr/bin will not work. Because the compiler in
$(HOST_DIR)/usr/bin is not actually the compiler, but a binary wrapper
built by Buildroot, for which the mtime will change at each build.

I must say I don't quite like the fact that the solution used here is
different between internal and external toolchain, but I haven't
thought too much of the problem.

How would ccache work with external toolchains?

Best regards,

Thomas
-- 
Thomas Petazzoni, CTO, Free Electrons
Embedded Linux, Kernel and Android engineering
http://free-electrons.com

[Buildroot] [PATCH 2/2] package/ccache: add wrapper for internal toolchain

[Károly Kasza]

Hello Thomas,

Well, even for external toolchains using the mtime on the compiler in
> $(HOST_DIR)/usr/bin will not work. Because the compiler in
> $(HOST_DIR)/usr/bin is not actually the compiler, but a binary wrapper
> built by Buildroot, for which the mtime will change at each build.
>

That is true, so the wrapper for external toolchains also needs the
extrafiles/hash solution. I simply missed this.


> I must say I don't quite like the fact that the solution used here is
> different between internal and external toolchain, but I haven't
> thought too much of the problem.
>

I thought this would be more efficient, but the above statement kills it.


> How would ccache work with external toolchains?
>

Well, using mtime is out of question because of the BR built wrapper, so it
requires the
same solution: a wrapper for ccache and using extrafiles/hash as for the
internal.

This means a wrapper will always be needed, and has to recognize the
toolchain - mtime for HOST_CC and EXTRAFILES for TARGET_CC always.

Kind regards,
Karoly
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.busybox.net/pipermail/buildroot/attachments/20150426/a70247b4/attachment.html>

[Buildroot] [PATCH 2/2] package/ccache: add wrapper for internal toolchain

[Thomas Petazzoni]

Dear K?roly Kasza,

On Sun, 26 Apr 2015 11:45:06 +0200, K?roly Kasza wrote:

> That is true, so the wrapper for external toolchains also needs the
> extrafiles/hash solution. I simply missed this.

No problem, but it would indeed be nice to have a solution that covers
both cases.

> > I must say I don't quite like the fact that the solution used here is
> > different between internal and external toolchain, but I haven't
> > thought too much of the problem.
> 
> I thought this would be more efficient, but the above statement kills it.

I certainly don't want to kill anything. This ccache problem has been
around for a long time, and it's definitely great to see someone taking
care of this relatively complicated issue.

> > How would ccache work with external toolchains?
> >
> 
> Well, using mtime is out of question because of the BR built wrapper, so it
> requires the
> same solution: a wrapper for ccache and using extrafiles/hash as for the
> internal.

Indeed. But maybe the existing external toolchain wrapper can be used.
We probably don't want to have a shell wrapper call a C wrapper
calling the compiler.

Thomas
-- 
Thomas Petazzoni, CTO, Free Electrons
Embedded Linux, Kernel and Android engineering
http://free-electrons.com

[Buildroot] [PATCH 2/2] package/ccache: add wrapper for internal toolchain

[Károly Kasza]

Dear Thomas,

No problem, but it would indeed be nice to have a solution that covers
> both cases.


Strongly agree.


> Indeed. But maybe the existing external toolchain wrapper can be used.
> We probably don't want to have a shell wrapper call a C wrapper
> calling the compiler.
>

I'll look into it!

Until then, I'll just resend the version bump with a more extensive commit
message, as Baruch suggested.

Regards,
Karoly
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.busybox.net/pipermail/buildroot/attachments/20150426/29b6bc2c/attachment.html>

[Buildroot] [PATCH 2/2] package/ccache: add wrapper for internal toolchain

[Károly Kasza]

Hello Thomas, all,


>
>> Indeed. But maybe the existing external toolchain wrapper can be used.
>> We probably don't want to have a shell wrapper call a C wrapper
>> calling the compiler.
>>
>
> I'll look into it!
>
>
Some overview as I understand:

By default ccache works by doing a hash on the compiler, it's arguments,
and the source file to choose a previously cached result instead of
compiling it. If no fitting hash found, it just falls back compiling (and
storing the result in cache).

In Buildroot, we can use 3+1 different compilers:

- The host compiler (mostly /usr/bin/gcc or whatever) - this changes only
if the host is updated, the default ccache checking method (mtime) can
efficiently used with it. No deep changes needed to make this work, only
applies for host-* packages. If we decide to use a ccache wrapper script
(see below), than this can be called from it. If we want to skip the
wrapper script here, then all occurrences of the $(CCACHE) variable
everywhere in BR needs to reviewed/modified, as $(CCACHE) $(HOSTCC) and
$(CCACHE) $(TARGET_CC) will have to be separated.

- The external toolchain target compiler - this is the same in terms of
binary, but the problem as ThomasP. pointed out, that it is called by a
wrapper that is recompiled every time, thus invalidating the cache if mtime
is used on it. Ccache can only hash it's first argument ($1) with mtime, so
the solution is to turn off this kind of hashing (compiler_check=none) and
use another file (extra_files) as the source of the hash. This file can be
an extract of the BR .config file, which only contains compiler/toolchain
related settings, thus recompiling different packages won't invalidate the
compiler's identity.
After roughly understanding the ext-toolchain-wrapper, I think there are
two paths to follow:
  1) Use a wrapper script to call ccache, which will call the wrapper
binary for the compiler. In this script, extrafiles can be set up as an
environment variable. This is far more easier, requires a lot less
modifications, but should (may) be a bit slower, as we have to call a shell
script before every compiler invocation. If we want a universal solution
for internal and external toolchains, this can be used.
  2) Modify BR to let the wrapper binary decide whether to use ccache
before calling the real compiler. This requires tampering with the
TARGET_CC and TARGET_CC_NOCCACHE variables in BR at least and modifying the
source code of ext-toolchain-wrapper, but should be a bit more faster, as
it won't call an additional shell script every time gcc/g++ is invoked.
This won't be universal naturally.

- The internal toolchain - this is recompiled every time, so mtime is out
of question. The external toolchain solution is also valid here, using
method 1) - ccache wrapper script. I could think of no other viable
solutions.

+1 the internal toolchain gcc-initial stage, for which the simple internal
toolchain's extrafiles setting should work.

My questions are:
 - Did I miss something?
 - Anybody has other solutions for hashing the external toolchain?
 - If not, which should be implemented?
 - Do we want an universal solution (wrapper script) or independent
solutions for every case?

Best regards,
Karoly
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.busybox.net/pipermail/buildroot/attachments/20150427/df1eb388/attachment.html>

[Buildroot] [PATCH 1/2] package/ccache: bump to version 3.2.1

[Baruch Siach]

Hi Karoly,
On Sun, Apr 26, 2015 at 11:21:01AM +0200, Karoly Kasza wrote:
> @@ -19,7 +19,7 @@ CCACHE_LICENSE_FILES = LICENSE.txt GPL-3.0.txt
>  # to use HOSTCC_NOCCACHE as the compiler. Instead, we take the easy
>  # path: tell ccache to use its internal copy of zlib, so that ccache
>  # has zero dependency besides the C library.
> -HOST_CCACHE_CONF_OPTS += ccache_cv_zlib_1_2_3=no
> +HOST_CCACHE_CONF_OPTS += --with-bundled-zlib
>  
>  # Patch host-ccache as follows:
>  #  - Use BR_CACHE_DIR instead of CCACHE_DIR, because CCACHE_DIR
> @@ -34,11 +34,11 @@ HOST_CCACHE_CONF_OPTS += ccache_cv_zlib_1_2_3=no
>  #    the need to specify BR_CACHE_DIR when invoking ccache directly.
>  define HOST_CCACHE_PATCH_CONFIGURATION
>  	sed -i 's,getenv("CCACHE_DIR"),getenv("BR_CACHE_DIR"),' $(@D)/ccache.c
> -	sed -i 's,getenv("CCACHE_COMPILERCHECK"),"none",' $(@D)/ccache.c
> -	sed -i 's,"%s/.ccache","$(BR_CACHE_DIR)",' $(@D)/ccache.c
> +	sed -i 's,conf->compiler_check = x_strdup("mtime"),conf->compiler_check = x_strdup("none"),' $(@D)/conf.c
> +	sed -i 's,"%s/.ccache","$(BR_CACHE_DIR)",' $(@D)/conf.c
>  endef

You are doing non trivial changes here and above. Please explain in a commit 
log what you are doing, and why this is needed as part of the version bump.

baruch

-- 
     http://baruch.siach.name/blog/                  ~. .~   Tk Open Systems
=}------------------------------------------------ooO--U--Ooo------------{=
   - baruch at tkos.co.il - tel: +972.2.679.5364, http://www.tkos.co.il -

[Buildroot] [PATCH 1/2] package/ccache: bump to version 3.2.1

[Károly Kasza]

Hello Baruch,

> -HOST_CCACHE_CONF_OPTS += ccache_cv_zlib_1_2_3=no
> > +HOST_CCACHE_CONF_OPTS += --with-bundled-zlib
>


> > -     sed -i 's,getenv("CCACHE_COMPILERCHECK"),"none",' $(@D)/ccache.c
> > -     sed -i 's,"%s/.ccache","$(BR_CACHE_DIR)",' $(@D)/ccache.c
> > +     sed -i 's,conf->compiler_check =
> x_strdup("mtime"),conf->compiler_check = x_strdup("none"),' $(@D)/conf.c
> > +     sed -i 's,"%s/.ccache","$(BR_CACHE_DIR)",' $(@D)/conf.c
>
>

> You are doing non trivial changes here and above. Please explain in a
> commit
> log what you are doing, and why this is needed as part of the version bump.
>

As you probably see, those lines do the exact same as before, just the
structure of the ccache sources changed with the major version bump. I'll
wait for some comments on the second patch regarding the introduction of a
wrapper script and resend with a more informative git log.

Kind regards,
Karoly
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.busybox.net/pipermail/buildroot/attachments/20150426/6a315ca3/attachment.html>

[Buildroot] [PATCH 1/2] package/ccache: bump to version 3.2.1

[Baruch Siach]

Hi K?roly,

On Sun, Apr 26, 2015 at 11:38:52AM +0200, K?roly Kasza wrote:
> > > -     sed -i 's,getenv("CCACHE_COMPILERCHECK"),"none",' $(@D)/ccache.c
> > > -     sed -i 's,"%s/.ccache","$(BR_CACHE_DIR)",' $(@D)/ccache.c
> > > +     sed -i 's,conf->compiler_check =
> > x_strdup("mtime"),conf->compiler_check = x_strdup("none"),' $(@D)/conf.c
> > > +     sed -i 's,"%s/.ccache","$(BR_CACHE_DIR)",' $(@D)/conf.c
> > 
> > You are doing non trivial changes here and above. Please explain in a
> > commit log what you are doing, and why this is needed as part of the 
> > version bump.
> 
> As you probably see, those lines do the exact same as before, just the
> structure of the ccache sources changed with the major version bump. I'll
> wait for some comments on the second patch regarding the introduction of a
> wrapper script and resend with a more informative git log.

Thanks. When doing so please comment also on the _HOOKS change.

baruch

-- 
     http://baruch.siach.name/blog/                  ~. .~   Tk Open Systems
=}------------------------------------------------ooO--U--Ooo------------{=
   - baruch at tkos.co.il - tel: +972.2.679.5364, http://www.tkos.co.il -