[Buildroot] [PATCH 0 of 5] Add find utility functions / exclude files from strip / reduce python

[Buildroot] [PATCH 0 of 5] Add find utility functions / exclude files from strip / reduce python

[Thomas De Schampheleire]

This patch series adds a few utility functions to Makefile.package.in that
help in composing find commands, and uses these functions to:
* exclude user-configured files and directories from being stripped in
  target-finalize, and
* reduce a python installation to the bare minimum.

Signed-off-by: Thomas De Schampheleire <thomas.de.schampheleire@gmail.com>

---
 Config.in                   |  19 +++++++++++++++++++
 Makefile                    |   9 +++++++--
 package/Makefile.package.in |  17 +++++++++++++++++
 package/python/Config.in    |  26 ++++++++++++++++++++++----
 package/python/python.mk    |  28 +++++++++++++++++++++++++++-
 5 files changed, 92 insertions(+), 7 deletions(-)

[Buildroot] [PATCH 1 of 5] Makefile.package.in: add utility functions find*clauses and notfirstword

[Thomas De Schampheleire]

This patch adds a few utility functions to Makefile.package.in.

Functions finddirclauses, finddirclauses_cont, findfileclauses and
findfileclauses_cont help in building a find command that skips a set of
directories and performs operations on a set of files.
This pattern can for example be used to keep certain files or directories from
being stripped, or to remove certain files from a package installation.

The notfirstword function is the inverse of the make 'firstword' function: it
returns all but the first word.

Signed-off-by: Thomas De Schampheleire <thomas.de.schampheleire@gmail.com>

---
 package/Makefile.package.in |  17 +++++++++++++++++
 1 files changed, 17 insertions(+), 0 deletions(-)

diff --git a/package/Makefile.package.in b/package/Makefile.package.in
--- a/package/Makefile.package.in
+++ b/package/Makefile.package.in
@@ -87,6 +87,23 @@ MESSAGE = echo "$(TERM_BOLD)>>> $($(PKG)
 TERM_BOLD := $(shell tput smso)
 TERM_RESET := $(shell tput rmso)
 
+# Utility functions for 'find'
+# findfileclauses_cont: creates "-o -name 'X' -o -name 'Y'"
+# [1:namelist, 2:prefix]
+findfileclauses_cont=$(patsubst %,$(2) -name '%',$(1))
+# findfileclauses: creates "-name 'X' -o -name 'Y'"
+# [1:namelist]
+findfileclauses=$(call findfileclauses_cont,$(firstword $(1))) $(call findfileclauses_cont,$(call notfirstword,$(1)),-o)
+# finddirclauses_cont: creates "-o -wholename 'basedir/dirX' -o -wholename 'basedir/dirY'"
+# [1:basedir, 2:namelist, 3:prefix]
+finddirclauses_cont=$(patsubst %,$(3) -wholename '$(1)/%',$(2))
+# finddirclauses: creates "-wholename 'basedir/dirX' -o -wholename 'basedir/dirY'"
+# [1:basedir, 2:namelist]
+finddirclauses=$(call finddirclauses_cont,$(1),$(firstword $(2))) $(call finddirclauses_cont,$(1),$(call notfirstword,$(2)),-o)
+
+# Miscellaneous utility functions
+notfirstword=$(wordlist 2,$(words $(1)),$(1))
+
 # Download method commands
 WGET:=$(call qstrip,$(BR2_WGET)) $(QUIET)
 SVN:=$(call qstrip,$(BR2_SVN))

[Buildroot] [PATCH 1 of 5] Makefile.package.in: add utility functions find*clauses and notfirstword

[Arnout Vandecappelle]

On Wednesday 19 October 2011 09:34:04, Thomas De Schampheleire wrote:
> +# findfileclauses_cont: creates "-o -name 'X' -o -name 'Y'"
> +# [1:namelist, 2:prefix]
> +findfileclauses_cont=$(patsubst %,$(2) -name '%',$(1))
> +# findfileclauses: creates "-name 'X' -o -name 'Y'"
> +# [1:namelist]
> +findfileclauses=$(call findfileclauses_cont,$(firstword $(1))) $(call findfileclauses_cont,$(call notfirstword,$(1)),-o)

 The _cont function is a bit redundant, no?

findfileclauses=-name '$(firstword $(1))' $(patsubst %,-o -name '%',$(call notfirstword,$(1)))

 I don't think the duplication of -name warrants the additional function
call.  It definitely doesn't make it easier to understand.

 Nevertheless, this is a nifty patch!


 Regards,
 Arnout
-- 
Arnout Vandecappelle                               arnout at mind be
Senior Embedded Software Architect                 +32-16-286540
Essensium/Mind                                     http://www.mind.be
G.Geenslaan 9, 3001 Leuven, Belgium                BE 872 984 063 RPR Leuven
LinkedIn profile: http://www.linkedin.com/in/arnoutvandecappelle
GPG fingerprint:  31BB CF53 8660 6F88 345D  54CC A836 5879 20D7 CF43

[Buildroot] [PATCH 1 of 5] Makefile.package.in: add utility functions find*clauses and notfirstword

[Thomas De Schampheleire]

Hi Arnout,

On Thu, Oct 20, 2011 at 12:32 AM, Arnout Vandecappelle <arnout@mind.be> wrote:
>
> On Wednesday 19 October 2011 09:34:04, Thomas De Schampheleire wrote:
>> +# findfileclauses_cont: creates "-o -name 'X' -o -name 'Y'"
>> +# [1:namelist, 2:prefix]
>> +findfileclauses_cont=$(patsubst %,$(2) -name '%',$(1))
>> +# findfileclauses: creates "-name 'X' -o -name 'Y'"
>> +# [1:namelist]
>> +findfileclauses=$(call findfileclauses_cont,$(firstword $(1))) $(call findfileclauses_cont,$(call notfirstword,$(1)),-o)
>
> ?The _cont function is a bit redundant, no?
>
> findfileclauses=-name '$(firstword $(1))' $(patsubst %,-o -name '%',$(call notfirstword,$(1)))
>
> ?I don't think the duplication of -name warrants the additional function
> call. ?It definitely doesn't make it easier to understand.

You're right. I got a little carried away with these functions... :-D

>
> ?Nevertheless, this is a nifty patch!
>

Thanks!
I'll wait a while for some more feedback on this series, and then I'll update.

Best regards,
 Thomas

[Buildroot] [PATCH 2 of 5] stripping: use findfileclauses utility function

[Thomas De Schampheleire]

Signed-off-by: Thomas De Schampheleire <thomas.de.schampheleire@gmail.com>

---
 Makefile |  2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/Makefile b/Makefile
--- a/Makefile
+++ b/Makefile
@@ -454,7 +454,7 @@ endif
 ifeq ($(BR2_PACKAGE_PYTHON_PYC_ONLY),y)
 	find $(TARGET_DIR)/usr/lib/ -name '*.py' -print0 | xargs -0 rm -f
 endif
-	find $(TARGET_DIR) -type f -perm +111 '!' -name 'libthread_db*.so*' | \
+	find $(TARGET_DIR) -type f -perm +111 '!' $(call findfileclauses,libthread_db*.so*) | \
 		xargs $(STRIPCMD) 2>/dev/null || true
 	find $(TARGET_DIR)/lib/modules -type f -name '*.ko' | \
 		xargs -r $(KSTRIPCMD) || true

[Buildroot] [PATCH 3 of 5] build: add option to exclude executables/dirs from being stripped

[Thomas De Schampheleire]

Sometimes it may be desirable to keep debug symbols for some binaries and
libraries on the target. This commit introduces the config option
BR2_STRIP_EXCLUDE_FILES, which is interpreted as a list of such binaries
and libraries, and the option BR2_STRIP_EXCLUDE_DIRS, which indicates
directories excluded from stripping entirely.
These exclusions are passed to the find command in the target-finalize step.

Signed-off-by: Sven Neumann <s.neumann@raumfeld.com>
Signed-off-by: Thomas De Schampheleire <thomas.de.schampheleire@gmail.com>

---
v2: add the possibility to exclude entire directories

 Config.in |  19 +++++++++++++++++++
 Makefile  |   9 +++++++--
 2 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/Config.in b/Config.in
--- a/Config.in
+++ b/Config.in
@@ -247,6 +247,25 @@ config BR2_STRIP_none
 	  none    do not strip (only for debugging!)
 endchoice
 
+config BR2_STRIP_EXCLUDE_FILES
+	string "executables that should not be stripped"
+	depends on !BR2_STRIP_none
+	default ""
+	help
+	  You may specify a space-separated list of binaries and libraries
+	  here that should not be stripped on the target.
+
+config BR2_STRIP_EXCLUDE_DIRS
+	string "directories that should be skipped when stripping"
+	depends on !BR2_STRIP_none
+	default ""
+	help
+	  You may specify a space-separated list of directories that should
+	  be skipped when stripping. Binaries and libraries in these
+	  directories will not be touched.
+	  The directories should be specified relative to the target directory,
+	  without leading slash.
+
 choice
 	prompt "gcc optimization level"
 	default BR2_OPTIMIZE_S
diff --git a/Makefile b/Makefile
--- a/Makefile
+++ b/Makefile
@@ -430,6 +430,12 @@ endif
 erase-fakeroots:
 	rm -f $(BUILD_DIR)/.fakeroot*
 
+STRIP_FIND_CMD=find $(TARGET_DIR)
+ifneq (,$(call qstrip,$(BR2_STRIP_EXCLUDE_DIRS)))
+STRIP_FIND_CMD+=\( $(call finddirclauses,$(TARGET_DIR),$(call qstrip,$(BR2_STRIP_EXCLUDE_DIRS))) \) -prune -o
+endif
+STRIP_FIND_CMD+=-type f -perm +111 -not \( $(call findfileclauses,libthread_db*.so* $(call qstrip,$(BR2_STRIP_EXCLUDE_FILES))) \) -print
+
 target-finalize:
 ifeq ($(BR2_HAVE_DEVFILES),y)
 	( support/scripts/copy.sh $(STAGING_DIR) $(TARGET_DIR) )
@@ -454,8 +460,7 @@ endif
 ifeq ($(BR2_PACKAGE_PYTHON_PYC_ONLY),y)
 	find $(TARGET_DIR)/usr/lib/ -name '*.py' -print0 | xargs -0 rm -f
 endif
-	find $(TARGET_DIR) -type f -perm +111 '!' $(call findfileclauses,libthread_db*.so*) | \
-		xargs $(STRIPCMD) 2>/dev/null || true
+	$(STRIP_FIND_CMD) | xargs $(STRIPCMD) 2>/dev/null || true
 	find $(TARGET_DIR)/lib/modules -type f -name '*.ko' | \
 		xargs -r $(KSTRIPCMD) || true
 

[Buildroot] [PATCH 4 of 5] python config: move configuration into menu

[Thomas De Schampheleire]

Group all Python-related configuration in a separate menuconfig.

Signed-off-by: Thomas De Schampheleire <thomas.de.schampheleire@gmail.com>

---
 package/python/Config.in |  8 ++++----
 1 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/package/python/Config.in b/package/python/Config.in
--- a/package/python/Config.in
+++ b/package/python/Config.in
@@ -1,4 +1,7 @@
-config BR2_PACKAGE_PYTHON
+comment "python requires a toolchain with WCHAR support"
+	depends on !BR2_USE_WCHAR
+
+menuconfig BR2_PACKAGE_PYTHON
 	bool "python"
 	depends on BR2_USE_WCHAR
 	select BR2_PACKAGE_LIBFFI
@@ -7,9 +10,6 @@ config BR2_PACKAGE_PYTHON
 
 	  http://www.python.org/
 
-comment "python requires a toolchain with WCHAR support"
-	depends on !BR2_USE_WCHAR
-
 if BR2_PACKAGE_PYTHON
 
 choice

[Buildroot] [PATCH 5 of 5] python: allow reduced installation

[Thomas De Schampheleire]

A default python installation takes more than 9M of disk space on target. A
usable installation doesn't need to be that big. A minimal configuration with
only those modules needed to start an interactive session requires only about
450K.

This patch adds an option to reduce the number of installed python modules to
the bare minimum. Additional modules to install can be specified in the config
file.

Signed-off-by: Thomas De Schampheleire <thomas.de.schampheleire@gmail.com>

---
As an example, to be able to run the 'dstat' tool, I had to add the following
extra modules:
BR2_PACKAGE_PYTHON_REDUCED_EXTRA_MODULES="__future__.pyc sched.pyc heapq.pyc
bisect.pyc collections.pyc keyword.pyc getopt.pyc fnmatch.pyc getpass.pyc
glob.pyc time.so itertools.so operator.so resource.so _collections.so"

 package/python/Config.in |  18 ++++++++++++++++++
 package/python/python.mk |  28 +++++++++++++++++++++++++++-
 2 files changed, 45 insertions(+), 1 deletions(-)

diff --git a/package/python/Config.in b/package/python/Config.in
--- a/package/python/Config.in
+++ b/package/python/Config.in
@@ -95,4 +95,22 @@ config BR2_PACKAGE_PYTHON_ZLIB
 
 endmenu
 
+
+config BR2_PACKAGE_PYTHON_REDUCED
+	bool "reduced python installation"
+	help
+	  Select this option to greatly reduce the number of python modules that are
+	  installed on target (from approx. 9M to 400K). Only the modules needed to
+	  start an interactive python session are installed by default.
+	  You can install additional modules by specifying them in
+	  BR2_PACKAGE_PYTHON_REDUCED_EXTRA_MODULES.
+
+config BR2_PACKAGE_PYTHON_REDUCED_EXTRA_MODULES
+	string "extra modules to include in reduced python installation"
+	depends on BR2_PACKAGE_PYTHON_REDUCED
+	help
+	  To install additional modules in a reduced Python installation,
+	  specify them here, space-separated. This list should only include the
+	  file name, not the directory part.
+
 endif
diff --git a/package/python/python.mk b/package/python/python.mk
--- a/package/python/python.mk
+++ b/package/python/python.mk
@@ -144,12 +144,14 @@ ifneq ($(BR2_HAVE_DEVFILES),y)
 PYTHON_POST_INSTALL_TARGET_HOOKS += PYTHON_REMOVE_DEVFILES
 endif
 
+PYTHON_TARGET_DIR=$(TARGET_DIR)/usr/lib/python$(PYTHON_VERSION_MAJOR)
+
 #
 # Remove useless files. In the config/ directory, only the Makefile
 # and the pyconfig.h files are needed at runtime.
 #
 define PYTHON_REMOVE_USELESS_FILES
-	for i in `find $(TARGET_DIR)/usr/lib/python$(PYTHON_VERSION_MAJOR)/config/ \
+	for i in `find $(PYTHON_TARGET_DIR)/config/ \
 		-type f -not -name pyconfig.h -a -not -name Makefile` ; do \
 		rm -f $$i ; \
 	done
@@ -157,6 +159,30 @@ endef
 
 PYTHON_POST_INSTALL_TARGET_HOOKS += PYTHON_REMOVE_USELESS_FILES
 
+#
+# Reduced Python installation: remove unneeded modules
+#
+ifeq ($(BR2_PACKAGE_PYTHON_REDUCED),y)
+PYTHON_REDUCED_MINIMAL_MODULES=$(patsubst %,%.pyc,\
+	site os stat posixpath genericpath warnings linecache types UserDict \
+	_abcoll abc _weakrefset copy_reg sysconfig re sre_compile sre_parse \
+	sre_constants) \
+	$(call qstrip,$(BR2_PACKAGE_PYTHON_REDUCED_EXTRA_MODULES))
+PYTHON_REDUCED_MINIMAL_DIRS=config
+
+PYTHON_REDUCED_FIND_CMD=find $(PYTHON_TARGET_DIR)
+PYTHON_REDUCED_FIND_CMD+=\( $(call finddirclauses,$(PYTHON_TARGET_DIR),$(PYTHON_REDUCED_MINIMAL_DIRS)) \) -prune -o
+PYTHON_REDUCED_FIND_CMD += -type f -not \( $(call findfileclauses,$(PYTHON_REDUCED_MINIMAL_MODULES)) \) -print
+
+define PYTHON_REDUCE_INSTALLATION
+	for i in `$(PYTHON_REDUCED_FIND_CMD)`; do \
+		rm -f $$i ; \
+	done
+endef
+
+PYTHON_POST_INSTALL_TARGET_HOOKS += PYTHON_REDUCE_INSTALLATION
+endif
+
 PYTHON_AUTORECONF = YES
 
 $(eval $(call AUTOTARGETS))