[PATCHv2] python: Unbreak Python third-party extensions

[PATCHv2] python: Unbreak Python third-party extensions

[Michael Lippautz]

This patch fixes compilation/linking of python third-party extensions, i.e.
Extensions that ship with C code.

Problem:
Python uses distutils(-native) to compile third-party extensions. distutils
uses its own sysconfig module to get the options for compiling and linking.
Since third-party extensions have to be linked against this libpython it
important that -L points into staging. This is not the case because
distutils.sysconfig uses a special Makefile that is shipped with python
determine the paths. The Makefile is the same that would be used on the
target to build third-party extensions. It therefore points into /usr/lib
instead of staging.

Solution:
Stage a modified version of the Makefile where the paths (incdir, libdir) have
been replaced by ones that point into staging.

Side-problem:
The recipe actually should not stage files itself in do_compile, but rather
handle everything that needs to be staged in do_install. This is currently not
possible because python compiles itself using distutils-native. Distutils on
the other hand does only allow to add a path, but not to substitute it,
requiring a staged Makefile and libpython.so before the actual python
compilation is triggered.

The second step to solve this would be to either patch distutils, or split
python into python-initial and python. The -initial part could create the
Makefile and the library, while the main part focuses on the target.

For further references see:
 http://lists.linuxtogo.org/pipermail/openembedded-core/2011-May/001752.html

Signed-off-by: Michael Lippautz <michael.lippautz@gmail.com>
Acked-by: Martin Jansa <Martin.Jansa@gmail.com>
---
 meta/recipes-devtools/python/python_2.6.6.bb |   41 +++++++++++++++++---------
 1 files changed, 27 insertions(+), 14 deletions(-)

diff --git a/meta/recipes-devtools/python/python_2.6.6.bb b/meta/recipes-devtools/python/python_2.6.6.bb
index 3a0a056..f438286 100644
--- a/meta/recipes-devtools/python/python_2.6.6.bb
+++ b/meta/recipes-devtools/python/python_2.6.6.bb
@@ -1,7 +1,7 @@
 require python.inc
 DEPENDS = "python-native db gdbm openssl readline sqlite3 zlib"
 DEPENDS_sharprom = "python-native db readline zlib gdbm openssl"
-PR = "${INC_PR}.2"
+PR = "${INC_PR}.3"
 LIC_FILES_CHKSUM = "file://LICENSE;md5=38fdd546420fab09ac6bd3d8a1c83eb6"
 
 DISTRO_SRC_URI ?= "file://sitecustomize.py"
@@ -34,23 +34,26 @@ do_configure_prepend() {
 	autoreconf -Wcross --verbose --install --force --exclude=autopoint Modules/_ctypes/libffi || oenote "_ctypes failed to autoreconf"
 }
 
-#
-# Copy config.h and an appropriate Makefile for distutils.sysconfig,
-# which laters uses the information out of these to compile extensions
-#
-do_compile_prepend() {
+do_compile() {
+	#
+	# Copy config.h and an appropriate Makefile for distutils.sysconfig,
+	# which laters uses the information out of these to compile extensions
+	#
+	# The following part (until python compilation) should probably moved to an
+	# -initial recipe to handle staging better
+	#
 	install -d ${STAGING_INCDIR}/python${PYTHON_MAJMIN}/
 	install -d ${STAGING_LIBDIR}/python${PYTHON_MAJMIN}/config/
 	install -m 0644 pyconfig.h ${STAGING_INCDIR}/python${PYTHON_MAJMIN}/
+
+	# remove hardcoded ccache, see http://bugs.openembedded.net/show_bug.cgi?id=4144
+	sed -i -e s,ccache,'$(CCACHE)', Makefile
+
 	install -m 0644 Makefile Makefile.orig
-	install -m 0644 Makefile Makefile.backup
-	sed -e 's,${includedir},${STAGING_INCDIR},' < Makefile.backup > Makefile
-	install -m 0644 Makefile Makefile.backup
-	sed -e 's,${libdir},${STAGING_LIBDIR},' < Makefile.backup > Makefile
+	sed -i -e 's,${includedir},${STAGING_INCDIR},' Makefile
+	sed -i -e 's,${libdir},${STAGING_LIBDIR},' Makefile
 	install -m 0644 Makefile ${STAGING_LIBDIR}/python${PYTHON_MAJMIN}/config/
-}
 
-do_compile() {
 	oe_runmake HOSTPGEN=${STAGING_BINDIR_NATIVE}/pgen \
 		HOSTPYTHON=${STAGING_BINDIR_NATIVE}/python \
 		STAGING_LIBDIR=${STAGING_LIBDIR} \
@@ -69,6 +72,9 @@ do_compile() {
 }
 
 do_install() {
+	# make install needs the original Makefile, or otherwise the inclues would
+	# go to ${D}${STAGING...}/...
+	install -m 0644 Makefile Makefile.sysroot
 	install -m 0644 Makefile.orig Makefile
 	
 	oe_runmake HOSTPGEN=${STAGING_BINDIR_NATIVE}/pgen \
@@ -78,12 +84,19 @@ do_install() {
 		BUILD_SYS=${BUILD_SYS} HOST_SYS=${HOST_SYS} \
 		DESTDIR=${D} LIBDIR=${libdir} install
 
+	install -m 0644 Makefile.sysroot ${D}/${libdir}/python${PYTHON_MAJMIN}/config/Makefile
+	rm Makefile.sysroot
+
 	if [ -e ${WORKDIR}/sitecustomize.py ]; then
 		install -m 0644 ${WORKDIR}/sitecustomize.py ${D}/${libdir}/python${PYTHON_MAJMIN}
 	fi
+}
 
-	# remove hardcoded ccache, see http://bugs.openembedded.net/show_bug.cgi?id=4144
-	sed -i -e s,ccache,'$(CCACHE)', ${D}/${libdir}/python${PYTHON_MAJMIN}/config/Makefile
+PACKAGE_PREPROCESS_FUNCS += "py_package_preprocess"
+
+py_package_preprocess () {
+	# copy back the old Makefile to fix target package
+	install -m 0644 Makefile.orig ${D}/${libdir}/python${PYTHON_MAJMIN}/config/Makefile
 }
 
 require python-${PYTHON_MAJMIN}-manifest.inc
-- 
1.7.3.4

Re: [PATCHv2] python: Unbreak Python third-party extensions

[Richard Purdie]

On Tue, 2011-05-03 at 18:40 +0200, Michael Lippautz wrote:
> This patch fixes compilation/linking of python third-party extensions, i.e.
> Extensions that ship with C code.
> 
> Problem:
> Python uses distutils(-native) to compile third-party extensions. distutils
> uses its own sysconfig module to get the options for compiling and linking.
> Since third-party extensions have to be linked against this libpython it
> important that -L points into staging. This is not the case because
> distutils.sysconfig uses a special Makefile that is shipped with python
> determine the paths. The Makefile is the same that would be used on the
> target to build third-party extensions. It therefore points into /usr/lib
> instead of staging.
> 
> Solution:
> Stage a modified version of the Makefile where the paths (incdir, libdir) have
> been replaced by ones that point into staging.
> 
> Side-problem:
> The recipe actually should not stage files itself in do_compile, but rather
> handle everything that needs to be staged in do_install. This is currently not
> possible because python compiles itself using distutils-native. Distutils on
> the other hand does only allow to add a path, but not to substitute it,
> requiring a staged Makefile and libpython.so before the actual python
> compilation is triggered.
> 
> The second step to solve this would be to either patch distutils, or split
> python into python-initial and python. The -initial part could create the
> Makefile and the library, while the main part focuses on the target.
> 
> For further references see:
>  http://lists.linuxtogo.org/pipermail/openembedded-core/2011-May/001752.html
> 
> Signed-off-by: Michael Lippautz <michael.lippautz@gmail.com>
> Acked-by: Martin Jansa <Martin.Jansa@gmail.com>
> ---
>  meta/recipes-devtools/python/python_2.6.6.bb |   41 +++++++++++++++++---------
>  1 files changed, 27 insertions(+), 14 deletions(-)

Merged to master, thanks.

Richard