[PATCH] perf tools: Add support to install perf python extension

[PATCH] perf tools: Add support to install perf python extension

[Jiri Olsa]

Changing the distutils to use their own default 'build'
directory.

Such setup makes the install command simple, because using
--build-lib and --build-temp does not seem to comply with
the way the distutils' install commands expect it.

Also keeping the python/perf.so file, so it could be used
for testing as of until now.

Signed-off-by: Jiri Olsa <jolsa@redhat.com>
---
 tools/perf/Makefile |   12 ++++++------
 1 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/tools/perf/Makefile b/tools/perf/Makefile
index 940257b..9c82a56 100644
--- a/tools/perf/Makefile
+++ b/tools/perf/Makefile
@@ -178,9 +178,8 @@ strip-libs = $(filter-out -l%,$(1))
 
 $(OUTPUT)python/perf.so: $(PYRF_OBJS)
 	$(QUIET_GEN)CFLAGS='$(BASIC_CFLAGS)' $(PYTHON_WORD) util/setup.py \
-	  --quiet build_ext \
-	  --build-lib='$(OUTPUT)python' \
-	  --build-temp='$(OUTPUT)python/temp'
+	  --quiet build_ext; \
+	cp $$(find build -name 'perf.so') python
 #
 # No Perl scripts right now:
 #
@@ -504,9 +503,7 @@ else
 
   PYTHON_WORD := $(call shell-wordify,$(PYTHON))
 
-  python-clean := $(PYTHON_WORD) util/setup.py clean \
-    --build-lib='$(OUTPUT)python' \
-    --build-temp='$(OUTPUT)python/temp'
+  python-clean := $(PYTHON_WORD) util/setup.py --quiet clean -a; $(RM) -f python/perf.so
 
   ifdef NO_LIBPYTHON
     $(call disable-python)
@@ -863,6 +860,9 @@ install: all
 	$(INSTALL) scripts/python/*.py -t '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/scripts/python'
 	$(INSTALL) scripts/python/bin/* -t '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/scripts/python/bin'
 
+install-python_ext:
+	$(PYTHON_WORD) util/setup.py --quiet install --root='/$(DESTDIR_SQ)'
+
 install-doc:
 	$(MAKE) -C Documentation install
 
-- 
1.7.1

Re: [PATCH] perf tools: Add support to install perf python extension

[Arnaldo Carvalho de Melo]

Em Thu, Jul 21, 2011 at 06:45:26PM +0200, Jiri Olsa escreveu:
> Changing the distutils to use their own default 'build'
> directory.
> 
> Such setup makes the install command simple, because using
> --build-lib and --build-temp does not seem to comply with
> the way the distutils' install commands expect it.
> 
> Also keeping the python/perf.so file, so it could be used
> for testing as of until now.

Yeah, I gave up when I couldn't get:

$ make O=/tmp/build-dir-for-perf/ -C tools/perf/ install

to work, does it now? Checking...

- Arnaldo
 
> Signed-off-by: Jiri Olsa <jolsa@redhat.com>
> ---
>  tools/perf/Makefile |   12 ++++++------
>  1 files changed, 6 insertions(+), 6 deletions(-)
> 
> diff --git a/tools/perf/Makefile b/tools/perf/Makefile
> index 940257b..9c82a56 100644
> --- a/tools/perf/Makefile
> +++ b/tools/perf/Makefile
> @@ -178,9 +178,8 @@ strip-libs = $(filter-out -l%,$(1))
>  
>  $(OUTPUT)python/perf.so: $(PYRF_OBJS)
>  	$(QUIET_GEN)CFLAGS='$(BASIC_CFLAGS)' $(PYTHON_WORD) util/setup.py \
> -	  --quiet build_ext \
> -	  --build-lib='$(OUTPUT)python' \
> -	  --build-temp='$(OUTPUT)python/temp'
> +	  --quiet build_ext; \
> +	cp $$(find build -name 'perf.so') python
>  #
>  # No Perl scripts right now:
>  #
> @@ -504,9 +503,7 @@ else
>  
>    PYTHON_WORD := $(call shell-wordify,$(PYTHON))
>  
> -  python-clean := $(PYTHON_WORD) util/setup.py clean \
> -    --build-lib='$(OUTPUT)python' \
> -    --build-temp='$(OUTPUT)python/temp'
> +  python-clean := $(PYTHON_WORD) util/setup.py --quiet clean -a; $(RM) -f python/perf.so
>  
>    ifdef NO_LIBPYTHON
>      $(call disable-python)
> @@ -863,6 +860,9 @@ install: all
>  	$(INSTALL) scripts/python/*.py -t '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/scripts/python'
>  	$(INSTALL) scripts/python/bin/* -t '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/scripts/python/bin'
>  
> +install-python_ext:
> +	$(PYTHON_WORD) util/setup.py --quiet install --root='/$(DESTDIR_SQ)'
> +
>  install-doc:
>  	$(MAKE) -C Documentation install
>  
> -- 
> 1.7.1

Re: [PATCH] perf tools: Add support to install perf python extension

[Jiri Olsa]

On Thu, Jul 21, 2011 at 02:11:48PM -0300, Arnaldo Carvalho de Melo wrote:
> Em Thu, Jul 21, 2011 at 06:45:26PM +0200, Jiri Olsa escreveu:
> > Changing the distutils to use their own default 'build'
> > directory.
> > 
> > Such setup makes the install command simple, because using
> > --build-lib and --build-temp does not seem to comply with
> > the way the distutils' install commands expect it.
> > 
> > Also keeping the python/perf.so file, so it could be used
> > for testing as of until now.
> 
> Yeah, I gave up when I couldn't get:
> 
> $ make O=/tmp/build-dir-for-perf/ -C tools/perf/ install
> 
> to work, does it now? Checking...

nope, I added install-python_ext since it installs to the
python system lib area, and it could break standard install

I think there's a way to install python modules under $HOME,
maybe we could do that if no DESTDIR is specified..
I can add it later ;)

thanks,
jirka

> 
> - Arnaldo
>  
> > Signed-off-by: Jiri Olsa <jolsa@redhat.com>
> > ---
> >  tools/perf/Makefile |   12 ++++++------
> >  1 files changed, 6 insertions(+), 6 deletions(-)
> > 
> > diff --git a/tools/perf/Makefile b/tools/perf/Makefile
> > index 940257b..9c82a56 100644
> > --- a/tools/perf/Makefile
> > +++ b/tools/perf/Makefile
> > @@ -178,9 +178,8 @@ strip-libs = $(filter-out -l%,$(1))
> >  
> >  $(OUTPUT)python/perf.so: $(PYRF_OBJS)
> >  	$(QUIET_GEN)CFLAGS='$(BASIC_CFLAGS)' $(PYTHON_WORD) util/setup.py \
> > -	  --quiet build_ext \
> > -	  --build-lib='$(OUTPUT)python' \
> > -	  --build-temp='$(OUTPUT)python/temp'
> > +	  --quiet build_ext; \
> > +	cp $$(find build -name 'perf.so') python
> >  #
> >  # No Perl scripts right now:
> >  #
> > @@ -504,9 +503,7 @@ else
> >  
> >    PYTHON_WORD := $(call shell-wordify,$(PYTHON))
> >  
> > -  python-clean := $(PYTHON_WORD) util/setup.py clean \
> > -    --build-lib='$(OUTPUT)python' \
> > -    --build-temp='$(OUTPUT)python/temp'
> > +  python-clean := $(PYTHON_WORD) util/setup.py --quiet clean -a; $(RM) -f python/perf.so
> >  
> >    ifdef NO_LIBPYTHON
> >      $(call disable-python)
> > @@ -863,6 +860,9 @@ install: all
> >  	$(INSTALL) scripts/python/*.py -t '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/scripts/python'
> >  	$(INSTALL) scripts/python/bin/* -t '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/scripts/python/bin'
> >  
> > +install-python_ext:
> > +	$(PYTHON_WORD) util/setup.py --quiet install --root='/$(DESTDIR_SQ)'
> > +
> >  install-doc:
> >  	$(MAKE) -C Documentation install
> >  
> > -- 
> > 1.7.1

Re: [PATCH] perf tools: Add support to install perf python extension

[Arnaldo Carvalho de Melo]

Em Thu, Jul 21, 2011 at 08:47:15PM +0200, Jiri Olsa escreveu:
> On Thu, Jul 21, 2011 at 02:11:48PM -0300, Arnaldo Carvalho de Melo wrote:
> > Em Thu, Jul 21, 2011 at 06:45:26PM +0200, Jiri Olsa escreveu:
> > > Changing the distutils to use their own default 'build'
> > > directory.
> > > 
> > > Such setup makes the install command simple, because using
> > > --build-lib and --build-temp does not seem to comply with
> > > the way the distutils' install commands expect it.
> > > 
> > > Also keeping the python/perf.so file, so it could be used
> > > for testing as of until now.
> > 
> > Yeah, I gave up when I couldn't get:
> > 
> > $ make O=/tmp/build-dir-for-perf/ -C tools/perf/ install
> > 
> > to work, does it now? Checking...
> 
> nope, I added install-python_ext since it installs to the
> python system lib area, and it could break standard install

But where is that it picks the binaries built? The problem I was
experiencing was that it wasn't obeying O=, just using the source dir
as the build dir, effectively building it again, in a different place.
 
> I think there's a way to install python modules under $HOME,
> maybe we could do that if no DESTDIR is specified..
> I can add it later ;)

Right, but this is not what I was talking about :-)

- Arnaldo

Re: [PATCH] perf tools: Add support to install perf python extension

[Jiri Olsa]

On Thu, Jul 21, 2011 at 04:04:07PM -0300, Arnaldo Carvalho de Melo wrote:
> Em Thu, Jul 21, 2011 at 08:47:15PM +0200, Jiri Olsa escreveu:
> > On Thu, Jul 21, 2011 at 02:11:48PM -0300, Arnaldo Carvalho de Melo wrote:
> > > Em Thu, Jul 21, 2011 at 06:45:26PM +0200, Jiri Olsa escreveu:
> > > > Changing the distutils to use their own default 'build'
> > > > directory.
> > > > 
> > > > Such setup makes the install command simple, because using
> > > > --build-lib and --build-temp does not seem to comply with
> > > > the way the distutils' install commands expect it.
> > > > 
> > > > Also keeping the python/perf.so file, so it could be used
> > > > for testing as of until now.
> > > 
> > > Yeah, I gave up when I couldn't get:
> > > 
> > > $ make O=/tmp/build-dir-for-perf/ -C tools/perf/ install
> > > 
> > > to work, does it now? Checking...
> > 
> > nope, I added install-python_ext since it installs to the
> > python system lib area, and it could break standard install
> 
> But where is that it picks the binaries built? The problem I was
> experiencing was that it wasn't obeying O=, just using the source dir

I haven't checked the 'O=' that much.. will do :)

> as the build dir, effectively building it again, in a different place.

I dont have that much experince with python, but here's what I found:
(might be missing something..)

build_ext command
- build_ext will by default create a 'build' directory and 
  put all the stuff in - temp and final dirs/binary
- you can overload this with build-lib/build-temp options

install command
- install does not have option to specify the build-lib directory
  (but it has the --root option that we need)
- so running just install assumes you have used default 'build'
  directory and if it does not exist it starts rebuild
- there's another command install_lib, which let's you specify
  the build-lib.. BUT it wont let you specify the --root option ;)

There's possibility to use setup.cfg file and generate/put options
for each command, so at the end we could choose what directory to use.

Also there might be a way to overload distutils class and put our
specific options to this new class..

As I checked some of python extension packages, some of them
do not use distutils at all.. can't imagine why ;)

jirka

>  
> > I think there's a way to install python modules under $HOME,
> > maybe we could do that if no DESTDIR is specified..
> > I can add it later ;)
> 
> Right, but this is not what I was talking about :-)
> 
> - Arnaldo

Re: [PATCH] perf tools: Add support to install perf python extension

[Arnaldo Carvalho de Melo]

Em Thu, Jul 21, 2011 at 09:41:50PM +0200, Jiri Olsa escreveu:
> On Thu, Jul 21, 2011 at 04:04:07PM -0300, Arnaldo Carvalho de Melo wrote:
> > Em Thu, Jul 21, 2011 at 08:47:15PM +0200, Jiri Olsa escreveu:
> > > nope, I added install-python_ext since it installs to the
> > > python system lib area, and it could break standard install

> > But where is that it picks the binaries built? The problem I was
> > experiencing was that it wasn't obeying O=, just using the source dir
 
> I haven't checked the 'O=' that much.. will do :)
 
> > as the build dir, effectively building it again, in a different place.
 
> I dont have that much experince with python, but here's what I found:
> (might be missing something..)

<SNIP>
 
> Also there might be a way to overload distutils class and put our
> specific options to this new class..
 
> As I checked some of python extension packages, some of them
> do not use distutils at all.. can't imagine why ;)

Ok, we're on the same page with that ';)' ;)

I'd love to use what is 'pythonic', i.e. whatever the python people
think we should use and don't bother with all the module
build/install details, but since we want to have O= working...

- Arnaldo

[PATCHv2] perf tools: Add support to install perf python extension

[Jiri Olsa]

On Thu, Jul 21, 2011 at 06:16:10PM -0300, Arnaldo Carvalho de Melo wrote:
> Em Thu, Jul 21, 2011 at 09:41:50PM +0200, Jiri Olsa escreveu:
> > On Thu, Jul 21, 2011 at 04:04:07PM -0300, Arnaldo Carvalho de Melo wrote:
> > > Em Thu, Jul 21, 2011 at 08:47:15PM +0200, Jiri Olsa escreveu:
> > > > nope, I added install-python_ext since it installs to the
> > > > python system lib area, and it could break standard install
> 
> > > But where is that it picks the binaries built? The problem I was
> > > experiencing was that it wasn't obeying O=, just using the source dir
>  
> > I haven't checked the 'O=' that much.. will do :)
>  
> > > as the build dir, effectively building it again, in a different place.
>  
> > I dont have that much experince with python, but here's what I found:
> > (might be missing something..)
> 
> <SNIP>
>  
> > Also there might be a way to overload distutils class and put our
> > specific options to this new class..
>  
> > As I checked some of python extension packages, some of them
> > do not use distutils at all.. can't imagine why ;)
> 
> Ok, we're on the same page with that ';)' ;)
> 
> I'd love to use what is 'pythonic', i.e. whatever the python people
> think we should use and don't bother with all the module
> build/install details, but since we want to have O= working...

attached new version, working for me with the O= settings

jirka


---
Subject: [PATCH] perf tools: Add support to install perf python extension

Adding install-python_ext target to install python extension
related files.  Installation directory is governed by python
distutils package and follows the DESTDIR variable settings.

Also moving python extension build output into '$(O)python_ext_build'
directory and making it configurable via PYTHON_EXTBUILD variable.

Keeping the '$(O)python/perf.so' file, so it could be used
for testing as of until now.

Signed-off-by: Jiri Olsa <jolsa@redhat.com>
---
 tools/perf/Makefile      |   19 +++++++++++++------
 tools/perf/util/setup.py |   21 ++++++++++++++++++++-
 2 files changed, 33 insertions(+), 7 deletions(-)

diff --git a/tools/perf/Makefile b/tools/perf/Makefile
index 940257b..2b44a53 100644
--- a/tools/perf/Makefile
+++ b/tools/perf/Makefile
@@ -178,9 +178,9 @@ strip-libs = $(filter-out -l%,$(1))
 
 $(OUTPUT)python/perf.so: $(PYRF_OBJS)
 	$(QUIET_GEN)CFLAGS='$(BASIC_CFLAGS)' $(PYTHON_WORD) util/setup.py \
-	  --quiet build_ext \
-	  --build-lib='$(OUTPUT)python' \
-	  --build-temp='$(OUTPUT)python/temp'
+	  --quiet build_ext; \
+	mkdir -p $(OUTPUT)python && \
+	cp $(PYTHON_EXTBUILD_LIB)perf.so $(OUTPUT)python/
 #
 # No Perl scripts right now:
 #
@@ -504,9 +504,13 @@ else
 
   PYTHON_WORD := $(call shell-wordify,$(PYTHON))
 
-  python-clean := $(PYTHON_WORD) util/setup.py clean \
-    --build-lib='$(OUTPUT)python' \
-    --build-temp='$(OUTPUT)python/temp'
+  # python extension build directories
+  PYTHON_EXTBUILD     := $(OUTPUT)python_ext_build/
+  PYTHON_EXTBUILD_LIB := $(PYTHON_EXTBUILD)lib/
+  PYTHON_EXTBUILD_TMP := $(PYTHON_EXTBUILD)tmp/
+  export PYTHON_EXTBUILD_LIB PYTHON_EXTBUILD_TMP
+
+  python-clean := rm -rf $(PYTHON_EXTBUILD) $(OUTPUT)python/perf.so
 
   ifdef NO_LIBPYTHON
     $(call disable-python)
@@ -863,6 +867,9 @@ install: all
 	$(INSTALL) scripts/python/*.py -t '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/scripts/python'
 	$(INSTALL) scripts/python/bin/* -t '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/scripts/python/bin'
 
+install-python_ext:
+	$(PYTHON_WORD) util/setup.py --quiet install --root='/$(DESTDIR_SQ)'
+
 install-doc:
 	$(MAKE) -C Documentation install
 
diff --git a/tools/perf/util/setup.py b/tools/perf/util/setup.py
index bbc982f..95d3700 100644
--- a/tools/perf/util/setup.py
+++ b/tools/perf/util/setup.py
@@ -3,9 +3,27 @@
 from distutils.core import setup, Extension
 from os import getenv
 
+from distutils.command.build_ext   import build_ext   as _build_ext
+from distutils.command.install_lib import install_lib as _install_lib
+
+class build_ext(_build_ext):
+    def finalize_options(self):
+        _build_ext.finalize_options(self)
+        self.build_lib  = build_lib
+        self.build_temp = build_tmp
+
+class install_lib(_install_lib):
+    def finalize_options(self):
+        _install_lib.finalize_options(self)
+        self.build_dir = build_lib
+
+
 cflags = ['-fno-strict-aliasing', '-Wno-write-strings']
 cflags += getenv('CFLAGS', '').split()
 
+build_lib = getenv('PYTHON_EXTBUILD_LIB')
+build_tmp = getenv('PYTHON_EXTBUILD_TMP')
+
 perf = Extension('perf',
 		  sources = ['util/python.c', 'util/ctype.c', 'util/evlist.c',
 			     'util/evsel.c', 'util/cpumap.c', 'util/thread_map.c',
@@ -21,4 +39,5 @@ setup(name='perf',
       author_email='acme@redhat.com',
       license='GPLv2',
       url='http://perf.wiki.kernel.org',
-      ext_modules=[perf])
+      ext_modules=[perf],
+      cmdclass={'build_ext': build_ext, 'install_lib': install_lib})
-- 
1.7.1

[tip:perf/urgent] perf tools: Add support to install perf python extension

[tip-bot for Jiri Olsa]

Commit-ID:  9941c96ad869d10f7e34e03990ce450ab8fcb83d
Gitweb:     http://git.kernel.org/tip/9941c96ad869d10f7e34e03990ce450ab8fcb83d
Author:     Jiri Olsa <jolsa@redhat.com>
AuthorDate: Fri, 22 Jul 2011 13:33:07 +0200
Committer:  Arnaldo Carvalho de Melo <acme@redhat.com>
CommitDate: Mon, 8 Aug 2011 12:54:26 -0300

perf tools: Add support to install perf python extension

Adding install-python_ext target to install python extension related
files.  Installation directory is governed by python distutils package
and follows the DESTDIR variable settings.

Also moving python extension build output into '$(O)python_ext_build'
directory and making it configurable via PYTHON_EXTBUILD variable.

Keeping the '$(O)python/perf.so' file, so it could be used for testing
as of until now.

Cc: Ingo Molnar <mingo@elte.hu>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Link: http://lkml.kernel.org/r/20110722113307.GA1931@jolsa.brq.redhat.com
Signed-off-by: Jiri Olsa <jolsa@redhat.com>
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
---
 tools/perf/Makefile      |   19 +++++++++++++------
 tools/perf/util/setup.py |   21 ++++++++++++++++++++-
 2 files changed, 33 insertions(+), 7 deletions(-)

diff --git a/tools/perf/Makefile b/tools/perf/Makefile
index 5bf48fc..822f967 100644
--- a/tools/perf/Makefile
+++ b/tools/perf/Makefile
@@ -178,9 +178,9 @@ strip-libs = $(filter-out -l%,$(1))
 
 $(OUTPUT)python/perf.so: $(PYRF_OBJS)
 	$(QUIET_GEN)CFLAGS='$(BASIC_CFLAGS)' $(PYTHON_WORD) util/setup.py \
-	  --quiet build_ext \
-	  --build-lib='$(OUTPUT)python' \
-	  --build-temp='$(OUTPUT)python/temp'
+	  --quiet build_ext; \
+	mkdir -p $(OUTPUT)python && \
+	cp $(PYTHON_EXTBUILD_LIB)perf.so $(OUTPUT)python/
 #
 # No Perl scripts right now:
 #
@@ -506,9 +506,13 @@ else
 
   PYTHON_WORD := $(call shell-wordify,$(PYTHON))
 
-  python-clean := $(PYTHON_WORD) util/setup.py clean \
-    --build-lib='$(OUTPUT)python' \
-    --build-temp='$(OUTPUT)python/temp'
+  # python extension build directories
+  PYTHON_EXTBUILD     := $(OUTPUT)python_ext_build/
+  PYTHON_EXTBUILD_LIB := $(PYTHON_EXTBUILD)lib/
+  PYTHON_EXTBUILD_TMP := $(PYTHON_EXTBUILD)tmp/
+  export PYTHON_EXTBUILD_LIB PYTHON_EXTBUILD_TMP
+
+  python-clean := rm -rf $(PYTHON_EXTBUILD) $(OUTPUT)python/perf.so
 
   ifdef NO_LIBPYTHON
     $(call disable-python)
@@ -865,6 +869,9 @@ install: all
 	$(INSTALL) scripts/python/*.py -t '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/scripts/python'
 	$(INSTALL) scripts/python/bin/* -t '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/scripts/python/bin'
 
+install-python_ext:
+	$(PYTHON_WORD) util/setup.py --quiet install --root='/$(DESTDIR_SQ)'
+
 install-doc:
 	$(MAKE) -C Documentation install
 
diff --git a/tools/perf/util/setup.py b/tools/perf/util/setup.py
index bbc982f..95d3700 100644
--- a/tools/perf/util/setup.py
+++ b/tools/perf/util/setup.py
@@ -3,9 +3,27 @@
 from distutils.core import setup, Extension
 from os import getenv
 
+from distutils.command.build_ext   import build_ext   as _build_ext
+from distutils.command.install_lib import install_lib as _install_lib
+
+class build_ext(_build_ext):
+    def finalize_options(self):
+        _build_ext.finalize_options(self)
+        self.build_lib  = build_lib
+        self.build_temp = build_tmp
+
+class install_lib(_install_lib):
+    def finalize_options(self):
+        _install_lib.finalize_options(self)
+        self.build_dir = build_lib
+
+
 cflags = ['-fno-strict-aliasing', '-Wno-write-strings']
 cflags += getenv('CFLAGS', '').split()
 
+build_lib = getenv('PYTHON_EXTBUILD_LIB')
+build_tmp = getenv('PYTHON_EXTBUILD_TMP')
+
 perf = Extension('perf',
 		  sources = ['util/python.c', 'util/ctype.c', 'util/evlist.c',
 			     'util/evsel.c', 'util/cpumap.c', 'util/thread_map.c',
@@ -21,4 +39,5 @@ setup(name='perf',
       author_email='acme@redhat.com',
       license='GPLv2',
       url='http://perf.wiki.kernel.org',
-      ext_modules=[perf])
+      ext_modules=[perf],
+      cmdclass={'build_ext': build_ext, 'install_lib': install_lib})