[PATCH] sstate.bbclass: Optimise sstate_hardcode_path

[PATCH] sstate.bbclass: Optimise sstate_hardcode_path

[Richard Purdie]

The sstate_hardcode_path() function triggered large numbers of exec()
calls when processing packages with large numbers of file relocations
(e.g. perl). This patch optimises those calls into longer single commands
which make the code significantly more efficient.

This reduced the do_package time for perl by 2 minutes (from 4.75 minutes)
for me.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
index 4bd3712..d4f95c1 100644
--- a/meta/classes/sstate.bbclass
+++ b/meta/classes/sstate.bbclass
@@ -317,19 +317,24 @@ def sstate_hardcode_path(d):
 	staging_host = d.getVar('STAGING_DIR_HOST', True)
 	sstate_builddir = d.getVar('SSTATE_BUILDDIR', True)
 
-	for i in file_list.split('\n'):
-		if not i:
-			continue           
-		if bb.data.inherits_class('native', d) or bb.data.inherits_class('nativesdk', d) or bb.data.inherits_class('crosssdk', d) or bb.data.inherits_class('cross-canadian', d):
-			cmd = "sed -i -e s:%s:FIXMESTAGINGDIR:g %s" % (staging, i)
-		elif bb.data.inherits_class('cross', d):
-			cmd = "sed -i -e s:%s:FIXMESTAGINGDIRTARGET:g %s \
-				sed -i -e s:%s:FIXMESTAGINGDIR:g %s" % (staging_target, i, staging, i)
-		else:
-			cmd = "sed -i -e s:%s:FIXMESTAGINGDIRHOST:g %s" % (staging_host, i)
+	files = " ".join(file_list.split('\n'))
 
+	if bb.data.inherits_class('native', d) or bb.data.inherits_class('nativesdk', d) or bb.data.inherits_class('crosssdk', d) or bb.data.inherits_class('cross-canadian', d):
+		cmd = "sed -i -e s:%s:FIXMESTAGINGDIR:g %s" % (staging, files)
+	elif bb.data.inherits_class('cross', d):
+		cmd = "sed -i -e s:%s:FIXMESTAGINGDIRTARGET:g %s \
+			sed -i -e s:%s:FIXMESTAGINGDIR:g %s" % (staging_target, files, staging, files)
+	else:
+		cmd = "sed -i -e s:%s:FIXMESTAGINGDIRHOST:g %s" % (staging_host, files)
+
+	if files:
 		os.system(cmd)
-		os.system("echo %s | sed -e 's:%s::' >> %sfixmepath" % (i, sstate_builddir, sstate_builddir))
+		fix = open("%sfixmepath" % (sstate_builddir), "w")
+		fixme = []
+		for f in file_list.split('\n'):
+			fixme.append(f.replace(sstate_builddir, ""))
+		fix.write("\n".join(fixme))
+		fix.close()
 	p.close()
 
 def sstate_package(ss, d):

Re: [PATCH] sstate.bbclass: Optimise sstate_hardcode_path

[Joshua Lock]

On 09/02/12 16:07, Richard Purdie wrote:
> The sstate_hardcode_path() function triggered large numbers of exec()
> calls when processing packages with large numbers of file relocations
> (e.g. perl). This patch optimises those calls into longer single commands
> which make the code significantly more efficient.
>
> This reduced the do_package time for perl by 2 minutes (from 4.75 minutes)
> for me.

Very nice speed up!

> Signed-off-by: Richard Purdie<richard.purdie@linuxfoundation.org>

A minor api/style nit below but:

Acked-by: Joshua Lock <josh@linux.intel.com>

> ---
> diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
> index 4bd3712..d4f95c1 100644
> --- a/meta/classes/sstate.bbclass
> +++ b/meta/classes/sstate.bbclass
> @@ -317,19 +317,24 @@ def sstate_hardcode_path(d):
>   	staging_host = d.getVar('STAGING_DIR_HOST', True)
>   	sstate_builddir = d.getVar('SSTATE_BUILDDIR', True)
>
> -	for i in file_list.split('\n'):
> -		if not i:
> -			continue
> -		if bb.data.inherits_class('native', d) or bb.data.inherits_class('nativesdk', d) or bb.data.inherits_class('crosssdk', d) or bb.data.inherits_class('cross-canadian', d):
> -			cmd = "sed -i -e s:%s:FIXMESTAGINGDIR:g %s" % (staging, i)
> -		elif bb.data.inherits_class('cross', d):
> -			cmd = "sed -i -e s:%s:FIXMESTAGINGDIRTARGET:g %s \
> -				sed -i -e s:%s:FIXMESTAGINGDIR:g %s" % (staging_target, i, staging, i)
> -		else:
> -			cmd = "sed -i -e s:%s:FIXMESTAGINGDIRHOST:g %s" % (staging_host, i)
> +	files = " ".join(file_list.split('\n'))
>
> +	if bb.data.inherits_class('native', d) or bb.data.inherits_class('nativesdk', d) or bb.data.inherits_class('crosssdk', d) or bb.data.inherits_class('cross-canadian', d):

I think this can be replaced by a single call to the inherits method of 
the oe.utils module:

oe.utils.inherits(d, 'native', 'nativesdk', 'crosssdk', 'cross-canadian')

which makes this a little more concise.

> +		cmd = "sed -i -e s:%s:FIXMESTAGINGDIR:g %s" % (staging, files)
> +	elif bb.data.inherits_class('cross', d):
> +		cmd = "sed -i -e s:%s:FIXMESTAGINGDIRTARGET:g %s \
> +			sed -i -e s:%s:FIXMESTAGINGDIR:g %s" % (staging_target, files, staging, files)
> +	else:
> +		cmd = "sed -i -e s:%s:FIXMESTAGINGDIRHOST:g %s" % (staging_host, files)
> +
> +	if files:
>   		os.system(cmd)
> -		os.system("echo %s | sed -e 's:%s::'>>  %sfixmepath" % (i, sstate_builddir, sstate_builddir))
> +		fix = open("%sfixmepath" % (sstate_builddir), "w")
> +		fixme = []
> +		for f in file_list.split('\n'):
> +			fixme.append(f.replace(sstate_builddir, ""))
> +		fix.write("\n".join(fixme))
> +		fix.close()
>   	p.close()
>
>   def sstate_package(ss, d):
>
>
>
> _______________________________________________
> Openembedded-core mailing list
> Openembedded-core@lists.openembedded.org
> http://lists.linuxtogo.org/cgi-bin/mailman/listinfo/openembedded-core

-- 
Joshua Lock
         Yocto Project "Johannes factotum"
         Intel Open Source Technology Centre