* [PATCH 01/22] package.bbclass: Multithread per file dependency generation code
2013-02-03 23:55 [PATCH 00/22] Packaging performance improvements, round 2 Richard Purdie
@ 2013-02-03 23:55 ` Richard Purdie
2013-02-03 23:55 ` [PATCH 02/22] package: Don't export PATH Richard Purdie
` (21 subsequent siblings)
22 siblings, 0 replies; 27+ messages in thread
From: Richard Purdie @ 2013-02-03 23:55 UTC (permalink / raw)
To: openembedded-core
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
meta/classes/base.bbclass | 2 +-
meta/classes/package.bbclass | 92 +++++++++++-------------------
meta/classes/update-alternatives.bbclass | 2 +-
meta/lib/oe/package.py | 51 +++++++++++++++++
4 files changed, 85 insertions(+), 62 deletions(-)
create mode 100644 meta/lib/oe/package.py
diff --git a/meta/classes/base.bbclass b/meta/classes/base.bbclass
index 4662d3b..5f43733 100644
--- a/meta/classes/base.bbclass
+++ b/meta/classes/base.bbclass
@@ -10,7 +10,7 @@ inherit utility-tasks
inherit metadata_scm
inherit logging
-OE_IMPORTS += "os sys time oe.path oe.utils oe.data oe.packagegroup oe.sstatesig oe.lsb"
+OE_IMPORTS += "os sys time oe.path oe.utils oe.data oe.package oe.packagegroup oe.sstatesig oe.lsb"
OE_IMPORTS[type] = "list"
def oe_import(d):
diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index fec3db6..cafab1e 100644
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -1236,83 +1236,55 @@ RPMDEPS = "${STAGING_LIBDIR_NATIVE}/rpm/bin/rpmdeps-oecore --macros ${STAGING_LI
# FILERDEPENDS_filepath_pkg - per file dep
python package_do_filedeps() {
- import re
-
if d.getVar('SKIP_FILEDEPS', True) == '1':
return
pkgdest = d.getVar('PKGDEST', True)
packages = d.getVar('PACKAGES', True)
-
rpmdeps = d.expand("${RPMDEPS}")
- r = re.compile(r'[<>=]+ +[^ ]*')
-
- def file_translate(file):
- ft = file.replace("@", "@at@")
- ft = ft.replace(" ", "@space@")
- ft = ft.replace("\t", "@tab@")
- ft = ft.replace("[", "@openbrace@")
- ft = ft.replace("]", "@closebrace@")
- ft = ft.replace("_", "@underscore@")
- return ft
-
- # Quick routine to process the results of the rpmdeps call...
- def process_deps(pipe, pkg, provides_files, requires_files):
- provides = {}
- requires = {}
-
- for line in pipe:
- f = line.split(" ", 1)[0].strip()
- line = line.split(" ", 1)[1].strip()
-
- if line.startswith("Requires:"):
- i = requires
- elif line.startswith("Provides:"):
- i = provides
- else:
- continue
-
- file = f.replace(pkgdest + "/" + pkg, "")
- file = file_translate(file)
- value = line.split(":", 1)[1].strip()
- value = r.sub(r'(\g<0>)', value)
-
- if value.startswith("rpmlib("):
- continue
- if value == "python":
- continue
- if file not in i:
- i[file] = []
- i[file].append(value)
-
- for file in provides:
- provides_files.append(file)
- key = "FILERPROVIDES_" + file + "_" + pkg
- d.setVar(key, " ".join(provides[file]))
-
- for file in requires:
- requires_files.append(file)
- key = "FILERDEPENDS_" + file + "_" + pkg
- d.setVar(key, " ".join(requires[file]))
def chunks(files, n):
return [files[i:i+n] for i in range(0, len(files), n)]
- # Determine dependencies
+ pkglist = []
for pkg in packages.split():
if pkg.endswith('-dbg') or pkg.endswith('-doc') or pkg.find('-locale-') != -1 or pkg.find('-localedata-') != -1 or pkg.find('-gconv-') != -1 or pkg.find('-charmap-') != -1 or pkg.startswith('kernel-module-'):
continue
+ for files in chunks(pkgfiles[pkg], 100):
+ pkglist.append((pkg, files, rpmdeps, pkgdest))
- provides_files = []
- requires_files = []
+ import multiprocessing
+ nproc = multiprocessing.cpu_count()
+ pool = multiprocessing.Pool(nproc)
+ processed = pool.imap(oe.package.filedeprunner, pkglist)
+ pool.close()
+ pool.join()
- for files in chunks(pkgfiles[pkg], 100):
- dep_pipe = os.popen(rpmdeps + " " + " ".join(files))
+ provides_files = {}
+ requires_files = {}
+
+ for result in processed:
+ (pkg, provides, requires) = result
+
+ if pkg not in provides_files:
+ provides_files[pkg] = []
+ if pkg not in requires_files:
+ requires_files[pkg] = []
- process_deps(dep_pipe, pkg, provides_files, requires_files)
+ for file in provides:
+ provides_files[pkg].append(file)
+ key = "FILERPROVIDES_" + file + "_" + pkg
+ d.setVar(key, " ".join(provides[file]))
+
+ for file in requires:
+ requires_files[pkg].append(file)
+ key = "FILERDEPENDS_" + file + "_" + pkg
+ d.setVar(key, " ".join(requires[file]))
- d.setVar("FILERDEPENDSFLIST_" + pkg, " ".join(requires_files))
- d.setVar("FILERPROVIDESFLIST_" + pkg, " ".join(provides_files))
+ for pkg in requires_files:
+ d.setVar("FILERDEPENDSFLIST_" + pkg, " ".join(requires_files[pkg]))
+ for pkg in provides_files:
+ d.setVar("FILERPROVIDESFLIST_" + pkg, " ".join(provides_files[pkg]))
}
def getshlibsdirs(d):
diff --git a/meta/classes/update-alternatives.bbclass b/meta/classes/update-alternatives.bbclass
index a75e282..90bc56b 100644
--- a/meta/classes/update-alternatives.bbclass
+++ b/meta/classes/update-alternatives.bbclass
@@ -353,7 +353,7 @@ python package_do_filedeps_append () {
continue
# Add file provide
- trans_target = file_translate(alt_target)
+ trans_target = oe.package.file_translate(alt_target)
d.appendVar('FILERPROVIDES_%s_%s' % (trans_target, pkg), " " + alt_link)
if not trans_target in (d.getVar('FILERPROVIDESFLIST_%s' % pkg, True) or ""):
d.appendVar('FILERPROVIDESFLIST_%s' % pkg, " " + trans_target)
diff --git a/meta/lib/oe/package.py b/meta/lib/oe/package.py
new file mode 100644
index 0000000..6b1c1f4
--- /dev/null
+++ b/meta/lib/oe/package.py
@@ -0,0 +1,51 @@
+
+def file_translate(file):
+ ft = file.replace("@", "@at@")
+ ft = ft.replace(" ", "@space@")
+ ft = ft.replace("\t", "@tab@")
+ ft = ft.replace("[", "@openbrace@")
+ ft = ft.replace("]", "@closebrace@")
+ ft = ft.replace("_", "@underscore@")
+ return ft
+
+def filedeprunner(arg):
+ import re
+
+ (pkg, pkgfiles, rpmdeps, pkgdest) = arg
+ provides = {}
+ requires = {}
+
+ r = re.compile(r'[<>=]+ +[^ ]*')
+
+ def process_deps(pipe, pkg, pkgdest, provides, requires):
+ for line in pipe:
+ f = line.split(" ", 1)[0].strip()
+ line = line.split(" ", 1)[1].strip()
+
+ if line.startswith("Requires:"):
+ i = requires
+ elif line.startswith("Provides:"):
+ i = provides
+ else:
+ continue
+
+ file = f.replace(pkgdest + "/" + pkg, "")
+ file = file_translate(file)
+ value = line.split(":", 1)[1].strip()
+ value = r.sub(r'(\g<0>)', value)
+
+ if value.startswith("rpmlib("):
+ continue
+ if value == "python":
+ continue
+ if file not in i:
+ i[file] = []
+ i[file].append(value)
+
+ return provides, requires
+
+ dep_pipe = os.popen(rpmdeps + " " + " ".join(pkgfiles))
+
+ provides, requires = process_deps(dep_pipe, pkg, pkgdest, provides, requires)
+
+ return (pkg, provides, requires)
--
1.7.10.4
^ permalink raw reply related [flat|nested] 27+ messages in thread
* [PATCH 02/22] package: Don't export PATH
2013-02-03 23:55 [PATCH 00/22] Packaging performance improvements, round 2 Richard Purdie
2013-02-03 23:55 ` [PATCH 01/22] package.bbclass: Multithread per file dependency generation code Richard Purdie
@ 2013-02-03 23:55 ` Richard Purdie
2013-02-03 23:55 ` [PATCH 03/22] package: Process package stripping in parallel Richard Purdie
` (20 subsequent siblings)
22 siblings, 0 replies; 27+ messages in thread
From: Richard Purdie @ 2013-02-03 23:55 UTC (permalink / raw)
To: openembedded-core
PATH is already exported, we don't need to do this each time we run
something, its just noise and overhead.
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
meta/classes/package.bbclass | 14 +++++---------
1 file changed, 5 insertions(+), 9 deletions(-)
diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index cafab1e..527ef31 100644
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -233,7 +233,6 @@ def splitdebuginfo(file, debugfile, debugsrcdir, d):
import commands, stat, subprocess
dvar = d.getVar('PKGD', True)
- pathprefix = "export PATH=%s; " % d.getVar('PATH', True)
objcopy = d.getVar("OBJCOPY", True)
debugedit = d.expand("${STAGING_LIBDIR_NATIVE}/rpm/bin/debugedit")
workdir = d.getVar("WORKDIR", True)
@@ -252,14 +251,14 @@ def splitdebuginfo(file, debugfile, debugsrcdir, d):
# We need to extract the debug src information here...
if debugsrcdir:
- subprocess.call("%s'%s' -b '%s' -d '%s' -i -l '%s' '%s'" % (pathprefix, debugedit, workparentdir, debugsrcdir, sourcefile, file), shell=True)
+ subprocess.call("'%s' -b '%s' -d '%s' -i -l '%s' '%s'" % (debugedit, workparentdir, debugsrcdir, sourcefile, file), shell=True)
bb.mkdirhier(os.path.dirname(debugfile))
- subprocess.call("%s'%s' --only-keep-debug '%s' '%s'" % (pathprefix, objcopy, file, debugfile), shell=True)
+ subprocess.call("'%s' --only-keep-debug '%s' '%s'" % (objcopy, file, debugfile), shell=True)
# Set the debuglink to have the view of the file path on the target
- subprocess.call("%s'%s' --add-gnu-debuglink='%s' '%s'" % (pathprefix, objcopy, debugfile, file), shell=True)
+ subprocess.call("'%s' --add-gnu-debuglink='%s' '%s'" % (objcopy, debugfile, file), shell=True)
if newmode:
os.chmod(file, origmode)
@@ -275,7 +274,6 @@ def copydebugsources(debugsrcdir, d):
sourcefile = d.expand("${WORKDIR}/debugsources.list")
if debugsrcdir and os.path.isfile(sourcefile):
dvar = d.getVar('PKGD', True)
- pathprefix = "export PATH=%s; " % d.getVar('PATH', True)
strip = d.getVar("STRIP", True)
objcopy = d.getVar("OBJCOPY", True)
debugedit = d.expand("${STAGING_LIBDIR_NATIVE}/rpm/bin/debugedit")
@@ -322,7 +320,6 @@ def runstrip(file, elftype, d):
import commands, stat, subprocess
- pathprefix = "export PATH=%s; " % d.getVar('PATH', True)
strip = d.getVar("STRIP", True)
newmode = None
@@ -345,7 +342,7 @@ def runstrip(file, elftype, d):
stripcmd = "'%s' %s '%s'" % (strip, extraflags, file)
bb.debug(1, "runstrip: %s" % stripcmd)
- ret = subprocess.call("%s%s" % (pathprefix, stripcmd), shell=True)
+ ret = subprocess.call(stripcmd, shell=True)
if newmode:
os.chmod(file, origmode)
@@ -747,8 +744,7 @@ python split_and_strip_files () {
# 16 - kernel module
def isELF(path):
type = 0
- pathprefix = "export PATH=%s; " % d.getVar('PATH', True)
- ret, result = commands.getstatusoutput("%sfile '%s'" % (pathprefix, path))
+ ret, result = commands.getstatusoutput("file '%s'" % path)
if ret:
bb.error("split_and_strip_files: 'file %s' failed" % path)
--
1.7.10.4
^ permalink raw reply related [flat|nested] 27+ messages in thread
* [PATCH 03/22] package: Process package stripping in parallel
2013-02-03 23:55 [PATCH 00/22] Packaging performance improvements, round 2 Richard Purdie
2013-02-03 23:55 ` [PATCH 01/22] package.bbclass: Multithread per file dependency generation code Richard Purdie
2013-02-03 23:55 ` [PATCH 02/22] package: Don't export PATH Richard Purdie
@ 2013-02-03 23:55 ` Richard Purdie
2013-02-04 17:23 ` Mark Hatle
2013-02-03 23:55 ` [PATCH 04/22] insane.bbclass: Add pkgvarcheck to check for suboptimal usages of variables Richard Purdie
` (19 subsequent siblings)
22 siblings, 1 reply; 27+ messages in thread
From: Richard Purdie @ 2013-02-03 23:55 UTC (permalink / raw)
To: openembedded-core
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
meta/classes/package.bbclass | 58 ++++++++++--------------------------------
meta/lib/oe/package.py | 45 ++++++++++++++++++++++++++++++++
2 files changed, 58 insertions(+), 45 deletions(-)
diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index 527ef31..9c8cdbc 100644
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -309,49 +309,6 @@ def copydebugsources(debugsrcdir, d):
if os.path.exists(p) and not os.listdir(p):
os.rmdir(p)
-def runstrip(file, elftype, d):
- # Function to strip a single file, called from split_and_strip_files below
- # A working 'file' (one which works on the target architecture)
- #
- # The elftype is a bit pattern (explained in split_and_strip_files) to tell
- # us what type of file we're processing...
- # 4 - executable
- # 8 - shared library
-
- import commands, stat, subprocess
-
- strip = d.getVar("STRIP", True)
-
- newmode = None
- if not os.access(file, os.W_OK) or os.access(file, os.R_OK):
- origmode = os.stat(file)[stat.ST_MODE]
- newmode = origmode | stat.S_IWRITE | stat.S_IREAD
- os.chmod(file, newmode)
-
- extraflags = ""
-
- # .so and shared library
- if elftype & 16:
- extraflags = "--strip-debug --remove-section=.comment --remove-section=.note --preserve-dates"
- elif ".so" in file and elftype & 8:
- extraflags = "--remove-section=.comment --remove-section=.note --strip-unneeded"
- # shared or executable:
- elif elftype & 8 or elftype & 4:
- extraflags = "--remove-section=.comment --remove-section=.note"
-
- stripcmd = "'%s' %s '%s'" % (strip, extraflags, file)
- bb.debug(1, "runstrip: %s" % stripcmd)
-
- ret = subprocess.call(stripcmd, shell=True)
-
- if newmode:
- os.chmod(file, origmode)
-
- if ret:
- bb.error("runstrip: '%s' strip command failed" % stripcmd)
-
- return 0
-
#
# Package data handling routines
#
@@ -902,13 +859,24 @@ python split_and_strip_files () {
# Now lets go back over things and strip them
#
if (d.getVar('INHIBIT_PACKAGE_STRIP', True) != '1'):
+ strip = d.getVar("STRIP", True)
+ sfiles = []
for file in file_list:
if file_list[file].startswith("ELF: "):
elf_file = int(file_list[file][5:])
#bb.note("Strip %s" % file)
- runstrip(file, elf_file, d)
+ sfiles.append((file, elf_file, strip))
for f in kernmods:
- runstrip(f, 16, d)
+ sfiles.append((f, 16, strip))
+
+
+ import multiprocessing
+ nproc = multiprocessing.cpu_count()
+ pool = multiprocessing.Pool(nproc)
+ processed = pool.imap(oe.package.runstrip, sfiles)
+ pool.close()
+ pool.join()
+
#
# End of strip
#
diff --git a/meta/lib/oe/package.py b/meta/lib/oe/package.py
index 6b1c1f4..9a0ddb8 100644
--- a/meta/lib/oe/package.py
+++ b/meta/lib/oe/package.py
@@ -1,3 +1,48 @@
+def runstrip(arg):
+ # Function to strip a single file, called from split_and_strip_files below
+ # A working 'file' (one which works on the target architecture)
+ #
+ # The elftype is a bit pattern (explained in split_and_strip_files) to tell
+ # us what type of file we're processing...
+ # 4 - executable
+ # 8 - shared library
+ # 16 - kernel module
+
+ import commands, stat, subprocess
+
+ (file, elftype, strip) = arg
+
+ newmode = None
+ if not os.access(file, os.W_OK) or os.access(file, os.R_OK):
+ origmode = os.stat(file)[stat.ST_MODE]
+ newmode = origmode | stat.S_IWRITE | stat.S_IREAD
+ os.chmod(file, newmode)
+
+ extraflags = ""
+
+ # kernel module
+ if elftype & 16:
+ extraflags = "--strip-debug --remove-section=.comment --remove-section=.note --preserve-dates"
+ # .so and shared library
+ elif ".so" in file and elftype & 8:
+ extraflags = "--remove-section=.comment --remove-section=.note --strip-unneeded"
+ # shared or executable:
+ elif elftype & 8 or elftype & 4:
+ extraflags = "--remove-section=.comment --remove-section=.note"
+
+ stripcmd = "'%s' %s '%s'" % (strip, extraflags, file)
+ bb.debug(1, "runstrip: %s" % stripcmd)
+
+ ret = subprocess.call(stripcmd, shell=True)
+
+ if newmode:
+ os.chmod(file, origmode)
+
+ if ret:
+ bb.error("runstrip: '%s' strip command failed" % stripcmd)
+
+ return
+
def file_translate(file):
ft = file.replace("@", "@at@")
--
1.7.10.4
^ permalink raw reply related [flat|nested] 27+ messages in thread
* [PATCH 04/22] insane.bbclass: Add pkgvarcheck to check for suboptimal usages of variables
2013-02-03 23:55 [PATCH 00/22] Packaging performance improvements, round 2 Richard Purdie
` (2 preceding siblings ...)
2013-02-03 23:55 ` [PATCH 03/22] package: Process package stripping in parallel Richard Purdie
@ 2013-02-03 23:55 ` Richard Purdie
2013-02-03 23:55 ` [PATCH 05/22] insane.bbclass: Add documentation headers for logical code blocks Richard Purdie
` (18 subsequent siblings)
22 siblings, 0 replies; 27+ messages in thread
From: Richard Purdie @ 2013-02-03 23:55 UTC (permalink / raw)
To: openembedded-core
Check through the variables:
'RDEPENDS', 'RRECOMMENDS', 'FILES', 'pkg_preinst', 'pkg_postinst', 'pkg_prerm', 'pkg_postrm'
and if there is a variable set which isn't package specific, inform the user
of this.
Using these variables without a package suffix is bad practise and complicates
dependencies of packages unnecessarily as well as complicates the code. Lets
convert the remaining issues and then we can take the small performance gain.
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
meta/classes/insane.bbclass | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
diff --git a/meta/classes/insane.bbclass b/meta/classes/insane.bbclass
index 5dfa5aa..57104ac 100644
--- a/meta/classes/insane.bbclass
+++ b/meta/classes/insane.bbclass
@@ -114,7 +114,7 @@ def package_qa_get_machine_dict():
# Currently not being used by default "desktop"
WARN_QA ?= "ldflags useless-rpaths rpaths staticdev libdir xorg-driver-abi textrel"
-ERROR_QA ?= "dev-so debug-deps dev-deps debug-files arch la2 pkgconfig la perms dep-cmp"
+ERROR_QA ?= "dev-so debug-deps dev-deps debug-files arch la2 pkgconfig la perms dep-cmp pkgvarcheck"
ALL_QA = "${WARN_QA} ${ERROR_QA}"
@@ -885,4 +885,12 @@ python () {
tests = d.getVar('ALL_QA', True).split()
if "desktop" in tests:
d.appendVar("PACKAGE_DEPENDS", "desktop-file-utils-native")
+
+ issues = []
+ if d.getVar('PACKAGES', True):
+ for var in 'RDEPENDS', 'RRECOMMENDS', 'FILES', 'pkg_preinst', 'pkg_postinst', 'pkg_prerm', 'pkg_postrm':
+ if d.getVar(var):
+ issues.append(var)
+ for i in issues:
+ package_qa_handle_error("pkgvarcheck", "%s: Variable %s is set as not being package specific, please fix this." % (d.getVar("FILE", True), i), d)
}
--
1.7.10.4
^ permalink raw reply related [flat|nested] 27+ messages in thread
* [PATCH 05/22] insane.bbclass: Add documentation headers for logical code blocks
2013-02-03 23:55 [PATCH 00/22] Packaging performance improvements, round 2 Richard Purdie
` (3 preceding siblings ...)
2013-02-03 23:55 ` [PATCH 04/22] insane.bbclass: Add pkgvarcheck to check for suboptimal usages of variables Richard Purdie
@ 2013-02-03 23:55 ` Richard Purdie
2013-02-03 23:55 ` [PATCH 06/22] staging/insane.bbclass: Move legacy do_stage check iinto insane.bbclass Richard Purdie
` (17 subsequent siblings)
22 siblings, 0 replies; 27+ messages in thread
From: Richard Purdie @ 2013-02-03 23:55 UTC (permalink / raw)
To: openembedded-core
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
meta/classes/insane.bbclass | 17 +++++++++++++++++
1 file changed, 17 insertions(+)
diff --git a/meta/classes/insane.bbclass b/meta/classes/insane.bbclass
index 57104ac..7e5634d 100644
--- a/meta/classes/insane.bbclass
+++ b/meta/classes/insane.bbclass
@@ -836,6 +836,10 @@ python do_qa_staging() {
python do_qa_configure() {
import subprocess
+ ###########################################################################
+ # Check config.log for cross compile issues
+ ###########################################################################
+
configs = []
workdir = d.getVar('WORKDIR', True)
bb.note("Checking autotools environment for common misconfiguration")
@@ -852,6 +856,10 @@ Rerun configure task after fixing this. The path was '%s'""" % root)
if "configure.in" in files:
configs.append(os.path.join(root, "configure.in"))
+ ###########################################################################
+ # Check gettext configuration and dependencies are correct
+ ###########################################################################
+
cnf = d.getVar('EXTRA_OECONF', True) or ""
if "gettext" not in d.getVar('P', True) and "gcc-runtime" not in d.getVar('P', True) and "--disable-nls" not in cnf:
ml = d.getVar("MLPREFIX", True) or ""
@@ -869,8 +877,13 @@ Rerun configure task after fixing this. The path was '%s'""" % root)
bb.fatal("""%s required but not in DEPENDS for file %s.
Missing inherit gettext?""" % (gt, config))
+ ###########################################################################
+ # Check license variables
+ ###########################################################################
+
if not package_qa_check_license(workdir, d):
bb.fatal("Licensing Error: LIC_FILES_CHKSUM does not match, please fix")
+
}
# The Staging Func, to check all staging
#addtask qa_staging after do_populate_sysroot before do_build
@@ -886,6 +899,10 @@ python () {
if "desktop" in tests:
d.appendVar("PACKAGE_DEPENDS", "desktop-file-utils-native")
+ ###########################################################################
+ # Check various variables
+ ###########################################################################
+
issues = []
if d.getVar('PACKAGES', True):
for var in 'RDEPENDS', 'RRECOMMENDS', 'FILES', 'pkg_preinst', 'pkg_postinst', 'pkg_prerm', 'pkg_postrm':
--
1.7.10.4
^ permalink raw reply related [flat|nested] 27+ messages in thread
* [PATCH 06/22] staging/insane.bbclass: Move legacy do_stage check iinto insane.bbclass
2013-02-03 23:55 [PATCH 00/22] Packaging performance improvements, round 2 Richard Purdie
` (4 preceding siblings ...)
2013-02-03 23:55 ` [PATCH 05/22] insane.bbclass: Add documentation headers for logical code blocks Richard Purdie
@ 2013-02-03 23:55 ` Richard Purdie
2013-02-03 23:55 ` [PATCH 07/22] staging.bbclass: Drop unused/legacy function Richard Purdie
` (16 subsequent siblings)
22 siblings, 0 replies; 27+ messages in thread
From: Richard Purdie @ 2013-02-03 23:55 UTC (permalink / raw)
To: openembedded-core
We might as well put all the sanity checks in one place.
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
meta/classes/insane.bbclass | 3 +++
meta/classes/staging.bbclass | 5 -----
2 files changed, 3 insertions(+), 5 deletions(-)
diff --git a/meta/classes/insane.bbclass b/meta/classes/insane.bbclass
index 7e5634d..504182c 100644
--- a/meta/classes/insane.bbclass
+++ b/meta/classes/insane.bbclass
@@ -903,6 +903,9 @@ python () {
# Check various variables
###########################################################################
+ if d.getVar('do_stage', True) is not None:
+ bb.fatal("Legacy staging found for %s as it has a do_stage function. This will need conversion to a do_install or often simply removal to work with OE-core" % d.getVar("FILE", True))
+
issues = []
if d.getVar('PACKAGES', True):
for var in 'RDEPENDS', 'RRECOMMENDS', 'FILES', 'pkg_preinst', 'pkg_postinst', 'pkg_prerm', 'pkg_postrm':
diff --git a/meta/classes/staging.bbclass b/meta/classes/staging.bbclass
index da90d31..643352d 100644
--- a/meta/classes/staging.bbclass
+++ b/meta/classes/staging.bbclass
@@ -114,9 +114,4 @@ python do_populate_sysroot_setscene () {
}
addtask do_populate_sysroot_setscene
-python () {
- if d.getVar('do_stage', True) is not None:
- bb.fatal("Legacy staging found for %s as it has a do_stage function. This will need conversion to a do_install or often simply removal to work with OE-core" % d.getVar("FILE", True))
-}
-
--
1.7.10.4
^ permalink raw reply related [flat|nested] 27+ messages in thread
* [PATCH 07/22] staging.bbclass: Drop unused/legacy function
2013-02-03 23:55 [PATCH 00/22] Packaging performance improvements, round 2 Richard Purdie
` (5 preceding siblings ...)
2013-02-03 23:55 ` [PATCH 06/22] staging/insane.bbclass: Move legacy do_stage check iinto insane.bbclass Richard Purdie
@ 2013-02-03 23:55 ` Richard Purdie
2013-02-03 23:55 ` [PATCH 08/22] update-rc.d: Drop OVERRIDES code Richard Purdie
` (15 subsequent siblings)
22 siblings, 0 replies; 27+ messages in thread
From: Richard Purdie @ 2013-02-03 23:55 UTC (permalink / raw)
To: openembedded-core
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
meta/classes/staging.bbclass | 4 ----
1 file changed, 4 deletions(-)
diff --git a/meta/classes/staging.bbclass b/meta/classes/staging.bbclass
index 643352d..b522c7d 100644
--- a/meta/classes/staging.bbclass
+++ b/meta/classes/staging.bbclass
@@ -1,8 +1,4 @@
-packagedstaging_fastpath () {
- :
-}
-
sysroot_stage_dir() {
src="$1"
dest="$2"
--
1.7.10.4
^ permalink raw reply related [flat|nested] 27+ messages in thread
* [PATCH 08/22] update-rc.d: Drop OVERRIDES code
2013-02-03 23:55 [PATCH 00/22] Packaging performance improvements, round 2 Richard Purdie
` (6 preceding siblings ...)
2013-02-03 23:55 ` [PATCH 07/22] staging.bbclass: Drop unused/legacy function Richard Purdie
@ 2013-02-03 23:55 ` Richard Purdie
2013-02-03 23:55 ` [PATCH 09/22] qemu: Set RDEPENDS on the specific package that needs it Richard Purdie
` (14 subsequent siblings)
22 siblings, 0 replies; 27+ messages in thread
From: Richard Purdie @ 2013-02-03 23:55 UTC (permalink / raw)
To: openembedded-core
The data store copy and overrides is overkill given the small number
of accesses that are being made. This simplifies the code.
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
meta/classes/update-rc.d_real.bbclass | 17 ++++++-----------
1 file changed, 6 insertions(+), 11 deletions(-)
diff --git a/meta/classes/update-rc.d_real.bbclass b/meta/classes/update-rc.d_real.bbclass
index 83816d6..0cd671f 100644
--- a/meta/classes/update-rc.d_real.bbclass
+++ b/meta/classes/update-rc.d_real.bbclass
@@ -50,32 +50,27 @@ python __anonymous() {
python populate_packages_prepend () {
def update_rcd_package(pkg):
bb.debug(1, 'adding update-rc.d calls to postinst/postrm for %s' % pkg)
- localdata = bb.data.createCopy(d)
- overrides = localdata.getVar("OVERRIDES", True)
- localdata.setVar("OVERRIDES", "%s:%s" % (pkg, overrides))
- bb.data.update_data(localdata)
-
"""
update_rc.d postinst is appended here because pkg_postinst may require to
execute on the target. Not doing so may cause update_rc.d postinst invoked
twice to cause unwanted warnings.
"""
- postinst = localdata.getVar('pkg_postinst', True)
+ postinst = d.getVar('pkg_postinst_%s' % pkg, True) or d.getVar('pkg_postinst', True)
if not postinst:
postinst = '#!/bin/sh\n'
- postinst += localdata.getVar('updatercd_postinst', True)
+ postinst += d.getVar('updatercd_postinst', True)
d.setVar('pkg_postinst_%s' % pkg, postinst)
- prerm = localdata.getVar('pkg_prerm', True)
+ prerm = d.getVar('pkg_prerm_%s' % pkg, True) or d.getVar('pkg_prerm', True)
if not prerm:
prerm = '#!/bin/sh\n'
- prerm += localdata.getVar('updatercd_prerm', True)
+ prerm += d.getVar('updatercd_prerm', True)
d.setVar('pkg_prerm_%s' % pkg, prerm)
- postrm = localdata.getVar('pkg_postrm', True)
+ postrm = d.getVar('pkg_postrm_%s' % pkg, True) or d.getVar('pkg_postrm', True)
if not postrm:
postrm = '#!/bin/sh\n'
- postrm += localdata.getVar('updatercd_postrm', True)
+ postrm += d.getVar('updatercd_postrm', True)
d.setVar('pkg_postrm_%s' % pkg, postrm)
pkgs = d.getVar('INITSCRIPT_PACKAGES', True)
--
1.7.10.4
^ permalink raw reply related [flat|nested] 27+ messages in thread
* [PATCH 09/22] qemu: Set RDEPENDS on the specific package that needs it
2013-02-03 23:55 [PATCH 00/22] Packaging performance improvements, round 2 Richard Purdie
` (7 preceding siblings ...)
2013-02-03 23:55 ` [PATCH 08/22] update-rc.d: Drop OVERRIDES code Richard Purdie
@ 2013-02-03 23:55 ` Richard Purdie
2013-02-03 23:55 ` [PATCH 10/22] gdb-cross-canadian: " Richard Purdie
` (13 subsequent siblings)
22 siblings, 0 replies; 27+ messages in thread
From: Richard Purdie @ 2013-02-03 23:55 UTC (permalink / raw)
To: openembedded-core
Set the RDEPENDS on the specific package that has the dependency and stop it
being applied to for example ${PN}-doc (and others).
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
meta/recipes-devtools/qemu/qemu.inc | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/meta/recipes-devtools/qemu/qemu.inc b/meta/recipes-devtools/qemu/qemu.inc
index ca411a6..2b60347 100644
--- a/meta/recipes-devtools/qemu/qemu.inc
+++ b/meta/recipes-devtools/qemu/qemu.inc
@@ -4,7 +4,7 @@ LICENSE = "GPLv2 & LGPLv2.1"
DEPENDS = "glib-2.0 zlib alsa-lib virtual/libx11 pixman"
DEPENDS_class-native = "zlib-native alsa-lib-native glib-2.0-native pixman-native"
DEPENDS_class-nativesdk = "nativesdk-zlib nativesdk-libsdl nativesdk-glib-2.0 nativesdk-pixman"
-RDEPENDS_class-nativesdk = "nativesdk-libsdl"
+RDEPENDS_${PN}_class-nativesdk = "nativesdk-libsdl"
require qemu-targets.inc
inherit autotools
--
1.7.10.4
^ permalink raw reply related [flat|nested] 27+ messages in thread
* [PATCH 10/22] gdb-cross-canadian: Set RDEPENDS on the specific package that needs it
2013-02-03 23:55 [PATCH 00/22] Packaging performance improvements, round 2 Richard Purdie
` (8 preceding siblings ...)
2013-02-03 23:55 ` [PATCH 09/22] qemu: Set RDEPENDS on the specific package that needs it Richard Purdie
@ 2013-02-03 23:55 ` Richard Purdie
2013-02-03 23:55 ` [PATCH 11/22] initramfs-live-boot: " Richard Purdie
` (12 subsequent siblings)
22 siblings, 0 replies; 27+ messages in thread
From: Richard Purdie @ 2013-02-03 23:55 UTC (permalink / raw)
To: openembedded-core
Set the RDEPENDS on the specific package that has the dependency and stop it
being applied to for example ${PN}-doc (and others).
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
meta/recipes-devtools/gdb/gdb-cross-canadian.inc | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/meta/recipes-devtools/gdb/gdb-cross-canadian.inc b/meta/recipes-devtools/gdb/gdb-cross-canadian.inc
index c947526..1f60322 100644
--- a/meta/recipes-devtools/gdb/gdb-cross-canadian.inc
+++ b/meta/recipes-devtools/gdb/gdb-cross-canadian.inc
@@ -6,8 +6,8 @@ PN = "gdb-cross-canadian-${TRANSLATED_TARGET_ARCH}"
BPN = "gdb"
DEPENDS = "nativesdk-ncurses nativesdk-expat nativesdk-gettext nativesdk-readline nativesdk-python"
-RDEPENDS += "nativesdk-python-core nativesdk-python-lang nativesdk-python-re \
- nativesdk-python-codecs nativesdk-python-netclient"
+RDEPENDS_${PN} += "nativesdk-python-core nativesdk-python-lang nativesdk-python-re \
+ nativesdk-python-codecs nativesdk-python-netclient"
EXTRA_OECONF_append = "--with-python=${WORKDIR}/python"
--
1.7.10.4
^ permalink raw reply related [flat|nested] 27+ messages in thread
* [PATCH 11/22] initramfs-live-boot: Set RDEPENDS on the specific package that needs it
2013-02-03 23:55 [PATCH 00/22] Packaging performance improvements, round 2 Richard Purdie
` (9 preceding siblings ...)
2013-02-03 23:55 ` [PATCH 10/22] gdb-cross-canadian: " Richard Purdie
@ 2013-02-03 23:55 ` Richard Purdie
2013-02-03 23:55 ` [PATCH 12/22] package.bbclass: Fix up bb.mkdirhier/bb.copyfile usage Richard Purdie
` (11 subsequent siblings)
22 siblings, 0 replies; 27+ messages in thread
From: Richard Purdie @ 2013-02-03 23:55 UTC (permalink / raw)
To: openembedded-core
Set the RDEPENDS on the specific package that has the dependency and stop it
being applied to for example ${PN}-doc (and others).
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
meta/recipes-core/initrdscripts/initramfs-live-boot_1.0.bb | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/meta/recipes-core/initrdscripts/initramfs-live-boot_1.0.bb b/meta/recipes-core/initrdscripts/initramfs-live-boot_1.0.bb
index 55a8600..1e674a4 100644
--- a/meta/recipes-core/initrdscripts/initramfs-live-boot_1.0.bb
+++ b/meta/recipes-core/initrdscripts/initramfs-live-boot_1.0.bb
@@ -1,8 +1,8 @@
DESCRIPTION = "A live image init script"
LICENSE = "MIT"
LIC_FILES_CHKSUM = "file://${COREBASE}/meta/COPYING.MIT;md5=3da9cfbcb788c80a0384361b4de20420"
-RDEPENDS = "udev"
DEPENDS = "virtual/kernel"
+RDEPENDS_${PN} = "udev"
SRC_URI = "file://init-live.sh"
PR = "r11"
--
1.7.10.4
^ permalink raw reply related [flat|nested] 27+ messages in thread
* [PATCH 12/22] package.bbclass: Fix up bb.mkdirhier/bb.copyfile usage
2013-02-03 23:55 [PATCH 00/22] Packaging performance improvements, round 2 Richard Purdie
` (10 preceding siblings ...)
2013-02-03 23:55 ` [PATCH 11/22] initramfs-live-boot: " Richard Purdie
@ 2013-02-03 23:55 ` Richard Purdie
2013-02-03 23:55 ` [PATCH 13/22] package.bbclass: Rewrite split_and_strip_files Richard Purdie
` (10 subsequent siblings)
22 siblings, 0 replies; 27+ messages in thread
From: Richard Purdie @ 2013-02-03 23:55 UTC (permalink / raw)
To: openembedded-core
These are in bb.utils so lets the correct function and avoid the overhead
of the fixup/warning for the deprecated usage.
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
meta/classes/package.bbclass | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index 9c8cdbc..bee6548 100644
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -253,7 +253,7 @@ def splitdebuginfo(file, debugfile, debugsrcdir, d):
if debugsrcdir:
subprocess.call("'%s' -b '%s' -d '%s' -i -l '%s' '%s'" % (debugedit, workparentdir, debugsrcdir, sourcefile, file), shell=True)
- bb.mkdirhier(os.path.dirname(debugfile))
+ bb.utils.mkdirhier(os.path.dirname(debugfile))
subprocess.call("'%s' --only-keep-debug '%s' '%s'" % (objcopy, file, debugfile), shell=True)
@@ -287,7 +287,7 @@ def copydebugsources(debugsrcdir, d):
basepath = basepath + "/" + p
if not os.path.exists(basepath):
nosuchdir.append(basepath)
- bb.mkdirhier(basepath)
+ bb.utils.mkdirhier(basepath)
processdebugsrc = "LC_ALL=C ; sort -z -u '%s' | egrep -v -z '(<internal>|<built-in>)$' | "
# We need to ignore files that are not actually ours
@@ -644,7 +644,7 @@ python fixup_perms () {
continue
# Create path to move directory to, move it, and then setup the symlink
- bb.mkdirhier(os.path.dirname(target))
+ bb.utils.mkdirhier(os.path.dirname(target))
#bb.note("Fixup Perms: Rename %s -> %s" % (dir, ptarget))
os.rename(origin, target)
#bb.note("Fixup Perms: Link %s -> %s" % (dir, link))
@@ -891,7 +891,7 @@ python populate_packages () {
packages = d.getVar('PACKAGES', True)
pn = d.getVar('PN', True)
- bb.mkdirhier(outdir)
+ bb.utils.mkdirhier(outdir)
os.chdir(dvar)
# Sanity check PACKAGES for duplicates and for LICENSE_EXCLUSION
@@ -914,7 +914,7 @@ python populate_packages () {
for pkg in package_list:
root = os.path.join(pkgdest, pkg)
- bb.mkdirhier(root)
+ bb.utils.mkdirhier(root)
filesvar = d.getVar('FILES_%s' % pkg, True) or d.getVar('FILES', True) or ""
if "//" in filesvar:
@@ -945,7 +945,7 @@ python populate_packages () {
def mkdir(src, dest, p):
src = os.path.join(src, p)
dest = os.path.join(dest, p)
- bb.mkdirhier(dest)
+ bb.utils.mkdirhier(dest)
fstat = os.stat(src)
os.chmod(dest, fstat.st_mode)
os.chown(dest, fstat.st_uid, fstat.st_gid)
@@ -975,7 +975,7 @@ python populate_packages () {
os.chmod(fpath, fstat.st_mode)
os.chown(fpath, fstat.st_uid, fstat.st_gid)
continue
- ret = bb.copyfile(file, fpath)
+ ret = bb.utils.copyfile(file, fpath)
if ret is False or ret == 0:
raise bb.build.FuncFailed("File population failed")
--
1.7.10.4
^ permalink raw reply related [flat|nested] 27+ messages in thread
* [PATCH 00/22] Packaging performance improvements, round 2
@ 2013-02-03 23:55 Richard Purdie
2013-02-03 23:55 ` [PATCH 01/22] package.bbclass: Multithread per file dependency generation code Richard Purdie
` (22 more replies)
0 siblings, 23 replies; 27+ messages in thread
From: Richard Purdie @ 2013-02-03 23:55 UTC (permalink / raw)
To: openembedded-core
The following patch series contains various performance improvements for
do_package.
It also includes changes to the sanity testing to start giving warnings
about packaging variables used without packages being specified, eg:
RDEPENDS = "x"
instead of
RDEPENDS_${PN} = "x"
The final patch in the series stops the packaging code using the non-package
versions in many cases. I'm not planning to merge that one right away, its just
included for illustration and benchmarking purposes now.
There were three cases I found using bad syntax in OE-Core and the series includes
patches to fix them.
There is also a useful performance improvement to sstate manipulations to use
hardlinks instead of file copies which should be universally helpful to
build performance of tasks.
The following changes are available in the git repository at:
git://git.openembedded.org/openembedded-core master-next
http://cgit.openembedded.org/cgit.cgi/openembedded-core/log/?h=master-next
Richard Purdie (22):
package.bbclass: Multithread per file dependency generation code
package: Don't export PATH
package: Process package stripping in parallel
insane.bbclass: Add pkgvarcheck to check for suboptimal usages of
variables
insane.bbclass: Add documentation headers for logical code blocks
staging/insane.bbclass: Move legacy do_stage check iinto
insane.bbclass
staging.bbclass: Drop unused/legacy function
update-rc.d: Drop OVERRIDES code
qemu: Set RDEPENDS on the specific package that needs it
gdb-cross-canadian: Set RDEPENDS on the specific package that needs
it
initramfs-live-boot: Set RDEPENDS on the specific package that needs
it
package.bbclass: Fix up bb.mkdirhier/bb.copyfile usage
package.bbclass: Rewrite split_and_strip_files
kernel.bbclass: Improve populate_packages_prepend
package.bbclass: Make use of cleandirs and dirs function flags
package.bbclass: Various minor performance tweaks
package.bbclass: Simplify empty directory removal
package.bbclass: Add PACKAGESPLITFUNCS variable
sstate/path.py: Add copyhardlinktree() function and use for
performance optimisation
package.bbclass: Better document the different phases of operation
package.bbclass: Pre-expand some variables to save time
classes: Drop none package specific packaging variable accesses
meta/classes/base.bbclass | 2 +-
meta/classes/gconf.bbclass | 9 +-
meta/classes/gtk-icon-cache.bbclass | 4 +-
meta/classes/gtk-immodules-cache.bbclass | 4 +-
meta/classes/insane.bbclass | 30 +-
meta/classes/kernel.bbclass | 52 +--
meta/classes/mime.bbclass | 8 +-
meta/classes/package.bbclass | 460 +++++++++-----------
meta/classes/sstate.bbclass | 4 +-
meta/classes/staging.bbclass | 9 -
meta/classes/update-alternatives.bbclass | 10 +-
meta/classes/update-rc.d_real.bbclass | 21 +-
meta/lib/oe/package.py | 96 ++++
meta/lib/oe/path.py | 4 +
.../initrdscripts/initramfs-live-boot_1.0.bb | 2 +-
meta/recipes-devtools/gdb/gdb-cross-canadian.inc | 4 +-
meta/recipes-devtools/qemu/qemu.inc | 2 +-
17 files changed, 390 insertions(+), 331 deletions(-)
create mode 100644 meta/lib/oe/package.py
--
1.7.10.4
^ permalink raw reply [flat|nested] 27+ messages in thread
* [PATCH 13/22] package.bbclass: Rewrite split_and_strip_files
2013-02-03 23:55 [PATCH 00/22] Packaging performance improvements, round 2 Richard Purdie
` (11 preceding siblings ...)
2013-02-03 23:55 ` [PATCH 12/22] package.bbclass: Fix up bb.mkdirhier/bb.copyfile usage Richard Purdie
@ 2013-02-03 23:55 ` Richard Purdie
2013-02-09 11:41 ` Enrico Scholz
2013-02-03 23:55 ` [PATCH 14/22] kernel.bbclass: Improve populate_packages_prepend Richard Purdie
` (9 subsequent siblings)
22 siblings, 1 reply; 27+ messages in thread
From: Richard Purdie @ 2013-02-03 23:55 UTC (permalink / raw)
To: openembedded-core
The split_and_strip_files funciton was hard to follow and its usage of prefixes
to strings was unusual. This rewrites it to use a list of hardlinks, symlinks and
elffiles where each list is iterated over at the correct point.
This means we can avoid creating dandling symlinks for example so we can simply
delete the cleanup code for this.
The isfile() check is also removed which gives a significant improvement in speed.
Its uneeded since os.walk will have already checked things in files are files.
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
meta/classes/package.bbclass | 205 ++++++++++++++++++++----------------------
1 file changed, 97 insertions(+), 108 deletions(-)
diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index bee6548..7e45655 100644
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -722,8 +722,9 @@ python split_and_strip_files () {
#
# First lets figure out all of the files we may have to process ... do this only once!
#
- file_list = {}
- file_links = {}
+ elffiles = {}
+ symlinks = {}
+ hardlinks = {}
kernmods = []
if (d.getVar('INHIBIT_PACKAGE_DEBUG_SPLIT', True) != '1') and \
(d.getVar('INHIBIT_PACKAGE_STRIP', True) != '1'):
@@ -734,119 +735,108 @@ python split_and_strip_files () {
kernmods.append(file)
continue
- # Only process files (and symlinks)... Skip files that are obviously debug files
- if not (debugappend != "" and file.endswith(debugappend)) and \
- not (debugdir != "" and debugdir in os.path.dirname(file[len(dvar):])) and \
- os.path.isfile(file):
- try:
- s = os.stat(file)
- except OSError, (err, strerror):
- if err != errno.ENOENT:
- raise
- # Skip broken symlinks
+ # Skip debug files
+ if debugappend and file.endswith(debugappend):
+ continue
+ if debugdir and debugdir in os.path.dirname(file[len(dvar):]):
+ continue
+
+ try:
+ s = os.stat(file)
+ except OSError, (err, strerror):
+ if err != errno.ENOENT:
+ raise
+ # Skip broken symlinks
+ continue
+ # Check its an excutable
+ if (s[stat.ST_MODE] & stat.S_IXUSR) or (s[stat.ST_MODE] & stat.S_IXGRP) or (s[stat.ST_MODE] & stat.S_IXOTH):
+ # If it's a symlink, and points to an ELF file, we capture the readlink target
+ if os.path.islink(file):
+ target = os.readlink(file)
+ if not os.path.isabs(target):
+ ltarget = os.path.join(os.path.dirname(file), target)
+ else:
+ ltarget = target
+
+ if isELF(ltarget):
+ #bb.note("Sym: %s (%d)" % (ltarget, isELF(ltarget)))
+ symlinks[file] = target
continue
- # Is the item excutable? Then we need to process it.
- if (s[stat.ST_MODE] & stat.S_IXUSR) or \
- (s[stat.ST_MODE] & stat.S_IXGRP) or \
- (s[stat.ST_MODE] & stat.S_IXOTH):
- # If it's a symlink, and points to an ELF file, we capture the readlink target
- if os.path.islink(file):
- target = os.readlink(file)
- if not os.path.isabs(target):
- ltarget = os.path.join(os.path.dirname(file), target)
- else:
- ltarget = target
-
- if isELF(ltarget):
- #bb.note("Sym: %s (%d)" % (ltarget, isELF(ltarget)))
- file_list[file] = "sym: " + target
+ # It's a file (or hardlink), not a link
+ # ...but is it ELF, and is it already stripped?
+ elf_file = isELF(file)
+ if elf_file & 1:
+ if elf_file & 2:
+ bb.warn("File '%s' from %s was already stripped, this will prevent future debugging!" % (src, pn))
continue
- # It's a file (or hardlink), not a link
- # ...but is it ELF, and is it already stripped?
- elf_file = isELF(file)
- if elf_file & 1:
- # Check if it's a hard link to something else
- if s.st_nlink > 1:
- file_reference = "%d_%d" % (s.st_dev, s.st_ino)
- # Hard link to something else
- file_list[file] = "hard: " + file_reference
- continue
-
- file_list[file] = "ELF: %d" % elf_file
-
+ # Check if it's a hard link to something else
+ if s.st_nlink > 1:
+ file_reference = "%d_%d" % (s.st_dev, s.st_ino)
+ # Hard link to something else
+ hardlinks[file] = file_reference
+ continue
+ elffiles[file] = elf_file
#
# First lets process debug splitting
#
if (d.getVar('INHIBIT_PACKAGE_DEBUG_SPLIT', True) != '1'):
- for file in file_list:
+ hardlinkmap = {}
+ # For hardlinks, process only one of the files
+ for file in hardlinks:
+ file_reference = hardlinks[file]
+ if file_reference not in hardlinkmap:
+ # If this is a new file, add it as a reference, and
+ # update it's type, so we can fall through and split
+ elffiles[file] = isELF(file)
+ hardlinkmap[file_reference] = file
+
+ for file in elffiles:
src = file[len(dvar):]
dest = debuglibdir + os.path.dirname(src) + debugdir + "/" + os.path.basename(src) + debugappend
fpath = dvar + dest
- # Preserve symlinks in debug area...
- if file_list[file].startswith("sym: "):
- ltarget = file_list[file][5:]
- lpath = os.path.dirname(ltarget)
- lbase = os.path.basename(ltarget)
- ftarget = ""
- if lpath and lpath != ".":
- ftarget += lpath + debugdir + "/"
- ftarget += lbase + debugappend
- if lpath.startswith(".."):
- ftarget = os.path.join("..", ftarget)
- bb.mkdirhier(os.path.dirname(fpath))
- #bb.note("Symlink %s -> %s" % (fpath, ftarget))
- os.symlink(ftarget, fpath)
- continue
- # Preserve hard links in debug area...
- file_reference = ""
- if file_list[file].startswith("hard: "):
- file_reference = file_list[file][6:]
- if file_reference not in file_links:
- # If this is a new file, add it as a reference, and
- # update it's type, so we can fall through and split
- file_list[file] = "ELF: %d" % (isELF(file))
- else:
- target = file_links[file_reference][len(dvar):]
- ftarget = dvar + debuglibdir + os.path.dirname(target) + debugdir + "/" + os.path.basename(target) + debugappend
- bb.mkdirhier(os.path.dirname(fpath))
- #bb.note("Link %s -> %s" % (fpath, ftarget))
- os.link(ftarget, fpath)
- continue
-
- # It's ELF...
- if file_list[file].startswith("ELF: "):
- elf_file = int(file_list[file][5:])
- if elf_file & 2:
- bb.warn("File '%s' from %s was already stripped, this will prevent future debugging!" % (src, pn))
- continue
+ # Split the file...
+ bb.utils.mkdirhier(os.path.dirname(fpath))
+ #bb.note("Split %s -> %s" % (file, fpath))
+ # Only store off the hard link reference if we successfully split!
+ splitdebuginfo(file, fpath, debugsrcdir, d)
+
+ # Hardlink our debug symbols to the other hardlink copies
+ for file in hardlinks:
+ if file not in elffiles:
+ file_reference = hardlinks[file]
+ target = hardlinkmap[file_reference][len(dvar):]
+ ftarget = dvar + debuglibdir + os.path.dirname(target) + debugdir + "/" + os.path.basename(target) + debugappend
+ bb.utils.mkdirhier(os.path.dirname(fpath))
+ #bb.note("Link %s -> %s" % (fpath, ftarget))
+ os.link(ftarget, fpath)
+
+ # Create symlinks for all cases we were able to split symbols
+ for file in symlinks:
+ src = file[len(dvar):]
+ dest = debuglibdir + os.path.dirname(src) + debugdir + "/" + os.path.basename(src) + debugappend
+ fpath = dvar + dest
+ # Skip it if the target doesn't exist
+ try:
+ s = os.stat(fpath)
+ except OSError, (err, strerror):
+ if err != errno.ENOENT:
+ raise
+ continue
- # Split the file...
- bb.mkdirhier(os.path.dirname(fpath))
- #bb.note("Split %s -> %s" % (file, fpath))
- # Only store off the hard link reference if we successfully split!
- if splitdebuginfo(file, fpath, debugsrcdir, d) == 0 and file_reference != "":
- file_links[file_reference] = file
-
- # The above may have generated dangling symlinks, remove them!
- # Dangling symlinks are a result of something NOT being split, such as a stripped binary.
- # This should be a rare occurance, but we want to clean up anyway.
- for file in file_list:
- if file_list[file].startswith("sym: "):
- src = file[len(dvar):]
- dest = debuglibdir + os.path.dirname(src) + debugdir + "/" + os.path.basename(src) + debugappend
- fpath = dvar + dest
- try:
- s = os.stat(fpath)
- except OSError, (err, strerror):
- if err != errno.ENOENT:
- raise
- #bb.note("Remove dangling link %s -> %s" % (fpath, os.readlink(fpath)))
- os.unlink(fpath)
- # This could leave an empty debug directory laying around
- # take care of the obvious case...
- subprocess.call("rmdir %s 2>/dev/null" % os.path.dirname(fpath), shell=True)
+ ltarget = symlinks[file]
+ lpath = os.path.dirname(ltarget)
+ lbase = os.path.basename(ltarget)
+ ftarget = ""
+ if lpath and lpath != ".":
+ ftarget += lpath + debugdir + "/"
+ ftarget += lbase + debugappend
+ if lpath.startswith(".."):
+ ftarget = os.path.join("..", ftarget)
+ bb.utils.mkdirhier(os.path.dirname(fpath))
+ #bb.note("Symlink %s -> %s" % (fpath, ftarget))
+ os.symlink(ftarget, fpath)
# Process the debugsrcdir if requested...
# This copies and places the referenced sources for later debugging...
@@ -861,11 +851,10 @@ python split_and_strip_files () {
if (d.getVar('INHIBIT_PACKAGE_STRIP', True) != '1'):
strip = d.getVar("STRIP", True)
sfiles = []
- for file in file_list:
- if file_list[file].startswith("ELF: "):
- elf_file = int(file_list[file][5:])
- #bb.note("Strip %s" % file)
- sfiles.append((file, elf_file, strip))
+ for file in elffiles:
+ elf_file = int(elffiles[file])
+ #bb.note("Strip %s" % file)
+ sfiles.append((file, elf_file, strip))
for f in kernmods:
sfiles.append((f, 16, strip))
--
1.7.10.4
^ permalink raw reply related [flat|nested] 27+ messages in thread
* [PATCH 14/22] kernel.bbclass: Improve populate_packages_prepend
2013-02-03 23:55 [PATCH 00/22] Packaging performance improvements, round 2 Richard Purdie
` (12 preceding siblings ...)
2013-02-03 23:55 ` [PATCH 13/22] package.bbclass: Rewrite split_and_strip_files Richard Purdie
@ 2013-02-03 23:55 ` Richard Purdie
2013-02-03 23:55 ` [PATCH 15/22] package.bbclass: Make use of cleandirs and dirs function flags Richard Purdie
` (8 subsequent siblings)
22 siblings, 0 replies; 27+ messages in thread
From: Richard Purdie @ 2013-02-03 23:55 UTC (permalink / raw)
To: openembedded-core
Small performance tweaks for populate_packages_prepend:
* Compile the regexps once at the start
* Don't keep importing a module which is already imported
* No need to check PKG is set, we'd have failed long before now if it wasn't
* Don't export PATH, bitbake takes care of this at the task level
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
meta/classes/kernel.bbclass | 48 +++++++++++++++++++------------------------
1 file changed, 21 insertions(+), 27 deletions(-)
diff --git a/meta/classes/kernel.bbclass b/meta/classes/kernel.bbclass
index cc61be6..4805944 100644
--- a/meta/classes/kernel.bbclass
+++ b/meta/classes/kernel.bbclass
@@ -302,71 +302,68 @@ fi
}
python populate_packages_prepend () {
+ import re
+
+ modinfoexp = re.compile("([^=]+)=(.*)")
+ kerverrexp = re.compile('^(.*-hh.*)[\.\+].*$')
+ depmodpat0 = re.compile("^(.*\.k?o):..*$")
+ depmodpat1 = re.compile("^(.*\.k?o):\s*(.*\.k?o)\s*$")
+ depmodpat2 = re.compile("^(.*\.k?o):\s*(.*\.k?o)\s*\\\$")
+ depmodpat3 = re.compile("^\t(.*\.k?o)\s*\\\$")
+ depmodpat4 = re.compile("^\t(.*\.k?o)\s*$")
+
def extract_modinfo(file):
- import tempfile, re, subprocess
+ import tempfile, subprocess
tempfile.tempdir = d.getVar("WORKDIR", True)
tf = tempfile.mkstemp()
tmpfile = tf[1]
- cmd = "PATH=\"%s\" %sobjcopy -j .modinfo -O binary %s %s" % (d.getVar("PATH", True), d.getVar("HOST_PREFIX", True) or "", file, tmpfile)
+ cmd = "%sobjcopy -j .modinfo -O binary %s %s" % (d.getVar("HOST_PREFIX", True) or "", file, tmpfile)
subprocess.call(cmd, shell=True)
f = open(tmpfile)
l = f.read().split("\000")
f.close()
os.close(tf[0])
os.unlink(tmpfile)
- exp = re.compile("([^=]+)=(.*)")
vals = {}
for i in l:
- m = exp.match(i)
+ m = modinfoexp.match(i)
if not m:
continue
vals[m.group(1)] = m.group(2)
return vals
def parse_depmod():
- import re
dvar = d.getVar('PKGD', True)
- if not dvar:
- bb.error("PKGD not defined")
- return
kernelver = d.getVar('KERNEL_VERSION', True)
kernelver_stripped = kernelver
- m = re.match('^(.*-hh.*)[\.\+].*$', kernelver)
+ m = kerverrexp.match(kernelver)
if m:
kernelver_stripped = m.group(1)
- path = d.getVar("PATH", True)
-
- cmd = "PATH=\"%s\" depmod -n -a -b %s -F %s/boot/System.map-%s %s" % (path, dvar, dvar, kernelver, kernelver_stripped)
+ cmd = "depmod -n -a -b %s -F %s/boot/System.map-%s %s" % (dvar, dvar, kernelver, kernelver_stripped)
f = os.popen(cmd, 'r')
deps = {}
- pattern0 = "^(.*\.k?o):..*$"
- pattern1 = "^(.*\.k?o):\s*(.*\.k?o)\s*$"
- pattern2 = "^(.*\.k?o):\s*(.*\.k?o)\s*\\\$"
- pattern3 = "^\t(.*\.k?o)\s*\\\$"
- pattern4 = "^\t(.*\.k?o)\s*$"
-
line = f.readline()
while line:
- if not re.match(pattern0, line):
+ if not depmodpat0.match(line):
line = f.readline()
continue
- m1 = re.match(pattern1, line)
+ m1 = depmodpat1.match(line)
if m1:
deps[m1.group(1)] = m1.group(2).split()
else:
- m2 = re.match(pattern2, line)
+ m2 = depmodpat2.match(line)
if m2:
deps[m2.group(1)] = m2.group(2).split()
line = f.readline()
- m3 = re.match(pattern3, line)
+ m3 = depmodpat3.match(line)
while m3:
deps[m2.group(1)].extend(m3.group(1).split())
line = f.readline()
- m3 = re.match(pattern3, line)
- m4 = re.match(pattern4, line)
+ m3 = depmodpat3.match(line)
+ m4 = depmodpat4.match(line)
deps[m2.group(1)].extend(m4.group(1).split())
line = f.readline()
f.close()
@@ -379,7 +376,6 @@ python populate_packages_prepend () {
file = file.replace("/lib/modules/%s/" % d.getVar('KERNEL_VERSION', True) or '', '', 1)
if module_deps.has_key(file):
- import re
dependencies = []
for i in module_deps[file]:
m = re.match(pattern, os.path.basename(i))
@@ -392,7 +388,6 @@ python populate_packages_prepend () {
return []
def frob_metadata(file, pkg, pattern, format, basename):
- import re
vals = extract_modinfo(file)
dvar = d.getVar('PKGD', True)
@@ -454,7 +449,6 @@ python populate_packages_prepend () {
if len(os.listdir(dir)) == 0:
os.rmdir(dir)
- import re
metapkg = "kernel-modules"
blacklist = [ 'kernel-dev', 'kernel-image', 'kernel-base', 'kernel-vmlinux' ]
for l in module_deps.values():
--
1.7.10.4
^ permalink raw reply related [flat|nested] 27+ messages in thread
* [PATCH 15/22] package.bbclass: Make use of cleandirs and dirs function flags
2013-02-03 23:55 [PATCH 00/22] Packaging performance improvements, round 2 Richard Purdie
` (13 preceding siblings ...)
2013-02-03 23:55 ` [PATCH 14/22] kernel.bbclass: Improve populate_packages_prepend Richard Purdie
@ 2013-02-03 23:55 ` Richard Purdie
2013-02-03 23:55 ` [PATCH 16/22] package.bbclass: Various minor performance tweaks Richard Purdie
` (7 subsequent siblings)
22 siblings, 0 replies; 27+ messages in thread
From: Richard Purdie @ 2013-02-03 23:55 UTC (permalink / raw)
To: openembedded-core
We can use the cleandirs and dirs flags for the fuctions to handle
directory cleaning and creation at the bitbake level rather than
using these calls within the functions
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
meta/classes/package.bbclass | 8 +++-----
1 file changed, 3 insertions(+), 5 deletions(-)
diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index 7e45655..22ca546 100644
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -429,11 +429,8 @@ python perform_packagecopy () {
dest = d.getVar('D', True)
dvar = d.getVar('PKGD', True)
- bb.mkdirhier(dvar)
-
# Start by package population by taking a copy of the installed
# files to operate on
- subprocess.call('rm -rf %s/*' % (dvar), shell=True)
# Preserve sparse files and hard links
subprocess.call('tar -cf - -C %s -ps . | tar -xf - -C %s' % (dest, dvar), shell=True)
@@ -441,6 +438,8 @@ python perform_packagecopy () {
if bb.data.inherits_class('nativesdk', d) or bb.data.inherits_class('cross-canadian', d):
rpath_replace (dvar, d)
}
+perform_packagecopy[cleandirs] = "${PKGD}"
+perform_packagecopy[dirs] = "${PKGD}"
# We generate a master list of directories to process, we start by
# seeding this list with reasonable defaults, then load from
@@ -897,7 +896,6 @@ python populate_packages () {
package_list.append(pkg)
d.setVar('PACKAGES', ' '.join(package_list))
pkgdest = d.getVar('PKGDEST', True)
- subprocess.call('rm -rf %s' % pkgdest, shell=True)
seen = []
@@ -1823,7 +1821,7 @@ addtask package before do_build after do_install
PACKAGELOCK = "${STAGING_DIR}/package-output.lock"
SSTATETASKS += "do_package"
do_package[sstate-name] = "package"
-do_package[cleandirs] = "${PKGDESTWORK}"
+do_package[cleandirs] = "${PKGDEST} ${PKGDESTWORK}"
do_package[sstate-plaindirs] = "${PKGD} ${PKGDEST} ${PKGDESTWORK}"
do_package[sstate-lockfile-shared] = "${PACKAGELOCK}"
do_package_setscene[dirs] = "${STAGING_DIR}"
--
1.7.10.4
^ permalink raw reply related [flat|nested] 27+ messages in thread
* [PATCH 16/22] package.bbclass: Various minor performance tweaks
2013-02-03 23:55 [PATCH 00/22] Packaging performance improvements, round 2 Richard Purdie
` (14 preceding siblings ...)
2013-02-03 23:55 ` [PATCH 15/22] package.bbclass: Make use of cleandirs and dirs function flags Richard Purdie
@ 2013-02-03 23:55 ` Richard Purdie
2013-02-03 23:55 ` [PATCH 17/22] package.bbclass: Simplify empty directory removal Richard Purdie
` (6 subsequent siblings)
22 siblings, 0 replies; 27+ messages in thread
From: Richard Purdie @ 2013-02-03 23:55 UTC (permalink / raw)
To: openembedded-core
* Call getVar outside the loop
* Drop unneeded PATH export (bitbake does this already)
* Drop unused variable
* Simplify if statement nesting
* Simplify variable expandion to a getVar call (expand would just call getVar)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
meta/classes/package.bbclass | 14 ++++++--------
1 file changed, 6 insertions(+), 8 deletions(-)
diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index 22ca546..6d2dd89 100644
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -557,11 +557,12 @@ python fixup_perms () {
# paths are resolved via BBPATH
def get_fs_perms_list(d):
str = ""
+ bbpath = d.getVar('BBPATH', True)
fs_perms_tables = d.getVar('FILESYSTEM_PERMS_TABLES', True)
if not fs_perms_tables:
fs_perms_tables = 'files/fs-perms.txt'
for conf_file in fs_perms_tables.split():
- str += " %s" % bb.which(d.getVar('BBPATH', True), conf_file)
+ str += " %s" % bb.utils.which(bbpath, conf_file)
return str
@@ -889,11 +890,10 @@ python populate_packages () {
for pkg in packages.split():
if d.getVar('LICENSE_EXCLUSION-' + pkg, True):
bb.warn("%s has an incompatible license. Excluding from packaging." % pkg)
+ elif pkg in package_list:
+ bb.error("%s is listed in PACKAGES multiple times, this leads to packaging errors." % pkg)
else:
- if pkg in package_list:
- bb.error("%s is listed in PACKAGES multiple times, this leads to packaging errors." % pkg)
- else:
- package_list.append(pkg)
+ package_list.append(pkg)
d.setVar('PACKAGES', ' '.join(package_list))
pkgdest = d.getVar('PKGDEST', True)
@@ -908,7 +908,6 @@ python populate_packages () {
bb.warn("FILES variable for package %s contains '//' which is invalid. Attempting to fix this but you should correct the metadata.\n" % pkg)
filesvar.replace("//", "/")
files = filesvar.split()
- file_links = {}
for file in files:
if os.path.isabs(file):
file = '.' + file
@@ -1192,7 +1191,7 @@ python package_do_filedeps() {
pkgdest = d.getVar('PKGDEST', True)
packages = d.getVar('PACKAGES', True)
- rpmdeps = d.expand("${RPMDEPS}")
+ rpmdeps = d.getVar('RPMDEPS', True)
def chunks(files, n):
return [files[i:i+n] for i in range(0, len(files), n)]
@@ -1282,7 +1281,6 @@ python package_do_shlibs() {
def linux_so(file):
needs_ldconfig = False
cmd = d.getVar('OBJDUMP', True) + " -p " + pipes.quote(file) + " 2>/dev/null"
- cmd = "PATH=\"%s\" %s" % (d.getVar('PATH', True), cmd)
fd = os.popen(cmd)
lines = fd.readlines()
fd.close()
--
1.7.10.4
^ permalink raw reply related [flat|nested] 27+ messages in thread
* [PATCH 17/22] package.bbclass: Simplify empty directory removal
2013-02-03 23:55 [PATCH 00/22] Packaging performance improvements, round 2 Richard Purdie
` (15 preceding siblings ...)
2013-02-03 23:55 ` [PATCH 16/22] package.bbclass: Various minor performance tweaks Richard Purdie
@ 2013-02-03 23:55 ` Richard Purdie
2013-02-03 23:55 ` [PATCH 18/22] package.bbclass: Add PACKAGESPLITFUNCS variable Richard Purdie
` (5 subsequent siblings)
22 siblings, 0 replies; 27+ messages in thread
From: Richard Purdie @ 2013-02-03 23:55 UTC (permalink / raw)
To: openembedded-core
Rather than an exec() per directory, we might as well exec one command and
be done with it.
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
meta/classes/package.bbclass | 6 +-----
1 file changed, 1 insertion(+), 5 deletions(-)
diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index 6d2dd89..c822c43 100644
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -298,11 +298,7 @@ def copydebugsources(debugsrcdir, d):
subprocess.call(processdebugsrc % (sourcefile, workbasedir, workparentdir, dvar, debugsrcdir), shell=True)
# The copy by cpio may have resulted in some empty directories! Remove these
- for root, dirs, files in os.walk("%s%s" % (dvar, debugsrcdir)):
- for d in dirs:
- dir = os.path.join(root, d)
- #bb.note("rmdir -p %s" % dir)
- subprocess.call("rmdir -p %s 2>/dev/null" % dir, shell=True)
+ subprocess.call("find %s%s -empty -type d -delete" % (dvar, debugsrcdir), shell=True)
# Also remove debugsrcdir if its empty
for p in nosuchdir[::-1]:
--
1.7.10.4
^ permalink raw reply related [flat|nested] 27+ messages in thread
* [PATCH 18/22] package.bbclass: Add PACKAGESPLITFUNCS variable
2013-02-03 23:55 [PATCH 00/22] Packaging performance improvements, round 2 Richard Purdie
` (16 preceding siblings ...)
2013-02-03 23:55 ` [PATCH 17/22] package.bbclass: Simplify empty directory removal Richard Purdie
@ 2013-02-03 23:55 ` Richard Purdie
2013-02-03 23:55 ` [PATCH 19/22] sstate/path.py: Add copyhardlinktree() function and use for performance optimisation Richard Purdie
` (4 subsequent siblings)
22 siblings, 0 replies; 27+ messages in thread
From: Richard Purdie @ 2013-02-03 23:55 UTC (permalink / raw)
To: openembedded-core
Prepending to populate_packages is rather ugly and means its hard to trace
errors and also profiling informaiton is summed together in one function.
This patch starts to split out the prepends to become separate functions
to avoid these issues. This is generally a neater way to write functions
than prepending to where there can sometimes be variable scope issues
and we've been bitten by whitespace issues in the past.
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
meta/classes/kernel.bbclass | 4 +++-
meta/classes/package.bbclass | 6 ++++++
meta/classes/update-alternatives.bbclass | 4 +++-
meta/classes/update-rc.d_real.bbclass | 4 +++-
4 files changed, 15 insertions(+), 3 deletions(-)
diff --git a/meta/classes/kernel.bbclass b/meta/classes/kernel.bbclass
index 4805944..00117a6 100644
--- a/meta/classes/kernel.bbclass
+++ b/meta/classes/kernel.bbclass
@@ -301,7 +301,9 @@ if [ x"$D" = "x" ]; then
fi
}
-python populate_packages_prepend () {
+PACKAGESPLITFUNCS_prepend = "split_kernel_packages "
+
+python split_kernel_packages () {
import re
modinfoexp = re.compile("([^=]+)=(.*)")
diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index c822c43..a09eb15 100644
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -1754,6 +1754,9 @@ PACKAGEBUILDPKGD ?= " \
${PACKAGE_PREPROCESS_FUNCS} \
split_and_strip_files \
fixup_perms \
+ "
+# Functions which split PKGD up into separate packages
+PACKAGESPLITFUNCS ?= " \
package_do_split_locales \
populate_packages"
# Functions which process metadata based on split packages
@@ -1793,6 +1796,9 @@ python do_package () {
for f in (d.getVar('PACKAGEBUILDPKGD', True) or '').split():
bb.build.exec_func(f, d)
+ for f in (d.getVar('PACKAGESPLITFUNCS', True) or '').split():
+ bb.build.exec_func(f, d)
+
# Build global list of files in each split package
global pkgfiles
pkgfiles = {}
diff --git a/meta/classes/update-alternatives.bbclass b/meta/classes/update-alternatives.bbclass
index 90bc56b..147d236 100644
--- a/meta/classes/update-alternatives.bbclass
+++ b/meta/classes/update-alternatives.bbclass
@@ -282,7 +282,9 @@ python perform_packagecopy_append () {
bb.warn('%s: Unable to resolve dangling symlink: %s' % (pn, alt_target))
}
-python populate_packages_prepend () {
+PACKAGESPLITFUNCS_prepend = "populate_packages_updateralternatives "
+
+python populate_packages_updatealternatives () {
pn = d.getVar('BPN', True)
# Do actual update alternatives processing
diff --git a/meta/classes/update-rc.d_real.bbclass b/meta/classes/update-rc.d_real.bbclass
index 0cd671f..f9d55fb 100644
--- a/meta/classes/update-rc.d_real.bbclass
+++ b/meta/classes/update-rc.d_real.bbclass
@@ -47,7 +47,9 @@ python __anonymous() {
update_rc_after_parse(d)
}
-python populate_packages_prepend () {
+PACKAGESPLITFUNCS_prepend = "populate_packages_updatercd "
+
+python populate_packages_updatercd () {
def update_rcd_package(pkg):
bb.debug(1, 'adding update-rc.d calls to postinst/postrm for %s' % pkg)
"""
--
1.7.10.4
^ permalink raw reply related [flat|nested] 27+ messages in thread
* [PATCH 19/22] sstate/path.py: Add copyhardlinktree() function and use for performance optimisation
2013-02-03 23:55 [PATCH 00/22] Packaging performance improvements, round 2 Richard Purdie
` (17 preceding siblings ...)
2013-02-03 23:55 ` [PATCH 18/22] package.bbclass: Add PACKAGESPLITFUNCS variable Richard Purdie
@ 2013-02-03 23:55 ` Richard Purdie
2013-02-03 23:55 ` [PATCH 20/22] package.bbclass: Better document the different phases of operation Richard Purdie
` (3 subsequent siblings)
22 siblings, 0 replies; 27+ messages in thread
From: Richard Purdie @ 2013-02-03 23:55 UTC (permalink / raw)
To: openembedded-core
Add a function which copys a tree as a set of hardlinks to the original
files, then use this in sstate to reduce some of the overhead of sstate
package creation since the file isn't actually copied.
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
meta/classes/sstate.bbclass | 4 ++--
meta/lib/oe/path.py | 4 ++++
2 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
index a79d2b5..6f77bb9 100644
--- a/meta/classes/sstate.bbclass
+++ b/meta/classes/sstate.bbclass
@@ -458,14 +458,14 @@ def sstate_package(ss, d):
dstpath = srcpath.replace(state[1], sstatebuild + state[0])
make_relative_symlink(srcpath, dstpath, d)
bb.debug(2, "Preparing tree %s for packaging at %s" % (state[1], sstatebuild + state[0]))
- oe.path.copytree(state[1], sstatebuild + state[0])
+ oe.path.copyhardlinktree(state[1], sstatebuild + state[0])
workdir = d.getVar('WORKDIR', True)
for plain in ss['plaindirs']:
pdir = plain.replace(workdir, sstatebuild)
bb.mkdirhier(plain)
bb.mkdirhier(pdir)
- oe.path.copytree(plain, pdir)
+ oe.path.copyhardlinktree(plain, pdir)
d.setVar('SSTATE_BUILDDIR', sstatebuild)
d.setVar('SSTATE_PKG', sstatepkg)
diff --git a/meta/lib/oe/path.py b/meta/lib/oe/path.py
index 7197b23..11d4694 100644
--- a/meta/lib/oe/path.py
+++ b/meta/lib/oe/path.py
@@ -83,6 +83,10 @@ def copytree(src, dst):
cmd = 'tar -cf - -C %s -ps . | tar -xf - -C %s' % (src, dst)
check_output(cmd, shell=True, stderr=subprocess.STDOUT)
+def copyhardlinktree(src, dst):
+ bb.utils.mkdirhier(dst)
+ cmd = 'cp -al %s %s' % (src, dst)
+ check_output(cmd, shell=True, stderr=subprocess.STDOUT)
def remove(path, recurse=True):
"""Equivalent to rm -f or rm -rf"""
--
1.7.10.4
^ permalink raw reply related [flat|nested] 27+ messages in thread
* [PATCH 20/22] package.bbclass: Better document the different phases of operation
2013-02-03 23:55 [PATCH 00/22] Packaging performance improvements, round 2 Richard Purdie
` (18 preceding siblings ...)
2013-02-03 23:55 ` [PATCH 19/22] sstate/path.py: Add copyhardlinktree() function and use for performance optimisation Richard Purdie
@ 2013-02-03 23:55 ` Richard Purdie
2013-02-03 23:55 ` [PATCH 21/22] package.bbclass: Pre-expand some variables to save time Richard Purdie
` (2 subsequent siblings)
22 siblings, 0 replies; 27+ messages in thread
From: Richard Purdie @ 2013-02-03 23:55 UTC (permalink / raw)
To: openembedded-core
Add headers to document the different phases of do_package and
make the steps clearer.
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
meta/classes/package.bbclass | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index a09eb15..2c3f173 100644
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -1778,6 +1778,10 @@ python do_package () {
# as any change to rpmdeps requires this to be rerun.
# PACKAGE_BBCLASS_VERSION = "1"
+ ###########################################################################
+ # Sanity test the setup
+ ###########################################################################
+
packages = (d.getVar('PACKAGES', True) or "").split()
if len(packages) < 1:
bb.debug(1, "No packages to build, skipping do_package")
@@ -1793,12 +1797,24 @@ python do_package () {
bb.error("WORKDIR, DEPLOY_DIR, D, PN and PKGD all must be defined, unable to package")
return
+ ###########################################################################
+ # Setup PKGD (from D)
+ ###########################################################################
+
for f in (d.getVar('PACKAGEBUILDPKGD', True) or '').split():
bb.build.exec_func(f, d)
+ ###########################################################################
+ # Split up PKGD into PKGDEST
+ ###########################################################################
+
for f in (d.getVar('PACKAGESPLITFUNCS', True) or '').split():
bb.build.exec_func(f, d)
+ ###########################################################################
+ # Process PKGDEST
+ ###########################################################################
+
# Build global list of files in each split package
global pkgfiles
pkgfiles = {}
--
1.7.10.4
^ permalink raw reply related [flat|nested] 27+ messages in thread
* [PATCH 21/22] package.bbclass: Pre-expand some variables to save time
2013-02-03 23:55 [PATCH 00/22] Packaging performance improvements, round 2 Richard Purdie
` (19 preceding siblings ...)
2013-02-03 23:55 ` [PATCH 20/22] package.bbclass: Better document the different phases of operation Richard Purdie
@ 2013-02-03 23:55 ` Richard Purdie
2013-02-03 23:55 ` [PATCH 22/22] classes: Drop none package specific packaging variable accesses Richard Purdie
2013-02-04 11:41 ` [PATCH 00/22] Packaging performance improvements, round 2 Richard Purdie
22 siblings, 0 replies; 27+ messages in thread
From: Richard Purdie @ 2013-02-03 23:55 UTC (permalink / raw)
To: openembedded-core
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
meta/classes/package.bbclass | 14 ++++++++++++++
1 file changed, 14 insertions(+)
diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index 2c3f173..2bae65b 100644
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -1798,6 +1798,20 @@ python do_package () {
return
###########################################################################
+ # Optimisations
+ ###########################################################################
+
+ # Contunually rexpanding complex expressions is inefficient, particularly when
+ # we write to the datastore and invalidate the expansion cache. This code
+ # pre-expands some frequently used variables
+
+ def expandVar(x, d):
+ d.setVar(x, d.getVar(x, True))
+
+ for x in 'PN', 'PV', 'BPN', 'TARGET_SYS', 'EXTENDPRAUTO':
+ expandVar(x, d)
+
+ ###########################################################################
# Setup PKGD (from D)
###########################################################################
--
1.7.10.4
^ permalink raw reply related [flat|nested] 27+ messages in thread
* [PATCH 22/22] classes: Drop none package specific packaging variable accesses
2013-02-03 23:55 [PATCH 00/22] Packaging performance improvements, round 2 Richard Purdie
` (20 preceding siblings ...)
2013-02-03 23:55 ` [PATCH 21/22] package.bbclass: Pre-expand some variables to save time Richard Purdie
@ 2013-02-03 23:55 ` Richard Purdie
2013-02-04 11:41 ` [PATCH 00/22] Packaging performance improvements, round 2 Richard Purdie
22 siblings, 0 replies; 27+ messages in thread
From: Richard Purdie @ 2013-02-03 23:55 UTC (permalink / raw)
To: openembedded-core
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
meta/classes/gconf.bbclass | 9 +++------
meta/classes/gtk-icon-cache.bbclass | 4 ++--
meta/classes/gtk-immodules-cache.bbclass | 4 ++--
meta/classes/mime.bbclass | 8 +++-----
meta/classes/package.bbclass | 19 ++++++++-----------
meta/classes/update-alternatives.bbclass | 4 ++--
meta/classes/update-rc.d_real.bbclass | 6 +++---
7 files changed, 23 insertions(+), 31 deletions(-)
diff --git a/meta/classes/gconf.bbclass b/meta/classes/gconf.bbclass
index cc01bca..e9076b2 100644
--- a/meta/classes/gconf.bbclass
+++ b/meta/classes/gconf.bbclass
@@ -56,18 +56,15 @@ python populate_packages_append () {
if schemas != []:
bb.note("adding gconf postinst and prerm scripts to %s" % pkg)
d.setVar('SCHEMA_FILES', " ".join(schemas))
- postinst = d.getVar('pkg_postinst_%s' % pkg, True) or d.getVar('pkg_postinst', True)
+ postinst = d.getVar('pkg_postinst_%s' % pkg, True)
if not postinst:
postinst = '#!/bin/sh\n'
postinst += d.getVar('gconf_postinst', True)
d.setVar('pkg_postinst_%s' % pkg, postinst)
- prerm = d.getVar('pkg_prerm_%s' % pkg, True) or d.getVar('pkg_prerm', True)
+ prerm = d.getVar('pkg_prerm_%s' % pkg, True)
if not prerm:
prerm = '#!/bin/sh\n'
prerm += d.getVar('gconf_prerm', True)
d.setVar('pkg_prerm_%s' % pkg, prerm)
- rdepends = d.getVar("RDEPENDS_%s" % pkg, True) or ""
- rdepends += ' ' + d.getVar('MLPREFIX') + 'gconf'
- d.setVar("RDEPENDS_%s" % pkg, rdepends)
-
+ d.appendVar("RDEPENDS_%s" % pkg, ' ' + d.getVar('MLPREFIX') + 'gconf')
}
diff --git a/meta/classes/gtk-icon-cache.bbclass b/meta/classes/gtk-icon-cache.bbclass
index cf33efd..4868081 100644
--- a/meta/classes/gtk-icon-cache.bbclass
+++ b/meta/classes/gtk-icon-cache.bbclass
@@ -72,13 +72,13 @@ python populate_packages_append () {
bb.note("adding gtk-icon-cache postinst and postrm scripts to %s" % pkg)
- postinst = d.getVar('pkg_postinst_%s' % pkg, True) or d.getVar('pkg_postinst', True)
+ postinst = d.getVar('pkg_postinst_%s' % pkg, True)
if not postinst:
postinst = '#!/bin/sh\n'
postinst += d.getVar('gtk_icon_cache_postinst', True)
d.setVar('pkg_postinst_%s' % pkg, postinst)
- postrm = d.getVar('pkg_postrm_%s' % pkg, True) or d.getVar('pkg_postrm', True)
+ postrm = d.getVar('pkg_postrm_%s' % pkg, True)
if not postrm:
postrm = '#!/bin/sh\n'
postrm += d.getVar('gtk_icon_cache_postrm', True)
diff --git a/meta/classes/gtk-immodules-cache.bbclass b/meta/classes/gtk-immodules-cache.bbclass
index a8855af..d8d9aa5 100644
--- a/meta/classes/gtk-immodules-cache.bbclass
+++ b/meta/classes/gtk-immodules-cache.bbclass
@@ -58,13 +58,13 @@ python populate_packages_append () {
for pkg in gtkimmodules_pkgs:
bb.note("adding gtk-immodule-cache postinst and postrm scripts to %s" % pkg)
- postinst = d.getVar('pkg_postinst_%s' % pkg, True) or d.getVar('pkg_postinst', True)
+ postinst = d.getVar('pkg_postinst_%s' % pkg, True)
if not postinst:
postinst = '#!/bin/sh\n'
postinst += d.getVar('gtk_immodule_cache_postinst', True)
d.setVar('pkg_postinst_%s' % pkg, postinst)
- postrm = d.getVar('pkg_postrm_%s' % pkg, True) or d.getVar('pkg_postrm', True)
+ postrm = d.getVar('pkg_postrm_%s' % pkg, True)
if not postrm:
postrm = '#!/bin/sh\n'
postrm += d.getVar('gtk_immodule_cache_postrm', True)
diff --git a/meta/classes/mime.bbclass b/meta/classes/mime.bbclass
index b669418..690610e 100644
--- a/meta/classes/mime.bbclass
+++ b/meta/classes/mime.bbclass
@@ -43,18 +43,16 @@ python populate_packages_append () {
mimes.append(f)
if mimes:
bb.note("adding mime postinst and postrm scripts to %s" % pkg)
- postinst = d.getVar('pkg_postinst_%s' % pkg, True) or d.getVar('pkg_postinst', True)
+ postinst = d.getVar('pkg_postinst_%s' % pkg, True)
if not postinst:
postinst = '#!/bin/sh\n'
postinst += d.getVar('mime_postinst', True)
d.setVar('pkg_postinst_%s' % pkg, postinst)
- postrm = d.getVar('pkg_postrm_%s' % pkg, True) or d.getVar('pkg_postrm', True)
+ postrm = d.getVar('pkg_postrm_%s' % pkg, True)
if not postrm:
postrm = '#!/bin/sh\n'
postrm += d.getVar('mime_postrm', True)
d.setVar('pkg_postrm_%s' % pkg, postrm)
bb.note("adding shared-mime-info-data dependency to %s" % pkg)
- rdepends = explode_deps(d.getVar('RDEPENDS_' + pkg, False) or d.getVar('RDEPENDS', False) or "" )
- rdepends.append("shared-mime-info-data")
- d.setVar('RDEPENDS_' + pkg, " " + " ".join(rdepends))
+ d.appendVar('RDEPENDS_' + pkg, " shared-mime-info-data")
}
diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index 2bae65b..cb7cceb 100644
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -415,7 +415,7 @@ python package_do_split_locales() {
# glibc-localedata-translit* won't install as a dependency
# for some other package which breaks meta-toolchain
# Probably breaks since virtual-locale- isn't provided anywhere
- #rdep = (d.getVar('RDEPENDS_%s' % pn, True) or d.getVar('RDEPENDS', True) or "").split()
+ #rdep = (d.getVar('RDEPENDS_%s' % pn, True) or "").split()
#rdep.append('%s-locale*' % pn)
#d.setVar('RDEPENDS_%s' % pn, ' '.join(rdep))
}
@@ -899,7 +899,7 @@ python populate_packages () {
root = os.path.join(pkgdest, pkg)
bb.utils.mkdirhier(root)
- filesvar = d.getVar('FILES_%s' % pkg, True) or d.getVar('FILES', True) or ""
+ filesvar = d.getVar('FILES_%s' % pkg, True) or ""
if "//" in filesvar:
bb.warn("FILES variable for package %s contains '//' which is invalid. Attempting to fix this but you should correct the metadata.\n" % pkg)
filesvar.replace("//", "/")
@@ -1027,7 +1027,7 @@ python package_fixsymlinks () {
bb.note("%s contains dangling symlink to %s" % (pkg, l))
for pkg in newrdepends:
- rdepends = bb.utils.explode_dep_versions2(d.getVar('RDEPENDS_' + pkg, True) or d.getVar('RDEPENDS', True) or "")
+ rdepends = bb.utils.explode_dep_versions2(d.getVar('RDEPENDS_' + pkg, True) or "")
for p in newrdepends[pkg]:
if p not in rdepends:
rdepends[p] = []
@@ -1410,7 +1410,7 @@ python package_do_shlibs() {
fd.close()
if needs_ldconfig and use_ldconfig:
bb.debug(1, 'adding ldconfig call to postinst for %s' % pkg)
- postinst = d.getVar('pkg_postinst_%s' % pkg, True) or d.getVar('pkg_postinst', True)
+ postinst = d.getVar('pkg_postinst_%s' % pkg, True)
if not postinst:
postinst = '#!/bin/sh\n'
postinst += d.getVar('ldconfig_postinst_fragment', True)
@@ -1597,7 +1597,7 @@ python read_shlibdeps () {
packages = d.getVar('PACKAGES', True).split()
for pkg in packages:
- rdepends = bb.utils.explode_dep_versions2(d.getVar('RDEPENDS_' + pkg, True) or d.getVar('RDEPENDS', True) or "")
+ rdepends = bb.utils.explode_dep_versions2(d.getVar('RDEPENDS_' + pkg, True) or "")
for dep in pkglibdeps[pkg]:
# Add the dep if it's not already there, or if no comparison is set
if dep not in rdepends:
@@ -1629,7 +1629,7 @@ python package_depchains() {
def pkg_adddeprrecs(pkg, base, suffix, getname, depends, d):
#bb.note('depends for %s is %s' % (base, depends))
- rreclist = bb.utils.explode_dep_versions2(d.getVar('RRECOMMENDS_' + pkg, True) or d.getVar('RRECOMMENDS', True) or "")
+ rreclist = bb.utils.explode_dep_versions2(d.getVar('RRECOMMENDS_' + pkg, True) or "")
for depend in depends:
if depend.find('-native') != -1 or depend.find('-cross') != -1 or depend.startswith('virtual/'):
@@ -1650,7 +1650,7 @@ python package_depchains() {
def pkg_addrrecs(pkg, base, suffix, getname, rdepends, d):
#bb.note('rdepends for %s is %s' % (base, rdepends))
- rreclist = bb.utils.explode_dep_versions2(d.getVar('RRECOMMENDS_' + pkg, True) or d.getVar('RRECOMMENDS', True) or "")
+ rreclist = bb.utils.explode_dep_versions2(d.getVar('RRECOMMENDS_' + pkg, True) or "")
for depend in rdepends:
if depend.find('virtual-locale-') != -1:
@@ -1677,9 +1677,6 @@ python package_depchains() {
add_dep(depends, dep)
rdepends = []
- for dep in bb.utils.explode_deps(d.getVar('RDEPENDS', True) or ""):
- add_dep(rdepends, dep)
-
for pkg in packages.split():
for dep in bb.utils.explode_deps(d.getVar('RDEPENDS_' + pkg, True) or ""):
add_dep(rdepends, dep)
@@ -1729,7 +1726,7 @@ python package_depchains() {
pkg_addrrecs(pkg, base, suffix, func, rdepends, d)
else:
rdeps = []
- for dep in bb.utils.explode_deps(d.getVar('RDEPENDS_' + base, True) or d.getVar('RDEPENDS', True) or ""):
+ for dep in bb.utils.explode_deps(d.getVar('RDEPENDS_' + base, True) or ""):
add_dep(rdeps, dep)
pkg_addrrecs(pkg, base, suffix, func, rdeps, d)
}
diff --git a/meta/classes/update-alternatives.bbclass b/meta/classes/update-alternatives.bbclass
index 147d236..8f4410b 100644
--- a/meta/classes/update-alternatives.bbclass
+++ b/meta/classes/update-alternatives.bbclass
@@ -327,12 +327,12 @@ python populate_packages_updatealternatives () {
bb.note('adding update-alternatives calls to postinst/postrm for %s' % pkg)
bb.note('%s' % alt_setup_links)
- postinst = (d.getVar('pkg_postinst_%s' % pkg, True) or d.getVar('pkg_postinst', True)) or '#!/bin/sh\n'
+ postinst = d.getVar('pkg_postinst_%s' % pkg, True) or '#!/bin/sh\n'
postinst += alt_setup_links
d.setVar('pkg_postinst_%s' % pkg, postinst)
bb.note('%s' % alt_remove_links)
- postrm = (d.getVar('pkg_postrm_%s' % pkg, True) or d.getVar('pkg_postrm', True)) or '#!/bin/sh\n'
+ postrm = d.getVar('pkg_postrm_%s' % pkg, True) or '#!/bin/sh\n'
postrm += alt_remove_links
d.setVar('pkg_postrm_%s' % pkg, postrm)
}
diff --git a/meta/classes/update-rc.d_real.bbclass b/meta/classes/update-rc.d_real.bbclass
index f9d55fb..30ae8e3 100644
--- a/meta/classes/update-rc.d_real.bbclass
+++ b/meta/classes/update-rc.d_real.bbclass
@@ -57,19 +57,19 @@ python populate_packages_updatercd () {
execute on the target. Not doing so may cause update_rc.d postinst invoked
twice to cause unwanted warnings.
"""
- postinst = d.getVar('pkg_postinst_%s' % pkg, True) or d.getVar('pkg_postinst', True)
+ postinst = d.getVar('pkg_postinst_%s' % pkg, True)
if not postinst:
postinst = '#!/bin/sh\n'
postinst += d.getVar('updatercd_postinst', True)
d.setVar('pkg_postinst_%s' % pkg, postinst)
- prerm = d.getVar('pkg_prerm_%s' % pkg, True) or d.getVar('pkg_prerm', True)
+ prerm = d.getVar('pkg_prerm_%s' % pkg, True)
if not prerm:
prerm = '#!/bin/sh\n'
prerm += d.getVar('updatercd_prerm', True)
d.setVar('pkg_prerm_%s' % pkg, prerm)
- postrm = d.getVar('pkg_postrm_%s' % pkg, True) or d.getVar('pkg_postrm', True)
+ postrm = d.getVar('pkg_postrm_%s' % pkg, True)
if not postrm:
postrm = '#!/bin/sh\n'
postrm += d.getVar('updatercd_postrm', True)
--
1.7.10.4
^ permalink raw reply related [flat|nested] 27+ messages in thread
* Re: [PATCH 00/22] Packaging performance improvements, round 2
2013-02-03 23:55 [PATCH 00/22] Packaging performance improvements, round 2 Richard Purdie
` (21 preceding siblings ...)
2013-02-03 23:55 ` [PATCH 22/22] classes: Drop none package specific packaging variable accesses Richard Purdie
@ 2013-02-04 11:41 ` Richard Purdie
22 siblings, 0 replies; 27+ messages in thread
From: Richard Purdie @ 2013-02-04 11:41 UTC (permalink / raw)
To: openembedded-core
On Sun, 2013-02-03 at 23:55 +0000, Richard Purdie wrote:
> The following patch series contains various performance improvements for
> do_package.
>
> It also includes changes to the sanity testing to start giving warnings
> about packaging variables used without packages being specified, eg:
>
> RDEPENDS = "x"
>
> instead of
>
> RDEPENDS_${PN} = "x"
>
> The final patch in the series stops the packaging code using the non-package
> versions in many cases. I'm not planning to merge that one right away, its just
> included for illustration and benchmarking purposes now.
>
> There were three cases I found using bad syntax in OE-Core and the series includes
> patches to fix them.
>
> There is also a useful performance improvement to sstate manipulations to use
> hardlinks instead of file copies which should be universally helpful to
> build performance of tasks.
>
> The following changes are available in the git repository at:
>
> git://git.openembedded.org/openembedded-core master-next
> http://cgit.openembedded.org/cgit.cgi/openembedded-core/log/?h=master-next
I found a few issues with the patches after more extensive testing and
have updated the patches in the branch. There are a few more RDEPENDS ->
RDEPENDS_${PN} fixes too.
Cheers,
Richard
^ permalink raw reply [flat|nested] 27+ messages in thread
* Re: [PATCH 03/22] package: Process package stripping in parallel
2013-02-03 23:55 ` [PATCH 03/22] package: Process package stripping in parallel Richard Purdie
@ 2013-02-04 17:23 ` Mark Hatle
0 siblings, 0 replies; 27+ messages in thread
From: Mark Hatle @ 2013-02-04 17:23 UTC (permalink / raw)
To: openembedded-core
On 2/3/13 5:55 PM, Richard Purdie wrote:
> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
> ---
> meta/classes/package.bbclass | 58 ++++++++++--------------------------------
> meta/lib/oe/package.py | 45 ++++++++++++++++++++++++++++++++
> 2 files changed, 58 insertions(+), 45 deletions(-)
>
> diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
> index 527ef31..9c8cdbc 100644
> --- a/meta/classes/package.bbclass
> +++ b/meta/classes/package.bbclass
> @@ -309,49 +309,6 @@ def copydebugsources(debugsrcdir, d):
> if os.path.exists(p) and not os.listdir(p):
> os.rmdir(p)
>
> -def runstrip(file, elftype, d):
> - # Function to strip a single file, called from split_and_strip_files below
> - # A working 'file' (one which works on the target architecture)
> - #
> - # The elftype is a bit pattern (explained in split_and_strip_files) to tell
> - # us what type of file we're processing...
> - # 4 - executable
> - # 8 - shared library
> -
> - import commands, stat, subprocess
> -
> - strip = d.getVar("STRIP", True)
> -
> - newmode = None
> - if not os.access(file, os.W_OK) or os.access(file, os.R_OK):
> - origmode = os.stat(file)[stat.ST_MODE]
> - newmode = origmode | stat.S_IWRITE | stat.S_IREAD
> - os.chmod(file, newmode)
> -
> - extraflags = ""
> -
> - # .so and shared library
> - if elftype & 16:
> - extraflags = "--strip-debug --remove-section=.comment --remove-section=.note --preserve-dates"
> - elif ".so" in file and elftype & 8:
> - extraflags = "--remove-section=.comment --remove-section=.note --strip-unneeded"
> - # shared or executable:
> - elif elftype & 8 or elftype & 4:
> - extraflags = "--remove-section=.comment --remove-section=.note"
> -
> - stripcmd = "'%s' %s '%s'" % (strip, extraflags, file)
> - bb.debug(1, "runstrip: %s" % stripcmd)
> -
> - ret = subprocess.call(stripcmd, shell=True)
> -
> - if newmode:
> - os.chmod(file, origmode)
> -
> - if ret:
> - bb.error("runstrip: '%s' strip command failed" % stripcmd)
> -
> - return 0
> -
> #
> # Package data handling routines
> #
> @@ -902,13 +859,24 @@ python split_and_strip_files () {
> # Now lets go back over things and strip them
> #
> if (d.getVar('INHIBIT_PACKAGE_STRIP', True) != '1'):
> + strip = d.getVar("STRIP", True)
> + sfiles = []
> for file in file_list:
> if file_list[file].startswith("ELF: "):
> elf_file = int(file_list[file][5:])
> #bb.note("Strip %s" % file)
> - runstrip(file, elf_file, d)
> + sfiles.append((file, elf_file, strip))
> for f in kernmods:
> - runstrip(f, 16, d)
> + sfiles.append((f, 16, strip))
> +
> +
> + import multiprocessing
> + nproc = multiprocessing.cpu_count()
> + pool = multiprocessing.Pool(nproc)
> + processed = pool.imap(oe.package.runstrip, sfiles)
> + pool.close()
> + pool.join()
> +
> #
> # End of strip
> #
> diff --git a/meta/lib/oe/package.py b/meta/lib/oe/package.py
> index 6b1c1f4..9a0ddb8 100644
> --- a/meta/lib/oe/package.py
> +++ b/meta/lib/oe/package.py
> @@ -1,3 +1,48 @@
> +def runstrip(arg):
> + # Function to strip a single file, called from split_and_strip_files below
Minor note, the comment above is now confusing as it's unclear where the
split_and_strip_files is. It might be reasonable at this point to simply remove
that -- or change the comment to indicate it's from the package.bbclass.
--Mark
> + # A working 'file' (one which works on the target architecture)
> + #
> + # The elftype is a bit pattern (explained in split_and_strip_files) to tell
> + # us what type of file we're processing...
> + # 4 - executable
> + # 8 - shared library
> + # 16 - kernel module
> +
> + import commands, stat, subprocess
> +
> + (file, elftype, strip) = arg
> +
> + newmode = None
> + if not os.access(file, os.W_OK) or os.access(file, os.R_OK):
> + origmode = os.stat(file)[stat.ST_MODE]
> + newmode = origmode | stat.S_IWRITE | stat.S_IREAD
> + os.chmod(file, newmode)
> +
> + extraflags = ""
> +
> + # kernel module
> + if elftype & 16:
> + extraflags = "--strip-debug --remove-section=.comment --remove-section=.note --preserve-dates"
> + # .so and shared library
> + elif ".so" in file and elftype & 8:
> + extraflags = "--remove-section=.comment --remove-section=.note --strip-unneeded"
> + # shared or executable:
> + elif elftype & 8 or elftype & 4:
> + extraflags = "--remove-section=.comment --remove-section=.note"
> +
> + stripcmd = "'%s' %s '%s'" % (strip, extraflags, file)
> + bb.debug(1, "runstrip: %s" % stripcmd)
> +
> + ret = subprocess.call(stripcmd, shell=True)
> +
> + if newmode:
> + os.chmod(file, origmode)
> +
> + if ret:
> + bb.error("runstrip: '%s' strip command failed" % stripcmd)
> +
> + return
> +
>
> def file_translate(file):
> ft = file.replace("@", "@at@")
>
^ permalink raw reply [flat|nested] 27+ messages in thread
* Re: [PATCH 13/22] package.bbclass: Rewrite split_and_strip_files
2013-02-03 23:55 ` [PATCH 13/22] package.bbclass: Rewrite split_and_strip_files Richard Purdie
@ 2013-02-09 11:41 ` Enrico Scholz
2013-02-10 13:50 ` Enrico Scholz
0 siblings, 1 reply; 27+ messages in thread
From: Enrico Scholz @ 2013-02-09 11:41 UTC (permalink / raw)
To: openembedded-core; +Cc: Richard Purdie
Richard Purdie <richard.purdie-hQyY1W1yCW8ekmWlsbkhG0B+6BGkLq7r@public.gmane.org> writes:
> - os.path.isfile(file):
> ...
> + try:
> + s = os.stat(file)
> + except OSError, (err, strerror):
> + if err != errno.ENOENT:
> + raise
Causes a regression (resp. triggers the previously hidden bug catched by
the os.path.isfile() check):
ERROR: Error executing a python function in .../org.openembedded.core/meta/recipes-core/systemd/systemd-serialgetty.bb:
OSError: [Errno 13] Permission denied: '.../systemd-serialgetty/1.0-r3/package/etc/systemd/system/getty.target.wants/serial-getty@ttySA0.service'
Enrico
^ permalink raw reply [flat|nested] 27+ messages in thread
* Re: [PATCH 13/22] package.bbclass: Rewrite split_and_strip_files
2013-02-09 11:41 ` Enrico Scholz
@ 2013-02-10 13:50 ` Enrico Scholz
0 siblings, 0 replies; 27+ messages in thread
From: Enrico Scholz @ 2013-02-10 13:50 UTC (permalink / raw)
To: openembedded-core
Enrico Scholz
<enrico.scholz-wttK6gPy29v+Hn7q9Vec/7NAH6kLmebB@public.gmane.org>
writes:
> Causes a regression (resp. triggers the previously hidden bug catched by
> the os.path.isfile() check):
>
> ERROR: Error executing a python function in .../org.openembedded.core/meta/recipes-core/systemd/systemd-serialgetty.bb:
> OSError: [Errno 13] Permission denied: '.../systemd-serialgetty/1.0-r3/package/etc/systemd/system/getty.target.wants/serial-getty-OjP1UEOGd4Fo45MUmRTO9Q@public.gmane.org'
I posted a patchset "lib: implemented oe.path.realpath()" which should
solve this issue.
Enrico
^ permalink raw reply [flat|nested] 27+ messages in thread
end of thread, other threads:[~2013-02-10 14:06 UTC | newest]
Thread overview: 27+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2013-02-03 23:55 [PATCH 00/22] Packaging performance improvements, round 2 Richard Purdie
2013-02-03 23:55 ` [PATCH 01/22] package.bbclass: Multithread per file dependency generation code Richard Purdie
2013-02-03 23:55 ` [PATCH 02/22] package: Don't export PATH Richard Purdie
2013-02-03 23:55 ` [PATCH 03/22] package: Process package stripping in parallel Richard Purdie
2013-02-04 17:23 ` Mark Hatle
2013-02-03 23:55 ` [PATCH 04/22] insane.bbclass: Add pkgvarcheck to check for suboptimal usages of variables Richard Purdie
2013-02-03 23:55 ` [PATCH 05/22] insane.bbclass: Add documentation headers for logical code blocks Richard Purdie
2013-02-03 23:55 ` [PATCH 06/22] staging/insane.bbclass: Move legacy do_stage check iinto insane.bbclass Richard Purdie
2013-02-03 23:55 ` [PATCH 07/22] staging.bbclass: Drop unused/legacy function Richard Purdie
2013-02-03 23:55 ` [PATCH 08/22] update-rc.d: Drop OVERRIDES code Richard Purdie
2013-02-03 23:55 ` [PATCH 09/22] qemu: Set RDEPENDS on the specific package that needs it Richard Purdie
2013-02-03 23:55 ` [PATCH 10/22] gdb-cross-canadian: " Richard Purdie
2013-02-03 23:55 ` [PATCH 11/22] initramfs-live-boot: " Richard Purdie
2013-02-03 23:55 ` [PATCH 12/22] package.bbclass: Fix up bb.mkdirhier/bb.copyfile usage Richard Purdie
2013-02-03 23:55 ` [PATCH 13/22] package.bbclass: Rewrite split_and_strip_files Richard Purdie
2013-02-09 11:41 ` Enrico Scholz
2013-02-10 13:50 ` Enrico Scholz
2013-02-03 23:55 ` [PATCH 14/22] kernel.bbclass: Improve populate_packages_prepend Richard Purdie
2013-02-03 23:55 ` [PATCH 15/22] package.bbclass: Make use of cleandirs and dirs function flags Richard Purdie
2013-02-03 23:55 ` [PATCH 16/22] package.bbclass: Various minor performance tweaks Richard Purdie
2013-02-03 23:55 ` [PATCH 17/22] package.bbclass: Simplify empty directory removal Richard Purdie
2013-02-03 23:55 ` [PATCH 18/22] package.bbclass: Add PACKAGESPLITFUNCS variable Richard Purdie
2013-02-03 23:55 ` [PATCH 19/22] sstate/path.py: Add copyhardlinktree() function and use for performance optimisation Richard Purdie
2013-02-03 23:55 ` [PATCH 20/22] package.bbclass: Better document the different phases of operation Richard Purdie
2013-02-03 23:55 ` [PATCH 21/22] package.bbclass: Pre-expand some variables to save time Richard Purdie
2013-02-03 23:55 ` [PATCH 22/22] classes: Drop none package specific packaging variable accesses Richard Purdie
2013-02-04 11:41 ` [PATCH 00/22] Packaging performance improvements, round 2 Richard Purdie
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox