[PATCH] sstate: Add a two character subdirectory to the sstate directory layout

* [PATCH] sstate: Add a two character subdirectory to the sstate directory layout
@ 2012-07-25 21:09 Richard Purdie
  2012-07-25 22:12 ` McClintock Matthew-B29882
  2012-08-02 13:53   ` Martin Jansa
  0 siblings, 2 replies; 18+ messages in thread
From: Richard Purdie @ 2012-07-25 21:09 UTC (permalink / raw)
  To: openembedded-devel

Currently all sstate files are placed into one directory. This does not scale and
causes a variety of filesystem issues. This patch adds a two character subdirectory
to the layout (based on the first two characters of the hash) so that files
can be split into several directories.

This should help performance of sstate in most cases by avoding creating directories with 
huge numbers of files.

The SSTATE_MIRRORS syntax needs updating to account for the extra path element by
the addition of a PATH item, for example:

SSTATE_MIRRORS = "file://.* file:///some/path/to/sstate-cache/PATH"
SSTATE_MIRRORS = "file://.* http://192.168.1.23/sstate-cache/PATH"

This change also sets the scene for using things like lsb-release in
the 

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---

diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
index 570b371..d00779a 100644
--- a/meta/classes/sstate.bbclass
+++ b/meta/classes/sstate.bbclass
@@ -4,16 +4,21 @@ SSTATE_MANIFESTS ?= "${TMPDIR}/sstate-control"
 SSTATE_MANFILEBASE = "${SSTATE_MANIFESTS}/manifest-${SSTATE_MANMACH}-"
 SSTATE_MANFILEPREFIX = "${SSTATE_MANFILEBASE}${PN}"
 
+def generate_sstatefn(spec, hash, d):
+    if not hash:
+        hash = "INVALID"
+    return hash[:2] + "/" + spec + hash
 
 SSTATE_PKGARCH    = "${PACKAGE_ARCH}"
 SSTATE_PKGSPEC    = "sstate-${PN}-${PACKAGE_ARCH}${TARGET_VENDOR}-${TARGET_OS}-${PV}-${PR}-${SSTATE_PKGARCH}-${SSTATE_VERSION}-"
-SSTATE_PKGNAME    = "${SSTATE_PKGSPEC}${BB_TASKHASH}"
+SSTATE_PKGNAME    = "${@generate_sstatefn(d.getVar('SSTATE_PKGSPEC', True), d.getVar('BB_TASKHASH', True), d)}"
 SSTATE_PKG        = "${SSTATE_DIR}/${SSTATE_PKGNAME}"
+SSTATE_PATHSPEC   = "${SSTATE_DIR}/*/${SSTATE_PKGSPEC}"
 
 SSTATE_SCAN_FILES ?= "*.la *-config *_config"
 SSTATE_SCAN_CMD ?= 'find ${SSTATE_BUILDDIR} \( -name "${@"\" -o -name \"".join(d.getVar("SSTATE_SCAN_FILES", True).split())}" \) -type f'
 
-BB_HASHFILENAME = "${SSTATE_PKGNAME}"
+BB_HASHFILENAME = "${SSTATE_PKGSPEC}"
 
 SSTATE_MANMACH ?= "${SSTATE_PKGARCH}"
 
@@ -158,10 +163,11 @@ def sstate_installpkg(ss, d):
         oe.path.remove(dir)
 
     sstateinst = d.expand("${WORKDIR}/sstate-install-%s/" % ss['name'])
+    sstatefetch = d.getVar('SSTATE_PKGNAME', True) + '_' + ss['name'] + ".tgz"
     sstatepkg = d.getVar('SSTATE_PKG', True) + '_' + ss['name'] + ".tgz"
 
     if not os.path.exists(sstatepkg):
-       pstaging_fetch(sstatepkg, d)
+       pstaging_fetch(sstatefetch, sstatepkg, d)
 
     if not os.path.isfile(sstatepkg):
         bb.note("Staging package %s does not exist" % sstatepkg)
@@ -223,8 +229,7 @@ def sstate_installpkg(ss, d):
 def sstate_clean_cachefile(ss, d):
     import oe.path
 
-    sstatepkgdir = d.getVar('SSTATE_DIR', True)
-    sstatepkgfile = sstatepkgdir + '/' + d.getVar('SSTATE_PKGSPEC', True) + "*_" + ss['name'] + ".tgz*"
+    sstatepkgfile = d.getVar('SSTATE_PATHSPEC', True) + "*_" + ss['name'] + ".tgz*"
     bb.note("Removing %s" % sstatepkgfile)
     oe.path.remove(sstatepkgfile)
 
@@ -417,7 +422,7 @@ def sstate_package(ss, d):
 
     return
 
-def pstaging_fetch(sstatepkg, d):
+def pstaging_fetch(sstatefetch, sstatepkg, d):
     import bb.fetch2
 
     # Only try and fetch if the user has configured a mirror
@@ -430,7 +435,7 @@ def pstaging_fetch(sstatepkg, d):
     bb.data.update_data(localdata)
 
     dldir = localdata.expand("${SSTATE_DIR}")
-    srcuri = "file://" + os.path.basename(sstatepkg)
+    srcuri = "file://" + sstatefetch
 
     bb.mkdirhier(dldir)
 
@@ -519,8 +524,7 @@ def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d):
     }
 
     for task in range(len(sq_fn)):
-        sstatefile = d.expand("${SSTATE_DIR}/" + sq_hashfn[task] + "_" + mapping[sq_task[task]] + ".tgz")
-        sstatefile = sstatefile.replace("${BB_TASKHASH}", sq_hash[task])
+        sstatefile = d.expand("${SSTATE_DIR}/" + generate_sstatefn(sq_hashfn[task], sq_hash[task], d) + "_" + mapping[sq_task[task]] + ".tgz")
         if os.path.exists(sstatefile):
             bb.debug(2, "SState: Found valid sstate file %s" % sstatefile)
             ret.append(task)
@@ -544,10 +548,9 @@ def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d):
             if task in ret:
                 continue
 
-            sstatefile = d.expand("${SSTATE_DIR}/" + sq_hashfn[task] + "_" + mapping[sq_task[task]] + ".tgz")
-            sstatefile = sstatefile.replace("${BB_TASKHASH}", sq_hash[task])
+            sstatefile = d.expand(generate_sstatefn(sq_hashfn[task], sq_hash[task], d) + "_" + mapping[sq_task[task]] + ".tgz")
 
-            srcuri = "file://" + os.path.basename(sstatefile)
+            srcuri = "file://" + sstatefile
             localdata.setVar('SRC_URI', srcuri)
             bb.debug(2, "SState: Attempting to fetch %s" % srcuri)
 






^ permalink raw reply related	[flat|nested] 18+ messages in thread