[PATCH 0/3] git fetcher unpack optimization V2

[PATCH 0/3] git fetcher unpack optimization V2

[Yu Ke]

From: Yu Ke <ke.yu@intel.com>

This is the remaining patches of last fetcher patch pull, including
- optimize git unpack method to use git local clone
- not to create the source tree mirror tar ball

With Richar and Bruce's comment, this patch is revised accordingly,
the major chagnes compared with V1 is:
- use git local clone instead of git repo copy to save disk space

it has been tested with "bitbake poky-image-sato" with empty DL_DIR

Pull URL: git://git.pokylinux.org/poky-contrib.git
  Branch: kyu3/git-unpack-2
  Browse: http://git.pokylinux.org/cgit.cgi/poky-contrib/log/?h=kyu3/git-unpack-2

Thanks,
    Yu Ke <ke.yu@intel.com>
---


Yu Ke (3):
  bb.fetch2: add git unpack
  kernel-yocto.bbclass: adjust the git dir to fit with git unpack
  git.py: remove the source tree tar ball

 bitbake/lib/bb/fetch2/git.py      |   66 ++++++++++--------------------------
 meta/classes/kernel-yocto.bbclass |    4 +-
 2 files changed, 21 insertions(+), 49 deletions(-)

[PATCH 1/3] bb.fetch2: add git unpack

[Yu Ke]

From: Yu Ke <ke.yu@intel.com>

git download will clone git repo to local, so git unpack can be simplified
to only checkouting the code to the work dir. For fullclone case, we also
need to manually copy all the ref info, which is needed by the later do_kernel_checkout()

Signed-off-by: Yu Ke <ke.yu@intel.com>
---
 bitbake/lib/bb/fetch2/git.py |   25 +++++++++++++++++++++++++
 1 files changed, 25 insertions(+), 0 deletions(-)

diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
index 08daa20..eadbf33 100644
--- a/bitbake/lib/bb/fetch2/git.py
+++ b/bitbake/lib/bb/fetch2/git.py
@@ -194,6 +194,31 @@ class Git(Fetch):
         os.chdir(ud.clonedir)
         bb.utils.prunedir(codir)
 
+    def unpack(self, ud, destdir, d):
+        """ unpack the downloaded src to destdir"""
+
+        subdir = ud.parm.get("subpath", "")
+        if subdir != "":
+            readpathspec = ":%s" % (subdir)
+        else:
+            readpathspec = ""
+
+        destdir = os.path.join(destdir, "git/")
+        if os.path.exists(destdir):
+            bb.utils.prunedir(destdir)
+
+        if 'fullclone' in ud.parm:
+            runfetchcmd("git clone -l -n %s %s" % (ud.clonedir, destdir), d)
+            if os.path.exists("%s/.git/refs/remotes/origin" % ud.clonedir):
+                runfetchcmd("cp -af %s/.git/refs/remotes/origin/* %s/.git/refs/remotes/origin/" %(ud.clonedir, destdir), d)
+            if os.path.exists("%s/.git/packed-refs" % ud.clonedir):
+                runfetchcmd("cp -af %s/.git/packed-refs %s/.git/" %(ud.clonedir, destdir), d)
+        else:
+            os.chdir(ud.clonedir)
+            runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.tag, readpathspec), d)
+            runfetchcmd("%s checkout-index -q -f --prefix=%s -a" % (ud.basecmd, destdir), d)
+        return True
+
     def supports_srcrev(self):
         return True
 
-- 
1.7.0.4

[PATCH 2/3] kernel-yocto.bbclass: adjust the git dir to fit with git unpack

[Yu Ke]

From: Yu Ke <ke.yu@intel.com>

Signed-off-by: Yu Ke <ke.yu@intel.com>
---
 meta/classes/kernel-yocto.bbclass |    4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/meta/classes/kernel-yocto.bbclass b/meta/classes/kernel-yocto.bbclass
index 8479b39..ca1fb15 100644
--- a/meta/classes/kernel-yocto.bbclass
+++ b/meta/classes/kernel-yocto.bbclass
@@ -89,11 +89,11 @@ do_patch() {
 }
 
 do_kernel_checkout() {
-	if [ -d ${WORKDIR}/.git/refs/remotes/origin ]; then
+	if [ -d ${WORKDIR}/git/.git/refs/remotes/origin ]; then
 		echo "Fixing up git directory for ${LINUX_KERNEL_TYPE}/${KMACHINE}"
 		rm -rf ${S}
 		mkdir ${S}
-		mv ${WORKDIR}/.git ${S}
+		mv ${WORKDIR}/git/.git ${S}
 	
 		if [ -e ${S}/.git/packed-refs ]; then
 			cd ${S}
-- 
1.7.0.4

[PATCH 3/3] git.py: remove the source tree tar ball

[Yu Ke]

From: Yu Ke <ke.yu@intel.com>

we already create repo tar ball, so no need to create the source tree tar ball

Signed-off-by: Yu Ke <ke.yu@intel.com>
---
 bitbake/lib/bb/fetch2/git.py |   63 +++--------------------------------------
 1 files changed, 5 insertions(+), 58 deletions(-)

diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
index eadbf33..25d86c4 100644
--- a/bitbake/lib/bb/fetch2/git.py
+++ b/bitbake/lib/bb/fetch2/git.py
@@ -66,18 +66,7 @@ class Git(Fetch):
         if not ud.tag or ud.tag == "master":
             ud.tag = self.latest_revision(url, ud, d)
 
-        subdir = ud.parm.get("subpath", "")
-        if subdir != "":
-            if subdir.endswith("/"):
-                subdir = subdir[:-1]
-            subdirpath = os.path.join(ud.path, subdir);
-        else:
-            subdirpath = ud.path;
-
-        if 'fullclone' in ud.parm:
-            ud.localfile = ud.mirrortarball
-        else:
-            ud.localfile = data.expand('git_%s%s_%s.tar.gz' % (ud.host, subdirpath.replace('/', '.'), ud.tag), d)
+        ud.localfile = ud.mirrortarball
 
         if 'noclone' in ud.parm:
             ud.localfile = None
@@ -90,8 +79,6 @@ class Git(Fetch):
             return True
         if 'noclone' in ud.parm:
             return False
-        if os.path.exists(ud.localpath):
-            return False
         if not self._contains_ref(ud.tag, d):
             return True
         return False
@@ -116,6 +103,8 @@ class Git(Fetch):
 
         repofile = os.path.join(data.getVar("DL_DIR", d, 1), ud.mirrortarball)
 
+        ud.repochanged = not os.path.exists(repofile)
+
         # If the checkout doesn't exist and the mirror tarball does, extract it
         if not os.path.exists(ud.clonedir) and os.path.exists(repofile):
             bb.mkdirhier(ud.clonedir)
@@ -140,60 +129,18 @@ class Git(Fetch):
             runfetchcmd("%s fetch --tags %s://%s%s%s" % (ud.basecmd, ud.proto, username, ud.host, ud.path), d)
             runfetchcmd("%s prune-packed" % ud.basecmd, d)
             runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d)
+            ud.repochanged = True
 
     def build_mirror_data(self, url, ud, d):
         # Generate a mirror tarball if needed
-        coname = '%s' % (ud.tag)
-        codir = os.path.join(ud.clonedir, coname)
         repofile = os.path.join(data.getVar("DL_DIR", d, 1), ud.mirrortarball)
 
         os.chdir(ud.clonedir)
         mirror_tarballs = data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True)
-        if mirror_tarballs != "0" or 'fullclone' in ud.parm:
+        if (mirror_tarballs != "0" or 'fullclone' in ud.parm) and ud.repochanged:
             logger.info("Creating tarball of git repository")
             runfetchcmd("tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*") ), d)
 
-        if 'fullclone' in ud.parm:
-            return
-
-        if os.path.exists(codir):
-            bb.utils.prunedir(codir)
-
-        subdir = ud.parm.get("subpath", "")
-        if subdir != "":
-            if subdir.endswith("/"):
-                subdirbase = os.path.basename(subdir[:-1])
-            else:
-                subdirbase = os.path.basename(subdir)
-        else:
-            subdirbase = ""
-
-        if subdir != "":
-            readpathspec = ":%s" % (subdir)
-            codir = os.path.join(codir, "git")
-            coprefix = os.path.join(codir, subdirbase, "")
-        else:
-            readpathspec = ""
-            coprefix = os.path.join(codir, "git", "")
-
-        scmdata = ud.parm.get("scmdata", "")
-        if scmdata == "keep":
-            runfetchcmd("%s clone -n %s %s" % (ud.basecmd, ud.clonedir, coprefix), d)
-            os.chdir(coprefix)
-            runfetchcmd("%s checkout -q -f %s%s" % (ud.basecmd, ud.tag, readpathspec), d)
-        else:
-            bb.mkdirhier(codir)
-            os.chdir(ud.clonedir)
-            runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.tag, readpathspec), d)
-            runfetchcmd("%s checkout-index -q -f --prefix=%s -a" % (ud.basecmd, coprefix), d)
-
-        os.chdir(codir)
-        logger.info("Creating tarball of git checkout")
-        runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.join(".", "*") ), d)
-
-        os.chdir(ud.clonedir)
-        bb.utils.prunedir(codir)
-
     def unpack(self, ud, destdir, d):
         """ unpack the downloaded src to destdir"""
 
-- 
1.7.0.4