* [PATCH 1/5] gitsm: Add call_process_submodules() to remove duplicated code
2024-10-26 9:39 [PATCH 0/5] Fix do_clean for git and gitsm liezhi.yang
@ 2024-10-26 9:39 ` liezhi.yang
2024-10-26 9:39 ` [PATCH 2/5] gitsm: Remove downloads/tmpdir when failed liezhi.yang
` (3 subsequent siblings)
4 siblings, 0 replies; 8+ messages in thread
From: liezhi.yang @ 2024-10-26 9:39 UTC (permalink / raw)
To: bitbake-devel
From: Robert Yang <liezhi.yang@windriver.com>
There are 14 lines can be removed, and can make it easy to maintain.
Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
---
lib/bb/fetch2/gitsm.py | 42 ++++++++++++++----------------------------
1 file changed, 14 insertions(+), 28 deletions(-)
diff --git a/lib/bb/fetch2/gitsm.py b/lib/bb/fetch2/gitsm.py
index f7f3af721..17aac6357 100644
--- a/lib/bb/fetch2/gitsm.py
+++ b/lib/bb/fetch2/gitsm.py
@@ -147,6 +147,17 @@ class GitSM(Git):
return submodules != []
+ def call_process_submodules(self, ud, d, extra_check, subfunc):
+ # If we're using a shallow mirror tarball it needs to be
+ # unpacked temporarily so that we can examine the .gitmodules file
+ if ud.shallow and os.path.exists(ud.fullshallow) and extra_check:
+ tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR"))
+ runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir)
+ self.process_submodules(ud, tmpdir, subfunc, d)
+ shutil.rmtree(tmpdir)
+ else:
+ self.process_submodules(ud, ud.clonedir, subfunc, d)
+
def need_update(self, ud, d):
if Git.need_update(self, ud, d):
return True
@@ -164,15 +175,7 @@ class GitSM(Git):
logger.error('gitsm: submodule update check failed: %s %s' % (type(e).__name__, str(e)))
need_update_result = True
- # If we're using a shallow mirror tarball it needs to be unpacked
- # temporarily so that we can examine the .gitmodules file
- if ud.shallow and os.path.exists(ud.fullshallow) and not os.path.exists(ud.clonedir):
- tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR"))
- runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir)
- self.process_submodules(ud, tmpdir, need_update_submodule, d)
- shutil.rmtree(tmpdir)
- else:
- self.process_submodules(ud, ud.clonedir, need_update_submodule, d)
+ self.call_process_submodules(ud, d, not os.path.exists(ud.clonedir), need_update_submodule)
if need_update_list:
logger.debug('gitsm: Submodules requiring update: %s' % (' '.join(need_update_list)))
@@ -195,16 +198,7 @@ class GitSM(Git):
raise
Git.download(self, ud, d)
-
- # If we're using a shallow mirror tarball it needs to be unpacked
- # temporarily so that we can examine the .gitmodules file
- if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d):
- tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR"))
- runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir)
- self.process_submodules(ud, tmpdir, download_submodule, d)
- shutil.rmtree(tmpdir)
- else:
- self.process_submodules(ud, ud.clonedir, download_submodule, d)
+ self.call_process_submodules(ud, d, self.need_update(ud, d), download_submodule)
def unpack(self, ud, destdir, d):
def unpack_submodules(ud, url, module, modpath, workdir, d):
@@ -263,14 +257,6 @@ class GitSM(Git):
newfetch = Fetch([url], d, cache=False)
urldata.extend(newfetch.expanded_urldata())
- # If we're using a shallow mirror tarball it needs to be unpacked
- # temporarily so that we can examine the .gitmodules file
- if ud.shallow and os.path.exists(ud.fullshallow) and ud.method.need_update(ud, d):
- tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR"))
- subprocess.check_call("tar -xzf %s" % ud.fullshallow, cwd=tmpdir, shell=True)
- self.process_submodules(ud, tmpdir, add_submodule, d)
- shutil.rmtree(tmpdir)
- else:
- self.process_submodules(ud, ud.clonedir, add_submodule, d)
+ self.call_process_submodules(ud, d, ud.method.need_update(ud, d), add_submodule)
return urldata
--
2.25.1
^ permalink raw reply related [flat|nested] 8+ messages in thread* [PATCH 2/5] gitsm: Remove downloads/tmpdir when failed
2024-10-26 9:39 [PATCH 0/5] Fix do_clean for git and gitsm liezhi.yang
2024-10-26 9:39 ` [PATCH 1/5] gitsm: Add call_process_submodules() to remove duplicated code liezhi.yang
@ 2024-10-26 9:39 ` liezhi.yang
2024-10-28 14:06 ` [bitbake-devel] " Richard Purdie
2024-10-26 9:39 ` [PATCH 3/5] gitsm: Add clean function liezhi.yang
` (2 subsequent siblings)
4 siblings, 1 reply; 8+ messages in thread
From: liezhi.yang @ 2024-10-26 9:39 UTC (permalink / raw)
To: bitbake-devel
From: Robert Yang <liezhi.yang@windriver.com>
The tmpdir such as downloads/tmplp3cnemv won't be removed without this fix.
Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
---
lib/bb/fetch2/gitsm.py | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/lib/bb/fetch2/gitsm.py b/lib/bb/fetch2/gitsm.py
index 17aac6357..fab4b1164 100644
--- a/lib/bb/fetch2/gitsm.py
+++ b/lib/bb/fetch2/gitsm.py
@@ -152,9 +152,11 @@ class GitSM(Git):
# unpacked temporarily so that we can examine the .gitmodules file
if ud.shallow and os.path.exists(ud.fullshallow) and extra_check:
tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR"))
- runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir)
- self.process_submodules(ud, tmpdir, subfunc, d)
- shutil.rmtree(tmpdir)
+ try:
+ runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir)
+ self.process_submodules(ud, tmpdir, subfunc, d)
+ finally:
+ shutil.rmtree(tmpdir)
else:
self.process_submodules(ud, ud.clonedir, subfunc, d)
--
2.25.1
^ permalink raw reply related [flat|nested] 8+ messages in thread* [PATCH 3/5] gitsm: Add clean function
2024-10-26 9:39 [PATCH 0/5] Fix do_clean for git and gitsm liezhi.yang
2024-10-26 9:39 ` [PATCH 1/5] gitsm: Add call_process_submodules() to remove duplicated code liezhi.yang
2024-10-26 9:39 ` [PATCH 2/5] gitsm: Remove downloads/tmpdir when failed liezhi.yang
@ 2024-10-26 9:39 ` liezhi.yang
2024-10-26 9:39 ` [PATCH 4/5] git: Clean shallow mirror tarball liezhi.yang
2024-10-26 9:39 ` [PATCH 5/5] git: Clean broken symlink liezhi.yang
4 siblings, 0 replies; 8+ messages in thread
From: liezhi.yang @ 2024-10-26 9:39 UTC (permalink / raw)
To: bitbake-devel
From: Robert Yang <liezhi.yang@windriver.com>
Fixed:
$ bitbake utfcpp -cfetch && bitbake utfcpp -ccleanall
The downloads/git2/github.com.nemtrif.ftest won't be cleaned without this fix.
Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
---
lib/bb/fetch2/gitsm.py | 13 +++++++++++++
1 file changed, 13 insertions(+)
diff --git a/lib/bb/fetch2/gitsm.py b/lib/bb/fetch2/gitsm.py
index fab4b1164..ba62517f0 100644
--- a/lib/bb/fetch2/gitsm.py
+++ b/lib/bb/fetch2/gitsm.py
@@ -249,6 +249,19 @@ class GitSM(Git):
# should also be skipped as these files were already smudged in the fetch stage if lfs
# was enabled.
runfetchcmd("GIT_LFS_SKIP_SMUDGE=1 %s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir)
+ def clean(self, ud, d):
+ def clean_submodule(ud, url, module, modpath, workdir, d):
+ url += ";bareclone=1;nobranch=1"
+ try:
+ newfetch = Fetch([url], d, cache=False)
+ newfetch.clean()
+ except Exception as e:
+ logger.warning('gitsm: submodule clean failed: %s %s' % (type(e).__name__, str(e)))
+
+ self.call_process_submodules(ud, d, True, clean_submodule)
+
+ # Clean top git dir
+ Git.clean(self, ud, d)
def implicit_urldata(self, ud, d):
import shutil, subprocess, tempfile
--
2.25.1
^ permalink raw reply related [flat|nested] 8+ messages in thread