[PATCH v2 0/2] devtool: handle nested git repos with submodules

[PATCH v2 0/2] devtool: handle nested git repos with submodules

[Clement Faure]

This patch set fixes devtool modify command when a recipe fetches a
git repository with submodules to BB_GIT_DEFAULT_DESTSUFFIX.
Devtool would parse BB_GIT_DEFAULT_DESTSUFFIX recursively and
attempt a duplicate git submodule add command resulting in a
devtool modify command failure.
The patch fixes that and stops descending once .gitmodules is
detected.
A selftest test case has been added to cover this scenario.

Tests ran:
- devtool.*

changes in v2:
- fix patch 1

Clement Faure (2):
  devtool: avoid recursion into nested git repos with submodules
  oeqa/selftest/devtool: add devtool modify testcase for nested gitsm

 .../devtool/devtool-test-git-gitsm_git.bb     | 10 ++++++++
 meta/lib/oeqa/selftest/cases/devtool.py       | 25 +++++++++++++++++++
 scripts/lib/devtool/__init__.py               |  4 +++
 3 files changed, 39 insertions(+)
 create mode 100644 meta-selftest/recipes-test/devtool/devtool-test-git-gitsm_git.bb

-- 
2.43.0

[PATCH v2 1/2] devtool: avoid recursion into nested git repos with submodules

[Clement Faure]

Prevent devtool from recursing into nested git repositories with
submodules to avoid double git submodule add operation.

Signed-off-by: Clement Faure <clement.faure@arm.com>
---
changes in v2:
- look for .gitmodules in files instead of dirs + files
---

 scripts/lib/devtool/__init__.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scripts/lib/devtool/__init__.py b/scripts/lib/devtool/__init__.py
index 969d6dc13a..1359473496 100644
--- a/scripts/lib/devtool/__init__.py
+++ b/scripts/lib/devtool/__init__.py
@@ -248,6 +248,10 @@ def setup_git_repo(repodir, version, devbranch, basetag='devtool-base', d=None):
                     remote_url = stdout.splitlines()[0]
                     logger.error(os.path.relpath(os.path.join(root, ".."), root))
                     bb.process.run('git submodule add %s %s' % (remote_url, os.path.relpath(root, os.path.join(root, ".."))), cwd=os.path.join(root, ".."))
+                    # Do not descend into nested git repos that have submodules themselves.
+                    if ".gitmodules" in files:
+                        logger.warning('Nested git repository with submodules %s; devtool will not recurse into it', root)
+                        dirs[:] = []
                     found = True
                 if found:
                     oe.patch.GitApplyTree.commitIgnored("Add additional submodule from SRC_URI", dir=os.path.join(root, ".."), d=d)
-- 
2.43.0

[PATCH v2 2/2] oeqa/selftest/devtool: add devtool modify testcase for nested gitsm

[Clement Faure]

Add a selftest that exercises devtool modify against git repositories
that contain nested git repositories with submodules.

Signed-off-by: Clement Faure <clement.faure@arm.com>
---
 .../devtool/devtool-test-git-gitsm_git.bb     | 10 ++++++++
 meta/lib/oeqa/selftest/cases/devtool.py       | 25 +++++++++++++++++++
 2 files changed, 35 insertions(+)
 create mode 100644 meta-selftest/recipes-test/devtool/devtool-test-git-gitsm_git.bb

diff --git a/meta-selftest/recipes-test/devtool/devtool-test-git-gitsm_git.bb b/meta-selftest/recipes-test/devtool/devtool-test-git-gitsm_git.bb
new file mode 100644
index 0000000000..ad50c7f13c
--- /dev/null
+++ b/meta-selftest/recipes-test/devtool/devtool-test-git-gitsm_git.bb
@@ -0,0 +1,10 @@
+SUMMARY = "Test recipe for fetching git submodules as second repo"
+HOMEPAGE = "http://git.yoctoproject.org/cgit/cgit.cgi/git-submodule-test/"
+LICENSE = "MIT"
+LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MIT;md5=0835ade698e0bcf8506ecda2f7b4f302"
+
+SRC_URI = "git://git.yoctoproject.org/git-submodule-test;branch=master;name=repo-git \
+    gitsm://git.yoctoproject.org/git-submodule-test;branch=master;name=repo-gitsm;destsuffix=${BB_GIT_DEFAULT_DESTSUFFIX}/nested/repo-gitsm"
+SRCREV_repo-git = "a2885dd7d25380d23627e7544b7bbb55014b16ee"
+SRCREV_repo-gitsm = "a2885dd7d25380d23627e7544b7bbb55014b16ee"
+SRCREV_FORMAT = "repo-git_repo-gitsm"
diff --git a/meta/lib/oeqa/selftest/cases/devtool.py b/meta/lib/oeqa/selftest/cases/devtool.py
index cf5ac6e9d7..cdfcf4f813 100644
--- a/meta/lib/oeqa/selftest/cases/devtool.py
+++ b/meta/lib/oeqa/selftest/cases/devtool.py
@@ -1153,6 +1153,31 @@ class DevtoolModifyTests(DevtoolBase):
         result = bitbake(testrecipe)
         self.assertEqual(result.status, 0, "Bitbake failed, exit code %s, output %s" % (result.status, result.output))
 
+    def test_devtool_modify_nested_gitsm(self):
+        """Checks that a recipe with multiple sources including a git repo with a nested git repo with
+        submodules can be used with devtool modify
+        """
+        testrecipe = 'devtool-test-git-gitsm'
+        src_uri = get_bb_var('SRC_URI', testrecipe)
+        self.assertIn('git://', src_uri, 'This test expects the %s recipe to fetch a git source' % testrecipe)
+        self.assertIn('gitsm://', src_uri, 'This test expects the %s recipe to fetch a gitsm source' % testrecipe)
+        # Try modifying a recipe
+        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
+        self.track_for_cleanup(tempdir)
+        self.track_for_cleanup(self.workspacedir)
+        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
+        result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir))
+        self.assertEqual(result.status, 0, "Could not modify recipe %s. Output: %s" % (testrecipe, result.output))
+        # Test devtool status
+        result = runCmd('devtool status')
+        self.assertIn(testrecipe, result.output)
+        self.assertIn(tempdir, result.output)
+        # Submodules in repo-gitsm should be extracted
+        source_repo_gitsm_gitmodules = os.path.join(tempdir, 'nested/repo-gitsm')
+        self.assertExists(source_repo_gitsm_gitmodules, 'Nested repo repo-gitsm not found')
+        self.assertExists(os.path.join(source_repo_gitsm_gitmodules, 'bitbake'), 'Submodule not found')
+        self.assertExists(os.path.join(source_repo_gitsm_gitmodules, 'bitbake-gitsm-test1'), 'Submodule not found')
+
 class DevtoolUpdateTests(DevtoolBase):
 
     def test_devtool_update_recipe(self):
-- 
2.43.0