Re: [PATCH 1/7] gitsm.py: Fix when a submodule is defined, but not initialized

[Mark Hatle <mark.hatle@windriver.com>]

On 1/18/19 9:07 AM, Khem Raj wrote:
>  I am seeing
> 
> http://errors.yoctoproject.org/Errors/Details/217408/
> http://errors.yoctoproject.org/Errors/Details/217412/

Do you know where these failing recipes are so that I can reproduce it locally?

rwmem looks like it might be meta-ti?
cli11 looks like something from meta-oe.

--Mark

> On Tue, Jan 15, 2019 at 1:33 PM Mark Hatle <mark.hatle@windriver.com> wrote:
>>
>> It is possible for a submodule to be defined in the .gitmodules file, but
>> never initialized in the repository itself.  This shows itself when searching
>> for the defined module hash you will get back a empty value.
>>
>> Similarly we need to identify and skip defined but not initialized submodules
>> during the unpack stages as well.
>>
>> Thanks to raphael.lisicki@siemens.com for their help is figuring out how
>> to resolve this issue.
>>
>> Additionally a problem was found where, while unlikely, it may be possible
>> for the wrong revision to have been searched using ls-tree.  This has been
>> resolved in the update_submodules function by keeping the correct revision
>> along with the submodule path.
>>
>> Signed-off-by: Mark Hatle <mark.hatle@windriver.com>
>> ---
>>  lib/bb/fetch2/gitsm.py | 32 ++++++++++++++++++++++++++++----
>>  1 file changed, 28 insertions(+), 4 deletions(-)
>>
>> diff --git a/lib/bb/fetch2/gitsm.py b/lib/bb/fetch2/gitsm.py
>> index 35729db..b7959ff 100644
>> --- a/lib/bb/fetch2/gitsm.py
>> +++ b/lib/bb/fetch2/gitsm.py
>> @@ -64,6 +64,7 @@ class GitSM(Git):
>>      def update_submodules(self, ud, d):
>>          submodules = []
>>          paths = {}
>> +        revision = {}
>>          uris = {}
>>          local_paths = {}
>>
>> @@ -77,6 +78,7 @@ class GitSM(Git):
>>              for m, md in self.parse_gitmodules(gitmodules).items():
>>                  submodules.append(m)
>>                  paths[m] = md['path']
>> +                revision[m] = ud.revisions[name]
>>                  uris[m] = md['url']
>>                  if uris[m].startswith('..'):
>>                      newud = copy.copy(ud)
>> @@ -84,7 +86,17 @@ class GitSM(Git):
>>                      uris[m] = Git._get_repo_url(self, newud)
>>
>>          for module in submodules:
>> -            module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], paths[module]), d, quiet=True, workdir=ud.clonedir)
>> +            try:
>> +                module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, revision[module], paths[module]), d, quiet=True, workdir=ud.clonedir)
>> +            except:
>> +                # If the command fails, we don't have a valid file to check.  If it doesn't
>> +                # fail -- it still might be a failure, see next check...
>> +                module_hash = ""
>> +
>> +            if not module_hash:
>> +                logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", module)
>> +                continue
>> +
>>              module_hash = module_hash.split()[2]
>>
>>              # Build new SRC_URI
>> @@ -143,7 +155,7 @@ class GitSM(Git):
>>          if not ud.shallow or ud.localpath != ud.fullshallow:
>>              self.update_submodules(ud, d)
>>
>> -    def copy_submodules(self, submodules, ud, destdir, d):
>> +    def copy_submodules(self, submodules, ud, name, destdir, d):
>>          if ud.bareclone:
>>              repo_conf = destdir
>>          else:
>> @@ -156,6 +168,18 @@ class GitSM(Git):
>>              srcpath = os.path.join(ud.clonedir, 'modules', md['path'])
>>              modpath = os.path.join(repo_conf, 'modules', md['path'])
>>
>> +            # Check if the module is initialized
>> +            try:
>> +                module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=ud.clonedir)
>> +            except:
>> +                # If the command fails, we don't have a valid file to check.  If it doesn't
>> +                # fail -- it still might be a failure, see next check...
>> +                module_hash = ""
>> +
>> +            if not module_hash:
>> +                logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", module)
>> +                continue
>> +
>>              if os.path.exists(srcpath):
>>                  if os.path.exists(os.path.join(srcpath, '.git')):
>>                      srcpath = os.path.join(srcpath, '.git')
>> @@ -188,7 +212,7 @@ class GitSM(Git):
>>                  continue
>>
>>              submodules = self.parse_gitmodules(gitmodules)
>> -            self.copy_submodules(submodules, ud, dest, d)
>> +            self.copy_submodules(submodules, ud, name, dest, d)
>>
>>      def unpack(self, ud, destdir, d):
>>          Git.unpack(self, ud, destdir, d)
>> @@ -211,7 +235,7 @@ class GitSM(Git):
>>                  continue
>>
>>              submodules = self.parse_gitmodules(gitmodules)
>> -            self.copy_submodules(submodules, ud, ud.destdir, d)
>> +            self.copy_submodules(submodules, ud, name, ud.destdir, d)
>>
>>              submodules_queue = [(module, os.path.join(repo_conf, 'modules', md['path'])) for module, md in submodules.items()]
>>              while len(submodules_queue) != 0:
>> --
>> 1.8.3.1
>>
>> --
>> _______________________________________________
>> bitbake-devel mailing list
>> bitbake-devel@lists.openembedded.org
>> http://lists.openembedded.org/mailman/listinfo/bitbake-devel