From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mail5.wrs.com (mail5.windriver.com [192.103.53.11]) by mail.openembedded.org (Postfix) with ESMTP id 8B3C57C288 for ; Fri, 18 Jan 2019 18:08:38 +0000 (UTC) Received: from ALA-HCA.corp.ad.wrs.com (ala-hca.corp.ad.wrs.com [147.11.189.40]) by mail5.wrs.com (8.15.2/8.15.2) with ESMTPS id x0II83ua030346 (version=TLSv1 cipher=AES128-SHA bits=128 verify=FAIL); Fri, 18 Jan 2019 10:08:13 -0800 Received: from soho-mhatle-m.local (147.11.119.42) by ALA-HCA.corp.ad.wrs.com (147.11.189.50) with Microsoft SMTP Server id 14.3.408.0; Fri, 18 Jan 2019 10:07:52 -0800 To: Khem Raj References: <20190115213137.113956-1-mark.hatle@windriver.com> <20190115213137.113956-2-mark.hatle@windriver.com> From: Mark Hatle Openpgp: preference=signencrypt Autocrypt: addr=mark.hatle@windriver.com; prefer-encrypt=mutual; keydata= mQENBFYKxFgBCACt/pzutBp6p/xVKTFJjHbM3KpQKCblyot/YP+bpTr51Hrc5xDXBQhoG7TC aIRvRIvbhEevEQK9y04gW3JK/5lobq5ORebolcsHlYBUvpNeIPjupLQwGvz/TPtrLRNGLqDC rvsM6OA2XbQ2bwzxWaSQS3ImE2O2iXOZn9HhThMGeDB4Nff3fgUvXOTDIrgWOn9K2DgLL7Yc zkUIlFdj+Nraksd/7BSk8oH6tjeBVhFqSFvKta9QxWgdr58oPaTYaW/xNqUjlLrbJuMw/MSe xzuYfdfDfm6J8kRjMOnwQ0n8svJElzqAk+d83ow38gpGQ+LkjGgnf8ZFJ4rUJFADroX3ABEB AAG0JU1hcmsgSGF0bGUgPG1hcmsuaGF0bGVAd2luZHJpdmVyLmNvbT6JATcEEwEIACEFAlYK xFgCGwMFCwkIBwIGFQgJCgsCBBYCAwECHgECF4AACgkQfv796/r0vvlvZAf9Gs+eN320yhRW V/fZCsngKhmOK4v3HrTwFrkSmoD9QHQiE/5IPdNacHwIPwZx07tNBohB8xOeNqCPRYRBwGhA AnxKOPyd0nnm6ZhPzbA57v4x3IGRQr4QzvcBTASJq91l3Ew4lpAslyx5w1DPPqRD7G8ycDKg peKyDwmdkvCunVisSAQI3XIMq2y230biTO98tDPEezg+lg+yTsz9ZT33F5KNuWrpf8VL5fG/ mt+kAv7wtsx/KTRbqhH3iFXF6eBSwMjAfTXFlkLfbM9riJGXrWEl9n2S2R3cDHNHug0lb8f4 whK370KEO4OwRKIYW/VUBmzk5XZUE9DTlDSV8ycsrrkBDQRWCsRYAQgAwK3FuHCE+HW3YWdH PUjeSn5p//xJ57u8g2rng8zm9zNjmYgpPv5UxozaD9i2jf4mlQLHGGOezhHae8K4Nj70oVcv 8AmwcrJa9i9WL1oy/9R3fHMWf/Ctt9VXTO0qlCuq6PDzaUfvsXR61aJIjTKNQTOjCLjY1vXm VSewUgARysmA8WrjTfwGBihMBxAX0+kIjx8nOlam0WvekMBXZ0AbS56oTLRxYao6DI3GeB/N oWPy/5DfuTKaSdM0Pf8al20x9RuNN5/HLMlyDH/k8bIa1xd9aAqW+Feiw5gC107V2E6ULyIy q6em2UrsmIRxrvpHqbNgQKqvTehJ+V/i4g/uOwARAQABiQEfBBgBCAAJBQJWCsRYAhsMAAoJ EH7+/ev69L755XAH/3ZcNhooqd9OBhFkvXm1iWZ8EoC7motWqVn2oEyxoonsg8AD9kFXiN+T dYp7dH99EZu9q4ptj56AXm4uHzOgywL/5/V2TY6twCGAjUGzDjAB5gzoi+JLIBlDiyOip0eL QswIhRk473xy3j8DA4oVamnSPWgyNJ+qsdt37YWDzoDFFvtDoRU7Eb+znfIMDKzlny0XU/8L cW1bNHJlpv/78GPdfP4tjysEd8MuA5jf5o5w4XqcwTqalffEJtQ/s3pbkstEi7qm5uPui5Kt gq6YYLSqcSNe0GWAF9/T+qwyo7burSTxUWCWtMmlXdAQLW9SynLhB3Jbch0nFAh0fCKi6yY= Organization: Wind River Systems Message-ID: Date: Fri, 18 Jan 2019 12:07:51 -0600 User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:60.0) Gecko/20100101 Thunderbird/60.4.0 MIME-Version: 1.0 In-Reply-To: Cc: bitbake-devel Subject: Re: [PATCH 1/7] gitsm.py: Fix when a submodule is defined, but not initialized X-BeenThere: bitbake-devel@lists.openembedded.org X-Mailman-Version: 2.1.12 Precedence: list List-Id: Patches and discussion that advance bitbake development List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 18 Jan 2019 18:08:38 -0000 Content-Type: text/plain; charset="utf-8" Content-Language: en-US Content-Transfer-Encoding: 7bit On 1/18/19 9:07 AM, Khem Raj wrote: > I am seeing > > http://errors.yoctoproject.org/Errors/Details/217408/ > http://errors.yoctoproject.org/Errors/Details/217412/ Do you know where these failing recipes are so that I can reproduce it locally? rwmem looks like it might be meta-ti? cli11 looks like something from meta-oe. --Mark > On Tue, Jan 15, 2019 at 1:33 PM Mark Hatle wrote: >> >> It is possible for a submodule to be defined in the .gitmodules file, but >> never initialized in the repository itself. This shows itself when searching >> for the defined module hash you will get back a empty value. >> >> Similarly we need to identify and skip defined but not initialized submodules >> during the unpack stages as well. >> >> Thanks to raphael.lisicki@siemens.com for their help is figuring out how >> to resolve this issue. >> >> Additionally a problem was found where, while unlikely, it may be possible >> for the wrong revision to have been searched using ls-tree. This has been >> resolved in the update_submodules function by keeping the correct revision >> along with the submodule path. >> >> Signed-off-by: Mark Hatle >> --- >> lib/bb/fetch2/gitsm.py | 32 ++++++++++++++++++++++++++++---- >> 1 file changed, 28 insertions(+), 4 deletions(-) >> >> diff --git a/lib/bb/fetch2/gitsm.py b/lib/bb/fetch2/gitsm.py >> index 35729db..b7959ff 100644 >> --- a/lib/bb/fetch2/gitsm.py >> +++ b/lib/bb/fetch2/gitsm.py >> @@ -64,6 +64,7 @@ class GitSM(Git): >> def update_submodules(self, ud, d): >> submodules = [] >> paths = {} >> + revision = {} >> uris = {} >> local_paths = {} >> >> @@ -77,6 +78,7 @@ class GitSM(Git): >> for m, md in self.parse_gitmodules(gitmodules).items(): >> submodules.append(m) >> paths[m] = md['path'] >> + revision[m] = ud.revisions[name] >> uris[m] = md['url'] >> if uris[m].startswith('..'): >> newud = copy.copy(ud) >> @@ -84,7 +86,17 @@ class GitSM(Git): >> uris[m] = Git._get_repo_url(self, newud) >> >> for module in submodules: >> - module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], paths[module]), d, quiet=True, workdir=ud.clonedir) >> + try: >> + module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, revision[module], paths[module]), d, quiet=True, workdir=ud.clonedir) >> + except: >> + # If the command fails, we don't have a valid file to check. If it doesn't >> + # fail -- it still might be a failure, see next check... >> + module_hash = "" >> + >> + if not module_hash: >> + logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", module) >> + continue >> + >> module_hash = module_hash.split()[2] >> >> # Build new SRC_URI >> @@ -143,7 +155,7 @@ class GitSM(Git): >> if not ud.shallow or ud.localpath != ud.fullshallow: >> self.update_submodules(ud, d) >> >> - def copy_submodules(self, submodules, ud, destdir, d): >> + def copy_submodules(self, submodules, ud, name, destdir, d): >> if ud.bareclone: >> repo_conf = destdir >> else: >> @@ -156,6 +168,18 @@ class GitSM(Git): >> srcpath = os.path.join(ud.clonedir, 'modules', md['path']) >> modpath = os.path.join(repo_conf, 'modules', md['path']) >> >> + # Check if the module is initialized >> + try: >> + module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=ud.clonedir) >> + except: >> + # If the command fails, we don't have a valid file to check. If it doesn't >> + # fail -- it still might be a failure, see next check... >> + module_hash = "" >> + >> + if not module_hash: >> + logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", module) >> + continue >> + >> if os.path.exists(srcpath): >> if os.path.exists(os.path.join(srcpath, '.git')): >> srcpath = os.path.join(srcpath, '.git') >> @@ -188,7 +212,7 @@ class GitSM(Git): >> continue >> >> submodules = self.parse_gitmodules(gitmodules) >> - self.copy_submodules(submodules, ud, dest, d) >> + self.copy_submodules(submodules, ud, name, dest, d) >> >> def unpack(self, ud, destdir, d): >> Git.unpack(self, ud, destdir, d) >> @@ -211,7 +235,7 @@ class GitSM(Git): >> continue >> >> submodules = self.parse_gitmodules(gitmodules) >> - self.copy_submodules(submodules, ud, ud.destdir, d) >> + self.copy_submodules(submodules, ud, name, ud.destdir, d) >> >> submodules_queue = [(module, os.path.join(repo_conf, 'modules', md['path'])) for module, md in submodules.items()] >> while len(submodules_queue) != 0: >> -- >> 1.8.3.1 >> >> -- >> _______________________________________________ >> bitbake-devel mailing list >> bitbake-devel@lists.openembedded.org >> http://lists.openembedded.org/mailman/listinfo/bitbake-devel