[PATCH 0/1] Minor npm fetcher optimisation

[PATCH 0/1] Minor npm fetcher optimisation

[Paul Eggleton]

The following changes since commit 57912de63fa83550c0ae658eb99b76e9cc91a8d1:

  cooker: record events on cooker exit (2016-09-06 10:36:34 +0100)

are available in the git repository at:

  git://git.yoctoproject.org/poky-contrib paule/bb-npm-download-fix
  http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=paule/bb-npm-download-fix

Paul Eggleton (1):
  fetch2/npm: don't download same URL multiple times

 lib/bb/fetch2/npm.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

-- 
2.5.5

[PATCH 1/1] fetch2/npm: don't download same URL multiple times

[Paul Eggleton]

If we've already fetched a particular URL then we do not need to do so
again within in the same operation. Maintain an internal list of fetched
URLs to avoid doing that.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 lib/bb/fetch2/npm.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/lib/bb/fetch2/npm.py b/lib/bb/fetch2/npm.py
index 43929ce..699ae72 100644
--- a/lib/bb/fetch2/npm.py
+++ b/lib/bb/fetch2/npm.py
@@ -165,7 +165,9 @@ class Npm(FetchMethod):
             pdata = json.loads('\n'.join(datalines))
         return pdata
 
-    def _getdependencies(self, pkg, data, version, d, ud, optional=False):
+    def _getdependencies(self, pkg, data, version, d, ud, optional=False, fetchedlist=None):
+        if fetchedlist is None:
+            fetchedlist = []
         pkgfullname = pkg
         if version != '*' and not '/' in version:
             pkgfullname += "@'%s'" % version
@@ -187,7 +189,9 @@ class Npm(FetchMethod):
         outputurl = pdata['dist']['tarball']
         data[pkg] = {}
         data[pkg]['tgz'] = os.path.basename(outputurl)
-        self._runwget(ud, d, "%s --directory-prefix=%s %s" % (self.basecmd, ud.prefixdir, outputurl), False)
+        if not outputurl in fetchedlist:
+            self._runwget(ud, d, "%s --directory-prefix=%s %s" % (self.basecmd, ud.prefixdir, outputurl), False)
+            fetchedlist.append(outputurl)
 
         dependencies = pdata.get('dependencies', {})
         optionalDependencies = pdata.get('optionalDependencies', {})
@@ -200,9 +204,9 @@ class Npm(FetchMethod):
             else:
                 depsfound[dep] = dependencies[dep]
         for dep, version in optdepsfound.items():
-            self._getdependencies(dep, data[pkg]['deps'], version, d, ud, optional=True)
+            self._getdependencies(dep, data[pkg]['deps'], version, d, ud, optional=True, fetchedlist=fetchedlist)
         for dep, version in depsfound.items():
-            self._getdependencies(dep, data[pkg]['deps'], version, d, ud)
+            self._getdependencies(dep, data[pkg]['deps'], version, d, ud, fetchedlist=fetchedlist)
 
     def _getshrinkeddependencies(self, pkg, data, version, d, ud, lockdown, manifest, toplevel=True):
         logger.debug(2, "NPM shrinkwrap file is %s" % data)
-- 
2.5.5