[PATCH 0/3] npm fetcher fixes

[PATCH 0/3] npm fetcher fixes

[Paul Eggleton]

The following changes since commit 19507b80f35d37dc4b1614bd390b8e261dd4a2bd:

  bitbake-user-manual: Added expand() function to list. (2016-03-03 17:33:59 +0000)

are available in the git repository at:

  git://git.yoctoproject.org/poky-contrib paule/bb-npm-fixes
  http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=paule/bb-npm-fixes

Paul Eggleton (3):
  fetch2/npm: fix indentation
  fetch2/npm: handle alternative dependency syntax
  fetch2/npm: ignore unknown headers in tarballs

 lib/bb/fetch2/npm.py | 50 ++++++++++++++++++++++++++------------------------
 1 file changed, 26 insertions(+), 24 deletions(-)

-- 
2.5.0

[PATCH 1/3] fetch2/npm: fix indentation

[Paul Eggleton]

No code changes, just fix to use four spaces.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 lib/bb/fetch2/npm.py | 44 ++++++++++++++++++++++----------------------
 1 file changed, 22 insertions(+), 22 deletions(-)

diff --git a/lib/bb/fetch2/npm.py b/lib/bb/fetch2/npm.py
index f99b308..59312f4 100644
--- a/lib/bb/fetch2/npm.py
+++ b/lib/bb/fetch2/npm.py
@@ -57,10 +57,10 @@ class Npm(FetchMethod):
         logger.debug(1, "NpmFetch: %s", msg)
 
     def clean(self, ud, d):
-         logger.debug(2, "Calling cleanup %s" % ud.pkgname)
-         bb.utils.remove(ud.localpath, False)
-         bb.utils.remove(ud.pkgdatadir, True)
-         bb.utils.remove(ud.fullmirror, False)
+        logger.debug(2, "Calling cleanup %s" % ud.pkgname)
+        bb.utils.remove(ud.localpath, False)
+        bb.utils.remove(ud.pkgdatadir, True)
+        bb.utils.remove(ud.fullmirror, False)
 
     def urldata_init(self, ud, d):
         """
@@ -105,32 +105,32 @@ class Npm(FetchMethod):
         runfetchcmd(command, d, quiet)
 
     def _unpackdep(self, ud, pkg, data, destdir, dldir, d):
-       file = data[pkg]['tgz']
-       logger.debug(2, "file to extract is %s" % file)
-       if file.endswith('.tgz') or file.endswith('.tar.gz') or file.endswith('.tar.Z'):
+        file = data[pkg]['tgz']
+        logger.debug(2, "file to extract is %s" % file)
+        if file.endswith('.tgz') or file.endswith('.tar.gz') or file.endswith('.tar.Z'):
             cmd = 'tar xz --strip 1 --no-same-owner -f %s/%s' % (dldir, file)
-       else:
+        else:
             bb.fatal("NPM package %s downloaded not a tarball!" % file)
 
-       # Change to subdir before executing command
-       save_cwd = os.getcwd()
-       if not os.path.exists(destdir):
-           os.makedirs(destdir)
-       os.chdir(destdir)
-       path = d.getVar('PATH', True)
-       if path:
+        # Change to subdir before executing command
+        save_cwd = os.getcwd()
+        if not os.path.exists(destdir):
+            os.makedirs(destdir)
+        os.chdir(destdir)
+        path = d.getVar('PATH', True)
+        if path:
             cmd = "PATH=\"%s\" %s" % (path, cmd)
-       bb.note("Unpacking %s to %s/" % (file, os.getcwd()))
-       ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True)
-       os.chdir(save_cwd)
+        bb.note("Unpacking %s to %s/" % (file, os.getcwd()))
+        ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True)
+        os.chdir(save_cwd)
 
-       if ret != 0:
+        if ret != 0:
             raise UnpackError("Unpack command %s failed with return value %s" % (cmd, ret), ud.url)
 
-       if 'deps' not in data[pkg]:
+        if 'deps' not in data[pkg]:
             return
-       for dep in data[pkg]['deps']:
-           self._unpackdep(ud, dep, data[pkg]['deps'], "%s/node_modules/%s" % (destdir, dep), dldir, d)
+        for dep in data[pkg]['deps']:
+            self._unpackdep(ud, dep, data[pkg]['deps'], "%s/node_modules/%s" % (destdir, dep), dldir, d)
 
 
     def unpack(self, ud, destdir, d):
-- 
2.5.0

[PATCH 2/3] fetch2/npm: handle alternative dependency syntax

[Paul Eggleton]

npm allows you to specify a dependency as a Github username+path, or
omit the version entirely. You can hit these if you don't use a
shrinkwrap file, with the result that the code later fails due to the
output of "npm view" being empty; so handle this lazily by just ignoring
this part of the dependency if it's not really a version.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 lib/bb/fetch2/npm.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/bb/fetch2/npm.py b/lib/bb/fetch2/npm.py
index 59312f4..761c2e0 100644
--- a/lib/bb/fetch2/npm.py
+++ b/lib/bb/fetch2/npm.py
@@ -144,13 +144,15 @@ class Npm(FetchMethod):
 
     def _getdependencies(self, pkg, data, version, d, ud):
         pkgfullname = pkg
-        if version:
+        if version != '*' and not '/' in version:
             pkgfullname += "@%s" % version
         logger.debug(2, "Calling getdeps on %s" % pkg)
         fetchcmd = "npm view %s dist.tarball --registry %s" % (pkgfullname, ud.registry)
         output = runfetchcmd(fetchcmd, d, True)
         # npm may resolve multiple versions
         outputarray = output.strip().splitlines()
+        if not outputarray:
+            raise FetchError("The command '%s' returned no output" % fetchcmd)
         # we just take the latest version npm resolved
         #logger.debug(2, "Output URL is %s - %s - %s" % (ud.basepath, ud.basename, ud.localfile))
         outputurl = outputarray[len(outputarray)-1].rstrip()
-- 
2.5.0

[PATCH 3/3] fetch2/npm: ignore unknown headers in tarballs

[Paul Eggleton]

Tarballs that are fetched down via npm repositories seem to often have
unknown headers. This doesn't affect our ability to extract the contents
though so we don't really care to see those warnings.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 lib/bb/fetch2/npm.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/bb/fetch2/npm.py b/lib/bb/fetch2/npm.py
index 761c2e0..457043f 100644
--- a/lib/bb/fetch2/npm.py
+++ b/lib/bb/fetch2/npm.py
@@ -108,7 +108,7 @@ class Npm(FetchMethod):
         file = data[pkg]['tgz']
         logger.debug(2, "file to extract is %s" % file)
         if file.endswith('.tgz') or file.endswith('.tar.gz') or file.endswith('.tar.Z'):
-            cmd = 'tar xz --strip 1 --no-same-owner -f %s/%s' % (dldir, file)
+            cmd = 'tar xz --strip 1 --no-same-owner --warning=no-unknown-keyword -f %s/%s' % (dldir, file)
         else:
             bb.fatal("NPM package %s downloaded not a tarball!" % file)
 
-- 
2.5.0