All of lore.kernel.org
 help / color / mirror / Atom feed
* [PATCH v2] bb/fetch2: fixes copying of file://dir; subdir=foo, bug 6128 and bug 6129
@ 2016-03-09 16:15 Alexander Shashkevich
  2016-03-09 16:54 ` Sasha Shashkevich
  0 siblings, 1 reply; 4+ messages in thread
From: Alexander Shashkevich @ 2016-03-09 16:15 UTC (permalink / raw)
  To: bitbake-devel

When in SRC_URI appears file://dir;subdir=foo unpacker copies 'dir' to ${WORKDIR}, not
${WORKDIR}/foo as it should be.

These changes are fixing following bugs as well:
Bug 6128 - Incorrect wildcard unpack behaviour in fetcher
Bug 6129 - Local directories unpack to a different location than local files

Signed-off-by: Alexander Shashkevich <alex@stunpix.com>
---
 bitbake/lib/bb/fetch2/__init__.py                  | 61 +++++++++-------------
 bitbake/lib/bb/tests/fetch.py                      | 36 ++++++++++---
 .../qemu/nativesdk-qemu-helper_1.0.bb              | 22 ++++----
 3 files changed, 63 insertions(+), 56 deletions(-)

diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index a9c044b..8b59598 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -1355,6 +1355,11 @@ class FetchMethod(object):
         iterate = False
         file = urldata.localpath
 
+        # Localpath can't deal with 'dir/*' entries, so it converts them to '.',
+        # but it must be corrected for local files copying
+        if urldata.basename == '*' and file.endswith('/.'):
+            file = '%s/%s' % (file.rstrip('/.'), urldata.path)
+
         try:
             unpack = bb.utils.to_boolean(urldata.parm.get('unpack'), True)
         except ValueError as exc:
@@ -1408,50 +1413,32 @@ class FetchMethod(object):
             elif file.endswith('.deb') or file.endswith('.ipk'):
                 cmd = 'ar -p %s data.tar.gz | zcat | tar --no-same-owner -xpf -' % file
 
+        # If 'subdir' param exists, create a dir and use it as destination for unpack cmd
+        if 'subdir' in urldata.parm:
+            unpackdir = '%s/%s' % (rootdir, urldata.parm.get('subdir'))
+            bb.utils.mkdirhier(unpackdir)
+        else:
+            unpackdir = rootdir
+
         if not unpack or not cmd:
             # If file == dest, then avoid any copies, as we already put the file into dest!
-            dest = os.path.join(rootdir, os.path.basename(file))
-            if (file != dest) and not (os.path.exists(dest) and os.path.samefile(file, dest)):
-                if os.path.isdir(file):
-                    # If for example we're asked to copy file://foo/bar, we need to unpack the result into foo/bar
-                    basepath = getattr(urldata, "basepath", None)
-                    destdir = "."
-                    if basepath and basepath.endswith("/"):
-                        basepath = basepath.rstrip("/")
-                    elif basepath:
-                        basepath = os.path.dirname(basepath)
-                    if basepath and basepath.find("/") != -1:
-                        destdir = basepath[:basepath.rfind('/')]
-                        destdir = destdir.strip('/')
-                    if destdir != "." and not os.access("%s/%s" % (rootdir, destdir), os.F_OK):
-                        os.makedirs("%s/%s" % (rootdir, destdir))
-                    cmd = 'cp -fpPR %s %s/%s/' % (file, rootdir, destdir)
-                    #cmd = 'tar -cf - -C "%d" -ps . | tar -xf - -C "%s/%s/"' % (file, rootdir, destdir)
-                else:
-                    # The "destdir" handling was specifically done for FILESPATH
-                    # items.  So, only do so for file:// entries.
-                    if urldata.type == "file" and urldata.path.find("/") != -1:
-                       destdir = urldata.path.rsplit("/", 1)[0]
-                       if urldata.parm.get('subdir') != None:
-                          destdir = urldata.parm.get('subdir') + "/" + destdir
-                    else:
-                       if urldata.parm.get('subdir') != None:
-                          destdir = urldata.parm.get('subdir')
-                       else:
-                          destdir = "."
-                    bb.utils.mkdirhier("%s/%s" % (rootdir, destdir))
-                    cmd = 'cp -f %s %s/%s/' % (file, rootdir, destdir)
+            dest = os.path.join(unpackdir, os.path.basename(file))
+            if file != dest and not (os.path.exists(dest) and os.path.samefile(file, dest)):
+                destdir = '.'
+                # For file:// entries all intermediate dirs in path must be created at destination
+                if urldata.type == "file" and not urldata.path.startswith("/"):
+                    urlpath = urldata.path.rstrip('/') # Trailing '/' does a copying to wrong place
+                    if urlpath.find("/") != -1:
+                        destdir = urlpath.rsplit("/", 1)[0] + '/'
+                        bb.utils.mkdirhier("%s/%s" % (unpackdir, destdir))
+                cmd = 'cp -fpPR %s %s' % (file, destdir)
 
         if not cmd:
             return
 
-        # Change to subdir before executing command
+        # Change to unpackdir before executing command
         save_cwd = os.getcwd();
-        os.chdir(rootdir)
-        if 'subdir' in urldata.parm:
-            newdir = ("%s/%s" % (rootdir, urldata.parm.get('subdir')))
-            bb.utils.mkdirhier(newdir)
-            os.chdir(newdir)
+        os.chdir(unpackdir)
 
         path = data.getVar('PATH', True)
         if path:
diff --git a/bitbake/lib/bb/tests/fetch.py b/bitbake/lib/bb/tests/fetch.py
index 94173c1..b3de37e 100644
--- a/bitbake/lib/bb/tests/fetch.py
+++ b/bitbake/lib/bb/tests/fetch.py
@@ -451,9 +451,7 @@ class FetcherLocalTest(FetcherTest):
 
     def test_local_wildcard(self):
         tree = self.fetchUnpack(['file://a', 'file://dir/*'])
-        # FIXME: this is broken - it should return ['a', 'dir/c', 'dir/d', 'dir/subdir/e']
-        # see https://bugzilla.yoctoproject.org/show_bug.cgi?id=6128
-        self.assertEqual(tree, ['a', 'b', 'dir/c', 'dir/d', 'dir/subdir/e'])
+        self.assertEqual(tree, ['a',  'dir/c', 'dir/d', 'dir/subdir/e'])
 
     def test_local_dir(self):
         tree = self.fetchUnpack(['file://a', 'file://dir'])
@@ -461,22 +459,44 @@ class FetcherLocalTest(FetcherTest):
 
     def test_local_subdir(self):
         tree = self.fetchUnpack(['file://dir/subdir'])
-        # FIXME: this is broken - it should return ['dir/subdir/e']
-        # see https://bugzilla.yoctoproject.org/show_bug.cgi?id=6129
-        self.assertEqual(tree, ['subdir/e'])
+        self.assertEqual(tree, ['dir/subdir/e'])
 
     def test_local_subdir_file(self):
         tree = self.fetchUnpack(['file://dir/subdir/e'])
         self.assertEqual(tree, ['dir/subdir/e'])
 
     def test_local_subdirparam(self):
-        tree = self.fetchUnpack(['file://a;subdir=bar'])
-        self.assertEqual(tree, ['bar/a'])
+        tree = self.fetchUnpack(['file://a;subdir=bar', 'file://dir;subdir=foo/moo'])
+        self.assertEqual(tree, ['bar/a', 'foo/moo/dir/c', 'foo/moo/dir/d', 'foo/moo/dir/subdir/e'])
 
     def test_local_deepsubdirparam(self):
         tree = self.fetchUnpack(['file://dir/subdir/e;subdir=bar'])
         self.assertEqual(tree, ['bar/dir/subdir/e'])
 
+    def test_local_abspathfiles(self):
+        abspath = 'file://%s' % self.localsrcdir
+        tree = self.fetchUnpack([abspath + '/a', abspath + '/dir/c'])
+        self.assertEqual(tree, ['a', 'c'])
+
+    def test_local_abspathdirs(self):
+        abspath = 'file://%s' % self.localsrcdir
+        tree = self.fetchUnpack([abspath + '/dir/subdir'])
+        self.assertEqual(tree, ['subdir/e'])
+
+    def test_local_abspathsubdirparam(self):
+        abspath = 'file://%s' % self.localsrcdir
+        tree = self.fetchUnpack([abspath + '/a;subdir=foo', abspath + '/dir/subdir;subdir=bar/foo'])
+        self.assertEqual(tree, ['bar/foo/subdir/e', 'foo/a'])
+
+    def test_local_trailingslash(self):
+        tree = self.fetchUnpack(['file://dir/subdir/', 'file://dir/subdir/;subdir=bar'])
+        self.assertEqual(tree, ['bar/dir/subdir/e', 'dir/subdir/e'])
+
+    def test_local_abspathtrailingslash(self):
+        abspath = 'file://%s' % self.localsrcdir
+        tree = self.fetchUnpack([abspath + '/dir/subdir/', abspath + '/dir/subdir/;subdir=bar'])
+        self.assertEqual(tree, ['bar/subdir/e', 'subdir/e'])
+
 class FetcherNetworkTest(FetcherTest):
 
     if os.environ.get("BB_SKIP_NETTESTS") == "yes":
diff --git a/meta/recipes-devtools/qemu/nativesdk-qemu-helper_1.0.bb b/meta/recipes-devtools/qemu/nativesdk-qemu-helper_1.0.bb
index 51d1c59..dc924f0 100644
--- a/meta/recipes-devtools/qemu/nativesdk-qemu-helper_1.0.bb
+++ b/meta/recipes-devtools/qemu/nativesdk-qemu-helper_1.0.bb
@@ -8,15 +8,15 @@ LIC_FILES_CHKSUM = "file://${WORKDIR}/tunctl.c;endline=4;md5=ff3a09996bc5fff6bc5
                     file://${COREBASE}/scripts/runqemu;endline=18;md5=77fbe442a88b1bcdc29c3ba67733b21b"
 
 
-SRC_URI = "file://${COREBASE}/scripts/runqemu \
-           file://${COREBASE}/scripts/runqemu-internal \
-           file://${COREBASE}/scripts/runqemu-addptable2image \
-           file://${COREBASE}/scripts/runqemu-gen-tapdevs \
-           file://${COREBASE}/scripts/runqemu-ifup \
-           file://${COREBASE}/scripts/runqemu-ifdown \
-           file://${COREBASE}/scripts/oe-find-native-sysroot \
-           file://${COREBASE}/scripts/runqemu-extract-sdk \
-           file://${COREBASE}/scripts/runqemu-export-rootfs \
+SRC_URI = "file://${COREBASE}/scripts/runqemu;subdir=scripts \
+           file://${COREBASE}/scripts/runqemu-internal;subdir=scripts \
+           file://${COREBASE}/scripts/runqemu-addptable2image;subdir=scripts \
+           file://${COREBASE}/scripts/runqemu-gen-tapdevs;subdir=scripts \
+           file://${COREBASE}/scripts/runqemu-ifup;subdir=scripts \
+           file://${COREBASE}/scripts/runqemu-ifdown;subdir=scripts \
+           file://${COREBASE}/scripts/oe-find-native-sysroot;subdir=scripts \
+           file://${COREBASE}/scripts/runqemu-extract-sdk;subdir=scripts \
+           file://${COREBASE}/scripts/runqemu-export-rootfs;subdir=scripts \
            file://tunctl.c \
            file://raw2flash.c \
           "
@@ -33,8 +33,8 @@ do_compile() {
 
 do_install() {
 	install -d ${D}${bindir}
-	install -m 0755 ${WORKDIR}${COREBASE}/scripts/oe-* ${D}${bindir}/
-	install -m 0755 ${WORKDIR}${COREBASE}/scripts/runqemu* ${D}${bindir}/
+	install -m 0755 ${WORKDIR}/scripts/oe-* ${D}${bindir}/
+	install -m 0755 ${WORKDIR}/scripts/runqemu* ${D}${bindir}/
 	install tunctl ${D}${bindir}/
 	install raw2flash.spitz ${D}${bindir}/
 	install flash2raw.spitz ${D}${bindir}/
-- 
2.5.0



^ permalink raw reply related	[flat|nested] 4+ messages in thread
end of thread, other threads:[~2016-03-10 14:40 UTC | newest]

Thread overview: 4+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2016-03-09 16:15 [PATCH v2] bb/fetch2: fixes copying of file://dir; subdir=foo, bug 6128 and bug 6129 Alexander Shashkevich
2016-03-09 16:54 ` Sasha Shashkevich
2016-03-09 22:42   ` Richard Purdie
2016-03-10 14:30     ` Sasha Shashkevich

This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.