[PATCH v2 0/3] Preserve extended attributes in sstate objects

[PATCH v2 0/3] Preserve extended attributes in sstate objects

[Joshua Lock]

This small series is part of a larger, on-going, effort to better support  extended attributes (xattr).

The goal of sending these three patches before the whole is complete is to improve support for distros with features that rely on xattr that wish to use meta-swupd.

meta-swupd creates sstate objects of update artefacts so that an OS update delta can be generated against a previous OS release without having to have the full build history in TMPDIR — this is especially useful for CI workflows.
Without these changes sstate objects don't preserve xattr and thus swupd updates artefacts are incorrect/incomplete.

Changes since v1:
* Drop extra addition of tar to buildtools tarball, it's already included
* Add two tests for oe.path.copytree(); one to ensure files with spaces and special characters are copied and one to ensure extended attributes are preserved.

Regards,

Joshua

The following changes since commit 384cf92ca9c3e66763c2c1ff2776c53d47ae25d6:

  init-install: Fixes the install script failing when not finding any mmcblk devices (2016-08-30 07:57:46 +0100)

are available in the git repository at:

  git://git.openembedded.org/openembedded-core-contrib joshuagl/xattr
  http://cgit.openembedded.org/cgit.cgi/openembedded-core-contrib/log/?h=joshuagl/xattr

Joshua Lock (3):
  oeqa.selftest: add a test for oe.path.copytree()
  oe.path: preserve xattr in copytree() and copyhardlinktree()
  oeqa.selftest.liboe: add test for xattr in copytree

 meta/lib/oe/path.py             | 11 +++++--
 meta/lib/oeqa/selftest/liboe.py | 63 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 71 insertions(+), 3 deletions(-)
 create mode 100644 meta/lib/oeqa/selftest/liboe.py

-- 
2.7.4

[PATCH v2 1/3] oeqa.selftest: add a test for oe.path.copytree()

[Joshua Lock]

One motivation for the use of cpio in oe.path.copytree() was to
ensure that files with spaces in their names were copied. Add a new
unittest module to test the OE module with a test case for copytree
with a spaces in a filename.

Signed-off-by: Joshua Lock <joshua.g.lock@intel.com>
---
 meta/lib/oeqa/selftest/liboe.py | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)
 create mode 100644 meta/lib/oeqa/selftest/liboe.py

diff --git a/meta/lib/oeqa/selftest/liboe.py b/meta/lib/oeqa/selftest/liboe.py
new file mode 100644
index 0000000..454b92a
--- /dev/null
+++ b/meta/lib/oeqa/selftest/liboe.py
@@ -0,0 +1,33 @@
+from oeqa.selftest.base import oeSelfTest
+from oeqa.utils.commands import get_bb_var
+import oe.path
+import glob
+import os
+import os.path
+
+class LibOE(oeSelfTest):
+    def test_copy_tree_special(self):
+        """
+        Summary:    oe.path.copytree() should copy files with special character
+        Expected:   'test file with sp£c!al @nd spaces' should exist in
+                    copy destination
+        Product:    OE-Core
+        Author:     Joshua Lock <joshua.g.lock@intel.com>
+        """
+        tmp_dir = get_bb_var('TMPDIR')
+        testloc = oe.path.join(tmp_dir, 'liboetests')
+        src = oe.path.join(testloc, 'src')
+        dst = oe.path.join(testloc, 'dst')
+        bb.utils.mkdirhier(testloc)
+        bb.utils.mkdirhier(src)
+        testfilename = 'test file with sp£c!al @nd spaces'
+
+        # create the test file and copy it
+        open(oe.path.join(src, testfilename), 'w+b').close()
+        oe.path.copytree(src, dst)
+
+        # ensure path exists in dest
+        fileindst = os.path.isfile(oe.path.join(dst, testfilename))
+        self.assertTrue(fileindst, "File with spaces doesn't exist in dst")
+
+        oe.path.remove(testloc)
-- 
2.7.4

[PATCH v2 2/3] oe.path: preserve xattr in copytree() and copyhardlinktree()

[Joshua Lock]

Pass appropriate options to tar invocations in copytree() and
copyhardlinktree() to ensure that any extended attributes on the files
are preserved during the copy.

We have to drop the use cpio in "Copy-pass" mode in copyhardlinktree()
because cpio doesn't support extended attributes on files. Instead we
revert back to using cp with different patterns depending on whether
or not the directory contains dot files.

Signed-off-by: Joshua Lock <joshua.g.lock@intel.com>
---
 meta/lib/oe/path.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/meta/lib/oe/path.py b/meta/lib/oe/path.py
index 3c07df3..631c3b4 100644
--- a/meta/lib/oe/path.py
+++ b/meta/lib/oe/path.py
@@ -65,7 +65,7 @@ def copytree(src, dst):
     # This way we also preserve hardlinks between files in the tree.
 
     bb.utils.mkdirhier(dst)
-    cmd = 'tar -cf - -C %s -p . | tar -xf - -C %s' % (src, dst)
+    cmd = "tar --xattrs --xattrs-include='*' -cf - -C %s -p . | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dst)
     subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
 
 def copyhardlinktree(src, dst):
@@ -77,9 +77,14 @@ def copyhardlinktree(src, dst):
     if (os.stat(src).st_dev ==  os.stat(dst).st_dev):
         # Need to copy directories only with tar first since cp will error if two 
         # writers try and create a directory at the same time
-        cmd = 'cd %s; find . -type d -print | tar -cf - -C %s -p --no-recursion --files-from - | tar -xf - -C %s' % (src, src, dst)
+        cmd = "cd %s; find . -type d -print | tar --xattrs --xattrs-include='*' -cf - -C %s -p --no-recursion --files-from - | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, src, dst)
         subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
-        cmd = 'cd %s; find . -print0 | cpio --null -pdlu %s' % (src, dst)
+        if os.path.isdir(src):
+            import glob
+            if len(glob.glob('%s/.??*' % src)) > 0:
+                src = src + '/.??* '
+            src = src + '/*'
+        cmd = 'cp -afl --preserve=xattr %s %s' % (src, dst)
         subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
     else:
         copytree(src, dst)
-- 
2.7.4

Re: [PATCH v2 2/3] oe.path: preserve xattr in copytree() and copyhardlinktree()

[Mark Hatle]

On 8/30/16 9:05 PM, Joshua Lock wrote:
> Pass appropriate options to tar invocations in copytree() and
> copyhardlinktree() to ensure that any extended attributes on the files
> are preserved during the copy.
> 
> We have to drop the use cpio in "Copy-pass" mode in copyhardlinktree()
> because cpio doesn't support extended attributes on files. Instead we
> revert back to using cp with different patterns depending on whether
> or not the directory contains dot files.
> 
> Signed-off-by: Joshua Lock <joshua.g.lock@intel.com>
> ---
>  meta/lib/oe/path.py | 11 ++++++++---
>  1 file changed, 8 insertions(+), 3 deletions(-)
> 
> diff --git a/meta/lib/oe/path.py b/meta/lib/oe/path.py
> index 3c07df3..631c3b4 100644
> --- a/meta/lib/oe/path.py
> +++ b/meta/lib/oe/path.py
> @@ -65,7 +65,7 @@ def copytree(src, dst):
>      # This way we also preserve hardlinks between files in the tree.
>  
>      bb.utils.mkdirhier(dst)
> -    cmd = 'tar -cf - -C %s -p . | tar -xf - -C %s' % (src, dst)
> +    cmd = "tar --xattrs --xattrs-include='*' -cf - -C %s -p . | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dst)
>      subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
>  
>  def copyhardlinktree(src, dst):
> @@ -77,9 +77,14 @@ def copyhardlinktree(src, dst):
>      if (os.stat(src).st_dev ==  os.stat(dst).st_dev):
>          # Need to copy directories only with tar first since cp will error if two 
>          # writers try and create a directory at the same time
> -        cmd = 'cd %s; find . -type d -print | tar -cf - -C %s -p --no-recursion --files-from - | tar -xf - -C %s' % (src, src, dst)
> +        cmd = "cd %s; find . -type d -print | tar --xattrs --xattrs-include='*' -cf - -C %s -p --no-recursion --files-from - | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, src, dst)
>          subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
> -        cmd = 'cd %s; find . -print0 | cpio --null -pdlu %s' % (src, dst)
> +        if os.path.isdir(src):
> +            import glob
> +            if len(glob.glob('%s/.??*' % src)) > 0:
> +                src = src + '/.??* '
> +            src = src + '/*'
> +        cmd = 'cp -afl --preserve=xattr %s %s' % (src, dst)

'cp -a' is a gnu-ism.  I'm not sure if this will cause any problems, but it has
in the past when people have used non coreutils 'cp'.

(I'm not sure if we're using 'cp -a' anywhere, but I know most of the time we
try to use the discrete set of arguments instead as they're more POSIX.)

(Otherwise the change looks good to me.)

>          subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
>      else:
>          copytree(src, dst)
> 

Re: [PATCH v2 2/3] oe.path: preserve xattr in copytree() and copyhardlinktree()

[Richard Purdie]

On Wed, 2016-08-31 at 08:41 +0800, Mark Hatle wrote:
> On 8/30/16 9:05 PM, Joshua Lock wrote:
> > Pass appropriate options to tar invocations in copytree() and
> > copyhardlinktree() to ensure that any extended attributes on the
> > files
> > are preserved during the copy.
> > 
> > We have to drop the use cpio in "Copy-pass" mode in
> > copyhardlinktree()
> > because cpio doesn't support extended attributes on files. Instead
> > we
> > revert back to using cp with different patterns depending on
> > whether
> > or not the directory contains dot files.
> > 
> > Signed-off-by: Joshua Lock <joshua.g.lock@intel.com>
> > ---
> >  meta/lib/oe/path.py | 11 ++++++++---
> >  1 file changed, 8 insertions(+), 3 deletions(-)
> > 
> > diff --git a/meta/lib/oe/path.py b/meta/lib/oe/path.py
> > index 3c07df3..631c3b4 100644
> > --- a/meta/lib/oe/path.py
> > +++ b/meta/lib/oe/path.py
> > @@ -65,7 +65,7 @@ def copytree(src, dst):
> >      # This way we also preserve hardlinks between files in the
> > tree.
> >  
> >      bb.utils.mkdirhier(dst)
> > -    cmd = 'tar -cf - -C %s -p . | tar -xf - -C %s' % (src, dst)
> > +    cmd = "tar --xattrs --xattrs-include='*' -cf - -C %s -p . |
> > tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dst)
> >      subprocess.check_output(cmd, shell=True,
> > stderr=subprocess.STDOUT)
> >  
> >  def copyhardlinktree(src, dst):
> > @@ -77,9 +77,14 @@ def copyhardlinktree(src, dst):
> >      if (os.stat(src).st_dev ==  os.stat(dst).st_dev):
> >          # Need to copy directories only with tar first since cp
> > will error if two 
> >          # writers try and create a directory at the same time
> > -        cmd = 'cd %s; find . -type d -print | tar -cf - -C %s -p -
> > -no-recursion --files-from - | tar -xf - -C %s' % (src, src, dst)
> > +        cmd = "cd %s; find . -type d -print | tar --xattrs -
> > -xattrs-include='*' -cf - -C %s -p --no-recursion --files-from - |
> > tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, src, dst)
> >          subprocess.check_output(cmd, shell=True,
> > stderr=subprocess.STDOUT)
> > -        cmd = 'cd %s; find . -print0 | cpio --null -pdlu %s' %
> > (src, dst)
> > +        if os.path.isdir(src):
> > +            import glob
> > +            if len(glob.glob('%s/.??*' % src)) > 0:
> > +                src = src + '/.??* '
> > +            src = src + '/*'
> > +        cmd = 'cp -afl --preserve=xattr %s %s' % (src, dst)
> 
> 'cp -a' is a gnu-ism.  I'm not sure if this will cause any problems,
> but it has
> in the past when people have used non coreutils 'cp'.
> 
> (I'm not sure if we're using 'cp -a' anywhere, but I know most of the
> time we
> try to use the discrete set of arguments instead as they're more
> POSIX.)

We have a number of cp -a calls around. I'm not against fixing that but
for now I'm going to take the patch as I have bigger problems to deal
with.

Cheers,

Richard

[PATCH v2 3/3] oeqa.selftest.liboe: add test for xattr in copytree

[Joshua Lock]

Add a test to ensure that oe.path.copytree() preserves extended
attributes on files.

Signed-off-by: Joshua Lock <joshua.g.lock@intel.com>
---
 meta/lib/oeqa/selftest/liboe.py | 32 +++++++++++++++++++++++++++++++-
 1 file changed, 31 insertions(+), 1 deletion(-)

diff --git a/meta/lib/oeqa/selftest/liboe.py b/meta/lib/oeqa/selftest/liboe.py
index 454b92a..2aeab61 100644
--- a/meta/lib/oeqa/selftest/liboe.py
+++ b/meta/lib/oeqa/selftest/liboe.py
@@ -1,5 +1,5 @@
 from oeqa.selftest.base import oeSelfTest
-from oeqa.utils.commands import get_bb_var
+from oeqa.utils.commands import get_bb_var, bitbake, runCmd
 import oe.path
 import glob
 import os
@@ -31,3 +31,33 @@ class LibOE(oeSelfTest):
         self.assertTrue(fileindst, "File with spaces doesn't exist in dst")
 
         oe.path.remove(testloc)
+
+    def test_copy_tree_xattr(self):
+        """
+        Summary:    oe.path.copytree() should preserve xattr on copied files
+        Expected:   testxattr file in destination should have user.oetest
+                    extended attribute
+        Product:    OE-Core
+        Author:     Joshua Lock <joshua.g.lock@intel.com>
+        """
+        tmp_dir = get_bb_var('TMPDIR')
+        testloc = oe.path.join(tmp_dir, 'liboetests')
+        src = oe.path.join(testloc, 'src')
+        dst = oe.path.join(testloc, 'dst')
+        bb.utils.mkdirhier(testloc)
+        bb.utils.mkdirhier(src)
+        testfilename = 'testxattr'
+
+        # ensure we have setfattr available
+        bitbake("attr-native")
+        
+        # create a file with xattr and copy it
+        open(oe.path.join(src, testfilename), 'w+b').close()
+        runCmd('setfattr -n user.oetest -v "testing liboe" %s' % oe.path.join(src, testfilename))
+        oe.path.copytree(src, dst)
+        
+        # ensure file in dest has user.oetest xattr
+        result = runCmd('getfattr -n user.oetest %s' % oe.path.join(dst, testfilename))
+        self.assertIn('user.oetest="testing liboe"', result.output, 'Extended attribute not sert in dst')
+
+        oe.path.remove(testloc)
-- 
2.7.4