[PATCH v3 0/3] Preserve extended attributes in sstate objects

[PATCH v3 0/3] Preserve extended attributes in sstate objects

[Joshua Lock]

This small series is part of a larger, on-going, effort to better support  extended attributes (xattr).

The goal of sending these three patches before the whole is complete is to improve support for distros with features that rely on xattr that wish to use meta-swupd.

meta-swupd creates sstate objects of update artefacts so that an OS update delta can be generated against a previous OS release without having to have the full build history in TMPDIR — this is especially useful for CI workflows.
Without these changes sstate objects don't preserve xattr and thus swupd updates artefacts are incorrect/incomplete.

Changes since v2:
* Call [s|g]etfattr with the full path in STAGING_BINDIR_NATIVE

Changes since v1:
* Drop extra addition of tar to buildtools tarball, it's already included
* Add two tests for oe.path.copytree(); one to ensure files with spaces and special characters are copied and one to ensure extended attributes are preserved.

Regards,

Joshua

The following changes since commit 384cf92ca9c3e66763c2c1ff2776c53d47ae25d6:

  init-install: Fixes the install script failing when not finding any mmcblk devices (2016-08-30 07:57:46 +0100)

are available in the git repository at:

  git://git.openembedded.org/openembedded-core-contrib joshuagl/xattr
  http://cgit.openembedded.org/cgit.cgi/openembedded-core-contrib/log/?h=joshuagl/xattr

Joshua Lock (3):
  oeqa.selftest: add a test for oe.path.copytree()
  oe.path: preserve xattr in copytree() and copyhardlinktree()
  oeqa.selftest.liboe: add test for xattr in copytree

 meta/lib/oe/path.py             | 11 +++++--
 meta/lib/oeqa/selftest/liboe.py | 64 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 72 insertions(+), 3 deletions(-)
 create mode 100644 meta/lib/oeqa/selftest/liboe.py

-- 
2.7.4

[PATCH v3 1/3] oeqa.selftest: add a test for oe.path.copytree()

[Joshua Lock]

One motivation for the use of cpio in oe.path.copytree() was to
ensure that files with spaces in their names were copied. Add a new
unittest module to test the OE module with a test case for copytree
with a spaces in a filename.

Signed-off-by: Joshua Lock <joshua.g.lock@intel.com>
---
 meta/lib/oeqa/selftest/liboe.py | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)
 create mode 100644 meta/lib/oeqa/selftest/liboe.py

diff --git a/meta/lib/oeqa/selftest/liboe.py b/meta/lib/oeqa/selftest/liboe.py
new file mode 100644
index 0000000..454b92a
--- /dev/null
+++ b/meta/lib/oeqa/selftest/liboe.py
@@ -0,0 +1,33 @@
+from oeqa.selftest.base import oeSelfTest
+from oeqa.utils.commands import get_bb_var
+import oe.path
+import glob
+import os
+import os.path
+
+class LibOE(oeSelfTest):
+    def test_copy_tree_special(self):
+        """
+        Summary:    oe.path.copytree() should copy files with special character
+        Expected:   'test file with sp£c!al @nd spaces' should exist in
+                    copy destination
+        Product:    OE-Core
+        Author:     Joshua Lock <joshua.g.lock@intel.com>
+        """
+        tmp_dir = get_bb_var('TMPDIR')
+        testloc = oe.path.join(tmp_dir, 'liboetests')
+        src = oe.path.join(testloc, 'src')
+        dst = oe.path.join(testloc, 'dst')
+        bb.utils.mkdirhier(testloc)
+        bb.utils.mkdirhier(src)
+        testfilename = 'test file with sp£c!al @nd spaces'
+
+        # create the test file and copy it
+        open(oe.path.join(src, testfilename), 'w+b').close()
+        oe.path.copytree(src, dst)
+
+        # ensure path exists in dest
+        fileindst = os.path.isfile(oe.path.join(dst, testfilename))
+        self.assertTrue(fileindst, "File with spaces doesn't exist in dst")
+
+        oe.path.remove(testloc)
-- 
2.7.4

[PATCH v3 2/3] oe.path: preserve xattr in copytree() and copyhardlinktree()

[Joshua Lock]

Pass appropriate options to tar invocations in copytree() and
copyhardlinktree() to ensure that any extended attributes on the files
are preserved during the copy.

We have to drop the use cpio in "Copy-pass" mode in copyhardlinktree()
because cpio doesn't support extended attributes on files. Instead we
revert back to using cp with different patterns depending on whether
or not the directory contains dot files.

Signed-off-by: Joshua Lock <joshua.g.lock@intel.com>
---
 meta/lib/oe/path.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/meta/lib/oe/path.py b/meta/lib/oe/path.py
index 3c07df3..631c3b4 100644
--- a/meta/lib/oe/path.py
+++ b/meta/lib/oe/path.py
@@ -65,7 +65,7 @@ def copytree(src, dst):
     # This way we also preserve hardlinks between files in the tree.
 
     bb.utils.mkdirhier(dst)
-    cmd = 'tar -cf - -C %s -p . | tar -xf - -C %s' % (src, dst)
+    cmd = "tar --xattrs --xattrs-include='*' -cf - -C %s -p . | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dst)
     subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
 
 def copyhardlinktree(src, dst):
@@ -77,9 +77,14 @@ def copyhardlinktree(src, dst):
     if (os.stat(src).st_dev ==  os.stat(dst).st_dev):
         # Need to copy directories only with tar first since cp will error if two 
         # writers try and create a directory at the same time
-        cmd = 'cd %s; find . -type d -print | tar -cf - -C %s -p --no-recursion --files-from - | tar -xf - -C %s' % (src, src, dst)
+        cmd = "cd %s; find . -type d -print | tar --xattrs --xattrs-include='*' -cf - -C %s -p --no-recursion --files-from - | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, src, dst)
         subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
-        cmd = 'cd %s; find . -print0 | cpio --null -pdlu %s' % (src, dst)
+        if os.path.isdir(src):
+            import glob
+            if len(glob.glob('%s/.??*' % src)) > 0:
+                src = src + '/.??* '
+            src = src + '/*'
+        cmd = 'cp -afl --preserve=xattr %s %s' % (src, dst)
         subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
     else:
         copytree(src, dst)
-- 
2.7.4

[PATCH v3 3/3] oeqa.selftest.liboe: add test for xattr in copytree

[Joshua Lock]

Add a test to ensure that oe.path.copytree() preserves extended
attributes on files.

Signed-off-by: Joshua Lock <joshua.g.lock@intel.com>
---
 meta/lib/oeqa/selftest/liboe.py | 33 ++++++++++++++++++++++++++++++++-
 1 file changed, 32 insertions(+), 1 deletion(-)

diff --git a/meta/lib/oeqa/selftest/liboe.py b/meta/lib/oeqa/selftest/liboe.py
index 454b92a..5c93069 100644
--- a/meta/lib/oeqa/selftest/liboe.py
+++ b/meta/lib/oeqa/selftest/liboe.py
@@ -1,5 +1,5 @@
 from oeqa.selftest.base import oeSelfTest
-from oeqa.utils.commands import get_bb_var
+from oeqa.utils.commands import get_bb_var, bitbake, runCmd
 import oe.path
 import glob
 import os
@@ -31,3 +31,34 @@ class LibOE(oeSelfTest):
         self.assertTrue(fileindst, "File with spaces doesn't exist in dst")
 
         oe.path.remove(testloc)
+
+    def test_copy_tree_xattr(self):
+        """
+        Summary:    oe.path.copytree() should preserve xattr on copied files
+        Expected:   testxattr file in destination should have user.oetest
+                    extended attribute
+        Product:    OE-Core
+        Author:     Joshua Lock <joshua.g.lock@intel.com>
+        """
+        tmp_dir = get_bb_var('TMPDIR')
+        testloc = oe.path.join(tmp_dir, 'liboetests')
+        src = oe.path.join(testloc, 'src')
+        dst = oe.path.join(testloc, 'dst')
+        bb.utils.mkdirhier(testloc)
+        bb.utils.mkdirhier(src)
+        testfilename = 'testxattr'
+
+        # ensure we have setfattr available
+        bitbake("attr-native")
+        bindir = get_bb_var('STAGING_BINDIR_NATIVE')
+
+        # create a file with xattr and copy it
+        open(oe.path.join(src, testfilename), 'w+b').close()
+        runCmd('%s/setfattr -n user.oetest -v "testing liboe" %s' % (bindir, oe.path.join(src, testfilename)))
+        oe.path.copytree(src, dst)
+
+        # ensure file in dest has user.oetest xattr
+        result = runCmd('%s/getfattr -n user.oetest %s' % (bindir, oe.path.join(dst, testfilename)))
+        self.assertIn('user.oetest="testing liboe"', result.output, 'Extended attribute not sert in dst')
+
+        oe.path.remove(testloc)
-- 
2.7.4