[PATCH v4 0/2] devtool deploy-target --strip option

[PATCH v4 0/2] devtool deploy-target --strip option

[Tobias Hagelborn]

New devtool deploy-target option --strip which enables deploying
stripped binaries, saving some space on target.

Updates in v4:
- Updated to correct the errors found by Autobilder.
- Verified devtool-deploy-target with poky-master

[YOCTO #11227]


Tobias Hagelborn (2):
  package.py: strip_execs: Support for .ko modules
  devtool: deploy-target: Support stripped libs and execs

 meta/lib/oe/package.py        | 44 ++++++++++++++++++++++++-------------------
 scripts/lib/devtool/deploy.py | 33 ++++++++++++++++++++++++++++----
 2 files changed, 54 insertions(+), 23 deletions(-)

-- 
2.1.4

[PATCH v4 1/2] package.py: strip_execs: Support for .ko modules

[Tobias Hagelborn]

* Support stripping of .ko modules verifying file extension and
  check of content "vermagic="
* Minor refactoring (removing lint errors)

Signed-off-by: Tobias Hagelborn <tobiasha@axis.com>
---
 meta/lib/oe/package.py | 44 +++++++++++++++++++++++++-------------------
 1 file changed, 25 insertions(+), 19 deletions(-)

diff --git a/meta/lib/oe/package.py b/meta/lib/oe/package.py
index a79c668..4daee59 100644
--- a/meta/lib/oe/package.py
+++ b/meta/lib/oe/package.py
@@ -56,9 +56,12 @@ def strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, qa_already_stripped=
     :param qa_already_stripped: Set to True if already-stripped' in ${INSANE_SKIP}
     This is for proper logging and messages only.
     """
-    import stat, errno, oe.path, oe.utils
+    import stat, errno, oe.path, oe.utils, mmap
 
-    os.chdir(dstdir)
+    # Detect .ko module by searching for "vermagic=" string
+    def is_kernel_module(path):
+        with open(path) as f:
+            return mmap.mmap(f.fileno(), 0, prot=mmap.PROT_READ).find(b"vermagic=") >= 0
 
     # Return type (bits):
     # 0 - not elf
@@ -67,31 +70,33 @@ def strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, qa_already_stripped=
     # 4 - executable
     # 8 - shared library
     # 16 - kernel module
-    def isELF(path):
-        type = 0
-        ret, result = oe.utils.getstatusoutput("file \"%s\"" % path.replace("\"", "\\\""))
+    def is_elf(path):
+        exec_type = 0
+        ret, result = oe.utils.getstatusoutput(
+            "file \"%s\"" % path.replace("\"", "\\\""))
 
         if ret:
             bb.error("split_and_strip_files: 'file %s' failed" % path)
-            return type
+            return exec_type
 
         # Not stripped
         if "ELF" in result:
-            type |= 1
+            exec_type |= 1
             if "not stripped" not in result:
-                type |= 2
+                exec_type |= 2
             if "executable" in result:
-                type |= 4
+                exec_type |= 4
             if "shared" in result:
-                type |= 8
-        return type
-
+                exec_type |= 8
+            if "relocatable" in result and is_kernel_module(path):
+                exec_type |= 16
+        return exec_type
 
     elffiles = {}
     inodes = {}
-    libdir = os.path.abspath(dstdir + os.sep + libdir)
-    base_libdir = os.path.abspath(dstdir + os.sep + base_libdir)
-
+    libdir = os.path.abspath(os.path.join(dstdir, libdir.lstrip(os.sep)))
+    base_libdir = os.path.abspath(os.path.join(dstdir, base_libdir.lstrip(os.sep)))
+    exec_mask = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
     #
     # First lets figure out all of the files we may have to process
     #
@@ -111,15 +116,16 @@ def strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, qa_already_stripped=
             if not s:
                 continue
             # Check its an excutable
-            if (s[stat.ST_MODE] & stat.S_IXUSR) or (s[stat.ST_MODE] & stat.S_IXGRP) or (s[stat.ST_MODE] & stat.S_IXOTH) \
-                    or ((file.startswith(libdir) or file.startswith(base_libdir)) and ".so" in f):
+            if s[stat.ST_MODE] & exec_mask \
+                    or ((file.startswith(libdir) or file.startswith(base_libdir)) and ".so" in f) \
+                    or file.endswith('.ko'):
                 # If it's a symlink, and points to an ELF file, we capture the readlink target
                 if os.path.islink(file):
                     continue
 
                 # It's a file (or hardlink), not a link
                 # ...but is it ELF, and is it already stripped?
-                elf_file = isELF(file)
+                elf_file = is_elf(file)
                 if elf_file & 1:
                     if elf_file & 2:
                         if qa_already_stripped:
@@ -132,6 +138,7 @@ def strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, qa_already_stripped=
                         os.unlink(file)
                         os.link(inodes[s.st_ino], file)
                     else:
+                        # break hardlinks so that we do not strip the original.
                         inodes[s.st_ino] = file
                         # break hardlink
                         bb.utils.copyfile(file, file)
@@ -143,7 +150,6 @@ def strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, qa_already_stripped=
     sfiles = []
     for file in elffiles:
         elf_file = int(elffiles[file])
-        #bb.note("Strip %s" % file)
         sfiles.append((file, elf_file, strip_cmd))
 
     oe.utils.multiprocess_exec(sfiles, runstrip)
-- 
2.1.4

[PATCH v4 2/2] devtool: deploy-target: Support stripped libs and execs

[Tobias Hagelborn]

New devtool deploy-target option --strip which enables deploying
stripped binaries, saving some space on target.

* Copies the files of ${D} into a new directory and strips them in place
* Used oe.package.strip_execs for stripping directory
* Added devtool.conf option "strip" for changing default behavior

Config example:
[Deploy]
strip = true

[YOCTO #11227]

Signed-off-by: Tobias Hagelborn <tobiasha@axis.com>
---
 scripts/lib/devtool/deploy.py | 33 +++++++++++++++++++++++++++++----
 1 file changed, 29 insertions(+), 4 deletions(-)

diff --git a/scripts/lib/devtool/deploy.py b/scripts/lib/devtool/deploy.py
index 04c34cb..7c57174 100644
--- a/scripts/lib/devtool/deploy.py
+++ b/scripts/lib/devtool/deploy.py
@@ -16,12 +16,16 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 """Devtool plugin containing the deploy subcommands"""
 
+import logging
 import os
+import shutil
 import subprocess
-import logging
 import tempfile
-import shutil
+
+import bb.utils
 import argparse_oe
+import oe.types
+
 from devtool import exec_fakeroot, setup_tinfoil, check_workspace_recipe, DevtoolError
 
 logger = logging.getLogger('devtool')
@@ -140,11 +144,12 @@ def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=Fals
     return '\n'.join(lines)
 
 
+
 def deploy(args, config, basepath, workspace):
     """Entry point for the devtool 'deploy' subcommand"""
-    import re
     import math
     import oe.recipeutils
+    import oe.package
 
     check_workspace_recipe(workspace, args.recipename, checksrc=False)
 
@@ -170,6 +175,17 @@ def deploy(args, config, basepath, workspace):
                             'recipe? If so, the install step has not installed '
                             'any files.' % args.recipename)
 
+        if args.strip and not args.dry_run:
+            # Fakeroot copy to new destination
+            srcdir = recipe_outdir
+            recipe_outdir = os.path.join(rd.getVar('WORKDIR', True), 'deploy-target-stripped')
+            if os.path.isdir(recipe_outdir):
+                bb.utils.remove(recipe_outdir, True)
+            exec_fakeroot(rd, "cp -af %s %s" % (os.path.join(srcdir, '.'), recipe_outdir), shell=True)
+            os.environ['PATH'] = ':'.join([os.environ['PATH'], rd.getVar('PATH', True) or ''])
+            oe.package.strip_execs(args.recipename, recipe_outdir, rd.getVar('STRIP', True), rd.getVar('libdir', True),
+                        rd.getVar('base_libdir', True))
+
         filelist = []
         ftotalsize = 0
         for root, _, files in os.walk(recipe_outdir):
@@ -189,7 +205,6 @@ def deploy(args, config, basepath, workspace):
                 print('  %s' % item)
             return 0
 
-
         extraoptions = ''
         if args.no_host_check:
             extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
@@ -301,6 +316,7 @@ def undeploy(args, config, basepath, workspace):
 
 def register_commands(subparsers, context):
     """Register devtool subcommands from the deploy plugin"""
+
     parser_deploy = subparsers.add_parser('deploy-target',
                                           help='Deploy recipe output files to live target machine',
                                           description='Deploys a recipe\'s build output (i.e. the output of the do_install task) to a live target machine over ssh. By default, any existing files will be preserved instead of being overwritten and will be restored if you run devtool undeploy-target. Note: this only deploys the recipe itself and not any runtime dependencies, so it is assumed that those have been installed on the target beforehand.',
@@ -313,6 +329,15 @@ def register_commands(subparsers, context):
     parser_deploy.add_argument('-p', '--no-preserve', help='Do not preserve existing files', action='store_true')
     parser_deploy.add_argument('--no-check-space', help='Do not check for available space before deploying', action='store_true')
     parser_deploy.add_argument('-P', '--port', default='22', help='Port to use for connection to the target')
+
+    strip_opts = parser_deploy.add_mutually_exclusive_group(required=False)
+    strip_opts.add_argument('-S', '--strip',
+                               help='Strip executables prior to deploying (default: %(default)s). '
+                                    'The default value of this option can be controlled by setting the strip option in the [Deploy] section to True or False.',
+                               default=oe.types.boolean(context.config.get('Deploy', 'strip', default='0')),
+                               action='store_true')
+    strip_opts.add_argument('--no-strip', help='Do not strip executables prior to deploy', dest='strip', action='store_false')
+
     parser_deploy.set_defaults(func=deploy)
 
     parser_undeploy = subparsers.add_parser('undeploy-target',
-- 
2.1.4

Re: [PATCH v4 1/2] package.py: strip_execs: Support for .ko modules

[Richard Purdie]

On Thu, 2017-08-24 at 16:13 +0200, Tobias Hagelborn wrote:
> * Support stripping of .ko modules verifying file extension and
>   check of content "vermagic="
> * Minor refactoring (removing lint errors)

Please don't do this. If you want to do "lint" cleanups please do it in
a separate patch. I can't decide if some of the cleanup is just that or
there was a real bug you were fixing.

I'm also a little worried about the overhead of mmaping all files and
then doing text searches on them. Could we avoid that unless its a
"*.ko" file?

I'm also not convinced:

-    libdir = os.path.abspath(dstdir + os.sep + libdir)
+    libdir = os.path.abspath(os.path.join(dstdir, libdir.lstrip(os.sep)))

is an improvement...


>                  # It's a file (or hardlink), not a link
>                  # ...but is it ELF, and is it already stripped?
> -                elf_file = isELF(file)
> +                elf_file = is_elf(file)
>                  if elf_file & 1:
>                      if elf_file & 2:
>                          if qa_already_stripped:
> @@ -132,6 +138,7 @@ def strip_execs(pn, dstdir, strip_cmd, libdir,
> base_libdir, qa_already_stripped=
>                          os.unlink(file)
>                          os.link(inodes[s.st_ino], file)
>                      else:
> +                        # break hardlinks so that we do not strip
> the original.
>                          inodes[s.st_ino] = file
>                          # break hardlink

Might as well delete the now unneeded second comment?

Cheers,

Richard