[PATCH] ipk/rootfs: run sanity test of multilib in parallel

[PATCH] ipk/rootfs: run sanity test of multilib in parallel

[seungkyun.kim]

From: "seungkyun.kim" <seungkyun.kim@lge.com>

For multilib type packages, there is an additional temporary
installation before the actual installation. It makes almost doubles
the do_rootfs time if having many multilib type packages.
To avoid this overhead, run sanity test in parallel.
Installing package groups through opkg takes much more time than
copying directory.

Signed-off-by: seungkyun.kim <seungkyun.kim@lge.com>
---
 meta/lib/oe/package_manager/ipk/rootfs.py | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/meta/lib/oe/package_manager/ipk/rootfs.py b/meta/lib/oe/package_manager/ipk/rootfs.py
index ba93eb62ea..db1cc1650e 100644
--- a/meta/lib/oe/package_manager/ipk/rootfs.py
+++ b/meta/lib/oe/package_manager/ipk/rootfs.py
@@ -6,6 +6,7 @@
 
 import re
 import filecmp
+import multiprocessing
 import shutil
 from oe.rootfs import Rootfs
 from oe.manifest import Manifest
@@ -198,10 +199,16 @@ class PkgRootfs(DpkgOpkgRootfs):
 
     def _multilib_test_install(self, pkgs):
         ml_temp = self.d.getVar("MULTILIB_TEMP_ROOTFS")
+        rootfs_temp = os.path.join(ml_temp, "rootfs")
         bb.utils.mkdirhier(ml_temp)
 
-        dirs = [self.image_rootfs]
+        bb.utils.remove(rootfs_temp, True)
+        shutil.copytree(self.image_rootfs, rootfs_temp)
+        dirs = [rootfs_temp]
+        return multiprocessing.Process(target=self._multilib_test_pkg_install, \
+                                       args=(pkgs, ml_temp, dirs, False))
 
+    def _multilib_test_pkg_install(self, pkgs, ml_temp, dirs):
         for variant in self.d.getVar("MULTILIB_VARIANTS").split():
             ml_target_rootfs = os.path.join(ml_temp, variant)
 
@@ -218,6 +225,8 @@ class PkgRootfs(DpkgOpkgRootfs):
             dirs.append(ml_target_rootfs)
 
         self._multilib_sanity_test(dirs)
+        rootfs_temp = os.path.join(ml_temp, "rootfs")
+        bb.utils.remove(rootfs_temp)
 
     '''
     While ipk incremental image generation is enabled, it will remove the
@@ -300,15 +309,20 @@ class PkgRootfs(DpkgOpkgRootfs):
 
         for pkg_type in self.install_order:
             if pkg_type in pkgs_to_install:
+                sanity_test = None
                 # For multilib, we perform a sanity test before final install
                 # If sanity test fails, it will automatically do a bb.fatal()
                 # and the installation will stop
                 if pkg_type == Manifest.PKG_TYPE_MULTILIB:
-                    self._multilib_test_install(pkgs_to_install[pkg_type])
+                    sanity_test= self._multilib_test_install(pkgs_to_install[pkg_type])
+                    sanity_test.start()
 
                 self.pm.install(pkgs_to_install[pkg_type],
                                 [False, True][pkg_type == Manifest.PKG_TYPE_ATTEMPT_ONLY])
 
+                if sanity_test is not None:
+                    sanity_test.join()
+
         if self.progress_reporter:
             self.progress_reporter.next_stage()
 
-- 
2.43.0

Re: [OE-core] [PATCH] ipk/rootfs: run sanity test of multilib in parallel

[Alexandre Belloni]

Hello,

This reliably fails on the autobuilders:

https://autobuilder.yoctoproject.org/typhoon/#/builders/44/builds/9008/steps/20/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/44/builds/9012/steps/20/logs/stdio

On 13/05/2024 13:37:52+0000, Seungkyun Kim wrote:
> From: "seungkyun.kim" <seungkyun.kim@lge.com>
> 
> For multilib type packages, there is an additional temporary
> installation before the actual installation. It makes almost doubles
> the do_rootfs time if having many multilib type packages.
> To avoid this overhead, run sanity test in parallel.
> Installing package groups through opkg takes much more time than
> copying directory.
> 
> Signed-off-by: seungkyun.kim <seungkyun.kim@lge.com>
> ---
>  meta/lib/oe/package_manager/ipk/rootfs.py | 18 ++++++++++++++++--
>  1 file changed, 16 insertions(+), 2 deletions(-)
> 
> diff --git a/meta/lib/oe/package_manager/ipk/rootfs.py b/meta/lib/oe/package_manager/ipk/rootfs.py
> index ba93eb62ea..db1cc1650e 100644
> --- a/meta/lib/oe/package_manager/ipk/rootfs.py
> +++ b/meta/lib/oe/package_manager/ipk/rootfs.py
> @@ -6,6 +6,7 @@
>  
>  import re
>  import filecmp
> +import multiprocessing
>  import shutil
>  from oe.rootfs import Rootfs
>  from oe.manifest import Manifest
> @@ -198,10 +199,16 @@ class PkgRootfs(DpkgOpkgRootfs):
>  
>      def _multilib_test_install(self, pkgs):
>          ml_temp = self.d.getVar("MULTILIB_TEMP_ROOTFS")
> +        rootfs_temp = os.path.join(ml_temp, "rootfs")
>          bb.utils.mkdirhier(ml_temp)
>  
> -        dirs = [self.image_rootfs]
> +        bb.utils.remove(rootfs_temp, True)
> +        shutil.copytree(self.image_rootfs, rootfs_temp)
> +        dirs = [rootfs_temp]
> +        return multiprocessing.Process(target=self._multilib_test_pkg_install, \
> +                                       args=(pkgs, ml_temp, dirs, False))
>  
> +    def _multilib_test_pkg_install(self, pkgs, ml_temp, dirs):
>          for variant in self.d.getVar("MULTILIB_VARIANTS").split():
>              ml_target_rootfs = os.path.join(ml_temp, variant)
>  
> @@ -218,6 +225,8 @@ class PkgRootfs(DpkgOpkgRootfs):
>              dirs.append(ml_target_rootfs)
>  
>          self._multilib_sanity_test(dirs)
> +        rootfs_temp = os.path.join(ml_temp, "rootfs")
> +        bb.utils.remove(rootfs_temp)
>  
>      '''
>      While ipk incremental image generation is enabled, it will remove the
> @@ -300,15 +309,20 @@ class PkgRootfs(DpkgOpkgRootfs):
>  
>          for pkg_type in self.install_order:
>              if pkg_type in pkgs_to_install:
> +                sanity_test = None
>                  # For multilib, we perform a sanity test before final install
>                  # If sanity test fails, it will automatically do a bb.fatal()
>                  # and the installation will stop
>                  if pkg_type == Manifest.PKG_TYPE_MULTILIB:
> -                    self._multilib_test_install(pkgs_to_install[pkg_type])
> +                    sanity_test= self._multilib_test_install(pkgs_to_install[pkg_type])
> +                    sanity_test.start()
>  
>                  self.pm.install(pkgs_to_install[pkg_type],
>                                  [False, True][pkg_type == Manifest.PKG_TYPE_ATTEMPT_ONLY])
>  
> +                if sanity_test is not None:
> +                    sanity_test.join()
> +
>          if self.progress_reporter:
>              self.progress_reporter.next_stage()
>  
> -- 
> 2.43.0
> 

> 
> -=-=-=-=-=-=-=-=-=-=-=-
> Links: You receive all messages sent to this group.
> View/Reply Online (#199239): https://lists.openembedded.org/g/openembedded-core/message/199239
> Mute This Topic: https://lists.openembedded.org/mt/106072581/3617179
> Group Owner: openembedded-core+owner@lists.openembedded.org
> Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [alexandre.belloni@bootlin.com]
> -=-=-=-=-=-=-=-=-=-=-=-
> 


-- 
Alexandre Belloni, co-owner and COO, Bootlin
Embedded Linux and Kernel engineering
https://bootlin.com