[PATCH 1/2] package_manager/ipk: skip checksums for unsigned local feeds

[PATCH 1/2] package_manager/ipk: skip checksums for unsigned local feeds

[ecordonnier]

From: Etienne Cordonnier <ecordonnier@snap.com>

Computing checksums in the opkg Packages index requires reading every
.ipk file in full.  For a large image with 7000+ packages (including
multi-gigabyte debug packages), this adds 150-300s to every do_rootfs run.

Checksums in the Packages index are only meaningful for signed feeds
(PACKAGE_FEED_SIGN=1): the GPG signature covers the Packages index
which contains the SHA256Sum of each .ipk, forming a chain of trust
that prevents tampered packages being swapped on a remote feed.

For unsigned local file:// feeds the packages are installed directly
from the build host filesystem where there is no tampering risk.  Skip
all checksum generation in that case by passing no --checksum flags to
opkg-make-index (the tool's default behaviour when given no flags).
Pass --force-checksum to opkg so it does not error on the absent
checksum fields.

On a test image with 7000+ packages (including a 2.3 GB debug
package): write_index time reduced from ~180s to ~22s (8x speedup)
when opkg-make-index is configured to produce no checksums by default.

See https://git.openembedded.org/openembedded-core/commit/?id=e462f47489f35902b6972f9837d9adfa542fc796
("Enable sha256 checksums in opkg indexer", 2019) for the original rationale.

AI-Generated: Claude Sonnet 4.6
Signed-off-by: Etienne Cordonnier <ecordonnier@snap.com>
---
 meta/lib/oe/package_manager/ipk/__init__.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/meta/lib/oe/package_manager/ipk/__init__.py b/meta/lib/oe/package_manager/ipk/__init__.py
index 344e0852177..6c0b896a086 100644
--- a/meta/lib/oe/package_manager/ipk/__init__.py
+++ b/meta/lib/oe/package_manager/ipk/__init__.py
@@ -12,7 +12,7 @@ from oe.package_manager import *
 from oe.package_manager.common_deb_ipk import OpkgDpkgPM
 
 class OpkgIndexer(Indexer):
-    def write_index(self):
+    def write_index(self, skip_checksums=False):
         arch_vars = ["ALL_MULTILIB_PACKAGE_ARCHS",
                      "SDK_PACKAGE_ARCHS",
                      ]
@@ -44,8 +44,9 @@ class OpkgIndexer(Indexer):
                 if not os.path.exists(pkgs_file):
                     open(pkgs_file, "w").close()
 
-                index_cmds.add('%s --checksum md5 --checksum sha256 -r %s -p %s -m %s %s' %
-                                  (opkg_index_cmd, pkgs_file, pkgs_file, pkgs_dir, opkg_index_cmd_extra_params))
+                checksum_args = '' if skip_checksums else '--checksum md5 --checksum sha256 '
+                index_cmds.add('%s %s-r %s -p %s -m %s %s' %
+                                  (opkg_index_cmd, checksum_args, pkgs_file, pkgs_file, pkgs_dir, opkg_index_cmd_extra_params))
 
                 index_sign_files.add(pkgs_file)
 
@@ -103,8 +104,12 @@ class OpkgPM(OpkgDpkgPM):
         self.deploy_dir = oe.path.join(self.d.getVar('WORKDIR'), ipk_repo_workdir)
         self.deploy_lock_file = os.path.join(self.deploy_dir, "deploy.lock")
         self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg")
+        self.from_feeds = (self.d.getVar('BUILD_IMAGES_FROM_FEEDS') or "") == "1"
+        self._skip_checksums = self.d.getVar('PACKAGE_FEED_SIGN') != '1' and not self.from_feeds
         self.opkg_args = ['--volatile-cache', '-f', config_file, '-t', self.d.expand('${T}/ipktemp/'), '-o', target_rootfs]
         self.opkg_args.extend(shlex.split(self.d.getVar("OPKG_ARGS")))
+        if self._skip_checksums:
+            self.opkg_args.append('--force-checksum')
 
         if prepare_index:
             create_packages_dir(self.d, self.deploy_dir, d.getVar("DEPLOY_DIR_IPK"), "package_write_ipk", filterbydependencies)
@@ -116,7 +121,6 @@ class OpkgPM(OpkgDpkgPM):
         if not os.path.exists(self.d.expand('${T}/saved')):
             bb.utils.mkdirhier(self.d.expand('${T}/saved'))
 
-        self.from_feeds = (self.d.getVar('BUILD_IMAGES_FROM_FEEDS') or "") == "1"
         if self.from_feeds:
             self._create_custom_config()
         else:
@@ -346,7 +350,7 @@ class OpkgPM(OpkgDpkgPM):
     def write_index(self):
         self.deploy_dir_lock()
 
-        result = self.indexer.write_index()
+        result = self.indexer.write_index(skip_checksums=self._skip_checksums)
 
         self.deploy_dir_unlock()
 
-- 
2.43.0

[PATCH 2/2] opkg: enable sha256 checksum verification by default

[ecordonnier]

From: Etienne Cordonnier <ecordonnier@snap.com>

WITH_SHA256=ON adds no external dependency: sha256.c is a self-contained
528-line pure C implementation already present in the source tree.

oe-core already generates SHA256Sum fields in the Packages index
(via --checksum sha256 in OpkgIndexer.write_index), but opkg silently
ignores them unless compiled with WITH_SHA256=ON.  Enabling sha256 by
default makes those checksums actually verified at install time,
improving package integrity checking at zero dependency cost.

Once SHA256 is universally available, --checksum md5 can also be
dropped from OpkgIndexer.write_index() -- MD5 is collision-prone and
should not be relied upon for integrity checking -- completing the
deprecation noted in https://git.openembedded.org/openembedded-core/commit/?id=e462f47489f35902b6972f9837d9adfa542fc796
(2019).

AI-Generated: Claude Sonnet 4.6
Signed-off-by: Etienne Cordonnier <ecordonnier@snap.com>
---
 meta/recipes-devtools/opkg/opkg_0.10.0.bb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/meta/recipes-devtools/opkg/opkg_0.10.0.bb b/meta/recipes-devtools/opkg/opkg_0.10.0.bb
index debcd94a300..e3c2dd790dd 100644
--- a/meta/recipes-devtools/opkg/opkg_0.10.0.bb
+++ b/meta/recipes-devtools/opkg/opkg_0.10.0.bb
@@ -30,7 +30,7 @@ inherit cmake pkgconfig ptest
 target_localstatedir := "${localstatedir}"
 OPKGLIBDIR ??= "${target_localstatedir}/lib"
 
-PACKAGECONFIG ??= "libsolv ${@bb.utils.filter('DISTRO_FEATURES', 'acl xattr', d)}"
+PACKAGECONFIG ??= "libsolv sha256 ${@bb.utils.filter('DISTRO_FEATURES', 'acl xattr', d)}"
 
 PACKAGECONFIG[gpg] = "-DWITH_GPGME=ON,-DWITH_GPGME=OFF,\
     gnupg gpgme libgpg-error,\
-- 
2.43.0

Re: [OE-core] [PATCH 1/2] package_manager/ipk: skip checksums for unsigned local feeds

[Jörg Sommer]

[-- Attachment #1: Type: text/plain, Size: 5142 bytes --]

Etienne Cordonnier via lists.openembedded.org schrieb am Mi 29. Jul, 17:29 (+0200):
> From: Etienne Cordonnier <ecordonnier@snap.com>
> 
> Computing checksums in the opkg Packages index requires reading every
> .ipk file in full.  For a large image with 7000+ packages (including
> multi-gigabyte debug packages), this adds 150-300s to every do_rootfs run.
> 
> Checksums in the Packages index are only meaningful for signed feeds
> (PACKAGE_FEED_SIGN=1): the GPG signature covers the Packages index
> which contains the SHA256Sum of each .ipk, forming a chain of trust
> that prevents tampered packages being swapped on a remote feed.
> 
> For unsigned local file:// feeds the packages are installed directly
> from the build host filesystem where there is no tampering risk.  Skip
> all checksum generation in that case by passing no --checksum flags to
> opkg-make-index (the tool's default behaviour when given no flags).
> Pass --force-checksum to opkg so it does not error on the absent
> checksum fields.
> 
> On a test image with 7000+ packages (including a 2.3 GB debug
> package): write_index time reduced from ~180s to ~22s (8x speedup)
> when opkg-make-index is configured to produce no checksums by default.
> 
> See https://git.openembedded.org/openembedded-core/commit/?id=e462f47489f35902b6972f9837d9adfa542fc796
> ("Enable sha256 checksums in opkg indexer", 2019) for the original rationale.
> 
> AI-Generated: Claude Sonnet 4.6
> Signed-off-by: Etienne Cordonnier <ecordonnier@snap.com>
> ---
>  meta/lib/oe/package_manager/ipk/__init__.py | 14 +++++++++-----
>  1 file changed, 9 insertions(+), 5 deletions(-)
> 
> diff --git a/meta/lib/oe/package_manager/ipk/__init__.py b/meta/lib/oe/package_manager/ipk/__init__.py
> index 344e0852177..6c0b896a086 100644
> --- a/meta/lib/oe/package_manager/ipk/__init__.py
> +++ b/meta/lib/oe/package_manager/ipk/__init__.py
> @@ -12,7 +12,7 @@ from oe.package_manager import *
>  from oe.package_manager.common_deb_ipk import OpkgDpkgPM
>  
>  class OpkgIndexer(Indexer):
> -    def write_index(self):
> +    def write_index(self, skip_checksums=False):

Suggestion: I would use the positive case (like with_checksums or
use_checksums or build_checksums), because reasoning about/reading code with
the inverse is more difficult.

>          arch_vars = ["ALL_MULTILIB_PACKAGE_ARCHS",
>                       "SDK_PACKAGE_ARCHS",
>                       ]
> @@ -44,8 +44,9 @@ class OpkgIndexer(Indexer):
>                  if not os.path.exists(pkgs_file):
>                      open(pkgs_file, "w").close()
>  
> -                index_cmds.add('%s --checksum md5 --checksum sha256 -r %s -p %s -m %s %s' %
> -                                  (opkg_index_cmd, pkgs_file, pkgs_file, pkgs_dir, opkg_index_cmd_extra_params))
> +                checksum_args = '' if skip_checksums else '--checksum md5 --checksum sha256 '
> +                index_cmds.add('%s %s-r %s -p %s -m %s %s' %
> +                                  (opkg_index_cmd, checksum_args, pkgs_file, pkgs_file, pkgs_dir, opkg_index_cmd_extra_params))
>  
>                  index_sign_files.add(pkgs_file)
>  
> @@ -103,8 +104,12 @@ class OpkgPM(OpkgDpkgPM):
>          self.deploy_dir = oe.path.join(self.d.getVar('WORKDIR'), ipk_repo_workdir)
>          self.deploy_lock_file = os.path.join(self.deploy_dir, "deploy.lock")
>          self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg")
> +        self.from_feeds = (self.d.getVar('BUILD_IMAGES_FROM_FEEDS') or "") == "1"

Or: bb.utils.to_boolean(...)

> +        self._skip_checksums = self.d.getVar('PACKAGE_FEED_SIGN') != '1' and not self.from_feeds

I would swap them because evaluation of the second one is cheaper.

>          self.opkg_args = ['--volatile-cache', '-f', config_file, '-t', self.d.expand('${T}/ipktemp/'), '-o', target_rootfs]
>          self.opkg_args.extend(shlex.split(self.d.getVar("OPKG_ARGS")))
> +        if self._skip_checksums:
> +            self.opkg_args.append('--force-checksum')
>  
>          if prepare_index:
>              create_packages_dir(self.d, self.deploy_dir, d.getVar("DEPLOY_DIR_IPK"), "package_write_ipk", filterbydependencies)
> @@ -116,7 +121,6 @@ class OpkgPM(OpkgDpkgPM):
>          if not os.path.exists(self.d.expand('${T}/saved')):
>              bb.utils.mkdirhier(self.d.expand('${T}/saved'))
>  
> -        self.from_feeds = (self.d.getVar('BUILD_IMAGES_FROM_FEEDS') or "") == "1"
>          if self.from_feeds:
>              self._create_custom_config()
>          else:
> @@ -346,7 +350,7 @@ class OpkgPM(OpkgDpkgPM):
>      def write_index(self):
>          self.deploy_dir_lock()
>  
> -        result = self.indexer.write_index()
> +        result = self.indexer.write_index(skip_checksums=self._skip_checksums)
>  
>          self.deploy_dir_unlock()


Jörg

-- 
Navimatix GmbH           T: 03641 - 327 99 0
Tatzendpromenade 2       F: 03641 - 526 306
07745 Jena               www.navimatix.de

Geschäftsführer: Steffen Späthe, Jan Rommeley
Registergericht: Amtsgericht Jena, HRB 501480

[-- Attachment #2: smime.p7s --]
[-- Type: application/pkcs7-signature, Size: 6684 bytes --]