[PATCH 1/4] libgcrypt: upgrade to 1.8.2

[PATCH 1/4] libgcrypt: upgrade to 1.8.2

[Alexander Kanavin]

Signed-off-by: Alexander Kanavin <alexander.kanavin@linux.intel.com>
---
 .../libgcrypt/{libgcrypt_1.8.1.bb => libgcrypt_1.8.2.bb}              | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
 rename meta/recipes-support/libgcrypt/{libgcrypt_1.8.1.bb => libgcrypt_1.8.2.bb} (92%)

diff --git a/meta/recipes-support/libgcrypt/libgcrypt_1.8.1.bb b/meta/recipes-support/libgcrypt/libgcrypt_1.8.2.bb
similarity index 92%
rename from meta/recipes-support/libgcrypt/libgcrypt_1.8.1.bb
rename to meta/recipes-support/libgcrypt/libgcrypt_1.8.2.bb
index 5bd815ae55a..b36e653a876 100644
--- a/meta/recipes-support/libgcrypt/libgcrypt_1.8.1.bb
+++ b/meta/recipes-support/libgcrypt/libgcrypt_1.8.2.bb
@@ -21,8 +21,8 @@ SRC_URI = "${GNUPG_MIRROR}/libgcrypt/libgcrypt-${PV}.tar.bz2 \
            file://0002-libgcrypt-fix-building-error-with-O2-in-sysroot-path.patch \
            file://0004-tests-Makefile.am-fix-undefined-reference-to-pthread.patch \
 "
-SRC_URI[md5sum] = "b21817f9d850064d2177285f1073ec55"
-SRC_URI[sha256sum] = "7a2875f8b1ae0301732e878c0cca2c9664ff09ef71408f085c50e332656a78b3"
+SRC_URI[md5sum] = "cfb0b5c79eab07686b6898160a407139"
+SRC_URI[sha256sum] = "c8064cae7558144b13ef0eb87093412380efa16c4ee30ad12ecb54886a524c07"
 
 BINCONFIG = "${bindir}/libgcrypt-config"
 
-- 
2.15.1

[PATCH 2/4] gnupg: upgrade to 2.2.4

[Alexander Kanavin]

Signed-off-by: Alexander Kanavin <alexander.kanavin@linux.intel.com>
---
 meta/recipes-support/gnupg/{gnupg_2.2.0.bb => gnupg_2.2.4.bb} | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
 rename meta/recipes-support/gnupg/{gnupg_2.2.0.bb => gnupg_2.2.4.bb} (91%)

diff --git a/meta/recipes-support/gnupg/gnupg_2.2.0.bb b/meta/recipes-support/gnupg/gnupg_2.2.4.bb
similarity index 91%
rename from meta/recipes-support/gnupg/gnupg_2.2.0.bb
rename to meta/recipes-support/gnupg/gnupg_2.2.4.bb
index 0176dddad24..e9f19ca8140 100644
--- a/meta/recipes-support/gnupg/gnupg_2.2.0.bb
+++ b/meta/recipes-support/gnupg/gnupg_2.2.4.bb
@@ -16,8 +16,8 @@ SRC_URI = "${GNUPG_MIRROR}/${BPN}/${BPN}-${PV}.tar.bz2 \
            file://0004-autogen.sh-fix-find-version-for-beta-checking.patch \
           "
 
-SRC_URI[md5sum] = "789f16949fae2d003d387f49e9da4b74"
-SRC_URI[sha256sum] = "d4514a0be0f7a1ff263193330019eb4b53c82f0f5e230af3c14df371271a45e6"
+SRC_URI[md5sum] = "709e5af5bba84d251c520222e720972f"
+SRC_URI[sha256sum] = "401a3e64780fdfa6d7670de0880aa5c9d589b3db7a7098979d7606cec546f2ec"
 
 EXTRA_OECONF = "--disable-ldap \
 		--disable-ccid-driver \
-- 
2.15.1

[PATCH 3/4] gnupg: enable native version

[Alexander Kanavin]

Signed-off-by: Alexander Kanavin <alexander.kanavin@linux.intel.com>
---
 meta/recipes-support/gnupg/gnupg_2.2.4.bb       | 2 ++
 meta/recipes-support/libksba/libksba_1.3.5.bb   | 2 ++
 meta/recipes-support/npth/npth_1.5.bb           | 2 ++
 meta/recipes-support/pinentry/pinentry_1.1.0.bb | 2 ++
 4 files changed, 8 insertions(+)

diff --git a/meta/recipes-support/gnupg/gnupg_2.2.4.bb b/meta/recipes-support/gnupg/gnupg_2.2.4.bb
index e9f19ca8140..e15bcf298ac 100644
--- a/meta/recipes-support/gnupg/gnupg_2.2.4.bb
+++ b/meta/recipes-support/gnupg/gnupg_2.2.4.bb
@@ -44,3 +44,5 @@ do_install_append() {
 PACKAGECONFIG ??= "gnutls"
 PACKAGECONFIG[gnutls] = "--enable-gnutls, --disable-gnutls, gnutls"
 PACKAGECONFIG[sqlite3] = "--enable-sqlite, --disable-sqlite, sqlite3"
+
+BBCLASSEXTEND = "native"
diff --git a/meta/recipes-support/libksba/libksba_1.3.5.bb b/meta/recipes-support/libksba/libksba_1.3.5.bb
index 36b66705634..a7ea53fed02 100644
--- a/meta/recipes-support/libksba/libksba_1.3.5.bb
+++ b/meta/recipes-support/libksba/libksba_1.3.5.bb
@@ -24,3 +24,5 @@ do_configure_prepend () {
 	# Else these could be used in preference to those in aclocal-copy
 	rm -f ${S}/m4/gpg-error.m4
 }
+
+BBCLASSEXTEND = "native"
diff --git a/meta/recipes-support/npth/npth_1.5.bb b/meta/recipes-support/npth/npth_1.5.bb
index 54de70c5c08..e7db6ae1b4b 100644
--- a/meta/recipes-support/npth/npth_1.5.bb
+++ b/meta/recipes-support/npth/npth_1.5.bb
@@ -19,3 +19,5 @@ inherit autotools binconfig-disabled
 
 FILES_${PN} = "${libdir}/libnpth.so.*"
 FILES_${PN}-dev += "${bindir}/npth-config"
+
+BBCLASSEXTEND = "native"
diff --git a/meta/recipes-support/pinentry/pinentry_1.1.0.bb b/meta/recipes-support/pinentry/pinentry_1.1.0.bb
index 3b77709ffb5..1eaa261f75a 100644
--- a/meta/recipes-support/pinentry/pinentry_1.1.0.bb
+++ b/meta/recipes-support/pinentry/pinentry_1.1.0.bb
@@ -34,3 +34,5 @@ PACKAGECONFIG[gtk2] = "--enable-pinentry-gtk2, --disable-pinentry-gtk2, gtk+ gli
 
 #To use libsecret, add meta-gnome
 PACKAGECONFIG[secret] = "--enable-libsecret, --disable-libsecret, libsecret"
+
+BBCLASSEXTEND = "native"
-- 
2.15.1

[PATCH 4/4] gnupg: use native version for signing, rather than one provided by host

[Alexander Kanavin]

Using host gpg has been problematic, and particularly this removes
the need to serialize package creation, as long as --auto-expand-secmem
is passed to gpg-agent, and gnupg >= 2.2.4 is in use
(https://dev.gnupg.org/T3530).

Sadly, gpg-agent itself is single-threaded, so in the longer run
we might want to seek alternatives:
https://lwn.net/Articles/742542/

(a smaller issue is that rpm itself runs the gpg fronted in a serial
fashion, which slows down the build in cases of recipes with very
large amount of packages, e.g. glibc-locale)

Note that sstate signing and verification continues to use host
gpg, as depending on native gpg would create circular dependencies.

[YOCTO #12022]

Signed-off-by: Alexander Kanavin <alexander.kanavin@linux.intel.com>
---
 meta/classes/sign_package_feed.bbclass | 2 +-
 meta/classes/sign_rpm.bbclass          | 6 +-----
 meta/lib/oe/gpg_sign.py                | 8 ++++++--
 meta/recipes-core/meta/signing-keys.bb | 1 +
 4 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/meta/classes/sign_package_feed.bbclass b/meta/classes/sign_package_feed.bbclass
index f03c4802d06..7ff3a35a2fa 100644
--- a/meta/classes/sign_package_feed.bbclass
+++ b/meta/classes/sign_package_feed.bbclass
@@ -43,4 +43,4 @@ python () {
 }
 
 do_package_index[depends] += "signing-keys:do_deploy"
-do_rootfs[depends] += "signing-keys:do_populate_sysroot"
+do_rootfs[depends] += "signing-keys:do_populate_sysroot gnupg-native:do_populate_sysroot"
diff --git a/meta/classes/sign_rpm.bbclass b/meta/classes/sign_rpm.bbclass
index 4961b03618f..64ae7ce30e3 100644
--- a/meta/classes/sign_rpm.bbclass
+++ b/meta/classes/sign_rpm.bbclass
@@ -68,8 +68,4 @@ python sign_rpm () {
 do_package_index[depends] += "signing-keys:do_deploy"
 do_rootfs[depends] += "signing-keys:do_populate_sysroot"
 
-# Newer versions of gpg (at least 2.1.5 and 2.2.1) have issues when signing occurs in parallel
-# so unfortunately the signing must be done serially. Once the upstream problem is fixed,
-# the following line must be removed otherwise we loose all the intrinsic parallelism from
-# bitbake.  For more information, check https://bugzilla.yoctoproject.org/show_bug.cgi?id=12022.
-do_package_write_rpm[lockfiles] += "${TMPDIR}/gpg.lock"
+PACKAGE_WRITE_DEPS += "gnupg-native"
diff --git a/meta/lib/oe/gpg_sign.py b/meta/lib/oe/gpg_sign.py
index 9cc88f020c1..b17272928fc 100644
--- a/meta/lib/oe/gpg_sign.py
+++ b/meta/lib/oe/gpg_sign.py
@@ -12,6 +12,7 @@ class LocalSigner(object):
         self.gpg_path = d.getVar('GPG_PATH')
         self.gpg_version = self.get_gpg_version()
         self.rpm_bin = bb.utils.which(os.getenv('PATH'), "rpmsign")
+        self.gpg_agent_bin = bb.utils.which(os.getenv('PATH'), "gpg-agent")
 
     def export_pubkey(self, output_file, keyid, armor=True):
         """Export GPG public key to a file"""
@@ -31,7 +32,7 @@ class LocalSigner(object):
         """Sign RPM files"""
 
         cmd = self.rpm_bin + " --addsign --define '_gpg_name %s'  " % keyid
-        gpg_args = '--no-permission-warning --batch --passphrase=%s' % passphrase
+        gpg_args = '--no-permission-warning --batch --passphrase=%s --agent-program=%s|--auto-expand-secmem' % (passphrase, self.gpg_agent_bin)
         if self.gpg_version > (2,1,):
             gpg_args += ' --pinentry-mode=loopback'
         cmd += "--define '_gpg_sign_cmd_extra_args %s' " % gpg_args
@@ -71,6 +72,9 @@ class LocalSigner(object):
         if self.gpg_version > (2,1,):
             cmd += ['--pinentry-mode', 'loopback']
 
+        if self.gpg_agent_bin:
+            cmd += ["--agent-program=%s|--auto-expand-secmem" % (self.gpg_agent_bin)]
+
         cmd += [input_file]
 
         try:
@@ -99,7 +103,7 @@ class LocalSigner(object):
         import subprocess
         try:
             ver_str = subprocess.check_output((self.gpg_bin, "--version", "--no-permission-warning")).split()[2].decode("utf-8")
-            return tuple([int(i) for i in ver_str.split('.')])
+            return tuple([int(i) for i in ver_str.split("-")[0].split('.')])
         except subprocess.CalledProcessError as e:
             raise bb.build.FuncFailed("Could not get gpg version: %s" % e)
 
diff --git a/meta/recipes-core/meta/signing-keys.bb b/meta/recipes-core/meta/signing-keys.bb
index 2c1cc3845ea..6387d90d474 100644
--- a/meta/recipes-core/meta/signing-keys.bb
+++ b/meta/recipes-core/meta/signing-keys.bb
@@ -41,6 +41,7 @@ python do_get_public_keys () {
 }
 do_get_public_keys[cleandirs] = "${B}"
 addtask get_public_keys before do_install
+do_get_public_keys[depends] += "gnupg-native:do_populate_sysroot"
 
 do_install () {
     if [ -f "${B}/rpm-key" ]; then
-- 
2.15.1

Re: [PATCH 4/4] gnupg: use native version for signing, rather than one provided by host

[Leonardo Sandoval]

Great that you figure out a solution.

So I belive we need to revert this commit:

commit 043d9ac0ae441e9a7e2ea8934bfc595a03ef9a52
Author: Leonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com>
Date:   Mon Sep 25 13:52:59 2017 -0700

    sign_rpm.bbclass: force rpm serial signing
    
    Newer versions of gpg (at least 2.1.5 and 2.2.1) have issues when signing occurs in parallel
    so (unfortunately) the signing must be done serially. Once the upstream problem is fixed,
    this patch must be reverted, otherwise we loose all the intrinsic parallelism from
    bitbake.
    
    [YOCTO #12022]
    
    (From OE-Core rev: 5301712f9735fcf8d3dec756772668de930e53fe)
    


On Wed, 10 Jan 2018 14:27:42 +0200
Alexander Kanavin <alexander.kanavin@linux.intel.com> wrote:

> Using host gpg has been problematic, and particularly this removes
> the need to serialize package creation, as long as --auto-expand-secmem
> is passed to gpg-agent, and gnupg >= 2.2.4 is in use
> (https://dev.gnupg.org/T3530).
> 
> Sadly, gpg-agent itself is single-threaded, so in the longer run
> we might want to seek alternatives:
> https://lwn.net/Articles/742542/
> 
> (a smaller issue is that rpm itself runs the gpg fronted in a serial
> fashion, which slows down the build in cases of recipes with very
> large amount of packages, e.g. glibc-locale)
> 
> Note that sstate signing and verification continues to use host
> gpg, as depending on native gpg would create circular dependencies.
> 
> [YOCTO #12022]
> 
> Signed-off-by: Alexander Kanavin <alexander.kanavin@linux.intel.com>
> ---
>  meta/classes/sign_package_feed.bbclass | 2 +-
>  meta/classes/sign_rpm.bbclass          | 6 +-----
>  meta/lib/oe/gpg_sign.py                | 8 ++++++--
>  meta/recipes-core/meta/signing-keys.bb | 1 +
>  4 files changed, 9 insertions(+), 8 deletions(-)
> 
> diff --git a/meta/classes/sign_package_feed.bbclass b/meta/classes/sign_package_feed.bbclass
> index f03c4802d06..7ff3a35a2fa 100644
> --- a/meta/classes/sign_package_feed.bbclass
> +++ b/meta/classes/sign_package_feed.bbclass
> @@ -43,4 +43,4 @@ python () {
>  }
>  
>  do_package_index[depends] += "signing-keys:do_deploy"
> -do_rootfs[depends] += "signing-keys:do_populate_sysroot"
> +do_rootfs[depends] += "signing-keys:do_populate_sysroot gnupg-native:do_populate_sysroot"
> diff --git a/meta/classes/sign_rpm.bbclass b/meta/classes/sign_rpm.bbclass
> index 4961b03618f..64ae7ce30e3 100644
> --- a/meta/classes/sign_rpm.bbclass
> +++ b/meta/classes/sign_rpm.bbclass
> @@ -68,8 +68,4 @@ python sign_rpm () {
>  do_package_index[depends] += "signing-keys:do_deploy"
>  do_rootfs[depends] += "signing-keys:do_populate_sysroot"
>  
> -# Newer versions of gpg (at least 2.1.5 and 2.2.1) have issues when signing occurs in parallel
> -# so unfortunately the signing must be done serially. Once the upstream problem is fixed,
> -# the following line must be removed otherwise we loose all the intrinsic parallelism from
> -# bitbake.  For more information, check https://bugzilla.yoctoproject.org/show_bug.cgi?id=12022.
> -do_package_write_rpm[lockfiles] += "${TMPDIR}/gpg.lock"
> +PACKAGE_WRITE_DEPS += "gnupg-native"
> diff --git a/meta/lib/oe/gpg_sign.py b/meta/lib/oe/gpg_sign.py
> index 9cc88f020c1..b17272928fc 100644
> --- a/meta/lib/oe/gpg_sign.py
> +++ b/meta/lib/oe/gpg_sign.py
> @@ -12,6 +12,7 @@ class LocalSigner(object):
>          self.gpg_path = d.getVar('GPG_PATH')
>          self.gpg_version = self.get_gpg_version()
>          self.rpm_bin = bb.utils.which(os.getenv('PATH'), "rpmsign")
> +        self.gpg_agent_bin = bb.utils.which(os.getenv('PATH'), "gpg-agent")
>  
>      def export_pubkey(self, output_file, keyid, armor=True):
>          """Export GPG public key to a file"""
> @@ -31,7 +32,7 @@ class LocalSigner(object):
>          """Sign RPM files"""
>  
>          cmd = self.rpm_bin + " --addsign --define '_gpg_name %s'  " % keyid
> -        gpg_args = '--no-permission-warning --batch --passphrase=%s' % passphrase
> +        gpg_args = '--no-permission-warning --batch --passphrase=%s --agent-program=%s|--auto-expand-secmem' % (passphrase, self.gpg_agent_bin)
>          if self.gpg_version > (2,1,):
>              gpg_args += ' --pinentry-mode=loopback'
>          cmd += "--define '_gpg_sign_cmd_extra_args %s' " % gpg_args
> @@ -71,6 +72,9 @@ class LocalSigner(object):
>          if self.gpg_version > (2,1,):
>              cmd += ['--pinentry-mode', 'loopback']
>  
> +        if self.gpg_agent_bin:
> +            cmd += ["--agent-program=%s|--auto-expand-secmem" % (self.gpg_agent_bin)]
> +
>          cmd += [input_file]
>  
>          try:
> @@ -99,7 +103,7 @@ class LocalSigner(object):
>          import subprocess
>          try:
>              ver_str = subprocess.check_output((self.gpg_bin, "--version", "--no-permission-warning")).split()[2].decode("utf-8")
> -            return tuple([int(i) for i in ver_str.split('.')])
> +            return tuple([int(i) for i in ver_str.split("-")[0].split('.')])
>          except subprocess.CalledProcessError as e:
>              raise bb.build.FuncFailed("Could not get gpg version: %s" % e)
>  
> diff --git a/meta/recipes-core/meta/signing-keys.bb b/meta/recipes-core/meta/signing-keys.bb
> index 2c1cc3845ea..6387d90d474 100644
> --- a/meta/recipes-core/meta/signing-keys.bb
> +++ b/meta/recipes-core/meta/signing-keys.bb
> @@ -41,6 +41,7 @@ python do_get_public_keys () {
>  }
>  do_get_public_keys[cleandirs] = "${B}"
>  addtask get_public_keys before do_install
> +do_get_public_keys[depends] += "gnupg-native:do_populate_sysroot"
>  
>  do_install () {
>      if [ -f "${B}/rpm-key" ]; then
> -- 
> 2.15.1
> 
> -- 
> _______________________________________________
> Openembedded-core mailing list
> Openembedded-core@lists.openembedded.org
> http://lists.openembedded.org/mailman/listinfo/openembedded-core

Re: [PATCH 4/4] gnupg: use native version for signing, rather than one provided by host

[Alexander Kanavin]

On 01/10/2018 05:01 PM, Leonardo Sandoval wrote:
> Great that you figure out a solution.
> 
> So I belive we need to revert this commit:
> 
> commit 043d9ac0ae441e9a7e2ea8934bfc595a03ef9a52
> Author: Leonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com>
> Date:   Mon Sep 25 13:52:59 2017 -0700
> 
>      sign_rpm.bbclass: force rpm serial signing

The revert is already included in the patch... :)

Alex