[PATCH v2 03/10] lib/hwfeatures-gcry: Enable SSE and AVX for x86_64 EFI

[Gary Lin via Grub-devel <grub-devel@gnu.org>]

Implement the necessary functions to dynamically enable SSE and AVX
on x86_64 EFI systems when the hardware is capable.

Signed-off-by: Gary Lin <glin@suse.com>
Reviewed-by: Daniel Kiper <daniel.kiper@oracle.com>
---
 grub-core/Makefile.core.def                |   1 +
 grub-core/lib/hwfeatures-gcry.c            |   9 +
 grub-core/lib/x86_64/efi/hwfeatures-gcry.c | 246 +++++++++++++++++++++
 include/grub/x86_64/cpuid.h                |   1 +
 include/grub/x86_64/efi/hwfeatures-gcry.h  |  25 +++
 5 files changed, 282 insertions(+)
 create mode 100644 grub-core/lib/x86_64/efi/hwfeatures-gcry.c
 create mode 100644 include/grub/x86_64/cpuid.h
 create mode 100644 include/grub/x86_64/efi/hwfeatures-gcry.h

diff --git a/grub-core/Makefile.core.def b/grub-core/Makefile.core.def
index 3173e66be..3d4b1a3b4 100644
--- a/grub-core/Makefile.core.def
+++ b/grub-core/Makefile.core.def
@@ -1721,6 +1721,7 @@ module = {
   name = crypto;
   common = lib/crypto.c;
   common = lib/hwfeatures-gcry.c;
+  x86_64_efi = lib/x86_64/efi/hwfeatures-gcry.c;
 
   extra_dist = lib/libgcrypt-grub/cipher/crypto.lst;
 };
diff --git a/grub-core/lib/hwfeatures-gcry.c b/grub-core/lib/hwfeatures-gcry.c
index 652e67c43..acfa7be33 100644
--- a/grub-core/lib/hwfeatures-gcry.c
+++ b/grub-core/lib/hwfeatures-gcry.c
@@ -19,6 +19,9 @@
 #include <grub/dl.h>
 
 #include <grub/hwfeatures-gcry.h>
+#if defined (__x86_64__) && defined (GRUB_MACHINE_EFI)
+#include <grub/x86_64/efi/hwfeatures-gcry.h>
+#endif
 
 GRUB_MOD_LICENSE ("GPLv3+");
 
@@ -33,11 +36,17 @@ grub_gcry_hwf_enabled (void)
 void
 grub_enable_gcry_hwf (void)
 {
+#if defined (__x86_64__) && defined (GRUB_MACHINE_EFI)
+  grub_enable_gcry_hwf_x86_64_efi ();
+#endif
   __gcry_use_hwf = true;
 }
 
 void
 grub_reset_gcry_hwf (void)
 {
+#if defined (__x86_64__) && defined (GRUB_MACHINE_EFI)
+  grub_reset_gcry_hwf_x86_64_efi ();
+#endif
   __gcry_use_hwf = false;
 }
diff --git a/grub-core/lib/x86_64/efi/hwfeatures-gcry.c b/grub-core/lib/x86_64/efi/hwfeatures-gcry.c
new file mode 100644
index 000000000..26cb0f512
--- /dev/null
+++ b/grub-core/lib/x86_64/efi/hwfeatures-gcry.c
@@ -0,0 +1,246 @@
+/*
+ *  GRUB  --  GRand Unified Bootloader
+ *  Copyright (C) 2025  Free Software Foundation, Inc.
+ *
+ *  GRUB is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  GRUB is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with GRUB.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <grub/types.h>
+#include <grub/x86_64/efi/hwfeatures-gcry.h>
+#include <grub/x86_64/cpuid.h>
+#include <grub/misc.h>
+
+/*
+ * Older versions of GCC may reorder the inline asm, which can lead to
+ * unexpected behavior when reading the Control Registers. The __FORCE_ORDER
+ * macro is used to prevent this.
+ *
+ * Ref: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=aa5cacdc29d76a005cbbee018a47faa6e724dd2d
+ */
+#define __FORCE_ORDER "m"(*(unsigned int *) 0x1000UL)
+
+#define HW_FEATURE_X86_64_SSE (1 << 0)
+#define HW_FEATURE_X86_64_AVX (1 << 1)
+
+static grub_uint32_t hw_features = 0;
+static grub_uint64_t old_cr0, old_cr4, old_xcr0;
+
+static inline grub_uint64_t
+read_cr0 (void)
+{
+  grub_uint64_t val;
+
+  asm volatile ("mov %%cr0, %0" : "=r" (val) : __FORCE_ORDER);
+  return val;
+}
+
+static inline grub_uint64_t
+read_cr4 (void)
+{
+  grub_uint64_t val;
+
+  asm volatile ("mov %%cr4,%0" : "=r" (val) : __FORCE_ORDER);
+  return val;
+}
+
+static inline void
+write_cr0 (grub_uint64_t val)
+{
+  asm volatile ("mov %0,%%cr4": "+r" (val) : : "memory");
+}
+
+static inline void
+write_cr4 (grub_uint64_t val)
+{
+  asm volatile ("mov %0,%%cr4": "+r" (val) : : "memory");
+}
+
+static grub_uint32_t
+get_cpuid_ecx (void)
+{
+  grub_uint32_t eax, ebx, ecx, edx;
+
+  grub_cpuid (1, eax, ebx, ecx, edx);
+
+  return ecx;
+}
+
+static grub_uint32_t
+get_cpuid_edx (void)
+{
+  grub_uint32_t eax, ebx, ecx, edx;
+
+  grub_cpuid (1, eax, ebx, ecx, edx);
+
+  return edx;
+}
+
+static bool
+enable_sse (void)
+{
+  grub_uint64_t cr0, cr4;
+  grub_uint32_t edx;
+
+  edx = get_cpuid_edx ();
+
+  /* Check CPUID.01H:EDX.FXSR[bit 24] and CPUID.01H:EDX.SSE[bit 25] */
+  if ((edx & (3 << 24)) != (3 << 24))
+    return false;
+
+  cr0 = old_cr0 = read_cr0 ();
+  cr4 = old_cr4 = read_cr4 ();
+
+  /* clear CR0.EM[bit 2] */
+  if ((cr0 & (1 << 2)) != 0)
+    cr0 &= ~(1 << 2);
+
+  /* Set CR0.MP[bit 1] */
+  if ((cr0 & (1 << 1)) == 0)
+    cr0 |= (1 << 1);
+
+  grub_dprintf ("hwfeatures", "CR0: 0x%"PRIxGRUB_UINT64_T" 0x%"PRIxGRUB_UINT64_T"\n", old_cr0, cr0);
+  if (old_cr0 != cr0)
+    write_cr0 (cr0);
+
+  /* Set CR4.OSFXSR[bit 9] and CR4.OSXMMEXCPT[bit 10] */
+  if ((cr4 & (3 << 9)) != (3 << 9))
+    cr4 |= (3 << 9);
+
+  grub_dprintf ("hwfeatures", "CR4: 0x%"PRIxGRUB_UINT64_T" 0x%"PRIxGRUB_UINT64_T"\n", old_cr4, cr4);
+  if (old_cr4 != cr4)
+    write_cr4 (cr4);
+
+  return true;
+}
+
+static inline grub_uint64_t
+xgetbv (grub_uint32_t index)
+{
+  grub_uint32_t eax, edx;
+
+  asm volatile ("xgetbv" : "=a" (eax), "=d" (edx) : "c" (index));
+
+  return eax + ((grub_uint64_t)edx << 32);
+}
+
+static inline void
+xsetbv (grub_uint32_t index, grub_uint64_t value)
+{
+  grub_uint32_t eax = (grub_uint32_t)value;
+  grub_uint32_t edx = (grub_uint32_t)(value >> 32);
+
+  asm volatile ("xsetbv" :: "a" (eax), "d" (edx), "c" (index));
+}
+
+static bool
+enable_avx (void)
+{
+  grub_uint64_t cr4;
+  grub_uint32_t ecx;
+  grub_uint64_t sse_avx_mask = (1 << 2) | (1 << 1);
+  grub_uint64_t xcr0;
+
+  ecx = get_cpuid_ecx ();
+
+  /* Check the following two bits
+   * -  CPUID.01H:ECX.XSAVE[bit 26]
+   *    If XSAVE is not supported, setting CR4.OSXSAVE will cause
+   *    general-protection fault (#GP).
+   * - CPUID.01H:ECX.AVX[bit 28]
+   */
+  grub_dprintf ("hwfeatures", "Check CPUID.01H:ECX 0x%"PRIuGRUB_UINT32_T"\n", ecx);
+  if ((ecx & (5 << 26)) != (5 << 26))
+    return false;
+
+  cr4 = read_cr4 ();
+
+  /* Set CR4.OSXSAVE[bit 18] */
+  if ((cr4 & (1 << 18)) == 0)
+    {
+      grub_dprintf ("hwfeatures", "Set CR4.OSXSAVE\n");
+      cr4 |= 1 << 18;
+      write_cr4 (cr4);
+    }
+
+  ecx = get_cpuid_ecx ();
+
+  /* Check CPUID.01H:ECX.OSXSAVE[bit 27] */
+  if ((ecx & (1 << 27)) == 0)
+    return false;
+
+  xcr0 = old_xcr0 = xgetbv (0);
+
+  /* Set XCR0[bit 1] and XCR0[bit 2] to enable SSE/AVX */
+  if ((xcr0 & sse_avx_mask) != sse_avx_mask)
+    {
+      grub_dprintf ("hwfeatures", "Set XCR0[2:1] to 11b\n");
+      xcr0 |= sse_avx_mask;
+      xsetbv (0, xcr0);
+    }
+
+  return true;
+}
+
+void
+grub_enable_gcry_hwf_x86_64_efi (void)
+{
+  if (enable_sse () == true)
+    hw_features |= HW_FEATURE_X86_64_SSE;
+
+  if (enable_avx () == true)
+    hw_features |= HW_FEATURE_X86_64_AVX;
+}
+
+void
+grub_reset_gcry_hwf_x86_64_efi (void)
+{
+  grub_uint64_t cr0, cr4, xcr0;
+
+  if ((hw_features & HW_FEATURE_X86_64_AVX) != 0)
+    {
+      xcr0 = xgetbv (0);
+      if (xcr0 != old_xcr0)
+	{
+	  /*
+	   * Reset the AVX state with 'vzeroupper' before clearing XCR0[bit 2].
+	   *
+	   * Ref: Intel 64 and IA-32 Architectures Software Developer's Manual
+	   *      - 13.3 ENABLING THE XSAVE FEATURE SET AND XSAVE-ENABLED FEATURES
+	   *
+	   * "As noted in Section 13.1, the processor will preserve AVX state
+	   *  unmodified if software clears XCR0[2]. However, clearing XCR0[2]
+	   *  while AVX state is not in its initial configuration may cause SSE
+	   *  instructions to incur a power and performance penalty."
+	   */
+	  asm volatile ("vzeroupper" ::: "memory");
+	  xsetbv (0, old_xcr0);
+	}
+    }
+
+  if ((hw_features & HW_FEATURE_X86_64_AVX) != 0 || (hw_features & HW_FEATURE_X86_64_SSE) != 0)
+    {
+      cr4 = read_cr4 ();
+      if (cr4 != old_cr4)
+	write_cr4 (old_cr4);
+    }
+
+  if ((hw_features & HW_FEATURE_X86_64_SSE) != 0)
+    {
+      cr0 = read_cr0 ();
+      if (cr0 != old_cr0)
+	write_cr0 (old_cr0);
+    }
+
+  hw_features = 0;
+}
diff --git a/include/grub/x86_64/cpuid.h b/include/grub/x86_64/cpuid.h
new file mode 100644
index 000000000..acd93e3a7
--- /dev/null
+++ b/include/grub/x86_64/cpuid.h
@@ -0,0 +1 @@
+#include <grub/i386/cpuid.h>
diff --git a/include/grub/x86_64/efi/hwfeatures-gcry.h b/include/grub/x86_64/efi/hwfeatures-gcry.h
new file mode 100644
index 000000000..a8e671058
--- /dev/null
+++ b/include/grub/x86_64/efi/hwfeatures-gcry.h
@@ -0,0 +1,25 @@
+/*
+ *  GRUB  --  GRand Unified Bootloader
+ *  Copyright (C) 2025  Free Software Foundation, Inc.
+ *
+ *  GRUB is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  GRUB is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with GRUB.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef HWF_GCRY_X86_64_EFI_HEADER
+#define HWF_GCRY_X86_64_EFI_HEADER 1
+
+extern void grub_enable_gcry_hwf_x86_64_efi (void);
+extern void grub_reset_gcry_hwf_x86_64_efi (void);
+
+#endif /* HWF_GCRY_X86_64_EFI_HEADER */
-- 
2.51.0


_______________________________________________
Grub-devel mailing list
Grub-devel@gnu.org
https://lists.gnu.org/mailman/listinfo/grub-devel