* [Patch v2 1/2] zstd: Add zstd decompression logic
2024-12-12 21:13 [Patch v2 0/2] ZBOOT zstd support Jeremy Linton
@ 2024-12-12 21:13 ` Jeremy Linton
2024-12-13 12:46 ` Simon Horman
2024-12-12 21:13 ` [Patch v2 2/2] kexec: Enable zstd in kexec decompression paths Jeremy Linton
1 sibling, 1 reply; 5+ messages in thread
From: Jeremy Linton @ 2024-12-12 21:13 UTC (permalink / raw)
To: kexec; +Cc: piliu, horms, arb, dyoung, Jeremy Linton
zstd is becoming a popular zlib replacement because it both tends to
get a better compression ratio as well performs considerably
better. As such, zstd is now one of the options that can be used to
compress the Linux kernel.
Lets enable it by using a system provided shared library, creating the
required boilerplate to match the existing zlib/lzma function
signatures, and creating build options to enable/disable it.
Signed-off-by: Jeremy Linton <jeremy.linton@arm.com>
Acked-by: Pingfan Liu <piliu@redhat.com>
---
configure.ac | 10 +++
kexec/Makefile | 4 +-
kexec/kexec-zstd.h | 7 ++
kexec/zstd.c | 181 +++++++++++++++++++++++++++++++++++++++++++++
4 files changed, 201 insertions(+), 1 deletion(-)
create mode 100644 kexec/kexec-zstd.h
create mode 100644 kexec/zstd.c
diff --git a/configure.ac b/configure.ac
index e030302..43cdb64 100644
--- a/configure.ac
+++ b/configure.ac
@@ -106,6 +106,9 @@ AC_ARG_WITH([zlib], AS_HELP_STRING([--without-zlib],[disable zlib support]),
AC_ARG_WITH([lzma], AS_HELP_STRING([--without-lzma],[disable lzma support]),
[ with_lzma="$withval"], [ with_lzma=yes ] )
+AC_ARG_WITH([zstd], AS_HELP_STRING([--without-zstd],[disable zstd support]),
+ [ with_zstd="$withval"], [ with_zstd=yes ] )
+
AC_ARG_WITH([xen], AS_HELP_STRING([--without-xen],
[disable extended xen support]), [ with_xen="$withval"], [ with_xen=yes ] )
@@ -180,6 +183,13 @@ if test "$with_lzma" = yes ; then
AC_MSG_NOTICE([lzma support disabled]))])
fi
+dnl See if I have a usable copy of zstd available
+if test "$with_zstd" = yes ; then
+ AC_CHECK_HEADER(zstd.h,
+ [AC_CHECK_LIB(zstd, ZSTD_decompress, ,
+ AC_MSG_NOTICE([zstd support disabled]))])
+fi
+
dnl find Xen control stack libraries
if test "$with_xen" = yes ; then
AC_CHECK_HEADER(xenctrl.h,
diff --git a/kexec/Makefile b/kexec/Makefile
index d4f26d7..e969d1e 100644
--- a/kexec/Makefile
+++ b/kexec/Makefile
@@ -26,6 +26,7 @@ KEXEC_SRCS_base += kexec/crashdump-xen.c
KEXEC_SRCS_base += kexec/phys_arch.c
KEXEC_SRCS_base += kexec/lzma.c
KEXEC_SRCS_base += kexec/zlib.c
+KEXEC_SRCS_base += kexec/zstd.c
KEXEC_SRCS_base += kexec/kexec-xen.c
KEXEC_SRCS_base += kexec/symbols.c
@@ -36,7 +37,8 @@ dist += kexec/Makefile \
kexec/crashdump.h kexec/firmware_memmap.h \
kexec/kexec-elf-boot.h \
kexec/kexec-elf.h kexec/kexec-sha256.h \
- kexec/kexec-zlib.h kexec/kexec-lzma.h \
+ kexec/kexec-zlib.h kexec/kexec-lzma.h \
+ kexec/kexec-zstd.h \
kexec/kexec-xen.h \
kexec/kexec-syscall.h kexec/kexec.h kexec/kexec.8
diff --git a/kexec/kexec-zstd.h b/kexec/kexec-zstd.h
new file mode 100644
index 0000000..c5dbb24
--- /dev/null
+++ b/kexec/kexec-zstd.h
@@ -0,0 +1,7 @@
+#ifndef __KEXEC_ZSTD_H
+#define __KEXEC_ZSTD_H
+
+#include <sys/types.h>
+
+char *zstd_decompress_file(const char *filename, off_t *r_size);
+#endif /* __KEXEC_ZSTD_H */
diff --git a/kexec/zstd.c b/kexec/zstd.c
new file mode 100644
index 0000000..79d5bf3
--- /dev/null
+++ b/kexec/zstd.c
@@ -0,0 +1,181 @@
+#include "kexec-zstd.h"
+#include "kexec.h"
+
+#include "config.h"
+
+#ifdef HAVE_LIBZSTD
+#include <stdio.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <limits.h>
+#include <stdbool.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <ctype.h>
+#include <zstd.h>
+
+
+/*
+ * Reimplementation of private function available if zstd is
+ * statically linked. Remove when it becomes public.
+ */
+unsigned ZSTD_isFrame(const void* buffer, size_t size)
+{
+ uint8_t *buf = (uint8_t *)buffer;
+ /* Magic zstd frame header value */
+ if ((buf[0] == 0x28) &&
+ (buf[1] == 0xB5) &&
+ (buf[2] == 0x2F) &&
+ (buf[3] == 0xFD))
+ return 1;
+
+ return 0;
+}
+
+/*
+ * A guess at the max compression ratio for buffer overallocation.
+ * Real values are frequently around 4:1, if this is wrong
+ * it simply results in buffer reallocation and the decompression
+ * operation being restarted from where it stopped.
+ */
+#define EXPECTED_RATIO 8
+
+/*
+ * This supports the streaming compression mode the kernel uses
+ * that can result in multiple zstd frames comprising a single
+ * compressed image. In order too be a bit more efficient than
+ * the libz/lzma implementations, we attempt to discover the input
+ * and output image sizes before performing the decompression.
+ * But, in streaming mode the first frame may not have a length
+ * or it seems the length could be incorrect if multiple frames
+ * are appended together. So, its written with buffer resize logic
+ * but a guess at the compression ratio is made to avoid the resize/copy
+ * operation. Ideally this code efficiently allocates the
+ * correct input buffer, and no more than 2-3x the output buffer
+ * size so that it can perform the decompress operation with a
+ * single decompress call.
+ */
+char *zstd_decompress_file(const char *filename, off_t *r_size)
+{
+ void *cBuff = NULL;
+ ZSTD_DCtx* dctx = NULL;
+ FILE *fp = NULL; /* use c streams to match gzip/lzma logic */
+ struct stat fp_stats;
+ uint8_t magic[4];
+ size_t ret;
+
+ ZSTD_outBuffer output = { NULL, 0, 0 };
+ ZSTD_inBuffer input = { NULL, 0, 0 };
+
+ dbgprintf("Try zstd decompression.\n");
+
+ *r_size = 0;
+ if (!filename) {
+ return NULL;
+ }
+ if (stat(filename, &fp_stats)) {
+ dbgprintf("Cannot stat `%s'\n", filename);
+ return NULL;
+ }
+ if (fp_stats.st_size < sizeof(magic)) {
+ dbgprintf("short file\n");
+ return NULL;
+ }
+ input.size = fp_stats.st_size;
+
+ fp = fopen(filename, "rb");
+ if (fp == 0) {
+ dbgprintf("Cannot open `%s'\n", filename);
+ goto fail;
+ }
+ /* before we read the whole buffer see if this looks like a zstd frame */
+ if (fread(&magic, 1, sizeof(magic), fp) != sizeof(magic)) {
+ dbgprintf("Unable to read zstd header\n");
+ goto fail;
+ }
+
+ if (!ZSTD_isFrame((void*)&magic, sizeof(magic))) {
+ dbgprintf("Not zstd compressed\n");
+ goto fail;
+ }
+
+ cBuff = xmalloc(input.size);
+ input.src = cBuff; /* use cBuff ptr to avoid const/mismatches */
+ rewind(fp);
+ if (fread(cBuff, 1, input.size, fp) != input.size) {
+ dbgprintf("Unable to read compressed data\n");
+ goto fail;
+ }
+ fclose(fp);
+ fp = NULL;
+
+
+ output.size = ZSTD_getFrameContentSize(input.src, input.size);
+ if (output.size == ZSTD_CONTENTSIZE_ERROR) {
+ dbgprintf("not compressed by zstd\n");
+ goto fail;
+ }
+
+ if (output.size == ZSTD_CONTENTSIZE_UNKNOWN) {
+ dbgprintf("original zstd size unknown!\n");
+ /*
+ * The compressed size is an optional field in the header
+ * So we guess at the compression ratio to avoid reallocating
+ * the buffer, but this can fail so we still have code to
+ * handle that case.
+ */
+ output.size = fp_stats.st_size * EXPECTED_RATIO;
+ }
+
+ output.dst = xmalloc(output.size);
+
+ dctx = ZSTD_createDCtx();
+ if (dctx == NULL) {
+ dbgprintf("zstd context allocation error\n");
+ goto fail;
+ }
+
+ do {
+ if (output.pos == output.size) {
+ output.size <<= 1;
+ output.dst = xrealloc(output.dst, output.size);
+ }
+
+ ret = ZSTD_decompressStream(dctx, &output , &input);
+ if (ZSTD_isError(ret)) {
+ dbgprintf("zstd error %s\n", ZSTD_getErrorName(ret));
+ goto fail;
+ }
+ dbgprintf("zstd decompressed input=%ld to output=%ld\n", input.pos, output.pos);
+
+ } while ((input.pos < input.size) || ret);
+
+ free(cBuff);
+ ZSTD_freeDCtx(dctx);
+
+ *r_size = output.pos;
+
+ return output.dst;
+
+fail:
+ if (fp)
+ fclose(fp);
+ if (dctx)
+ ZSTD_freeDCtx(dctx);
+ if (output.dst)
+ free(output.dst);
+ if (cBuff)
+ free(cBuff);
+
+ return NULL;
+
+}
+#else
+
+char *zstd_decompress_file(const char *filename, off_t *r_size)
+{
+ return NULL;
+}
+#endif
--
2.47.0
^ permalink raw reply related [flat|nested] 5+ messages in thread* [Patch v2 2/2] kexec: Enable zstd in kexec decompression paths
2024-12-12 21:13 [Patch v2 0/2] ZBOOT zstd support Jeremy Linton
2024-12-12 21:13 ` [Patch v2 1/2] zstd: Add zstd decompression logic Jeremy Linton
@ 2024-12-12 21:13 ` Jeremy Linton
2024-12-13 12:46 ` Simon Horman
1 sibling, 1 reply; 5+ messages in thread
From: Jeremy Linton @ 2024-12-12 21:13 UTC (permalink / raw)
To: kexec; +Cc: piliu, horms, arb, dyoung, Jeremy Linton
The kexec decompression routines support detecting zlib or lzma
compressed images. Lets add the ability to automatically decompress
zstd ones as well. The zstd attempt is made after gzip because its
fairly inexpensive relative lzma, and gzip remains a popular option.
Signed-off-by: Jeremy Linton <jeremy.linton@arm.com>
Acked-by: Pingfan Liu <piliu@redhat.com>
---
kexec/kexec-pe-zboot.c | 3 ++-
kexec/kexec.c | 10 +++++++---
2 files changed, 9 insertions(+), 4 deletions(-)
diff --git a/kexec/kexec-pe-zboot.c b/kexec/kexec-pe-zboot.c
index 3abd17d..0a82097 100644
--- a/kexec/kexec-pe-zboot.c
+++ b/kexec/kexec-pe-zboot.c
@@ -59,8 +59,9 @@ int pez_prepare(const char *crude_buf, off_t buf_sz, int *kernel_fd,
* algorithms than are supported here, error out if we detect that.
*/
if (memcmp(&z->compress_type, "gzip", 4) &&
+ memcmp(&z->compress_type, "zstd", 4) &&
memcmp(&z->compress_type, "lzma", 4)) {
- dbgprintf("%s: kexec can only decompress gziped and lzma images.\n", __func__);
+ dbgprintf("%s: kexec can only decompress gziped, lzma and zstd images\n", __func__);
return -1;
}
diff --git a/kexec/kexec.c b/kexec/kexec.c
index 4f51987..70097ae 100644
--- a/kexec/kexec.c
+++ b/kexec/kexec.c
@@ -55,6 +55,7 @@
#include "kexec-sha256.h"
#include "kexec-zlib.h"
#include "kexec-lzma.h"
+#include "kexec-zstd.h"
#include <arch/options.h>
#define KEXEC_LOADED_PATH "/sys/kernel/kexec_loaded"
@@ -639,9 +640,12 @@ char *slurp_decompress_file(const char *filename, off_t *r_size)
kernel_buf = zlib_decompress_file(filename, r_size);
if (!kernel_buf) {
- kernel_buf = lzma_decompress_file(filename, r_size);
- if (!kernel_buf)
- return slurp_file(filename, r_size);
+ kernel_buf = zstd_decompress_file(filename, r_size);
+ if (!kernel_buf) {
+ kernel_buf = lzma_decompress_file(filename, r_size);
+ if (!kernel_buf)
+ return slurp_file(filename, r_size);
+ }
}
return kernel_buf;
}
--
2.47.0
^ permalink raw reply related [flat|nested] 5+ messages in thread