Re: [PATCH 1/4] io: Implement zstdio decompression

[Avnish Chouhan <avnish@linux.ibm.com>]

On 2025-09-24 01:49, grub-devel-request@gnu.org wrote:
> Message: 4
> Date: Tue, 23 Sep 2025 14:12:57 -0600
> From: Logan Gunthorpe <logang@deltatee.com>
> To: grub-devel@gnu.org, Daniel Kiper <daniel.kiper@oracle.com>, Alex
> 	Burmashev <alexander.burmashev@oracle.com>, Vladimir 'phcoder'
> 	Serbinenko <phcoder@gmail.com>
> Cc: Nagy Elemér Károly  <nagy.elemer.karoly@gmail.com>, Logan
> 	Gunthorpe <logang@deltatee.com>
> Subject: [PATCH 1/4] io: Implement zstdio decompression
> Message-ID: <20250923201300.3079-2-logang@deltatee.com>
> 
> Add zstd based io decompression.
> 
> Based largely on the existing xzio, implement the same features using
> the zstd library already included in the project.
> 
> Signed-off-by: Logan Gunthorpe <logang@deltatee.com>
> ---
>  Makefile.util.def           |   1 +
>  grub-core/Makefile.core.def |   9 +-
>  grub-core/io/zstdio.c       | 237 ++++++++++++++++++++++++++++++++++++
>  include/grub/file.h         |   3 +-
>  4 files changed, 248 insertions(+), 2 deletions(-)
>  create mode 100644 grub-core/io/zstdio.c
> 
> diff --git a/Makefile.util.def b/Makefile.util.def
> index 038253b37a42..74786177f908 100644
> --- a/Makefile.util.def
> +++ b/Makefile.util.def
> @@ -162,6 +162,7 @@ library = {
>    common = grub-core/io/gzio.c;
>    common = grub-core/io/xzio.c;
>    common = grub-core/io/lzopio.c;
> +  common = grub-core/io/zstdio.c;
>    common = grub-core/kern/ia64/dl_helper.c;
>    common = grub-core/kern/arm/dl_helper.c;
>    common = grub-core/kern/arm64/dl_helper.c;
> diff --git a/grub-core/Makefile.core.def b/grub-core/Makefile.core.def
> index 0fcf67f9df7d..4c8005782e31 100644
> --- a/grub-core/Makefile.core.def
> +++ b/grub-core/Makefile.core.def
> @@ -473,7 +473,7 @@ image = {
>    i386_pc = boot/i386/pc/boot.S;
> 
>    cppflags = '-DHYBRID_BOOT=1';
> -
> +
>    i386_pc_ldflags = '$(TARGET_IMG_LDFLAGS)';
>    i386_pc_ldflags = '$(TARGET_IMG_BASE_LDOPT),0x7C00';
> 
> @@ -2486,6 +2486,13 @@ module = {
>    cppflags = '-I$(srcdir)/lib/posix_wrap -I$(srcdir)/lib/minilzo
> -DMINILZO_HAVE_CONFIG_H';
>  };
> 
> +module = {
> +  name = zstdio;
> +  common = io/zstdio.c;
> +  cppflags = '-I$(srcdir)/lib/posix_wrap -I$(srcdir)/lib/zstd';
> +  cflags='-Wno-unreachable-code';
> +};
> +
>  module = {
>    name = testload;
>    common = commands/testload.c;
> diff --git a/grub-core/io/zstdio.c b/grub-core/io/zstdio.c
> new file mode 100644
> index 000000000000..b64655aa63e3
> --- /dev/null
> +++ b/grub-core/io/zstdio.c
> @@ -0,0 +1,237 @@
> +/* zstdio.c - decompression support for zstd */
> +/*
> + *  GRUB  --  GRand Unified Bootloader
> + *  Copyright (C) 2010  Free Software Foundation, Inc.
> + *
> + *  GRUB is free software: you can redistribute it and/or modify
> + *  it under the terms of the GNU General Public License as published 
> by
> + *  the Free Software Foundation, either version 3 of the License, or
> + *  (at your option) any later version.
> + *
> + *  GRUB is distributed in the hope that it will be useful,
> + *  but WITHOUT ANY WARRANTY; without even the implied warranty of
> + *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> + *  GNU General Public License for more details.
> + *
> + *  You should have received a copy of the GNU General Public License
> + *  along with GRUB.  If not, see <http://www.gnu.org/licenses/>.
> + */
> +
> +#include <grub/err.h>
> +#include <grub/mm.h>
> +#include <grub/misc.h>
> +#include <grub/file.h>
> +#include <grub/fs.h>
> +#include <grub/dl.h>
> +
> +GRUB_MOD_LICENSE ("GPLv3+");
> +
> +#include "zstd.h"
> +
> +#define STREAM_HEADER_SIZE 16
> +
> +struct grub_zstdio
> +{
> +  grub_file_t file;
> +  ZSTD_DCtx *dctx;
> +  grub_size_t insize;
> +  grub_size_t outsize;
> +
> +  ZSTD_outBuffer output;
> +  ZSTD_inBuffer input;
> +
> +  grub_off_t saved_offset;
> +  grub_uint8_t bufs[];
> +};
> +
> +typedef struct grub_zstdio *grub_zstdio_t;
> +static struct grub_fs grub_zstdio_fs;
> +
> +static int
> +test_header (grub_file_t file)
> +{
> +  grub_zstdio_t zstdio = file->data;
> +  size_t zret;
> +
> +  zstdio->input.pos = 0;
> +  zstdio->output.pos = 0;
> +  zstdio->output.size = zstdio->outsize;
> +  zstdio->input.size = grub_file_read (zstdio->file, zstdio->bufs,
> +				       STREAM_HEADER_SIZE);

Hi Logan,

Indentation seems little off in line "STREAM_HEADER_SIZE);" . Please 
recheck!

> +  if (zstdio->input.size != STREAM_HEADER_SIZE)
> +    return 0;
> +
> +  zret = ZSTD_decompressStream(zstdio->dctx, &zstdio->output, 
> &zstdio->input);

Missing empty space, "ZSTD_decompressStream (zstdio->dctx...."

> +  if (ZSTD_isError(zret))

Same as above. Missing empty space!

> +    return 0;
> +
> +  return 1;
> +}
> +
> +static grub_file_t
> +grub_zstdio_open (grub_file_t io, enum grub_file_type type)
> +{
> +  grub_file_t file;
> +  grub_zstdio_t zstdio;
> +
> +  if (type & GRUB_FILE_TYPE_NO_DECOMPRESS)
> +    return io;
> +
> +  file = (grub_file_t) grub_zalloc (sizeof (*file));

"grub_zalloc (sizeof (grub_file_t));" would be better!

> +  if (!file)

if (file == NULL)

> +    return 0;
> +
> +  zstdio = grub_zalloc (sizeof (*zstdio) + ZSTD_DStreamInSize() +
> +			ZSTD_DStreamOutSize());

zstdio = grub_zalloc (sizeof (grub_zstdio_t) + ZSTD_DStreamInSize () + 
ZSTD_DStreamOutSize ());

> +  if (!zstdio)

if (zstdio == NULL)

> +    {
> +      grub_free (file);
> +      return 0;
> +    }
> +
> +  zstdio->file = io;
> +  zstdio->insize = ZSTD_DStreamInSize();
> +  zstdio->outsize = ZSTD_DStreamOutSize();

zstdio->insize = ZSTD_DStreamInSize ();
zstdio->outsize = ZSTD_DStreamOutSize ();

Please add the empty space wherever missing!

> +  zstdio->input.src = zstdio->bufs;
> +  zstdio->output.dst = &zstdio->bufs[zstdio->insize];
> +
> +  file->device = io->device;
> +  file->data = zstdio;
> +  file->fs = &grub_zstdio_fs;
> +  file->size = GRUB_FILE_SIZE_UNKNOWN;
> +  file->not_easily_seekable = 1;
> +
> +  if (grub_file_tell (zstdio->file) != 0)
> +    grub_file_seek (zstdio->file, 0);
> +
> +  zstdio->dctx = ZSTD_createDCtx();
> +  if (!zstdio->dctx)

if (zstdio->dctx == NULL)

> +    {
> +      grub_free (file);
> +      grub_free (zstdio);
> +      return 0;
> +    }
> +
> +  if (!test_header (file))
> +    {
> +      grub_errno = GRUB_ERR_NONE;
> +      grub_file_seek (io, 0);
> +      ZSTD_freeDCtx(zstdio->dctx);

ZSTD_freeDCtx (zstdio->dctx);

> +      grub_free (zstdio);
> +      grub_free (file);
> +
> +      return io;
> +    }
> +
> +  return file;
> +}
> +
> +static grub_ssize_t
> +grub_zstdio_read (grub_file_t file, char *buf, grub_size_t len)
> +{
> +  grub_zstdio_t zstdio = file->data;
> +  grub_ssize_t ret = 0;
> +  grub_ssize_t readret;
> +  grub_off_t current_offset;
> +  grub_size_t zret;
> +
> +  /* If seek backward need to reset decoder and start from beginning
> of file. */
> +  if (file->offset < zstdio->saved_offset)
> +    {
> +      ZSTD_initDStream(zstdio->dctx);

ZSTD_initDStream (zstdio->dctx);

> +      zstdio->input.pos = 0;
> +      zstdio->input.size = 0;
> +      zstdio->output.pos = 0;
> +      zstdio->saved_offset = 0;
> +      grub_file_seek (zstdio->file, 0);
> +    }
> +
> +  current_offset = zstdio->saved_offset;
> +
> +  while (len > 0)
> +    {
> +      zstdio->output.size = file->offset + ret + len - current_offset;
> +      if (zstdio->output.size > zstdio->outsize)
> +	zstdio->output.size = zstdio->outsize;
> +      if (zstdio->input.pos == zstdio->input.size)
> +	{
> +	  readret = grub_file_read (zstdio->file, zstdio->bufs,
> +				    zstdio->insize);
> +	  if (readret < 0)
> +	    return -1;

Adding an empty line would be good here! And could you please recheck 
the indentation in this while loop. Seems little off.

> +	  zstdio->input.size = readret;
> +	  zstdio->input.pos = 0;
> +	}
> +
> +      zret = ZSTD_decompressStream(zstdio->dctx, &zstdio->output,

...ZSTD_decompressStream (zstdio...

> +				   &zstdio->input);
> +      if (ZSTD_isError(zret))

if (ZSTD_isError (zret))

> +	{
> +	  grub_error (GRUB_ERR_BAD_COMPRESSED_DATA,
> +		      N_("zstd file corrupted or unsupported block options"));
> +	  return -1;
> +        }
> +
> +      grub_off_t new_offset = current_offset + zstdio->output.pos;
> +
> +      if (file->offset <= new_offset)
> +	  /* Store first chunk of data in buffer.  */

/* Store first chunk of data in buffer.  */
if (file->offset <= new_offset)

> +        {
> +	  grub_size_t delta = new_offset - (file->offset + ret);
> +	  grub_memmove (buf, (grub_uint8_t *)zstdio->output.dst +

Empty space missing! "..., (grub_uint8_t *) zstdio..."

> +			(zstdio->output.pos - delta),
> +	                delta);
> +	  len -= delta;
> +	  buf += delta;
> +	  ret += delta;
> +	}
> +	current_offset = new_offset;
> +
> +	zstdio->output.pos = 0;
> +
> +	if (zstdio->input.pos == 0 && zstdio->output.pos == 0)
> +	  break;
> +    }
> +
> +  if (ret >= 0)
> +    zstdio->saved_offset = file->offset + ret;
> +
> +  return ret;
> +}
> +
> +/* Release everything, including the underlying file object.  */
> +static grub_err_t
> +grub_zstdio_close (grub_file_t file)
> +{
> +  grub_zstdio_t zstdio = file->data;
> +
> +  ZSTD_freeDCtx(zstdio->dctx);

ZSTD_freeDCtx (zstdio->dctx);

Thank you!

Regards,
Avnish Chouhan

> +
> +  grub_file_close (zstdio->file);
> +  grub_free (zstdio);
> +
> +  /* Device must not be closed twice.  */
> +  file->device = 0;
> +  file->name = 0;
> +  return grub_errno;
> +}
> +
> +static struct grub_fs grub_zstdio_fs = {
> +  .name = "zstdio",
> +  .fs_dir = 0,
> +  .fs_open = 0,
> +  .fs_read = grub_zstdio_read,
> +  .fs_close = grub_zstdio_close,
> +  .fs_label = 0,
> +  .next = 0
> +};
> +
> +GRUB_MOD_INIT (zstdio)
> +{
> +  grub_file_filter_register (GRUB_FILE_FILTER_ZSTDIO, 
> grub_zstdio_open);
> +}
> +
> +GRUB_MOD_FINI (zstdio)
> +{
> +  grub_file_filter_unregister (GRUB_FILE_FILTER_ZSTDIO);
> +}
> diff --git a/include/grub/file.h b/include/grub/file.h
> index a5bf3a792d6f..7c665730a090 100644
> --- a/include/grub/file.h
> +++ b/include/grub/file.h
> @@ -186,9 +186,10 @@ typedef enum grub_file_filter_id
>      GRUB_FILE_FILTER_GZIO,
>      GRUB_FILE_FILTER_XZIO,
>      GRUB_FILE_FILTER_LZOPIO,
> +    GRUB_FILE_FILTER_ZSTDIO,
>      GRUB_FILE_FILTER_MAX,
>      GRUB_FILE_FILTER_COMPRESSION_FIRST = GRUB_FILE_FILTER_GZIO,
> -    GRUB_FILE_FILTER_COMPRESSION_LAST = GRUB_FILE_FILTER_LZOPIO,
> +    GRUB_FILE_FILTER_COMPRESSION_LAST = GRUB_FILE_FILTER_ZSTDIO,
>    } grub_file_filter_id_t;
> 
>  typedef grub_file_t (*grub_file_filter_t) (grub_file_t in, enum
> grub_file_type type);
> --
> 2.47.3
> 
> 
> 
> 
> ------------------------------
> 
> Subject: Digest Footer
> 
> _______________________________________________
> Grub-devel mailing list
> Grub-devel@gnu.org
> https://lists.gnu.org/mailman/listinfo/grub-devel
> 
> 
> ------------------------------
> 
> End of Grub-devel Digest, Vol 259, Issue 83
> *******************************************

_______________________________________________
Grub-devel mailing list
Grub-devel@gnu.org
https://lists.gnu.org/mailman/listinfo/grub-devel