[PATCH] btrfs: add test for encoded reads

[PATCH] btrfs: add test for encoded reads

[Mark Harmstone]

Add btrfs/333 and its helper programs btrfs_encoded_read and
btrfs_encoded_write, in order to test encoded reads.

We use the BTRFS_IOC_ENCODED_WRITE ioctl to write random data into a
compressed extent, then use the BTRFS_IOC_ENCODED_READ ioctl to check
that it matches what we've written. If the new io_uring interface for
encoded reads is supported, we also check that that matches the ioctl.

Note that what we write isn't valid compressed data, so any non-encoded
reads on these files will fail.

Signed-off-by: Mark Harmstone <maharmstone@fb.com>
---
 .gitignore                |   2 +
 src/Makefile              |   3 +-
 src/btrfs_encoded_read.c  | 175 ++++++++++++++++++++++++++++++
 src/btrfs_encoded_write.c | 206 +++++++++++++++++++++++++++++++++++
 tests/btrfs/333           | 220 ++++++++++++++++++++++++++++++++++++++
 tests/btrfs/333.out       |   2 +
 6 files changed, 607 insertions(+), 1 deletion(-)
 create mode 100644 src/btrfs_encoded_read.c
 create mode 100644 src/btrfs_encoded_write.c
 create mode 100755 tests/btrfs/333
 create mode 100644 tests/btrfs/333.out

diff --git a/.gitignore b/.gitignore
index f16173d9..efd47773 100644
--- a/.gitignore
+++ b/.gitignore
@@ -62,6 +62,8 @@ tags
 /src/attr_replace_test
 /src/attr-list-by-handle-cursor-test
 /src/bstat
+/src/btrfs_encoded_read
+/src/btrfs_encoded_write
 /src/bulkstat_null_ocount
 /src/bulkstat_unlink_test
 /src/bulkstat_unlink_test_modified
diff --git a/src/Makefile b/src/Makefile
index a0396332..b42b8147 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -34,7 +34,8 @@ LINUX_TARGETS = xfsctl bstat t_mtab getdevicesize preallo_rw_pattern_reader \
 	attr_replace_test swapon mkswap t_attr_corruption t_open_tmpfiles \
 	fscrypt-crypt-util bulkstat_null_ocount splice-test chprojid_fail \
 	detached_mounts_propagation ext4_resize t_readdir_3 splice2pipe \
-	uuid_ioctl t_snapshot_deleted_subvolume fiemap-fault min_dio_alignment
+	uuid_ioctl t_snapshot_deleted_subvolume fiemap-fault min_dio_alignment \
+	btrfs_encoded_read btrfs_encoded_write
 
 EXTRA_EXECS = dmerror fill2attr fill2fs fill2fs_check scaleread.sh \
 	      btrfs_crc32c_forged_name.py popdir.pl popattr.py \
diff --git a/src/btrfs_encoded_read.c b/src/btrfs_encoded_read.c
new file mode 100644
index 00000000..a5082f70
--- /dev/null
+++ b/src/btrfs_encoded_read.c
@@ -0,0 +1,175 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/uio.h>
+#include <sys/ioctl.h>
+#include <linux/btrfs.h>
+#include "config.h"
+
+#ifdef HAVE_LIBURING_H
+#include <liburing.h>
+#endif
+
+#define BTRFS_MAX_COMPRESSED 131072
+#define QUEUE_DEPTH 1
+
+static int encoded_read_ioctl(const char *filename, long long offset)
+{
+	int ret, fd;
+	char buf[BTRFS_MAX_COMPRESSED];
+	struct iovec iov;
+	struct btrfs_ioctl_encoded_io_args enc;
+
+	fd = open(filename, O_RDONLY);
+	if (fd < 0) {
+		fprintf(stderr, "open failed for %s\n", filename);
+		return 1;
+	}
+
+	iov.iov_base = buf;
+	iov.iov_len = sizeof(buf);
+
+	enc.iov = &iov;
+	enc.iovcnt = 1;
+	enc.offset = offset;
+	enc.flags = 0;
+
+	ret = ioctl(fd, BTRFS_IOC_ENCODED_READ, &enc);
+
+	if (ret < 0) {
+		printf("%i\n", -errno);
+		close(fd);
+		return 0;
+	}
+
+	close(fd);
+
+	printf("%i\n", ret);
+	printf("%llu\n", enc.len);
+	printf("%llu\n", enc.unencoded_len);
+	printf("%llu\n", enc.unencoded_offset);
+	printf("%u\n", enc.compression);
+	printf("%u\n", enc.encryption);
+
+	fwrite(buf, ret, 1, stdout);
+
+	return 0;
+}
+
+static int encoded_read_io_uring(const char *filename, long long offset)
+{
+#ifdef HAVE_LIBURING_H
+	int ret, fd;
+	char buf[BTRFS_MAX_COMPRESSED];
+	struct iovec iov;
+	struct btrfs_ioctl_encoded_io_args enc;
+	struct io_uring ring;
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe;
+
+	io_uring_queue_init(QUEUE_DEPTH, &ring, 0);
+
+	fd = open(filename, O_RDONLY);
+	if (fd < 0) {
+		fprintf(stderr, "open failed for %s\n", filename);
+		ret = 1;
+		goto out_uring;
+	}
+
+	iov.iov_base = buf;
+	iov.iov_len = sizeof(buf);
+
+	enc.iov = &iov;
+	enc.iovcnt = 1;
+	enc.offset = offset;
+	enc.flags = 0;
+
+	sqe = io_uring_get_sqe(&ring);
+	if (!sqe) {
+		fprintf(stderr, "io_uring_get_sqe failed\n");
+		ret = 1;
+		goto out_close;
+	}
+
+	io_uring_prep_rw(IORING_OP_URING_CMD, sqe, fd, &enc, sizeof(enc), 0);
+	sqe->cmd_op = BTRFS_IOC_ENCODED_READ;
+
+	io_uring_submit(&ring);
+
+	ret = io_uring_wait_cqe(&ring, &cqe);
+	if (ret < 0) {
+		fprintf(stderr, "io_uring_wait_cqe returned %i\n", ret);
+		ret = 1;
+		goto out_close;
+	}
+
+	io_uring_cqe_seen(&ring, cqe);
+
+	if (cqe->res < 0) {
+		printf("%i\n", cqe->res);
+		ret = 0;
+		goto out_close;
+	}
+
+	printf("%i\n", cqe->res);
+	printf("%llu\n", enc.len);
+	printf("%llu\n", enc.unencoded_len);
+	printf("%llu\n", enc.unencoded_offset);
+	printf("%u\n", enc.compression);
+	printf("%u\n", enc.encryption);
+
+	fwrite(buf, cqe->res, 1, stdout);
+
+	ret = 0;
+
+out_close:
+	close(fd);
+
+out_uring:
+	io_uring_queue_exit(&ring);
+
+	return ret;
+#else
+	fprintf(stderr, "liburing not linked in\n");
+	return 1;
+#endif
+}
+
+static void usage()
+{
+	fprintf(stderr, "Usage: btrfs_encoded_read ioctl|io_uring filename offset\n");
+}
+
+int main(int argc, char *argv[])
+{
+	const char *filename;
+	long long offset;
+
+	if (argc != 4) {
+		usage();
+		return 1;
+	}
+
+	filename = argv[2];
+
+	offset = atoll(argv[3]);
+	if (offset == 0 && errno != 0) {
+		usage();
+		return 1;
+	}
+
+	if (!strcmp(argv[1], "ioctl")) {
+		return encoded_read_ioctl(filename, offset);
+	} else if (!strcmp(argv[1], "io_uring")) {
+		return encoded_read_io_uring(filename, offset);
+	} else {
+		usage();
+		return 1;
+	}
+}
diff --git a/src/btrfs_encoded_write.c b/src/btrfs_encoded_write.c
new file mode 100644
index 00000000..dbb62722
--- /dev/null
+++ b/src/btrfs_encoded_write.c
@@ -0,0 +1,206 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/uio.h>
+#include <sys/ioctl.h>
+#include <linux/btrfs.h>
+#include "config.h"
+
+#ifdef HAVE_LIBURING_H
+#include <liburing.h>
+#endif
+
+#define BTRFS_MAX_COMPRESSED 131072
+#define QUEUE_DEPTH 1
+
+static int encoded_write_ioctl(const char *filename, long long offset,
+			       long long len, long long unencoded_len,
+			       long long unencoded_offset, int compression,
+			       char *buf, size_t size)
+{
+	int ret, fd;
+	struct iovec iov;
+	struct btrfs_ioctl_encoded_io_args enc;
+
+	fd = open(filename, O_CREAT | O_TRUNC | O_WRONLY, 0644);
+	if (fd < 0) {
+		fprintf(stderr, "open failed for %s\n", filename);
+		return 1;
+	}
+
+	iov.iov_base = buf;
+	iov.iov_len = size;
+
+	memset(&enc, 0, sizeof(enc));
+	enc.iov = &iov;
+	enc.iovcnt = 1;
+	enc.offset = offset;
+	enc.len = len;
+	enc.unencoded_len = unencoded_len;
+	enc.unencoded_offset = unencoded_offset;
+	enc.compression = compression;
+
+	ret = ioctl(fd, BTRFS_IOC_ENCODED_WRITE, &enc);
+
+	if (ret < 0) {
+		printf("%i\n", -errno);
+		close(fd);
+		return 0;
+	}
+
+	printf("%i\n", ret);
+
+	close(fd);
+
+	return 0;
+}
+
+static int encoded_write_io_uring(const char *filename, long long offset,
+				  long long len, long long unencoded_len,
+				  long long unencoded_offset, int compression,
+				  char *buf, size_t size)
+{
+#ifdef HAVE_LIBURING_H
+	int ret, fd;
+	struct iovec iov;
+	struct btrfs_ioctl_encoded_io_args enc;
+	struct io_uring ring;
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe;
+
+	io_uring_queue_init(QUEUE_DEPTH, &ring, 0);
+
+	fd = open(filename, O_CREAT | O_TRUNC | O_WRONLY, 0644);
+	if (fd < 0) {
+		fprintf(stderr, "open failed for %s\n", filename);
+		ret = 1;
+		goto out_uring;
+	}
+
+	iov.iov_base = buf;
+	iov.iov_len = size;
+
+	memset(&enc, 0, sizeof(enc));
+	enc.iov = &iov;
+	enc.iovcnt = 1;
+	enc.offset = offset;
+	enc.len = len;
+	enc.unencoded_len = unencoded_len;
+	enc.unencoded_offset = unencoded_offset;
+	enc.compression = compression;
+
+	sqe = io_uring_get_sqe(&ring);
+	if (!sqe) {
+		fprintf(stderr, "io_uring_get_sqe failed\n");
+		ret = 1;
+		goto out_close;
+	}
+
+	io_uring_prep_rw(IORING_OP_URING_CMD, sqe, fd, &enc, sizeof(enc), 0);
+	sqe->cmd_op = BTRFS_IOC_ENCODED_WRITE;
+
+	io_uring_submit(&ring);
+
+	ret = io_uring_wait_cqe(&ring, &cqe);
+	if (ret < 0) {
+		fprintf(stderr, "io_uring_wait_cqe returned %i\n", ret);
+		ret = 1;
+		goto out_close;
+	}
+
+	io_uring_cqe_seen(&ring, cqe);
+
+	if (cqe->res < 0) {
+		printf("%i\n", cqe->res);
+		ret = 0;
+		goto out_close;
+	}
+
+	printf("%i\n", cqe->res);
+
+	ret = 0;
+
+out_close:
+	close(fd);
+
+out_uring:
+	io_uring_queue_exit(&ring);
+
+	return ret;
+#else
+	fprintf(stderr, "liburing not linked in\n");
+	return 1;
+#endif
+}
+
+static void usage()
+{
+	fprintf(stderr, "Usage: btrfs_encoded_write ioctl|io_uring filename offset len unencoded_len unencoded_offset compression\n");
+}
+
+int main(int argc, char *argv[])
+{
+	const char *filename;
+	long long offset, len, unencoded_len, unencoded_offset;
+	int compression;
+	char buf[BTRFS_MAX_COMPRESSED];
+	size_t size;
+
+	if (argc != 8) {
+		usage();
+		return 1;
+	}
+
+	filename = argv[2];
+
+	offset = atoll(argv[3]);
+	if (offset == 0 && errno != 0) {
+		usage();
+		return 1;
+	}
+
+	len = atoll(argv[4]);
+	if (len == 0 && errno != 0) {
+		usage();
+		return 1;
+	}
+
+	unencoded_len = atoll(argv[5]);
+	if (unencoded_len == 0 && errno != 0) {
+		usage();
+		return 1;
+	}
+
+	unencoded_offset = atoll(argv[6]);
+	if (unencoded_offset == 0 && errno != 0) {
+		usage();
+		return 1;
+	}
+
+	compression = atoi(argv[7]);
+	if (compression == 0 && errno != 0) {
+		usage();
+		return 1;
+	}
+
+	size = fread(buf, 1, BTRFS_MAX_COMPRESSED, stdin);
+
+	if (!strcmp(argv[1], "ioctl")) {
+		return encoded_write_ioctl(filename, offset, len, unencoded_len,
+					   unencoded_offset, compression, buf,
+					   size);
+	} else if (!strcmp(argv[1], "io_uring")) {
+		return encoded_write_io_uring(filename, offset, len,
+					      unencoded_len, unencoded_offset,
+					      compression, buf, size);
+	} else {
+		usage();
+		return 1;
+	}
+}
diff --git a/tests/btrfs/333 b/tests/btrfs/333
new file mode 100755
index 00000000..8e4de4c0
--- /dev/null
+++ b/tests/btrfs/333
@@ -0,0 +1,220 @@
+#! /bin/bash
+# SPDX-License-Identifier: GPL-2.0
+# Copyright (c) 2024 Meta Platforms, Inc.  All Rights Reserved.
+#
+# FS QA Test No. btrfs/333
+#
+# Test btrfs encoded reads
+
+. ./common/preamble
+_begin_fstest auto quick compress rw
+
+. ./common/filter
+. ./common/btrfs
+
+_supported_fs btrfs
+
+do_encoded_read() {
+    local fn=$1
+    local type=$2
+    local exp_ret=$3
+    local exp_len=$4
+    local exp_unencoded_len=$5
+    local exp_unencoded_offset=$6
+    local exp_compression=$7
+    local exp_md5=$8
+
+    local tmpfile=`mktemp`
+
+    echo "running btrfs_encoded_read $type $fn 0 > $tmpfile" >>$seqres.full
+    src/btrfs_encoded_read $type $fn 0 > $tmpfile
+
+    if [[ $? -ne 0 ]]; then
+        echo "btrfs_encoded_read failed" >>$seqres.full
+        rm $tmpfile
+        return 1
+    fi
+
+    exec {FD}< $tmpfile
+
+    read -u ${FD} ret
+
+    if [[ $ret == -95 && $type -eq "io_uring" ]]; then
+        echo "btrfs io_uring encoded read failed with -EOPNOTSUPP, skipping" >>$seqres.full
+        exec {FD}<&-
+        return 1
+    fi
+
+    if [[ $ret -lt 0 ]]; then
+        if [[ $ret == -1 ]]; then
+            echo "btrfs encoded read failed with -EPERM; are you running as root?" >>$seqres.full
+        else
+            echo "btrfs encoded read failed (errno $ret)" >>$seqres.full
+        fi
+        exec {FD}<&-
+        return 1
+    fi
+
+    local status=0
+
+    if [[ $ret -ne $exp_ret ]]; then
+        echo "$fn: btrfs encoded read returned $ret, expected $exp_ret" >>$seqres.full
+        status=1
+    fi
+
+    read -u ${FD} len
+    read -u ${FD} unencoded_len
+    read -u ${FD} unencoded_offset
+    read -u ${FD} compression
+    read -u ${FD} encryption
+
+    local filesize=`stat -c%s $tmpfile`
+    local datafile=`mktemp`
+
+    dd if=$tmpfile of=$datafile bs=1 count=$ret skip=$(($filesize-$ret)) status=none
+
+    exec {FD}<&-
+    rm $tmpfile
+
+    local md5=`md5sum $datafile | cut -d ' ' -f 1`
+    rm $datafile
+
+    if [[ $len -ne $exp_len ]]; then
+        echo "$fn: btrfs encoded read had len of $len, expected $exp_len" >>$seqres.full
+        status=1
+    fi
+
+    if [[ $unencoded_len -ne $exp_unencoded_len ]]; then
+        echo "$fn: btrfs encoded read had unencoded_len of $unencoded_len, expected $exp_unencoded_len" >>$seqres.full
+        status=1
+    fi
+
+    if [[ $unencoded_offset -ne $exp_unencoded_offset ]]; then
+        echo "$fn: btrfs encoded read had unencoded_offset of $unencoded_offset, expected $exp_unencoded_offset" >>$seqres.full
+        status=1
+    fi
+
+    if [[ $compression -ne $exp_compression ]]; then
+        echo "$fn: btrfs encoded read had compression of $compression, expected $exp_compression" >>$seqres.full
+        status=1
+    fi
+
+    if [[ $encryption -ne 0 ]]; then
+        echo "$fn: btrfs encoded read had encryption of $encryption, expected 0" >>$seqres.full
+        status=1
+    fi
+
+    if [[ $md5 != $exp_md5 ]]; then
+        echo "$fn: data returned had hash of $md5, expected $exp_md5" >>$seqres.full
+        status=1
+    fi
+
+    return $status
+}
+
+do_encoded_write() {
+    local fn=$1
+    local exp_ret=$2
+    local len=$3
+    local unencoded_len=$4
+    local unencoded_offset=$5
+    local compression=$6
+    local data_file=$7
+
+    local tmpfile=`mktemp`
+
+    echo "running btrfs_encoded_write ioctl $fn 0 $len $unencoded_len $unencoded_offset $compression < $data_file > $tmpfile" >>$seqres.full
+    src/btrfs_encoded_write ioctl $fn 0 $len $unencoded_len $unencoded_offset $compression < $data_file > $tmpfile
+
+    if [[ $? -ne 0 ]]; then
+        echo "btrfs_encoded_write failed" >>$seqres.full
+        rm $tmpfile
+        return 1
+    fi
+
+    exec {FD}< $tmpfile
+
+    read -u ${FD} ret
+
+    if [[ $ret -lt 0 ]]; then
+        if [[ $ret == -1 ]]; then
+            echo "btrfs encoded write failed with -EPERM; are you running as root?" >>$seqres.full
+        else
+            echo "btrfs encoded write failed (errno $ret)" >>$seqres.full
+        fi
+        exec {FD}<&-
+        return 1
+    fi
+
+    exec {FD}<&-
+    rm $tmpfile
+
+    return 0
+}
+
+test_file() {
+    local size=$1
+    local len=$2
+    local unencoded_len=$3
+    local unencoded_offset=$4
+    local compression=$5
+
+    local tmpfile=`mktemp -p $SCRATCH_MNT`
+    local randfile=`mktemp`
+
+    dd if=/dev/urandom of=$randfile bs=1 count=$size status=none
+    local md5=`md5sum $randfile | cut -d ' ' -f 1`
+
+    do_encoded_write $tmpfile $size $len $unencoded_len $unencoded_offset \
+        $compression $randfile || _fail "encoded write ioctl failed"
+
+    rm $randfile
+
+    do_encoded_read $tmpfile ioctl $size $len $unencoded_len \
+        $unencoded_offset $compression $md5 || _fail "encoded read ioctl failed"
+    do_encoded_read $tmpfile io_uring $size $len $unencoded_len \
+        $unencoded_offset $compression $md5 || _fail "encoded read io_uring failed"
+
+    rm $tmpfile
+}
+
+_scratch_mkfs >> $seqres.full 2>&1 || _fail "mkfs failed"
+sector_size=$(_scratch_btrfs_sectorsize)
+
+if [[ $sector_size -ne 4096 && $sector_size -ne 65536 ]]; then
+    _notrun "sector size $sector_size not supported by this test"
+fi
+
+_scratch_mount "-o max_inline=2048"
+
+if [[ $sector_size -eq 4096 ]]; then
+    test_file 40960 97966 98304 0 1 # zlib
+    test_file 40960 97966 98304 0 2 # zstd
+    test_file 40960 97966 98304 0 3 # lzo 4k
+    test_file 40960 97966 110592 4096 1 # bookended zlib
+    test_file 40960 97966 110592 4096 2 # bookended zstd
+    test_file 40960 97966 110592 4096 3 # bookended lzo 4k
+elif [[ $sector_size -eq 65536 ]]; then
+    test_file 65536 97966 131072 0 1 # zlib
+    test_file 65536 97966 131072 0 2 # zstd
+    test_file 65536 97966 131072 0 7 # lzo 64k
+    # can't test bookended extents on 64k, as max is only 2 sectors long
+fi
+
+# btrfs won't create inline files unless PAGE_SIZE == sector size
+if [[ "$(_get_page_size)" -eq $sector_size ]]; then
+    test_file 892 1931 1931 0 1 # inline zlib
+    test_file 892 1931 1931 0 2 # inline zstd
+
+    if [[ $sector_size -eq 4096 ]]; then
+        test_file 892 1931 1931 0 3 # inline lzo 4k
+    elif [[ $sector_size -eq 65536 ]]; then
+        test_file 892 1931 1931 0 7 # inline lzo 64k
+    fi
+fi
+
+_scratch_unmount
+
+echo Silence is golden
+status=0
+exit
diff --git a/tests/btrfs/333.out b/tests/btrfs/333.out
new file mode 100644
index 00000000..60a15898
--- /dev/null
+++ b/tests/btrfs/333.out
@@ -0,0 +1,2 @@
+QA output created by 333
+Silence is golden
-- 
2.45.2

Re: [PATCH] btrfs: add test for encoded reads

[Johannes Thumshirn]

On 11.11.24 15:56, Mark Harmstone wrote:
> Add btrfs/333 and its helper programs btrfs_encoded_read and
> btrfs_encoded_write, in order to test encoded reads.
> 
> We use the BTRFS_IOC_ENCODED_WRITE ioctl to write random data into a
> compressed extent, then use the BTRFS_IOC_ENCODED_READ ioctl to check
> that it matches what we've written. If the new io_uring interface for
> encoded reads is supported, we also check that that matches the ioctl.
> 
> Note that what we write isn't valid compressed data, so any non-encoded
> reads on these files will fail.
> 
> Signed-off-by: Mark Harmstone <maharmstone@fb.com>
> ---
>   .gitignore                |   2 +
>   src/Makefile              |   3 +-
>   src/btrfs_encoded_read.c  | 175 ++++++++++++++++++++++++++++++
>   src/btrfs_encoded_write.c | 206 +++++++++++++++++++++++++++++++++++
>   tests/btrfs/333           | 220 ++++++++++++++++++++++++++++++++++++++
>   tests/btrfs/333.out       |   2 +
>   6 files changed, 607 insertions(+), 1 deletion(-)
>   create mode 100644 src/btrfs_encoded_read.c
>   create mode 100644 src/btrfs_encoded_write.c
>   create mode 100755 tests/btrfs/333
>   create mode 100644 tests/btrfs/333.out
> 
> diff --git a/.gitignore b/.gitignore
> index f16173d9..efd47773 100644
> --- a/.gitignore
> +++ b/.gitignore
> @@ -62,6 +62,8 @@ tags
>   /src/attr_replace_test
>   /src/attr-list-by-handle-cursor-test
>   /src/bstat
> +/src/btrfs_encoded_read
> +/src/btrfs_encoded_write
>   /src/bulkstat_null_ocount
>   /src/bulkstat_unlink_test
>   /src/bulkstat_unlink_test_modified
> diff --git a/src/Makefile b/src/Makefile
> index a0396332..b42b8147 100644
> --- a/src/Makefile
> +++ b/src/Makefile
> @@ -34,7 +34,8 @@ LINUX_TARGETS = xfsctl bstat t_mtab getdevicesize preallo_rw_pattern_reader \
>   	attr_replace_test swapon mkswap t_attr_corruption t_open_tmpfiles \
>   	fscrypt-crypt-util bulkstat_null_ocount splice-test chprojid_fail \
>   	detached_mounts_propagation ext4_resize t_readdir_3 splice2pipe \
> -	uuid_ioctl t_snapshot_deleted_subvolume fiemap-fault min_dio_alignment
> +	uuid_ioctl t_snapshot_deleted_subvolume fiemap-fault min_dio_alignment \
> +	btrfs_encoded_read btrfs_encoded_write
>   
>   EXTRA_EXECS = dmerror fill2attr fill2fs fill2fs_check scaleread.sh \
>   	      btrfs_crc32c_forged_name.py popdir.pl popattr.py \
> diff --git a/src/btrfs_encoded_read.c b/src/btrfs_encoded_read.c
> new file mode 100644
> index 00000000..a5082f70
> --- /dev/null
> +++ b/src/btrfs_encoded_read.c
> @@ -0,0 +1,175 @@
> +// SPDX-License-Identifier: GPL-2.0
> +// Copyright (c) Meta Platforms, Inc. and affiliates.
> +
> +#include <stdio.h>
> +#include <stdlib.h>
> +#include <string.h>
> +#include <errno.h>
> +#include <fcntl.h>
> +#include <unistd.h>
> +#include <sys/uio.h>
> +#include <sys/ioctl.h>
> +#include <linux/btrfs.h>

For this I need

+#include <linux/io_uring.h>

otherwise I get:

     [CC]    btrfs_encoded_read
/bin/sh ../libtool --quiet --tag=CC --mode=link /usr/bin/gcc-13 
btrfs_encoded_read.c -o btrfs_encoded_read -g -O2 -g -O2 -DDEBUG 
-I../include -DVERSION=\"1.1.1\" -D_GNU_SOURCE -D_FILE_OFFSET_BITS=64 
-funsigned-char -fno-strict-aliasing -Wall -DHAVE_FALLOCATE 
-DNEED_INTERNAL_XFS_IOC_EXCHANGE_RANGE   -lhandle -lacl -lpthread -lrt 
-luuid -lgdbm_compat -lgdbm -laio
  -luring   ../lib/libtest.la
btrfs_encoded_read.c: In function 'encoded_read_io_uring':
btrfs_encoded_read.c:100:26: error: 'IORING_OP_URING_CMD' undeclared 
(first use in this function); did you mean 'IORING_OP_LINKAT'?
   100 |         io_uring_prep_rw(IORING_OP_URING_CMD, sqe, fd, &enc, 
sizeof(enc), 0);
       |                          ^~~~~~~~~~~~~~~~~~~
       |                          IORING_OP_LINKAT
btrfs_encoded_read.c:100:26: note: each undeclared identifier is 
reported only once for each function it appears in
btrfs_encoded_read.c:101:12: error: 'struct io_uring_sqe' has no member 
named 'cmd_op'
   101 |         sqe->cmd_op = BTRFS_IOC_ENCODED_READ;
       |            ^~

during compilation.

Not sure if a ./configure macro thingy should/would solve this.

Re: [PATCH] btrfs: add test for encoded reads

[Johannes Thumshirn]

And another thing (sorry for spamming you)

When running the test on for-next (HEAD == e82c936293a) with lockdep 
enabled I get the following:


johannes@nuc:ci$ cat ../fstests/results/btrfs/333.dmesg
[    5.826816] run fstests btrfs/333 at 2024-11-12 09:43:20
[    6.992664] BTRFS: device fsid c92bd0ac-9334-40e5-8c01-75a45093c706 
devid 1 transid 6 /dev/nvme1n1 (259:2) scanned by mount (659)
[    6.994951] BTRFS info (device nvme1n1): first mount of filesystem 
c92bd0ac-9334-40e5-8c01-75a45093c706
[    6.996139] BTRFS info (device nvme1n1): using crc32c (crc32c-intel) 
checksum algorithm
[    6.997099] BTRFS info (device nvme1n1): using free-space-tree
[    7.000920] BTRFS info (device nvme1n1): checking UUID tree

[    7.465790] ================================================
[    7.466415] WARNING: lock held when returning to user space!
[    7.467024] 6.12.0-rc7+ #1044 Not tainted
[    7.467470] ------------------------------------------------
[    7.468135] btrfs_encoded_r/703 is leaving the kernel with locks 
still held!
[    7.468955] 1 lock held by btrfs_encoded_r/703:
[    7.469486]  #0: ffff8881163ca4e0 
(&sb->s_type->i_mutex_key#14){....}-{3:3}, at: btrfs_inode_lock+0x2a/0x70
[   11.346356] BTRFS info (device nvme1n1): last unmount of filesystem 
c92bd0ac-9334-40e5-8c01-75a45093c706
[   11.375284] BTRFS info (device nvme0n1): last unmount of filesystem 
5c716421-ae86-49ee-b283-13cc4758d395

Byte,
	Johannes

Re: [PATCH] btrfs: add test for encoded reads

[Mark Harmstone]

On 12/11/24 09:47, Johannes Thumshirn wrote:
> > 
> And another thing (sorry for spamming you)
> 
> When running the test on for-next (HEAD == e82c936293a) with lockdep
> enabled I get the following:
> 
> 
> johannes@nuc:ci$ cat ../fstests/results/btrfs/333.dmesg
> [    5.826816] run fstests btrfs/333 at 2024-11-12 09:43:20
> [    6.992664] BTRFS: device fsid c92bd0ac-9334-40e5-8c01-75a45093c706
> devid 1 transid 6 /dev/nvme1n1 (259:2) scanned by mount (659)
> [    6.994951] BTRFS info (device nvme1n1): first mount of filesystem
> c92bd0ac-9334-40e5-8c01-75a45093c706
> [    6.996139] BTRFS info (device nvme1n1): using crc32c (crc32c-intel)
> checksum algorithm
> [    6.997099] BTRFS info (device nvme1n1): using free-space-tree
> [    7.000920] BTRFS info (device nvme1n1): checking UUID tree
> 
> [    7.465790] ================================================
> [    7.466415] WARNING: lock held when returning to user space!
> [    7.467024] 6.12.0-rc7+ #1044 Not tainted
> [    7.467470] ------------------------------------------------
> [    7.468135] btrfs_encoded_r/703 is leaving the kernel with locks
> still held!
> [    7.468955] 1 lock held by btrfs_encoded_r/703:
> [    7.469486]  #0: ffff8881163ca4e0
> (&sb->s_type->i_mutex_key#14){....}-{3:3}, at: btrfs_inode_lock+0x2a/0x70
> [   11.346356] BTRFS info (device nvme1n1): last unmount of filesystem
> c92bd0ac-9334-40e5-8c01-75a45093c706
> [   11.375284] BTRFS info (device nvme0n1): last unmount of filesystem
> 5c716421-ae86-49ee-b283-13cc4758d395
> 
> Byte,
> 	Johannes

Thanks Johannes, I'll send a patch for this.

Mark

Re: [PATCH] btrfs: add test for encoded reads

[Mark Harmstone]

On 12/11/24 09:37, Johannes Thumshirn wrote:
> > 
> On 11.11.24 15:56, Mark Harmstone wrote:
>> Add btrfs/333 and its helper programs btrfs_encoded_read and
>> btrfs_encoded_write, in order to test encoded reads.
>>
>> We use the BTRFS_IOC_ENCODED_WRITE ioctl to write random data into a
>> compressed extent, then use the BTRFS_IOC_ENCODED_READ ioctl to check
>> that it matches what we've written. If the new io_uring interface for
>> encoded reads is supported, we also check that that matches the ioctl.
>>
>> Note that what we write isn't valid compressed data, so any non-encoded
>> reads on these files will fail.
>>
>> Signed-off-by: Mark Harmstone <maharmstone@fb.com>
>> ---
>>    .gitignore                |   2 +
>>    src/Makefile              |   3 +-
>>    src/btrfs_encoded_read.c  | 175 ++++++++++++++++++++++++++++++
>>    src/btrfs_encoded_write.c | 206 +++++++++++++++++++++++++++++++++++
>>    tests/btrfs/333           | 220 ++++++++++++++++++++++++++++++++++++++
>>    tests/btrfs/333.out       |   2 +
>>    6 files changed, 607 insertions(+), 1 deletion(-)
>>    create mode 100644 src/btrfs_encoded_read.c
>>    create mode 100644 src/btrfs_encoded_write.c
>>    create mode 100755 tests/btrfs/333
>>    create mode 100644 tests/btrfs/333.out
>>
>> diff --git a/.gitignore b/.gitignore
>> index f16173d9..efd47773 100644
>> --- a/.gitignore
>> +++ b/.gitignore
>> @@ -62,6 +62,8 @@ tags
>>    /src/attr_replace_test
>>    /src/attr-list-by-handle-cursor-test
>>    /src/bstat
>> +/src/btrfs_encoded_read
>> +/src/btrfs_encoded_write
>>    /src/bulkstat_null_ocount
>>    /src/bulkstat_unlink_test
>>    /src/bulkstat_unlink_test_modified
>> diff --git a/src/Makefile b/src/Makefile
>> index a0396332..b42b8147 100644
>> --- a/src/Makefile
>> +++ b/src/Makefile
>> @@ -34,7 +34,8 @@ LINUX_TARGETS = xfsctl bstat t_mtab getdevicesize preallo_rw_pattern_reader \
>>    	attr_replace_test swapon mkswap t_attr_corruption t_open_tmpfiles \
>>    	fscrypt-crypt-util bulkstat_null_ocount splice-test chprojid_fail \
>>    	detached_mounts_propagation ext4_resize t_readdir_3 splice2pipe \
>> -	uuid_ioctl t_snapshot_deleted_subvolume fiemap-fault min_dio_alignment
>> +	uuid_ioctl t_snapshot_deleted_subvolume fiemap-fault min_dio_alignment \
>> +	btrfs_encoded_read btrfs_encoded_write
>>    
>>    EXTRA_EXECS = dmerror fill2attr fill2fs fill2fs_check scaleread.sh \
>>    	      btrfs_crc32c_forged_name.py popdir.pl popattr.py \
>> diff --git a/src/btrfs_encoded_read.c b/src/btrfs_encoded_read.c
>> new file mode 100644
>> index 00000000..a5082f70
>> --- /dev/null
>> +++ b/src/btrfs_encoded_read.c
>> @@ -0,0 +1,175 @@
>> +// SPDX-License-Identifier: GPL-2.0
>> +// Copyright (c) Meta Platforms, Inc. and affiliates.
>> +
>> +#include <stdio.h>
>> +#include <stdlib.h>
>> +#include <string.h>
>> +#include <errno.h>
>> +#include <fcntl.h>
>> +#include <unistd.h>
>> +#include <sys/uio.h>
>> +#include <sys/ioctl.h>
>> +#include <linux/btrfs.h>
> 
> For this I need
> 
> +#include <linux/io_uring.h>
> 
> otherwise I get:
> 
>       [CC]    btrfs_encoded_read
> /bin/sh ../libtool --quiet --tag=CC --mode=link /usr/bin/gcc-13
> btrfs_encoded_read.c -o btrfs_encoded_read -g -O2 -g -O2 -DDEBUG
> -I../include -DVERSION=\"1.1.1\" -D_GNU_SOURCE -D_FILE_OFFSET_BITS=64
> -funsigned-char -fno-strict-aliasing -Wall -DHAVE_FALLOCATE
> -DNEED_INTERNAL_XFS_IOC_EXCHANGE_RANGE   -lhandle -lacl -lpthread -lrt
> -luuid -lgdbm_compat -lgdbm -laio
>    -luring   ../lib/libtest.la
> btrfs_encoded_read.c: In function 'encoded_read_io_uring':
> btrfs_encoded_read.c:100:26: error: 'IORING_OP_URING_CMD' undeclared
> (first use in this function); did you mean 'IORING_OP_LINKAT'?
>     100 |         io_uring_prep_rw(IORING_OP_URING_CMD, sqe, fd, &enc,
> sizeof(enc), 0);
>         |                          ^~~~~~~~~~~~~~~~~~~
>         |                          IORING_OP_LINKAT
> btrfs_encoded_read.c:100:26: note: each undeclared identifier is
> reported only once for each function it appears in
> btrfs_encoded_read.c:101:12: error: 'struct io_uring_sqe' has no member
> named 'cmd_op'
>     101 |         sqe->cmd_op = BTRFS_IOC_ENCODED_READ;
>         |            ^~
> 
> during compilation.
> 
> Not sure if a ./configure macro thingy should/would solve this.

We could do that, but elsewhere we're using liburing.h rather than the 
kernel version.

It looks like IORING_OP_URING_CMD was added to liburing with version 
2.2, which came out in June 2022. I don't know whether that's old enough 
that we can just declare it as our minimum version, whether we should be 
probing for the liburing version, whether we should be working round 
this somehow, or what.

Zorro, what do you think?

Mark

Re: [PATCH] btrfs: add test for encoded reads

[Zorro Lang]

On Wed, Nov 13, 2024 at 12:28:33PM +0000, Mark Harmstone wrote:
> On 12/11/24 09:37, Johannes Thumshirn wrote:
> > > 
> > On 11.11.24 15:56, Mark Harmstone wrote:
> >> Add btrfs/333 and its helper programs btrfs_encoded_read and
> >> btrfs_encoded_write, in order to test encoded reads.
> >>
> >> We use the BTRFS_IOC_ENCODED_WRITE ioctl to write random data into a
> >> compressed extent, then use the BTRFS_IOC_ENCODED_READ ioctl to check
> >> that it matches what we've written. If the new io_uring interface for
> >> encoded reads is supported, we also check that that matches the ioctl.
> >>
> >> Note that what we write isn't valid compressed data, so any non-encoded
> >> reads on these files will fail.
> >>
> >> Signed-off-by: Mark Harmstone <maharmstone@fb.com>
> >> ---
> >>    .gitignore                |   2 +
> >>    src/Makefile              |   3 +-
> >>    src/btrfs_encoded_read.c  | 175 ++++++++++++++++++++++++++++++
> >>    src/btrfs_encoded_write.c | 206 +++++++++++++++++++++++++++++++++++
> >>    tests/btrfs/333           | 220 ++++++++++++++++++++++++++++++++++++++
> >>    tests/btrfs/333.out       |   2 +
> >>    6 files changed, 607 insertions(+), 1 deletion(-)
> >>    create mode 100644 src/btrfs_encoded_read.c
> >>    create mode 100644 src/btrfs_encoded_write.c
> >>    create mode 100755 tests/btrfs/333
> >>    create mode 100644 tests/btrfs/333.out
> >>
> >> diff --git a/.gitignore b/.gitignore
> >> index f16173d9..efd47773 100644
> >> --- a/.gitignore
> >> +++ b/.gitignore
> >> @@ -62,6 +62,8 @@ tags
> >>    /src/attr_replace_test
> >>    /src/attr-list-by-handle-cursor-test
> >>    /src/bstat
> >> +/src/btrfs_encoded_read
> >> +/src/btrfs_encoded_write
> >>    /src/bulkstat_null_ocount
> >>    /src/bulkstat_unlink_test
> >>    /src/bulkstat_unlink_test_modified
> >> diff --git a/src/Makefile b/src/Makefile
> >> index a0396332..b42b8147 100644
> >> --- a/src/Makefile
> >> +++ b/src/Makefile
> >> @@ -34,7 +34,8 @@ LINUX_TARGETS = xfsctl bstat t_mtab getdevicesize preallo_rw_pattern_reader \
> >>    	attr_replace_test swapon mkswap t_attr_corruption t_open_tmpfiles \
> >>    	fscrypt-crypt-util bulkstat_null_ocount splice-test chprojid_fail \
> >>    	detached_mounts_propagation ext4_resize t_readdir_3 splice2pipe \
> >> -	uuid_ioctl t_snapshot_deleted_subvolume fiemap-fault min_dio_alignment
> >> +	uuid_ioctl t_snapshot_deleted_subvolume fiemap-fault min_dio_alignment \
> >> +	btrfs_encoded_read btrfs_encoded_write
> >>    
> >>    EXTRA_EXECS = dmerror fill2attr fill2fs fill2fs_check scaleread.sh \
> >>    	      btrfs_crc32c_forged_name.py popdir.pl popattr.py \
> >> diff --git a/src/btrfs_encoded_read.c b/src/btrfs_encoded_read.c
> >> new file mode 100644
> >> index 00000000..a5082f70
> >> --- /dev/null
> >> +++ b/src/btrfs_encoded_read.c
> >> @@ -0,0 +1,175 @@
> >> +// SPDX-License-Identifier: GPL-2.0
> >> +// Copyright (c) Meta Platforms, Inc. and affiliates.
> >> +
> >> +#include <stdio.h>
> >> +#include <stdlib.h>
> >> +#include <string.h>
> >> +#include <errno.h>
> >> +#include <fcntl.h>
> >> +#include <unistd.h>
> >> +#include <sys/uio.h>
> >> +#include <sys/ioctl.h>
> >> +#include <linux/btrfs.h>
> > 
> > For this I need
> > 
> > +#include <linux/io_uring.h>
> > 
> > otherwise I get:
> > 
> >       [CC]    btrfs_encoded_read
> > /bin/sh ../libtool --quiet --tag=CC --mode=link /usr/bin/gcc-13
> > btrfs_encoded_read.c -o btrfs_encoded_read -g -O2 -g -O2 -DDEBUG
> > -I../include -DVERSION=\"1.1.1\" -D_GNU_SOURCE -D_FILE_OFFSET_BITS=64
> > -funsigned-char -fno-strict-aliasing -Wall -DHAVE_FALLOCATE
> > -DNEED_INTERNAL_XFS_IOC_EXCHANGE_RANGE   -lhandle -lacl -lpthread -lrt
> > -luuid -lgdbm_compat -lgdbm -laio
> >    -luring   ../lib/libtest.la
> > btrfs_encoded_read.c: In function 'encoded_read_io_uring':
> > btrfs_encoded_read.c:100:26: error: 'IORING_OP_URING_CMD' undeclared
> > (first use in this function); did you mean 'IORING_OP_LINKAT'?
> >     100 |         io_uring_prep_rw(IORING_OP_URING_CMD, sqe, fd, &enc,
> > sizeof(enc), 0);
> >         |                          ^~~~~~~~~~~~~~~~~~~
> >         |                          IORING_OP_LINKAT
> > btrfs_encoded_read.c:100:26: note: each undeclared identifier is
> > reported only once for each function it appears in
> > btrfs_encoded_read.c:101:12: error: 'struct io_uring_sqe' has no member
> > named 'cmd_op'
> >     101 |         sqe->cmd_op = BTRFS_IOC_ENCODED_READ;
> >         |            ^~
> > 
> > during compilation.
> > 
> > Not sure if a ./configure macro thingy should/would solve this.
> 
> We could do that, but elsewhere we're using liburing.h rather than the 
> kernel version.
> 
> It looks like IORING_OP_URING_CMD was added to liburing with version 
> 2.2, which came out in June 2022. I don't know whether that's old enough 
> that we can just declare it as our minimum version, whether we should be 
> probing for the liburing version, whether we should be working round 
> this somehow, or what.
> 
> Zorro, what do you think?

2022 was just 2 years ago, some downstream distributions might use old version.
I think that might be too early to leave a "2 years ago" system out of the using of
latest xfstests :)

Thanks,
Zorro

> 
> Mark
> 

Re: [PATCH] btrfs: add test for encoded reads

[Mark Harmstone]

On 14/11/24 05:06, Zorro Lang wrote:
...

>> It looks like IORING_OP_URING_CMD was added to liburing with version
>> 2.2, which came out in June 2022. I don't know whether that's old enough
>> that we can just declare it as our minimum version, whether we should be
>> probing for the liburing version, whether we should be working round
>> this somehow, or what.
>>
>> Zorro, what do you think?
> 
> 2022 was just 2 years ago, some downstream distributions might use old version.
> I think that might be too early to leave a "2 years ago" system out of the using of
> latest xfstests :)
> 
> Thanks,
> Zorro

Okay, no worries. I can change it so that it uses the raw syscalls 
rather than the liburing helpers. It'll be a lot uglier, but at least 
it'll work.

Johannes, what distro and version are you on? I'll make sure it works on 
that.

Mark

Re: [PATCH] btrfs: add test for encoded reads

[Johannes Thumshirn]

On 20.11.24 17:14, Mark Harmstone wrote:
> On 14/11/24 05:06, Zorro Lang wrote:
> ...
> 
>>> It looks like IORING_OP_URING_CMD was added to liburing with version
>>> 2.2, which came out in June 2022. I don't know whether that's old enough
>>> that we can just declare it as our minimum version, whether we should be
>>> probing for the liburing version, whether we should be working round
>>> this somehow, or what.
>>>
>>> Zorro, what do you think?
>>
>> 2022 was just 2 years ago, some downstream distributions might use old version.
>> I think that might be too early to leave a "2 years ago" system out of the using of
>> latest xfstests :)
>>
>> Thanks,
>> Zorro
> 
> Okay, no worries. I can change it so that it uses the raw syscalls
> rather than the liburing helpers. It'll be a lot uglier, but at least
> it'll work.
> 
> Johannes, what distro and version are you on? I'll make sure it works on
> that.

I'm on openSUSE 15.6 atm.

Thanks,
	Johannes

Re: [PATCH] btrfs: add test for encoded reads

[Anand Jain]

On 21/11/24 00:31, Johannes Thumshirn wrote:
> On 20.11.24 17:14, Mark Harmstone wrote:
>> On 14/11/24 05:06, Zorro Lang wrote:
>> ...
>>
>>>> It looks like IORING_OP_URING_CMD was added to liburing with version
>>>> 2.2, which came out in June 2022. I don't know whether that's old enough
>>>> that we can just declare it as our minimum version, whether we should be
>>>> probing for the liburing version, whether we should be working round
>>>> this somehow, or what.
>>>>
>>>> Zorro, what do you think?
>>>
>>> 2022 was just 2 years ago, some downstream distributions might use old version.
>>> I think that might be too early to leave a "2 years ago" system out of the using of
>>> latest xfstests :)
>>>
>>> Thanks,
>>> Zorro
>>
>> Okay, no worries. I can change it so that it uses the raw syscalls
>> rather than the liburing helpers. It'll be a lot uglier, but at least
>> it'll work.
>>
>> Johannes, what distro and version are you on? I'll make sure it works on
>> that.
> 
> I'm on openSUSE 15.6 atm.
> 


Sorry for the delayed comment. Systems without the latest headers files
(missing `BTRFS_IOC_ENCODED_READ`) will fail to compile, e.g., OL9 with 
UEK7.

    27 |         struct btrfs_ioctl_encoded_io_args enc;
btrfs_encoded_read.c:27:44: error: storage size of 'enc' isn't known


btrfs_encoded_read.c:43:25: error: 'BTRFS_IOC_ENCODED_READ' undeclared 
(first use in this function); did you mean 'BTRFS_IOC_DEVICES_READY'?
    43 |         ret = ioctl(fd, BTRFS_IOC_ENCODED_READ, &enc);


fstests ./configure may need to manage feature enable?.


Thanks, Anand


> Thanks,
> 	Johannes