* [PATCH 0/2] fstests: add two tests for the precontent fanotify work
@ 2024-09-04 20:32 Josef Bacik
2024-09-04 20:32 ` [PATCH 1/2] fstests: add a test for the precontent fanotify hooks Josef Bacik
2024-09-04 20:32 ` [PATCH 2/2] fstests: add a test for executing from a precontent watch directory Josef Bacik
0 siblings, 2 replies; 5+ messages in thread
From: Josef Bacik @ 2024-09-04 20:32 UTC (permalink / raw)
To: fstests, linux-fsdevel
Hello,
These are for the fanotify pre-content hooks feature which is posted here
https://lore.kernel.org/linux-fsdevel/cover.1725481503.git.josef@toxicpanda.com/
It adds a couple of c files to do the work necessary to setup the directories
and do the watches, and has two tests, one to validate we get the right values
on mmap and reads, and another to validate that executables work properly.
I've tested them to make sure they work properly with xfs, btrfs, ext4, and
bcachefs with my patches applied. Thanks,
Josef
Josef Bacik (2):
fstests: add a test for the precontent fanotify hooks
fstests: add a test for executing from a precontent watch directory
doc/group-names.txt | 1 +
src/Makefile | 2 +-
src/precontent/Makefile | 26 ++
src/precontent/mmap-validate.c | 227 +++++++++++++++++
src/precontent/populate.c | 188 ++++++++++++++
src/precontent/remote-fetch.c | 441 +++++++++++++++++++++++++++++++++
tests/generic/800 | 68 +++++
tests/generic/800.out | 2 +
tests/generic/801 | 64 +++++
tests/generic/801.out | 2 +
10 files changed, 1020 insertions(+), 1 deletion(-)
create mode 100644 src/precontent/Makefile
create mode 100644 src/precontent/mmap-validate.c
create mode 100644 src/precontent/populate.c
create mode 100644 src/precontent/remote-fetch.c
create mode 100644 tests/generic/800
create mode 100644 tests/generic/800.out
create mode 100644 tests/generic/801
create mode 100644 tests/generic/801.out
--
2.43.0
^ permalink raw reply [flat|nested] 5+ messages in thread
* [PATCH 1/2] fstests: add a test for the precontent fanotify hooks
2024-09-04 20:32 [PATCH 0/2] fstests: add two tests for the precontent fanotify work Josef Bacik
@ 2024-09-04 20:32 ` Josef Bacik
2024-09-05 7:17 ` Amir Goldstein
2024-09-04 20:32 ` [PATCH 2/2] fstests: add a test for executing from a precontent watch directory Josef Bacik
1 sibling, 1 reply; 5+ messages in thread
From: Josef Bacik @ 2024-09-04 20:32 UTC (permalink / raw)
To: fstests, linux-fsdevel
This is a test to validate the precontent hooks work properly for reads
and page faults.
Signed-off-by: Josef Bacik <josef@toxicpanda.com>
---
doc/group-names.txt | 1 +
src/Makefile | 2 +-
src/precontent/Makefile | 26 ++
src/precontent/mmap-validate.c | 227 +++++++++++++++++
src/precontent/populate.c | 188 ++++++++++++++
src/precontent/remote-fetch.c | 441 +++++++++++++++++++++++++++++++++
tests/generic/800 | 68 +++++
tests/generic/800.out | 2 +
8 files changed, 954 insertions(+), 1 deletion(-)
create mode 100644 src/precontent/Makefile
create mode 100644 src/precontent/mmap-validate.c
create mode 100644 src/precontent/populate.c
create mode 100644 src/precontent/remote-fetch.c
create mode 100644 tests/generic/800
create mode 100644 tests/generic/800.out
diff --git a/doc/group-names.txt b/doc/group-names.txt
index 6cf71796..93ba6a23 100644
--- a/doc/group-names.txt
+++ b/doc/group-names.txt
@@ -56,6 +56,7 @@ fiexchange XFS_IOC_EXCHANGE_RANGE ioctl
freeze filesystem freeze tests
fsck general fsck tests
fsmap FS_IOC_GETFSMAP ioctl
+fsnotify fsnotify tests
fsr XFS free space reorganizer
fuzzers filesystem fuzz tests
growfs increasing the size of a filesystem
diff --git a/src/Makefile b/src/Makefile
index 559209be..6dae892e 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -40,7 +40,7 @@ EXTRA_EXECS = dmerror fill2attr fill2fs fill2fs_check scaleread.sh \
btrfs_crc32c_forged_name.py popdir.pl popattr.py \
soak_duration.awk
-SUBDIRS = log-writes perf
+SUBDIRS = log-writes perf precontent
LLDLIBS = $(LIBHANDLE) $(LIBACL) -lpthread -lrt -luuid
diff --git a/src/precontent/Makefile b/src/precontent/Makefile
new file mode 100644
index 00000000..367a34bb
--- /dev/null
+++ b/src/precontent/Makefile
@@ -0,0 +1,26 @@
+# SPDX-License-Identifier: GPL-2.0
+
+TOPDIR = ../..
+include $(TOPDIR)/include/builddefs
+
+TARGETS = $(basename $(wildcard *.c))
+
+CFILES = $(TARGETS:=.c)
+LDIRT = $(TARGETS)
+
+default: depend $(TARGETS)
+
+depend: .dep
+
+include $(BUILDRULES)
+
+$(TARGETS): $(CFILES)
+ @echo " [CC] $@"
+ $(Q)$(LTLINK) $@.c -o $@ $(CFLAGS) $(LDFLAGS) $(LDLIBS)
+
+install:
+ $(INSTALL) -m 755 -d $(PKG_LIB_DIR)/src/precontent
+ $(INSTALL) -m 755 $(TARGETS) $(PKG_LIB_DIR)/src/precontent
+
+-include .dep
+
diff --git a/src/precontent/mmap-validate.c b/src/precontent/mmap-validate.c
new file mode 100644
index 00000000..af606d5b
--- /dev/null
+++ b/src/precontent/mmap-validate.c
@@ -0,0 +1,227 @@
+// SPDX-License-Identifier: GPL-2.0
+#define _FILE_OFFSET_BITS 64
+#include <errno.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+/* 1 MiB. */
+#define FILE_SIZE (1 * 1024 * 1024)
+#define PAGE_SIZE 4096
+
+#define __free(func) __attribute__((cleanup(func)))
+
+static void freep(void *ptr)
+{
+ void *real_ptr = *(void **)ptr;
+ if (real_ptr == NULL)
+ return;
+ free(real_ptr);
+}
+
+static void close_fd(int *fd)
+{
+ if (*fd < 0)
+ return;
+ close(*fd);
+}
+
+static void unmap(void *ptr)
+{
+ void *real_ptr = *(void **)ptr;
+ if (real_ptr == NULL)
+ return;
+ munmap(real_ptr, PAGE_SIZE);
+}
+
+static void print_buffer(const char *buf, off_t buf_off, off_t off, size_t len)
+{
+ for (int i = 0; i <= (len / 32); i++) {
+ printf("%lu:", off + (i * 32));
+
+ for (int c = 0; c < 32; c++) {
+ if (!(c % 8))
+ printf(" ");
+ printf("%c", buf[buf_off++]);
+ }
+ printf("\n");
+ }
+}
+
+static int validate_buffer(const char *type, const char *buf,
+ const char *pattern, off_t bufoff, off_t off,
+ size_t len)
+{
+
+ if (memcmp(buf + bufoff, pattern + off, len)) {
+ printf("Buffers do not match at off %lu size %lu after %s\n",
+ off, len, type);
+ printf("read buffer\n");
+ print_buffer(buf, bufoff, off, len);
+ printf("valid buffer\n");
+ print_buffer(pattern, off, off, len);
+ return 1;
+ }
+
+ return 0;
+}
+
+static int validate_range_fd(int fd, char *pattern, off_t off, size_t len)
+{
+ char *buf __free(freep) = NULL;
+ ssize_t ret;
+ size_t readin = 0;
+
+ buf = malloc(len);
+ if (!buf) {
+ perror("malloc buf");
+ return 1;
+ }
+
+ while ((ret = pread(fd, buf + readin, len - readin, off + readin)) > 0) {
+ readin += ret;
+ if (readin == len)
+ break;
+ }
+
+ if (ret < 0) {
+ perror("read");
+ return 1;
+ }
+
+ return validate_buffer("read", buf, pattern, 0, off, len);
+}
+
+static int validate_file(const char *file, char *pattern)
+{
+ int fd __free(close_fd) = -EBADF;
+ char *buf __free(unmap) = NULL;
+ int ret;
+
+ fd = open(file, O_RDONLY);
+ if (fd < 0) {
+ perror("open file");
+ return 1;
+ }
+
+ /* Cycle through the file and do some random reads and validate them. */
+ for (int i = 0; i < 5; i++) {
+ off_t off = random() % FILE_SIZE;
+ size_t len = random() % PAGE_SIZE;
+
+ while ((off + len) > FILE_SIZE) {
+ len = FILE_SIZE - off;
+ if (len)
+ break;
+ len = random() % PAGE_SIZE;
+ }
+
+ ret = validate_range_fd(fd, pattern, off, len);
+ if (ret)
+ return ret;
+ }
+
+ buf = mmap(NULL, FILE_SIZE, PROT_READ|PROT_EXEC, MAP_PRIVATE, fd, 0);
+ if (!buf) {
+ perror("mmap");
+ return 1;
+ }
+
+ /* Validate random ranges of the mmap buffer. */
+ for (int i = 0; i < 5; i++) {
+ off_t off = random() % FILE_SIZE;
+ size_t len = random() % PAGE_SIZE;
+
+ while ((off + len) > FILE_SIZE) {
+ len = FILE_SIZE - off;
+ if (len)
+ break;
+ len = random() % PAGE_SIZE;
+ }
+
+ ret = validate_buffer("mmap", buf, pattern, off, off, len);
+ if (ret)
+ return ret;
+ }
+
+ /* Now check the whole thing, one page at a time. */
+ for (int i = 0; i < (FILE_SIZE / PAGE_SIZE); i++) {
+ ret = validate_buffer("mmap", buf, pattern, i * PAGE_SIZE,
+ i * PAGE_SIZE, PAGE_SIZE);
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
+
+static int create_file(const char *file, char *pattern)
+{
+ ssize_t ret;
+ size_t written = 0;
+ int fd __free(close_fd) = -EBADF;
+
+ fd = open(file, O_RDWR | O_CREAT, 0644);
+ if (fd < 0) {
+ perror("opening file");
+ return 1;
+ }
+
+ while ((ret = write(fd, pattern + written, FILE_SIZE - written)) > 0) {
+ written += ret;
+ if (written == FILE_SIZE)
+ break;
+ }
+
+ if (ret < 0) {
+ perror("writing to the file");
+ return 1;
+ }
+
+ return 0;
+}
+
+static void generate_pattern(char *pattern)
+{
+ for (int i = 0; i < (FILE_SIZE / PAGE_SIZE); i++) {
+ char fill = 'a' + (i % 26);
+
+ memset(pattern + (i * PAGE_SIZE), fill, PAGE_SIZE);
+ }
+}
+
+static void usage(void)
+{
+ fprintf(stderr, "Usage: mmap-validate <create|validate> <file>\n");
+}
+
+int main(int argc, char **argv)
+{
+ char *pattern __free(freep) = NULL;
+
+ if (argc != 3) {
+ usage();
+ return 1;
+ }
+
+ pattern = malloc(FILE_SIZE * sizeof(char));
+ if (!pattern) {
+ perror("malloc pattern");
+ return 1;
+ }
+
+ generate_pattern(pattern);
+
+ if (!strcmp(argv[1], "create"))
+ return create_file(argv[2], pattern);
+
+ if (strcmp(argv[1], "validate")) {
+ usage();
+ return 1;
+ }
+
+ return validate_file(argv[2], pattern);
+}
diff --git a/src/precontent/populate.c b/src/precontent/populate.c
new file mode 100644
index 00000000..9c935427
--- /dev/null
+++ b/src/precontent/populate.c
@@ -0,0 +1,188 @@
+// SPDX-License-Identifier: GPL-2.0
+#define _FILE_OFFSET_BITS 64
+#include <dirent.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <poll.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#define __free(func) __attribute__((cleanup(func)))
+
+static void close_fd(int *fd)
+{
+ if (*fd < 0)
+ return;
+ close(*fd);
+}
+
+static void close_dir(DIR **dir)
+{
+ if (*dir == NULL)
+ return;
+ closedir(*dir);
+}
+
+static void freep(void *ptr)
+{
+ void *real_ptr = *(void **)ptr;
+ if (real_ptr == NULL)
+ return;
+ free(real_ptr);
+}
+
+/*
+ * Dup a path, make sure there's a trailing '/' to make path concat easier.
+ */
+static char *pathdup(const char *orig)
+{
+ char *ret;
+ size_t len = strlen(orig);
+
+ /* Easy path, we have a trailing '/'. */
+ if (orig[len - 1] == '/')
+ return strdup(orig);
+
+ ret = malloc((len + 2) * sizeof(char));
+ if (!ret)
+ return ret;
+
+ memcpy(ret, orig, len);
+ ret[len] = '/';
+ len++;
+ ret[len] = '\0';
+ return ret;
+}
+
+static int process_directory(DIR *srcdir, char *srcpath, char *dstpath)
+{
+ char *src __free(freep) = NULL;
+ char *dst __free(freep) = NULL;
+ size_t srclen = strlen(srcpath) + 256;
+ size_t dstlen = strlen(dstpath) + 256;
+ struct dirent *dirent;
+
+ src = malloc(srclen * sizeof(char));
+ dst = malloc(dstlen * sizeof(char));
+ if (!src || !dst) {
+ perror("allocating path buf");
+ return -1;
+ }
+
+ errno = 0;
+ while ((dirent = readdir(srcdir)) != NULL) {
+ if (!strcmp(dirent->d_name, ".") ||
+ !strcmp(dirent->d_name, ".."))
+ continue;
+
+ if (dirent->d_type == DT_DIR) {
+ DIR *nextdir __free(close_dir) = NULL;
+ struct stat st;
+ int ret;
+
+ snprintf(src, srclen, "%s%s/", srcpath, dirent->d_name);
+ snprintf(dst, dstlen, "%s%s/", dstpath, dirent->d_name);
+
+ nextdir = opendir(src);
+ if (!nextdir) {
+ fprintf(stderr, "Couldn't open directory %s: %s (%d)\n",
+ src, strerror(errno), errno);
+ return -1;
+ }
+
+ if (stat(src, &st)) {
+ fprintf(stderr, "Couldn't stat directory %s: %s (%d)\n",
+ src, strerror(errno), errno);
+ return -1;
+ }
+
+ if (mkdir(dst, st.st_mode)) {
+ fprintf(stderr, "Couldn't mkdir %s: %s (%d)\n",
+ dst, strerror(errno), errno);
+ return -1;
+ }
+
+ ret = process_directory(nextdir, src, dst);
+ if (ret)
+ return ret;
+ } else if (dirent->d_type == DT_REG) {
+ int fd __free(close_fd) = -EBADF;
+ struct stat st;
+
+ snprintf(src, srclen, "%s%s", srcpath, dirent->d_name);
+ snprintf(dst, dstlen, "%s%s", dstpath, dirent->d_name);
+
+ if (stat(src, &st)) {
+ fprintf(stderr, "Couldn't stat file %s: %s (%d)\n",
+ src, strerror(errno), errno);
+ return -1;
+ }
+
+ fd = open(dst, O_WRONLY|O_CREAT, st.st_mode);
+ if (fd < 0) {
+ fprintf(stderr, "Couldn't create file %s: %s (%d)\n",
+ dst, strerror(errno), errno);
+ return -1;
+ }
+
+ if (truncate(dst, st.st_size)) {
+ fprintf(stderr, "Couldn't truncate file %s: %s (%d)\n",
+ dst, strerror(errno), errno);
+ return -1;
+ }
+
+
+ if (fsync(fd)) {
+ fprintf(stderr, "Couldn't fsync file %s: %s (%d)\n",
+ dst, strerror(errno), errno);
+ return -1;
+ }
+
+ if (posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED)) {
+ fprintf(stderr, "Couldn't clear cache on file %s: %s (%d)\n",
+ dst, strerror(errno), errno);
+ return -1;
+ }
+ }
+ errno = 0;
+ }
+
+ if (errno) {
+ perror("readdir");
+ return -1;
+ }
+ return 0;
+}
+
+int main(int argc, char **argv)
+{
+ DIR *srcdir __free(close_dir) = NULL;
+ char *dstpath __free(freep) = NULL;
+ char *srcpath __free(freep) = NULL;
+ int ret;
+
+ if (argc != 3) {
+ fprintf(stderr, "Usage: populate <src directory> <dest directory>\n");
+ return 1;
+ }
+
+ srcpath = pathdup(argv[1]);
+ dstpath = pathdup(argv[2]);
+ if (!dstpath || !srcpath) {
+ perror("allocating paths");
+ return 1;
+ }
+
+ srcdir = opendir(srcpath);
+ if (!srcdir) {
+ perror("open src directory");
+ return 1;
+ }
+
+ ret = process_directory(srcdir, srcpath, dstpath);
+ return ret ? 1 : 0;
+}
diff --git a/src/precontent/remote-fetch.c b/src/precontent/remote-fetch.c
new file mode 100644
index 00000000..2e35b9f8
--- /dev/null
+++ b/src/precontent/remote-fetch.c
@@ -0,0 +1,441 @@
+// SPDX-License-Identifier: GPL-2.0
+#define _FILE_OFFSET_BITS 64
+#include <dirent.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <poll.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/fanotify.h>
+#include <sys/sendfile.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#ifndef FAN_ERRNO_BITS
+#define FAN_ERRNO_BITS 8
+#define FAN_ERRNO_SHIFT (32 - FAN_ERRNO_BITS)
+#define FAN_ERRNO_MASK ((1 << FAN_ERRNO_BITS) - 1)
+#define FAN_DENY_ERRNO(err) \
+ (FAN_DENY | ((((__u32)(err)) & FAN_ERRNO_MASK) << FAN_ERRNO_SHIFT))
+#endif
+
+#ifndef FAN_PRE_ACCESS
+#define FAN_PRE_ACCESS 0x00080000
+#endif
+
+#ifndef FAN_PRE_MODIFY
+#define FAN_PRE_MODIFY 0x00100000
+#endif
+
+#ifndef FAN_EVENT_INFO_TYPE_RANGE
+#define FAN_EVENT_INFO_TYPE_RANGE 6
+struct fanotify_event_info_range {
+ struct fanotify_event_info_header hdr;
+ __u32 pad;
+ __u64 offset;
+ __u64 count;
+};
+#endif
+
+#define FAN_EVENTS (FAN_PRE_ACCESS | FAN_PRE_MODIFY)
+
+#define __round_mask(x, y) ((__typeof__(x))((y)-1))
+#define round_up(x, y) ((((x)-1) | __round_mask(x, y))+1)
+#define round_down(x, y) ((x) & ~__round_mask(x, y))
+
+static const char *srcpath;
+static const char *dstpath;
+static int pagesize;
+static bool use_sendfile = false;
+
+#define __free(func) __attribute__((cleanup(func)))
+
+static void close_dir(DIR **dir)
+{
+ if (*dir == NULL)
+ return;
+ closedir(*dir);
+}
+
+static void close_fd(int *fd)
+{
+ if (*fd < 0)
+ return;
+ close(*fd);
+}
+
+static void freep(void *ptr)
+{
+ void *real_ptr = *(void **)ptr;
+ if (real_ptr == NULL)
+ return;
+ free(real_ptr);
+}
+
+static int strip_dstpath(char *path)
+{
+ size_t remaining;
+
+ if (strlen(path) <= strlen(dstpath)) {
+ fprintf(stderr, "'%s' not in the path '%s'", path, dstpath);
+ return -1;
+ }
+
+ if (strncmp(path, dstpath, strlen(dstpath))) {
+ fprintf(stderr, "path '%s' doesn't start with the source path '%s'\n",
+ path, dstpath);
+ return -1;
+ }
+
+ remaining = strlen(path) - strlen(dstpath);
+ memmove(path, path + strlen(dstpath), remaining);
+
+ /* strip any leading / in order to make it easier to concat. */
+ while (*path == '/') {
+ if (remaining == 0) {
+ fprintf(stderr, "you gave us a weird ass string\n");
+ return -1;
+ }
+ remaining--;
+ memmove(path, path + 1, remaining);
+ }
+ path[remaining] = '\0';
+ return 0;
+}
+
+/*
+ * Dup a path, make sure there's a trailing '/' to make path concat easier.
+ */
+static char *pathdup(const char *orig)
+{
+ char *ret;
+ size_t len = strlen(orig);
+
+ /* Easy path, we have a trailing '/'. */
+ if (orig[len - 1] == '/')
+ return strdup(orig);
+
+ ret = malloc((len + 2) * sizeof(char));
+ if (!ret)
+ return ret;
+
+ memcpy(ret, orig, len);
+ ret[len] = '/';
+ len++;
+ ret[len] = '\0';
+ return ret;
+}
+
+static char *get_relpath(int fd)
+{
+ char procfd_path[PATH_MAX];
+ char abspath[PATH_MAX];
+ ssize_t path_len;
+ int ret;
+
+ /* readlink doesn't NULL terminate. */
+ memset(abspath, 0, sizeof(abspath));
+
+ snprintf(procfd_path, sizeof(procfd_path), "/proc/self/fd/%d", fd);
+ path_len = readlink(procfd_path, abspath, sizeof(abspath) - 1);
+ if (path_len < 0) {
+ perror("readlink");
+ return NULL;
+ }
+
+ ret = strip_dstpath(abspath);
+ if (ret < 0)
+ return NULL;
+
+ return strdup(abspath);
+}
+
+static int copy_range(int src_fd, int fd, off_t offset, size_t count)
+{
+ off_t src_offset = offset;
+ ssize_t copied;
+
+ if (use_sendfile)
+ goto slow;
+
+ while ((copied = copy_file_range(src_fd, &src_offset, fd, &offset,
+ count, 0)) >= 0) {
+ if (copied == 0)
+ return 0;
+
+ count -= copied;
+ if (count == 0)
+ return 0;
+ }
+
+ if (errno != EXDEV) {
+ perror("copy_file_range");
+ return -1;
+ }
+ use_sendfile = true;
+
+slow:
+ /* I love linux interfaces. */
+ if (lseek(fd, offset, SEEK_SET) == (off_t)-1) {
+ perror("seeking");
+ return -1;
+ }
+
+ while ((copied = sendfile(fd, src_fd, &src_offset, count)) >= 0) {
+ if (copied == 0)
+ return 0;
+
+ count -= copied;
+ if (count == 0)
+ return 0;
+ }
+
+ perror("sendfile");
+ return -1;
+}
+
+static int handle_event(int fanotify_fd, int fd, off_t offset, size_t count)
+{
+ char path[PATH_MAX];
+ char *relpath __free(freep) = NULL;
+ int src_fd __free(close_fd) = -1;
+ off_t end = offset + count;
+ blkcnt_t src_blocks;
+ struct stat st;
+
+ relpath = get_relpath(fd);
+ if (!relpath)
+ return -1;
+
+ offset = round_down(offset, pagesize);
+ end = round_up(end, pagesize);
+ count = end - offset;
+
+ snprintf(path, sizeof(path), "%s%s", srcpath, relpath);
+ src_fd = open(path, O_RDONLY);
+ if (src_fd < 0) {
+ fprintf(stderr, "srcpath %s relpath %s\n", srcpath, relpath);
+ fprintf(stderr, "error opening file %s: %s (%d)\n", path, strerror(errno), errno);
+ return -1;
+ }
+
+ if (fstat(src_fd, &st)) {
+ perror("src fd is fucked");
+ return -1;
+ }
+
+ src_blocks = st.st_blocks;
+
+ if (fstat(fd, &st)) {
+ perror("fd is fucked");
+ return -1;
+ }
+
+ /*
+ * If we are the same size or larger (which can happen if we copy zero's
+ * instead of inserting a hole) then just assume we're full. This is
+ * approximation can fall over, but its good enough for a PoC.
+ */
+ if (st.st_blocks >= src_blocks) {
+ int ret;
+
+ snprintf(path, sizeof(path), "%s%s", dstpath, relpath);
+ ret = fanotify_mark(fanotify_fd, FAN_MARK_REMOVE,
+ FAN_EVENTS, -1, path);
+ if (ret < 0) {
+ /* We already removed the mark, carry on. */
+ if (errno == ENOENT) {
+ errno = 0;
+ return 0;
+ }
+ perror("removing fanotify mark");
+ return -1;
+ }
+ return 0;
+ }
+
+
+ return copy_range(src_fd, fd, offset, count);
+}
+
+static int handle_events(int fd)
+{
+ const struct fanotify_event_metadata *metadata;
+ struct fanotify_event_metadata buf[200];
+ ssize_t len;
+ struct fanotify_response response;
+ int ret = 0;
+
+ len = read(fd, (void *)buf, sizeof(buf));
+ if (len <= 0 && errno != EINTR) {
+ perror("reading fanotify events");
+ return -1;
+ }
+
+ metadata = buf;
+ while(FAN_EVENT_OK(metadata, len)) {
+ off_t offset = 0;
+ size_t count = 0;
+
+ if (metadata->vers != FANOTIFY_METADATA_VERSION) {
+ fprintf(stderr, "invalid metadata version, have %d, expect %d\n",
+ metadata->vers, FANOTIFY_METADATA_VERSION);
+ return -1;
+ }
+ if (metadata->fd < 0) {
+ fprintf(stderr, "metadata fd is an error\n");
+ return -1;
+ }
+ if (!(metadata->mask & FAN_EVENTS)) {
+ fprintf(stderr, "metadata mask incorrect %llu\n",
+ metadata->mask);
+ return -1;
+ }
+
+ /*
+ * We have a specific range, load that instead of filling the
+ * entire file in.
+ */
+ if (metadata->event_len > FAN_EVENT_METADATA_LEN) {
+ const struct fanotify_event_info_range *range;
+ range = (const struct fanotify_event_info_range *)(metadata + 1);
+ if (range->hdr.info_type == FAN_EVENT_INFO_TYPE_RANGE) {
+ count = range->count;
+ offset = range->offset;
+ if (count == 0) {
+ ret = 0;
+ goto next;
+ }
+ }
+ }
+
+ /* We don't have a range, pre-fill the whole file. */
+ if (count == 0) {
+ struct stat st;
+
+ if (fstat(metadata->fd, &st)) {
+ perror("stat() on opened file");
+ return -1;
+ }
+
+ count = st.st_size;
+ }
+
+ ret = handle_event(fd, metadata->fd, offset, count);
+next:
+ response.fd = metadata->fd;
+ if (ret)
+ response.response = FAN_DENY_ERRNO(errno);
+ else
+ response.response = FAN_ALLOW;
+ write(fd, &response, sizeof(response));
+ close(metadata->fd);
+ metadata = FAN_EVENT_NEXT(metadata, len);
+ }
+
+ return ret;
+}
+
+static int add_marks(const char *src, int fanotify_fd)
+{
+ char *path __free(freep) = NULL;
+ DIR *dir __free(close_dir) = NULL;
+ size_t pathlen = strlen(src) + 256;
+ struct dirent *dirent;
+
+ path = malloc(pathlen * sizeof(char));
+ if (!path) {
+ perror("allocating path buf");
+ return -1;
+ }
+
+ dir = opendir(src);
+ if (!dir) {
+ fprintf(stderr, "Couldn't open directory %s: %s (%d)\n",
+ src, strerror(errno), errno);
+ return -1;
+ }
+
+ errno = 0;
+ while ((dirent = readdir(dir)) != NULL) {
+ int ret;
+
+ if (!strcmp(dirent->d_name, ".") ||
+ !strcmp(dirent->d_name, ".."))
+ continue;
+
+ if (dirent->d_type == DT_DIR) {
+ snprintf(path, pathlen, "%s%s/", src, dirent->d_name);
+ ret = add_marks(path, fanotify_fd);
+ if (ret)
+ return ret;
+ } else if (dirent->d_type == DT_REG) {
+ ret = fanotify_mark(fanotify_fd, FAN_MARK_ADD,
+ FAN_EVENTS, dirfd(dir),
+ dirent->d_name);
+ if (ret < 0) {
+ perror("fanotify_mark");
+ return -1;
+ }
+ }
+ errno = 0;
+ }
+ return 0;
+}
+
+static void usage(void)
+{
+ fprintf(stderr, "Usage: remote-fetch <src directory> <dest directory>\n");
+}
+
+int main(int argc, char **argv)
+{
+ int fd __free(close_fd) = -1;
+ int dirfd __free(close_fd) = -1;
+ int ret;
+
+ if (argc != 3) {
+ usage();
+ return 1;
+ }
+
+ pagesize = sysconf(_SC_PAGESIZE);
+ if (pagesize < 0) {
+ perror("sysconf");
+ return 1;
+ }
+
+ srcpath = pathdup(argv[1]);
+ dstpath = pathdup(argv[2]);
+ if (!srcpath || !dstpath) {
+ perror("allocate paths");
+ return 1;
+ }
+
+ dirfd = open(dstpath, O_DIRECTORY | O_RDONLY);
+ if (dirfd < 0) {
+ perror("open dstpath");
+ return 1;
+ }
+
+ fd = fanotify_init(FAN_CLASS_PRE_CONTENT | FAN_UNLIMITED_MARKS, O_WRONLY | O_LARGEFILE);
+ if (fd < 0) {
+ perror("fanotify_init");
+ return 1;
+ }
+
+ ret = add_marks(dstpath, fd);
+ if (ret < 0)
+ return 1;
+
+ for (;;) {
+ ret = handle_events(fd);
+ if (ret)
+ break;
+ }
+
+ return (ret < 0) ? 1 : 0;
+}
diff --git a/tests/generic/800 b/tests/generic/800
new file mode 100644
index 00000000..08ac5b26
--- /dev/null
+++ b/tests/generic/800
@@ -0,0 +1,68 @@
+#! /bin/bash
+# SPDX-License-Identifier: GPL-2.0
+# Copyright (c) 2024 Meta Platforms, Inc. All Rights Reserved.
+#
+# FS QA Test 800
+#
+# Validate the pre-content related fanotify features
+#
+# The mmap-verify pre-content tool generates a file and then validates that the
+# pre-content watched directory properly fills it in with a mixture of page
+# faults and normal reads.
+#
+. ./common/preamble
+_begin_fstest quick auto fsnotify
+
+_cleanup()
+{
+ cd /
+ rm -rf $TEST_DIR/dst-$seq
+ rm -rf $TEST_DIR/src-$seq
+}
+
+# real QA test starts here
+_supported_fs generic
+_require_test
+_require_test_program "precontent/mmap-validate"
+_require_test_program "precontent/populate"
+_require_test_program "precontent/remote-fetch"
+
+dstdir=$TEST_DIR/dst-$seq
+srcdir=$TEST_DIR/src-$seq
+
+POPULATE=$here/src/precontent/populate
+REMOTE_FETCH=$here/src/precontent/remote-fetch
+MMAP_VALIDATE=$here/src/precontent/mmap-validate
+
+mkdir $dstdir $srcdir
+
+# Generate the test file
+$MMAP_VALIDATE create $srcdir/validate
+
+# Generate the stub file in the watch directory
+$POPULATE $srcdir $dstdir
+
+# Start the remote watcher
+$REMOTE_FETCH $srcdir $dstdir &
+
+FETCH_PID=$!
+
+# We may not support fanotify, give it a second to start and then make sure the
+# fetcher is running before we try to validate the buffer
+sleep 1
+
+if ! ps -p $FETCH_PID > /dev/null
+then
+ _notrun "precontent watches not supported"
+fi
+
+$MMAP_VALIDATE validate $dstdir/validate
+
+kill -9 $FETCH_PID &> /dev/null
+wait $FETCH_PID &> /dev/null
+
+echo "Silence is golden"
+
+# success, all done
+status=$?
+exit
diff --git a/tests/generic/800.out b/tests/generic/800.out
new file mode 100644
index 00000000..bdfaa2ce
--- /dev/null
+++ b/tests/generic/800.out
@@ -0,0 +1,2 @@
+QA output created by 800
+Silence is golden
--
2.43.0
^ permalink raw reply related [flat|nested] 5+ messages in thread
* [PATCH 2/2] fstests: add a test for executing from a precontent watch directory
2024-09-04 20:32 [PATCH 0/2] fstests: add two tests for the precontent fanotify work Josef Bacik
2024-09-04 20:32 ` [PATCH 1/2] fstests: add a test for the precontent fanotify hooks Josef Bacik
@ 2024-09-04 20:32 ` Josef Bacik
2024-09-05 7:21 ` Amir Goldstein
1 sibling, 1 reply; 5+ messages in thread
From: Josef Bacik @ 2024-09-04 20:32 UTC (permalink / raw)
To: fstests, linux-fsdevel
The main purpose of putting precontent hooks in the page fault path is
to enable running an executable from a precontent watch. Add a test to
create a precontent watched directory with bash in it, and then execute
that copy of bash to validate that we fill in the pages properly and are
able to execute.
Signed-off-by: Josef Bacik <josef@toxicpanda.com>
---
tests/generic/801 | 64 +++++++++++++++++++++++++++++++++++++++++++
tests/generic/801.out | 2 ++
2 files changed, 66 insertions(+)
create mode 100644 tests/generic/801
create mode 100644 tests/generic/801.out
diff --git a/tests/generic/801 b/tests/generic/801
new file mode 100644
index 00000000..7a1cc653
--- /dev/null
+++ b/tests/generic/801
@@ -0,0 +1,64 @@
+#! /bin/bash
+# SPDX-License-Identifier: GPL-2.0
+# Copyright (c) 2024 Meta Platforms, Inc. All Rights Reserved.
+#
+# FS QA Test 801
+#
+# Validate the pre-content hooks work properly with exec
+#
+# Copy bash into our source directory and then setup the HSM daemon to mirror
+# into the destination directory, and execute bash from the destination
+# directory to make sure it loads properly.
+#
+. ./common/preamble
+_begin_fstest quick auto fsnotify
+
+_cleanup()
+{
+ cd /
+ rm -rf $TEST_DIR/dst-$seq
+ rm -rf $TEST_DIR/src-$seq
+}
+
+# real QA test starts here
+_supported_fs generic
+_require_test
+_require_test_program "precontent/populate"
+_require_test_program "precontent/remote-fetch"
+
+dstdir=$TEST_DIR/dst-$seq
+srcdir=$TEST_DIR/src-$seq
+
+POPULATE=$here/src/precontent/populate
+REMOTE_FETCH=$here/src/precontent/remote-fetch
+
+mkdir $dstdir $srcdir
+
+# Copy bash into our source dir
+cp $(which bash) $srcdir
+
+# Generate the stub file in the watch directory
+$POPULATE $srcdir $dstdir
+
+# Start the remote watcher
+$REMOTE_FETCH $srcdir $dstdir &
+
+FETCH_PID=$!
+
+# We may not support fanotify, give it a second to start and then make sure the
+# fetcher is running before we try to run our test
+sleep 1
+
+if ! ps -p $FETCH_PID > /dev/null
+then
+ _notrun "precontent watches not supported"
+fi
+
+$dstdir/bash -c "echo 'Hello!'"
+
+kill -9 $FETCH_PID &> /dev/null
+wait $FETCH_PID &> /dev/null
+
+# success, all done
+status=0
+exit
diff --git a/tests/generic/801.out b/tests/generic/801.out
new file mode 100644
index 00000000..98e6a16c
--- /dev/null
+++ b/tests/generic/801.out
@@ -0,0 +1,2 @@
+QA output created by 801
+Hello!
--
2.43.0
^ permalink raw reply related [flat|nested] 5+ messages in thread
* Re: [PATCH 1/2] fstests: add a test for the precontent fanotify hooks
2024-09-04 20:32 ` [PATCH 1/2] fstests: add a test for the precontent fanotify hooks Josef Bacik
@ 2024-09-05 7:17 ` Amir Goldstein
0 siblings, 0 replies; 5+ messages in thread
From: Amir Goldstein @ 2024-09-05 7:17 UTC (permalink / raw)
To: Josef Bacik; +Cc: fstests, linux-fsdevel
On Wed, Sep 4, 2024 at 10:33 PM Josef Bacik <josef@toxicpanda.com> wrote:
>
> This is a test to validate the precontent hooks work properly for reads
> and page faults.
>
> Signed-off-by: Josef Bacik <josef@toxicpanda.com>
Very cool!
Some minor comments below.
I will publicly commit to duplicating your test to LTP
in the hope of shaming myself to actually do it...
> ---
> doc/group-names.txt | 1 +
> src/Makefile | 2 +-
> src/precontent/Makefile | 26 ++
> src/precontent/mmap-validate.c | 227 +++++++++++++++++
> src/precontent/populate.c | 188 ++++++++++++++
> src/precontent/remote-fetch.c | 441 +++++++++++++++++++++++++++++++++
> tests/generic/800 | 68 +++++
> tests/generic/800.out | 2 +
> 8 files changed, 954 insertions(+), 1 deletion(-)
> create mode 100644 src/precontent/Makefile
> create mode 100644 src/precontent/mmap-validate.c
> create mode 100644 src/precontent/populate.c
> create mode 100644 src/precontent/remote-fetch.c
> create mode 100644 tests/generic/800
> create mode 100644 tests/generic/800.out
>
> diff --git a/doc/group-names.txt b/doc/group-names.txt
> index 6cf71796..93ba6a23 100644
> --- a/doc/group-names.txt
> +++ b/doc/group-names.txt
> @@ -56,6 +56,7 @@ fiexchange XFS_IOC_EXCHANGE_RANGE ioctl
> freeze filesystem freeze tests
> fsck general fsck tests
> fsmap FS_IOC_GETFSMAP ioctl
> +fsnotify fsnotify tests
> fsr XFS free space reorganizer
> fuzzers filesystem fuzz tests
> growfs increasing the size of a filesystem
> diff --git a/src/Makefile b/src/Makefile
> index 559209be..6dae892e 100644
> --- a/src/Makefile
> +++ b/src/Makefile
> @@ -40,7 +40,7 @@ EXTRA_EXECS = dmerror fill2attr fill2fs fill2fs_check scaleread.sh \
> btrfs_crc32c_forged_name.py popdir.pl popattr.py \
> soak_duration.awk
>
> -SUBDIRS = log-writes perf
> +SUBDIRS = log-writes perf precontent
>
> LLDLIBS = $(LIBHANDLE) $(LIBACL) -lpthread -lrt -luuid
>
> diff --git a/src/precontent/Makefile b/src/precontent/Makefile
> new file mode 100644
> index 00000000..367a34bb
> --- /dev/null
> +++ b/src/precontent/Makefile
> @@ -0,0 +1,26 @@
> +# SPDX-License-Identifier: GPL-2.0
> +
> +TOPDIR = ../..
> +include $(TOPDIR)/include/builddefs
> +
> +TARGETS = $(basename $(wildcard *.c))
> +
> +CFILES = $(TARGETS:=.c)
> +LDIRT = $(TARGETS)
> +
> +default: depend $(TARGETS)
> +
> +depend: .dep
> +
> +include $(BUILDRULES)
> +
> +$(TARGETS): $(CFILES)
> + @echo " [CC] $@"
> + $(Q)$(LTLINK) $@.c -o $@ $(CFLAGS) $(LDFLAGS) $(LDLIBS)
> +
> +install:
> + $(INSTALL) -m 755 -d $(PKG_LIB_DIR)/src/precontent
> + $(INSTALL) -m 755 $(TARGETS) $(PKG_LIB_DIR)/src/precontent
> +
> +-include .dep
> +
> diff --git a/src/precontent/mmap-validate.c b/src/precontent/mmap-validate.c
> new file mode 100644
> index 00000000..af606d5b
> --- /dev/null
> +++ b/src/precontent/mmap-validate.c
> @@ -0,0 +1,227 @@
> +// SPDX-License-Identifier: GPL-2.0
> +#define _FILE_OFFSET_BITS 64
> +#include <errno.h>
> +#include <fcntl.h>
> +#include <stdio.h>
> +#include <stdlib.h>
> +#include <string.h>
> +#include <sys/mman.h>
> +#include <unistd.h>
> +
> +/* 1 MiB. */
> +#define FILE_SIZE (1 * 1024 * 1024)
> +#define PAGE_SIZE 4096
> +
> +#define __free(func) __attribute__((cleanup(func)))
> +
> +static void freep(void *ptr)
> +{
> + void *real_ptr = *(void **)ptr;
> + if (real_ptr == NULL)
> + return;
> + free(real_ptr);
> +}
> +
> +static void close_fd(int *fd)
> +{
> + if (*fd < 0)
> + return;
> + close(*fd);
> +}
> +
> +static void unmap(void *ptr)
> +{
> + void *real_ptr = *(void **)ptr;
> + if (real_ptr == NULL)
> + return;
> + munmap(real_ptr, PAGE_SIZE);
> +}
> +
> +static void print_buffer(const char *buf, off_t buf_off, off_t off, size_t len)
> +{
> + for (int i = 0; i <= (len / 32); i++) {
> + printf("%lu:", off + (i * 32));
> +
> + for (int c = 0; c < 32; c++) {
> + if (!(c % 8))
> + printf(" ");
> + printf("%c", buf[buf_off++]);
> + }
> + printf("\n");
> + }
> +}
> +
> +static int validate_buffer(const char *type, const char *buf,
> + const char *pattern, off_t bufoff, off_t off,
> + size_t len)
> +{
> +
> + if (memcmp(buf + bufoff, pattern + off, len)) {
> + printf("Buffers do not match at off %lu size %lu after %s\n",
> + off, len, type);
> + printf("read buffer\n");
> + print_buffer(buf, bufoff, off, len);
> + printf("valid buffer\n");
> + print_buffer(pattern, off, off, len);
> + return 1;
> + }
> +
> + return 0;
> +}
> +
> +static int validate_range_fd(int fd, char *pattern, off_t off, size_t len)
> +{
> + char *buf __free(freep) = NULL;
> + ssize_t ret;
> + size_t readin = 0;
> +
> + buf = malloc(len);
> + if (!buf) {
> + perror("malloc buf");
> + return 1;
> + }
> +
> + while ((ret = pread(fd, buf + readin, len - readin, off + readin)) > 0) {
> + readin += ret;
> + if (readin == len)
> + break;
> + }
> +
> + if (ret < 0) {
> + perror("read");
> + return 1;
> + }
> +
> + return validate_buffer("read", buf, pattern, 0, off, len);
> +}
> +
> +static int validate_file(const char *file, char *pattern)
> +{
> + int fd __free(close_fd) = -EBADF;
> + char *buf __free(unmap) = NULL;
> + int ret;
> +
> + fd = open(file, O_RDONLY);
> + if (fd < 0) {
> + perror("open file");
> + return 1;
> + }
> +
> + /* Cycle through the file and do some random reads and validate them. */
> + for (int i = 0; i < 5; i++) {
> + off_t off = random() % FILE_SIZE;
> + size_t len = random() % PAGE_SIZE;
> +
> + while ((off + len) > FILE_SIZE) {
> + len = FILE_SIZE - off;
> + if (len)
> + break;
> + len = random() % PAGE_SIZE;
> + }
> +
> + ret = validate_range_fd(fd, pattern, off, len);
> + if (ret)
> + return ret;
> + }
> +
> + buf = mmap(NULL, FILE_SIZE, PROT_READ|PROT_EXEC, MAP_PRIVATE, fd, 0);
> + if (!buf) {
> + perror("mmap");
> + return 1;
> + }
> +
> + /* Validate random ranges of the mmap buffer. */
> + for (int i = 0; i < 5; i++) {
> + off_t off = random() % FILE_SIZE;
> + size_t len = random() % PAGE_SIZE;
> +
> + while ((off + len) > FILE_SIZE) {
> + len = FILE_SIZE - off;
> + if (len)
> + break;
> + len = random() % PAGE_SIZE;
> + }
> +
> + ret = validate_buffer("mmap", buf, pattern, off, off, len);
> + if (ret)
> + return ret;
> + }
> +
> + /* Now check the whole thing, one page at a time. */
> + for (int i = 0; i < (FILE_SIZE / PAGE_SIZE); i++) {
> + ret = validate_buffer("mmap", buf, pattern, i * PAGE_SIZE,
> + i * PAGE_SIZE, PAGE_SIZE);
> + if (ret)
> + return ret;
> + }
> +
> + return 0;
> +}
> +
> +static int create_file(const char *file, char *pattern)
> +{
> + ssize_t ret;
> + size_t written = 0;
> + int fd __free(close_fd) = -EBADF;
> +
> + fd = open(file, O_RDWR | O_CREAT, 0644);
> + if (fd < 0) {
> + perror("opening file");
> + return 1;
> + }
> +
> + while ((ret = write(fd, pattern + written, FILE_SIZE - written)) > 0) {
> + written += ret;
> + if (written == FILE_SIZE)
> + break;
> + }
> +
> + if (ret < 0) {
> + perror("writing to the file");
> + return 1;
> + }
> +
> + return 0;
> +}
> +
> +static void generate_pattern(char *pattern)
> +{
> + for (int i = 0; i < (FILE_SIZE / PAGE_SIZE); i++) {
> + char fill = 'a' + (i % 26);
> +
> + memset(pattern + (i * PAGE_SIZE), fill, PAGE_SIZE);
> + }
> +}
> +
> +static void usage(void)
> +{
> + fprintf(stderr, "Usage: mmap-validate <create|validate> <file>\n");
> +}
> +
> +int main(int argc, char **argv)
> +{
> + char *pattern __free(freep) = NULL;
> +
> + if (argc != 3) {
> + usage();
> + return 1;
> + }
> +
> + pattern = malloc(FILE_SIZE * sizeof(char));
> + if (!pattern) {
> + perror("malloc pattern");
> + return 1;
> + }
> +
> + generate_pattern(pattern);
> +
> + if (!strcmp(argv[1], "create"))
> + return create_file(argv[2], pattern);
> +
> + if (strcmp(argv[1], "validate")) {
> + usage();
> + return 1;
> + }
> +
> + return validate_file(argv[2], pattern);
> +}
> diff --git a/src/precontent/populate.c b/src/precontent/populate.c
> new file mode 100644
> index 00000000..9c935427
> --- /dev/null
> +++ b/src/precontent/populate.c
> @@ -0,0 +1,188 @@
> +// SPDX-License-Identifier: GPL-2.0
> +#define _FILE_OFFSET_BITS 64
> +#include <dirent.h>
> +#include <errno.h>
> +#include <fcntl.h>
> +#include <limits.h>
> +#include <poll.h>
> +#include <stdio.h>
> +#include <stdlib.h>
> +#include <string.h>
> +#include <sys/stat.h>
> +#include <unistd.h>
> +
> +#define __free(func) __attribute__((cleanup(func)))
> +
> +static void close_fd(int *fd)
> +{
> + if (*fd < 0)
> + return;
> + close(*fd);
> +}
> +
> +static void close_dir(DIR **dir)
> +{
> + if (*dir == NULL)
> + return;
> + closedir(*dir);
> +}
> +
> +static void freep(void *ptr)
> +{
> + void *real_ptr = *(void **)ptr;
> + if (real_ptr == NULL)
> + return;
> + free(real_ptr);
> +}
> +
> +/*
> + * Dup a path, make sure there's a trailing '/' to make path concat easier.
> + */
> +static char *pathdup(const char *orig)
> +{
> + char *ret;
> + size_t len = strlen(orig);
> +
> + /* Easy path, we have a trailing '/'. */
> + if (orig[len - 1] == '/')
> + return strdup(orig);
> +
> + ret = malloc((len + 2) * sizeof(char));
> + if (!ret)
> + return ret;
> +
> + memcpy(ret, orig, len);
> + ret[len] = '/';
> + len++;
> + ret[len] = '\0';
> + return ret;
> +}
> +
> +static int process_directory(DIR *srcdir, char *srcpath, char *dstpath)
> +{
> + char *src __free(freep) = NULL;
> + char *dst __free(freep) = NULL;
> + size_t srclen = strlen(srcpath) + 256;
> + size_t dstlen = strlen(dstpath) + 256;
> + struct dirent *dirent;
> +
> + src = malloc(srclen * sizeof(char));
> + dst = malloc(dstlen * sizeof(char));
> + if (!src || !dst) {
> + perror("allocating path buf");
> + return -1;
> + }
> +
> + errno = 0;
> + while ((dirent = readdir(srcdir)) != NULL) {
> + if (!strcmp(dirent->d_name, ".") ||
> + !strcmp(dirent->d_name, ".."))
> + continue;
> +
> + if (dirent->d_type == DT_DIR) {
> + DIR *nextdir __free(close_dir) = NULL;
> + struct stat st;
> + int ret;
> +
> + snprintf(src, srclen, "%s%s/", srcpath, dirent->d_name);
> + snprintf(dst, dstlen, "%s%s/", dstpath, dirent->d_name);
> +
> + nextdir = opendir(src);
> + if (!nextdir) {
> + fprintf(stderr, "Couldn't open directory %s: %s (%d)\n",
> + src, strerror(errno), errno);
> + return -1;
> + }
> +
> + if (stat(src, &st)) {
> + fprintf(stderr, "Couldn't stat directory %s: %s (%d)\n",
> + src, strerror(errno), errno);
> + return -1;
> + }
> +
> + if (mkdir(dst, st.st_mode)) {
> + fprintf(stderr, "Couldn't mkdir %s: %s (%d)\n",
> + dst, strerror(errno), errno);
> + return -1;
> + }
> +
> + ret = process_directory(nextdir, src, dst);
> + if (ret)
> + return ret;
> + } else if (dirent->d_type == DT_REG) {
> + int fd __free(close_fd) = -EBADF;
> + struct stat st;
> +
> + snprintf(src, srclen, "%s%s", srcpath, dirent->d_name);
> + snprintf(dst, dstlen, "%s%s", dstpath, dirent->d_name);
> +
> + if (stat(src, &st)) {
> + fprintf(stderr, "Couldn't stat file %s: %s (%d)\n",
> + src, strerror(errno), errno);
> + return -1;
> + }
> +
> + fd = open(dst, O_WRONLY|O_CREAT, st.st_mode);
> + if (fd < 0) {
> + fprintf(stderr, "Couldn't create file %s: %s (%d)\n",
> + dst, strerror(errno), errno);
> + return -1;
> + }
> +
> + if (truncate(dst, st.st_size)) {
> + fprintf(stderr, "Couldn't truncate file %s: %s (%d)\n",
> + dst, strerror(errno), errno);
> + return -1;
> + }
> +
> +
> + if (fsync(fd)) {
> + fprintf(stderr, "Couldn't fsync file %s: %s (%d)\n",
> + dst, strerror(errno), errno);
> + return -1;
> + }
> +
> + if (posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED)) {
> + fprintf(stderr, "Couldn't clear cache on file %s: %s (%d)\n",
> + dst, strerror(errno), errno);
> + return -1;
> + }
> + }
> + errno = 0;
> + }
> +
> + if (errno) {
> + perror("readdir");
> + return -1;
> + }
> + return 0;
> +}
> +
> +int main(int argc, char **argv)
> +{
> + DIR *srcdir __free(close_dir) = NULL;
> + char *dstpath __free(freep) = NULL;
> + char *srcpath __free(freep) = NULL;
> + int ret;
> +
> + if (argc != 3) {
> + fprintf(stderr, "Usage: populate <src directory> <dest directory>\n");
> + return 1;
> + }
> +
> + srcpath = pathdup(argv[1]);
> + dstpath = pathdup(argv[2]);
> + if (!dstpath || !srcpath) {
> + perror("allocating paths");
> + return 1;
> + }
> +
> + srcdir = opendir(srcpath);
> + if (!srcdir) {
> + perror("open src directory");
> + return 1;
> + }
> +
> + ret = process_directory(srcdir, srcpath, dstpath);
> + return ret ? 1 : 0;
> +}
> diff --git a/src/precontent/remote-fetch.c b/src/precontent/remote-fetch.c
> new file mode 100644
> index 00000000..2e35b9f8
> --- /dev/null
> +++ b/src/precontent/remote-fetch.c
> @@ -0,0 +1,441 @@
> +// SPDX-License-Identifier: GPL-2.0
> +#define _FILE_OFFSET_BITS 64
> +#include <dirent.h>
> +#include <errno.h>
> +#include <fcntl.h>
> +#include <limits.h>
> +#include <poll.h>
> +#include <stdbool.h>
> +#include <stdio.h>
> +#include <stdlib.h>
> +#include <string.h>
> +#include <sys/fanotify.h>
> +#include <sys/sendfile.h>
> +#include <sys/stat.h>
> +#include <unistd.h>
> +
> +#ifndef FAN_ERRNO_BITS
> +#define FAN_ERRNO_BITS 8
> +#define FAN_ERRNO_SHIFT (32 - FAN_ERRNO_BITS)
> +#define FAN_ERRNO_MASK ((1 << FAN_ERRNO_BITS) - 1)
> +#define FAN_DENY_ERRNO(err) \
> + (FAN_DENY | ((((__u32)(err)) & FAN_ERRNO_MASK) << FAN_ERRNO_SHIFT))
> +#endif
> +
> +#ifndef FAN_PRE_ACCESS
> +#define FAN_PRE_ACCESS 0x00080000
> +#endif
> +
> +#ifndef FAN_PRE_MODIFY
> +#define FAN_PRE_MODIFY 0x00100000
> +#endif
> +
> +#ifndef FAN_EVENT_INFO_TYPE_RANGE
> +#define FAN_EVENT_INFO_TYPE_RANGE 6
> +struct fanotify_event_info_range {
> + struct fanotify_event_info_header hdr;
> + __u32 pad;
> + __u64 offset;
> + __u64 count;
> +};
> +#endif
> +
> +#define FAN_EVENTS (FAN_PRE_ACCESS | FAN_PRE_MODIFY)
> +
> +#define __round_mask(x, y) ((__typeof__(x))((y)-1))
> +#define round_up(x, y) ((((x)-1) | __round_mask(x, y))+1)
> +#define round_down(x, y) ((x) & ~__round_mask(x, y))
> +
> +static const char *srcpath;
> +static const char *dstpath;
> +static int pagesize;
> +static bool use_sendfile = false;
> +
> +#define __free(func) __attribute__((cleanup(func)))
> +
> +static void close_dir(DIR **dir)
> +{
> + if (*dir == NULL)
> + return;
> + closedir(*dir);
> +}
> +
> +static void close_fd(int *fd)
> +{
> + if (*fd < 0)
> + return;
> + close(*fd);
> +}
> +
> +static void freep(void *ptr)
> +{
> + void *real_ptr = *(void **)ptr;
> + if (real_ptr == NULL)
> + return;
> + free(real_ptr);
> +}
> +
> +static int strip_dstpath(char *path)
> +{
> + size_t remaining;
> +
> + if (strlen(path) <= strlen(dstpath)) {
> + fprintf(stderr, "'%s' not in the path '%s'", path, dstpath);
> + return -1;
> + }
> +
> + if (strncmp(path, dstpath, strlen(dstpath))) {
> + fprintf(stderr, "path '%s' doesn't start with the source path '%s'\n",
> + path, dstpath);
> + return -1;
> + }
> +
> + remaining = strlen(path) - strlen(dstpath);
> + memmove(path, path + strlen(dstpath), remaining);
> +
> + /* strip any leading / in order to make it easier to concat. */
> + while (*path == '/') {
> + if (remaining == 0) {
> + fprintf(stderr, "you gave us a weird ass string\n");
> + return -1;
> + }
> + remaining--;
> + memmove(path, path + 1, remaining);
> + }
> + path[remaining] = '\0';
> + return 0;
> +}
> +
> +/*
> + * Dup a path, make sure there's a trailing '/' to make path concat easier.
> + */
> +static char *pathdup(const char *orig)
> +{
> + char *ret;
> + size_t len = strlen(orig);
> +
> + /* Easy path, we have a trailing '/'. */
> + if (orig[len - 1] == '/')
> + return strdup(orig);
> +
> + ret = malloc((len + 2) * sizeof(char));
> + if (!ret)
> + return ret;
> +
> + memcpy(ret, orig, len);
> + ret[len] = '/';
> + len++;
> + ret[len] = '\0';
> + return ret;
> +}
> +
> +static char *get_relpath(int fd)
> +{
> + char procfd_path[PATH_MAX];
> + char abspath[PATH_MAX];
> + ssize_t path_len;
> + int ret;
> +
> + /* readlink doesn't NULL terminate. */
> + memset(abspath, 0, sizeof(abspath));
> +
> + snprintf(procfd_path, sizeof(procfd_path), "/proc/self/fd/%d", fd);
> + path_len = readlink(procfd_path, abspath, sizeof(abspath) - 1);
> + if (path_len < 0) {
> + perror("readlink");
> + return NULL;
> + }
> +
> + ret = strip_dstpath(abspath);
> + if (ret < 0)
> + return NULL;
> +
> + return strdup(abspath);
> +}
> +
> +static int copy_range(int src_fd, int fd, off_t offset, size_t count)
> +{
> + off_t src_offset = offset;
> + ssize_t copied;
> +
> + if (use_sendfile)
> + goto slow;
> +
> + while ((copied = copy_file_range(src_fd, &src_offset, fd, &offset,
> + count, 0)) >= 0) {
> + if (copied == 0)
> + return 0;
> +
> + count -= copied;
> + if (count == 0)
> + return 0;
> + }
> +
> + if (errno != EXDEV) {
> + perror("copy_file_range");
> + return -1;
> + }
> + use_sendfile = true;
> +
> +slow:
> + /* I love linux interfaces. */
> + if (lseek(fd, offset, SEEK_SET) == (off_t)-1) {
> + perror("seeking");
> + return -1;
> + }
> +
> + while ((copied = sendfile(fd, src_fd, &src_offset, count)) >= 0) {
> + if (copied == 0)
> + return 0;
> +
> + count -= copied;
> + if (count == 0)
> + return 0;
> + }
> +
> + perror("sendfile");
> + return -1;
> +}
> +
> +static int handle_event(int fanotify_fd, int fd, off_t offset, size_t count)
> +{
> + char path[PATH_MAX];
> + char *relpath __free(freep) = NULL;
> + int src_fd __free(close_fd) = -1;
> + off_t end = offset + count;
> + blkcnt_t src_blocks;
> + struct stat st;
> +
> + relpath = get_relpath(fd);
> + if (!relpath)
> + return -1;
> +
> + offset = round_down(offset, pagesize);
> + end = round_up(end, pagesize);
> + count = end - offset;
> +
> + snprintf(path, sizeof(path), "%s%s", srcpath, relpath);
> + src_fd = open(path, O_RDONLY);
> + if (src_fd < 0) {
> + fprintf(stderr, "srcpath %s relpath %s\n", srcpath, relpath);
> + fprintf(stderr, "error opening file %s: %s (%d)\n", path, strerror(errno), errno);
> + return -1;
> + }
> +
> + if (fstat(src_fd, &st)) {
> + perror("src fd is fucked");
> + return -1;
> + }
> +
> + src_blocks = st.st_blocks;
> +
> + if (fstat(fd, &st)) {
> + perror("fd is fucked");
> + return -1;
> + }
> +
> + /*
> + * If we are the same size or larger (which can happen if we copy zero's
> + * instead of inserting a hole) then just assume we're full. This is
> + * approximation can fall over, but its good enough for a PoC.
> + */
> + if (st.st_blocks >= src_blocks) {
> + int ret;
> +
> + snprintf(path, sizeof(path), "%s%s", dstpath, relpath);
> + ret = fanotify_mark(fanotify_fd, FAN_MARK_REMOVE,
> + FAN_EVENTS, -1, path);
> + if (ret < 0) {
> + /* We already removed the mark, carry on. */
> + if (errno == ENOENT) {
> + errno = 0;
> + return 0;
> + }
> + perror("removing fanotify mark");
> + return -1;
> + }
> + return 0;
> + }
> +
> +
> + return copy_range(src_fd, fd, offset, count);
> +}
> +
> +static int handle_events(int fd)
> +{
> + const struct fanotify_event_metadata *metadata;
> + struct fanotify_event_metadata buf[200];
> + ssize_t len;
> + struct fanotify_response response;
> + int ret = 0;
> +
> + len = read(fd, (void *)buf, sizeof(buf));
> + if (len <= 0 && errno != EINTR) {
> + perror("reading fanotify events");
> + return -1;
> + }
> +
> + metadata = buf;
> + while(FAN_EVENT_OK(metadata, len)) {
> + off_t offset = 0;
> + size_t count = 0;
> +
> + if (metadata->vers != FANOTIFY_METADATA_VERSION) {
> + fprintf(stderr, "invalid metadata version, have %d, expect %d\n",
> + metadata->vers, FANOTIFY_METADATA_VERSION);
> + return -1;
> + }
> + if (metadata->fd < 0) {
> + fprintf(stderr, "metadata fd is an error\n");
> + return -1;
> + }
> + if (!(metadata->mask & FAN_EVENTS)) {
> + fprintf(stderr, "metadata mask incorrect %llu\n",
> + metadata->mask);
> + return -1;
> + }
> +
> + /*
> + * We have a specific range, load that instead of filling the
> + * entire file in.
> + */
> + if (metadata->event_len > FAN_EVENT_METADATA_LEN) {
> + const struct fanotify_event_info_range *range;
> + range = (const struct fanotify_event_info_range *)(metadata + 1);
> + if (range->hdr.info_type == FAN_EVENT_INFO_TYPE_RANGE) {
> + count = range->count;
> + offset = range->offset;
> + if (count == 0) {
> + ret = 0;
> + goto next;
> + }
> + }
> + }
> +
> + /* We don't have a range, pre-fill the whole file. */
> + if (count == 0) {
> + struct stat st;
> +
> + if (fstat(metadata->fd, &st)) {
> + perror("stat() on opened file");
> + return -1;
> + }
> +
> + count = st.st_size;
> + }
> +
> + ret = handle_event(fd, metadata->fd, offset, count);
> +next:
> + response.fd = metadata->fd;
> + if (ret)
> + response.response = FAN_DENY_ERRNO(errno);
> + else
> + response.response = FAN_ALLOW;
> + write(fd, &response, sizeof(response));
This write() can fail if errno is not on the white list.
If you ignore this failure, the pre-content event will block.
> + close(metadata->fd);
> + metadata = FAN_EVENT_NEXT(metadata, len);
> + }
> +
> + return ret;
> +}
> +
> +static int add_marks(const char *src, int fanotify_fd)
> +{
> + char *path __free(freep) = NULL;
> + DIR *dir __free(close_dir) = NULL;
> + size_t pathlen = strlen(src) + 256;
> + struct dirent *dirent;
> +
> + path = malloc(pathlen * sizeof(char));
> + if (!path) {
> + perror("allocating path buf");
> + return -1;
> + }
> +
> + dir = opendir(src);
> + if (!dir) {
> + fprintf(stderr, "Couldn't open directory %s: %s (%d)\n",
> + src, strerror(errno), errno);
> + return -1;
> + }
> +
> + errno = 0;
> + while ((dirent = readdir(dir)) != NULL) {
> + int ret;
> +
> + if (!strcmp(dirent->d_name, ".") ||
> + !strcmp(dirent->d_name, ".."))
> + continue;
> +
> + if (dirent->d_type == DT_DIR) {
> + snprintf(path, pathlen, "%s%s/", src, dirent->d_name);
> + ret = add_marks(path, fanotify_fd);
> + if (ret)
> + return ret;
> + } else if (dirent->d_type == DT_REG) {
> + ret = fanotify_mark(fanotify_fd, FAN_MARK_ADD,
> + FAN_EVENTS, dirfd(dir),
> + dirent->d_name);
> + if (ret < 0) {
> + perror("fanotify_mark");
> + return -1;
> + }
> + }
> + errno = 0;
> + }
> + return 0;
> +}
> +
> +static void usage(void)
> +{
> + fprintf(stderr, "Usage: remote-fetch <src directory> <dest directory>\n");
> +}
> +
> +int main(int argc, char **argv)
> +{
> + int fd __free(close_fd) = -1;
> + int dirfd __free(close_fd) = -1;
> + int ret;
> +
> + if (argc != 3) {
> + usage();
> + return 1;
> + }
> +
> + pagesize = sysconf(_SC_PAGESIZE);
> + if (pagesize < 0) {
> + perror("sysconf");
> + return 1;
> + }
> +
> + srcpath = pathdup(argv[1]);
> + dstpath = pathdup(argv[2]);
> + if (!srcpath || !dstpath) {
> + perror("allocate paths");
> + return 1;
> + }
> +
> + dirfd = open(dstpath, O_DIRECTORY | O_RDONLY);
> + if (dirfd < 0) {
> + perror("open dstpath");
> + return 1;
> + }
> +
> + fd = fanotify_init(FAN_CLASS_PRE_CONTENT | FAN_UNLIMITED_MARKS, O_WRONLY | O_LARGEFILE);
> + if (fd < 0) {
> + perror("fanotify_init");
> + return 1;
> + }
> +
> + ret = add_marks(dstpath, fd);
> + if (ret < 0)
> + return 1;
> +
> + for (;;) {
> + ret = handle_events(fd);
> + if (ret)
> + break;
> + }
> +
> + return (ret < 0) ? 1 : 0;
> +}
> diff --git a/tests/generic/800 b/tests/generic/800
> new file mode 100644
> index 00000000..08ac5b26
> --- /dev/null
> +++ b/tests/generic/800
> @@ -0,0 +1,68 @@
> +#! /bin/bash
> +# SPDX-License-Identifier: GPL-2.0
> +# Copyright (c) 2024 Meta Platforms, Inc. All Rights Reserved.
> +#
> +# FS QA Test 800
> +#
> +# Validate the pre-content related fanotify features
> +#
> +# The mmap-verify pre-content tool generates a file and then validates that the
> +# pre-content watched directory properly fills it in with a mixture of page
> +# faults and normal reads.
> +#
> +. ./common/preamble
> +_begin_fstest quick auto fsnotify
> +
> +_cleanup()
> +{
> + cd /
> + rm -rf $TEST_DIR/dst-$seq
> + rm -rf $TEST_DIR/src-$seq
> +}
> +
> +# real QA test starts here
> +_supported_fs generic
> +_require_test
> +_require_test_program "precontent/mmap-validate"
> +_require_test_program "precontent/populate"
> +_require_test_program "precontent/remote-fetch"
> +
> +dstdir=$TEST_DIR/dst-$seq
> +srcdir=$TEST_DIR/src-$seq
> +
> +POPULATE=$here/src/precontent/populate
> +REMOTE_FETCH=$here/src/precontent/remote-fetch
> +MMAP_VALIDATE=$here/src/precontent/mmap-validate
> +
> +mkdir $dstdir $srcdir
For tests on the test partition, should also cleanup
prior leftovers at the beginning of the test.
> +
> +# Generate the test file
> +$MMAP_VALIDATE create $srcdir/validate
> +
> +# Generate the stub file in the watch directory
> +$POPULATE $srcdir $dstdir
> +
> +# Start the remote watcher
> +$REMOTE_FETCH $srcdir $dstdir &
> +
> +FETCH_PID=$!
> +
> +# We may not support fanotify, give it a second to start and then make sure the
> +# fetcher is running before we try to validate the buffer
> +sleep 1
> +
> +if ! ps -p $FETCH_PID > /dev/null
> +then
> + _notrun "precontent watches not supported"
That's not so nice and prone to races.
Let's do a deterministic test for precontent support,
i.e. $here/src/precontent/test-support
or $REMOTE_FETCH - $dstdir
and abstract this test inside _require_fsnotify_precontent
Thanks,
Amir.
^ permalink raw reply [flat|nested] 5+ messages in thread
* Re: [PATCH 2/2] fstests: add a test for executing from a precontent watch directory
2024-09-04 20:32 ` [PATCH 2/2] fstests: add a test for executing from a precontent watch directory Josef Bacik
@ 2024-09-05 7:21 ` Amir Goldstein
0 siblings, 0 replies; 5+ messages in thread
From: Amir Goldstein @ 2024-09-05 7:21 UTC (permalink / raw)
To: Josef Bacik; +Cc: fstests, linux-fsdevel
On Wed, Sep 4, 2024 at 10:33 PM Josef Bacik <josef@toxicpanda.com> wrote:
>
> The main purpose of putting precontent hooks in the page fault path is
> to enable running an executable from a precontent watch. Add a test to
> create a precontent watched directory with bash in it, and then execute
> that copy of bash to validate that we fill in the pages properly and are
> able to execute.
>
> Signed-off-by: Josef Bacik <josef@toxicpanda.com>
Nice!
Same comments as previous patch.
Thanks,
Amir.
> ---
> tests/generic/801 | 64 +++++++++++++++++++++++++++++++++++++++++++
> tests/generic/801.out | 2 ++
> 2 files changed, 66 insertions(+)
> create mode 100644 tests/generic/801
> create mode 100644 tests/generic/801.out
>
> diff --git a/tests/generic/801 b/tests/generic/801
> new file mode 100644
> index 00000000..7a1cc653
> --- /dev/null
> +++ b/tests/generic/801
> @@ -0,0 +1,64 @@
> +#! /bin/bash
> +# SPDX-License-Identifier: GPL-2.0
> +# Copyright (c) 2024 Meta Platforms, Inc. All Rights Reserved.
> +#
> +# FS QA Test 801
> +#
> +# Validate the pre-content hooks work properly with exec
> +#
> +# Copy bash into our source directory and then setup the HSM daemon to mirror
> +# into the destination directory, and execute bash from the destination
> +# directory to make sure it loads properly.
> +#
> +. ./common/preamble
> +_begin_fstest quick auto fsnotify
> +
> +_cleanup()
> +{
> + cd /
> + rm -rf $TEST_DIR/dst-$seq
> + rm -rf $TEST_DIR/src-$seq
> +}
> +
> +# real QA test starts here
> +_supported_fs generic
> +_require_test
> +_require_test_program "precontent/populate"
> +_require_test_program "precontent/remote-fetch"
> +
> +dstdir=$TEST_DIR/dst-$seq
> +srcdir=$TEST_DIR/src-$seq
> +
> +POPULATE=$here/src/precontent/populate
> +REMOTE_FETCH=$here/src/precontent/remote-fetch
> +
> +mkdir $dstdir $srcdir
> +
> +# Copy bash into our source dir
> +cp $(which bash) $srcdir
> +
> +# Generate the stub file in the watch directory
> +$POPULATE $srcdir $dstdir
> +
> +# Start the remote watcher
> +$REMOTE_FETCH $srcdir $dstdir &
> +
> +FETCH_PID=$!
> +
> +# We may not support fanotify, give it a second to start and then make sure the
> +# fetcher is running before we try to run our test
> +sleep 1
> +
> +if ! ps -p $FETCH_PID > /dev/null
> +then
> + _notrun "precontent watches not supported"
> +fi
> +
> +$dstdir/bash -c "echo 'Hello!'"
> +
> +kill -9 $FETCH_PID &> /dev/null
> +wait $FETCH_PID &> /dev/null
> +
> +# success, all done
> +status=0
> +exit
> diff --git a/tests/generic/801.out b/tests/generic/801.out
> new file mode 100644
> index 00000000..98e6a16c
> --- /dev/null
> +++ b/tests/generic/801.out
> @@ -0,0 +1,2 @@
> +QA output created by 801
> +Hello!
> --
> 2.43.0
>
>
^ permalink raw reply [flat|nested] 5+ messages in thread
end of thread, other threads:[~2024-09-05 7:21 UTC | newest]
Thread overview: 5+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2024-09-04 20:32 [PATCH 0/2] fstests: add two tests for the precontent fanotify work Josef Bacik
2024-09-04 20:32 ` [PATCH 1/2] fstests: add a test for the precontent fanotify hooks Josef Bacik
2024-09-05 7:17 ` Amir Goldstein
2024-09-04 20:32 ` [PATCH 2/2] fstests: add a test for executing from a precontent watch directory Josef Bacik
2024-09-05 7:21 ` Amir Goldstein
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).