Linux XFS filesystem development
 help / color / mirror / Atom feed
* Really bad I/O lock contention between non-overlapping reader and writer
@ 2026-10-12 17:47 Vitali Lovich
  2026-10-12 18:23 ` Vitali Lovich
                   ` (2 more replies)
  0 siblings, 3 replies; 9+ messages in thread
From: Vitali Lovich @ 2026-10-12 17:47 UTC (permalink / raw)
  To: linux-xfs

[-- Attachment #1: Type: text/plain, Size: 1844 bytes --]

I think I encountered suboptimal locking behavior with concurrent
readers and writers on a file, which resulted in huge latency spikes
for the reader even though the writers and readers are touching
unrelated ranges within the file. The writer/hole puncher hold an
exclusive lock on the inode while forcing a flush of outstanding
writes to disk which creates huge latency spikes for reads.

I've observed read's taking >> 500ms even though none of the writes
could be overlapping and traces seem to indicate it's the shared inode
lock contending with the exclusive inode lock that writes / hole
punching is holding. In my real-world test case traces indicated that
the cause might be inode_dio_wait while holding the exclusive inode
lock.

I've observed this behavior on XFS, EXT4 and BTRFS so it seems like a
general VFS challenge. Generally, Btrfs seems to have less read
latencies created than Ext4 (~2.5x more at the 0-1ms read than EXT4),
which in turn has smaller and fewer tail latencies than XFS  (~2x more
at 0-1ms and extreme is smaller) although all filesystems exhibit the
issue. Starting with the XFS list in case I'm way off base that this
is a problem.

Attaching a simplified reproduction of what I'm doing with one writer
appending data to the file, a separate writer punching holes in
unrelated ranges, and an O_DIRECT reader that's reading data. Reads
are doing random 4kib I/O in [0, 64MiB), hole punching is punching
4kib holes in [64MiB, 576Mib] and writing is always appending after
577MiB of the file. The directory the anonymous file is created in is
CWD by default unless you pass a different path. I'd expect read
latencies to not be impacted by waiting for writes to flush yet it
seems like clearly there is such a serialization.

cc -O2 -g -std=gnu11 -Wall -Wextra -Werror repro-syscall.c -pthread -o
syscall-repro

[-- Attachment #2: pessimistic_locking_between_readers_and_writers.c --]
[-- Type: text/x-csrc, Size: 5089 bytes --]

/*
 * File layout (MiB; ranges are half-open):
 *   [0, 64)     random 4 KiB reads
 *   [64, 576)   one-time 4 KiB hole punches
 *   [576, 577)  unused guard
 *   [577, ...)  repeated 512 MiB writes
 *
 * Before starting the worker threads, initialization writes and fsyncs the
 * entire [0, 577) range in 1 MiB chunks. Thus every punched range was written
 * first. The reader chooses aligned blocks only within [0, 64), and each
 * pread ends no later than 64 MiB. Punches stay within [64, 576); the writer
 * starts at 577 MiB and appends, so none of the three operations overlap.
 *
 * The read, write, and punch descriptors are opened on the same inode. The
 * pathname is unlinked after opening; the descriptors keep that file alive.
 */
#define _GNU_SOURCE
#include <errno.h>
#include <fcntl.h>
#include <pthread.h>
#include <stdatomic.h>
#include <stdint.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <time.h>
#include <unistd.h>

#define K (1024ULL)
#define M (1024ULL * K)
/* File layout: readable data | punched data | guard | appended writes. */
#define READ_MIB 64
#define HOLE_MIB 512
#define WRITE_MIB 512
#define PUNCH_KIB 4
#define BUCKETS 1001

static int rfd, wfd, pfd;
static atomic_bool stop;
static atomic_ullong writes, punches, reads;
static uint64_t hist[BUCKETS], max_read_ns, over100ms;
static inline uint64_t ns(void) {
    struct timespec t;
    clock_gettime(CLOCK_MONOTONIC, &t);
    return (uint64_t)t.tv_sec * 1000000000 + t.tv_nsec;
}
static void die(const char *s) { perror(s); exit(1); }
static void *aligned(size_t n) {
    void *p;
    int e = posix_memalign(&p, 4096, n);
    if (e) { errno = e; die("posix_memalign"); }
    memset(p, 0x5a, n);
    return p;
}
static void *reader(void *unused) {
    (void)unused;
    char *b = aligned(4096);
    uint64_t x = 0x9e3779b97f4a7c15ULL;
    while (!atomic_load(&stop)) {
        x ^= x << 13; x ^= x >> 7; x ^= x << 17;
        off_t off = (off_t)((x % (READ_MIB * M / 4096)) * 4096);
        uint64_t a = ns();
        ssize_t n = pread(rfd, b, 4096, off);
        uint64_t d = ns() - a;
        if (n != 4096) die("pread");
        if (d > max_read_ns) max_read_ns = d;
        unsigned bucket = d / 1000000;
        hist[bucket < BUCKETS ? bucket : BUCKETS - 1]++;
        over100ms += d >= 100000000;
        atomic_fetch_add(&reads, 1);
    }
    free(b); return NULL;
}
static void *writer(void *unused) {
    (void)unused;
    char *b = aligned(WRITE_MIB * M);
    off_t off = (READ_MIB + HOLE_MIB + 1) * M;
    while (!atomic_load(&stop)) {
        size_t done = 0;
        while (done < WRITE_MIB * M) {
            ssize_t n = pwrite(wfd, b + done, WRITE_MIB * M - done, off + done);
            if (n < 0) die("pwrite");
            if (!n) { errno = EIO; die("zero pwrite"); }
            done += (size_t)n;
        }
        off += WRITE_MIB * M;
        atomic_fetch_add(&writes, 1);
    }
    free(b); return NULL;
}
static void *puncher(void *unused) {
    (void)unused;
    off_t off = READ_MIB * M;
    const off_t end = (READ_MIB + HOLE_MIB) * M;
    while (!atomic_load(&stop) && off < end) {
        if (fallocate(pfd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, off, PUNCH_KIB * K)) die("fallocate punch");
        off += PUNCH_KIB * K;
        atomic_fetch_add(&punches, 1);
    }
    return NULL;
}
int main(int argc, char **argv) {
    const char *dir = argc > 1 ? argv[1] : ".";
    unsigned seconds = argc > 2 ? strtoul(argv[2], NULL, 10) : 10;
    char path[4096];
    if (snprintf(path, sizeof(path), "%s/syscall-repro-XXXXXX", dir) >= (int)sizeof(path)) { errno = ENAMETOOLONG; die("path"); }
    int fd = mkstemp(path); if (fd < 0) die("mkstemp");
    rfd = open(path, O_RDONLY | O_DIRECT); if (rfd < 0) die("open read");
    wfd = open(path, O_RDWR | O_DIRECT); if (wfd < 0) die("open write");
    pfd = open(path, O_RDWR); if (pfd < 0) die("open punch");
    unlink(path);
    char *init = aligned(M);
    off_t end = (READ_MIB + HOLE_MIB + 1) * M;
    for (off_t o = 0; o < end; o += M)
        if (pwrite(fd, init, M, o) != M) die("initialize");
    if (fsync(fd)) die("fsync init");
    free(init); close(fd);
    printf("direct pread/pwrite; fallocate punch; read=[0,%d MiB), punch=[%d,%d MiB), write starts at %lld MiB; seconds=%u\n", READ_MIB, READ_MIB, READ_MIB + HOLE_MIB, (long long)(READ_MIB + HOLE_MIB + 1), seconds);
    pthread_t rt, wt, pt;
    if (pthread_create(&rt, NULL, reader, NULL) || pthread_create(&wt, NULL, writer, NULL) || pthread_create(&pt, NULL, puncher, NULL)) die("pthread_create");
    sleep(seconds);
    atomic_store(&stop, true);
    pthread_join(rt, NULL); pthread_join(wt, NULL); pthread_join(pt, NULL);
    printf("summary reads=%llu writes=%llu punches=%llu max-read=%.1f ms >100ms=%llu\n", atomic_load(&reads), atomic_load(&writes), atomic_load(&punches), max_read_ns / 1e6, (unsigned long long)over100ms);
    puts("latency-ms,count");
    for (unsigned i = 0; i < BUCKETS; ++i) if (hist[i]) printf("%u,%llu\n", i, (unsigned long long)hist[i]);
    return 0;
}

^ permalink raw reply	[flat|nested] 9+ messages in thread
end of thread, other threads:[~2026-10-14  3:01 UTC | newest]

Thread overview: 9+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2026-10-12 17:47 Really bad I/O lock contention between non-overlapping reader and writer Vitali Lovich
2026-10-12 18:23 ` Vitali Lovich
2026-10-12 19:37 ` Alan Huang
2026-10-12 19:53   ` Vitali Lovich
2026-10-12 20:27     ` Alan Huang
2026-10-12 20:47 ` Dave Chinner
2026-10-13 23:49   ` Vitali Lovich
2026-10-14  3:01     ` Dave Chinner
     [not found]   ` <CAF8PYMi5DX0AkzbSTFnCchHnr3QeB-S7J+2FNMCqB8zrtR9ojA@mail.gmail.com>
2026-10-14  2:34     ` Dave Chinner

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox