[Bug] XFS: DIO random write + BufferIO read

* [Bug] XFS: DIO random write + BufferIO read
@ 2014-04-04 15:09 Zhi Yong Wu
  2014-04-08  1:59 ` Zhi Yong Wu
  2014-04-08 14:28 ` Jeff Moyer
  0 siblings, 2 replies; 4+ messages in thread
From: Zhi Yong Wu @ 2014-04-04 15:09 UTC (permalink / raw)
  To: xfstests
  Cc: Dave Chinner, jack, linux-fsdevel@vger.kernel.org,
	Theodore Ts'o

HI

When i try something on XFS filesytem, i hit one issue as below:

One main task create multiple threads at first, Then it will dio
random write some files with random offset and length for each thread.
When those files get ready, those multiple threads will bufferio read
them and check if the data are same as their corresponding buffer. In
theory, they should be same, but the actual result isn't what we
expect

By the way, i did the same try on ext3 filesystem, but didn't get any
such issue.

Below is the test code:

#include <errno.h>
#include <string.h>
#include <time.h>
#include <pthread.h>
#include <assert.h>
#include <stdlib.h>
#include <stdio.h>
#include <libaio.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <syscall.h>
#include <fcntl.h>
#include <libaio.h>
#include <string>
#include <vector>

using namespace std;

#define AIO_BLKSIZE  4096
#define AIO_MAXIO 64

static long gettid()
{
    return static_cast<long>(pthread_self());
}

static void* aioThreadWrapper(void* arg);

class DioTest;
struct cbParam {
    cbParam(DioTest* p, int fileseq) :p_(p), fileseq_(fileseq) {}

    DioTest* p_;
    int fileseq_;
};

class DioTest
{
    public:
        DioTest(const char* name) {
            filename = name;
            memset(&myctx, 0, sizeof(myctx));
            io_queue_init(AIO_MAXIO, &myctx);
            pthread_t tid;
            pthread_create(&tid, NULL, aioThreadWrapper, this);
        }

        void wr_done(int fileseq, struct iocb *iocb, long res, long res2) {
            close(iocb->aio_fildes);

            if (res2 != 0) {
                printf("aio write error n");
                abort();
            }
            if (res != iocb->u.c.nbytes) {
                printf("write missed bytes expect %ld got %ld n",
                        iocb->u.c.nbytes, res);
                abort();
            }
            size_t length = iocb->u.c.nbytes;
            size_t offset = iocb->u.c.offset;
            char path[1024];
            snprintf(path, sizeof(path), "%s%d", filename, fileseq);
            int fd = open(path, O_RDONLY);
            assert(fd >= 0);
            char* readbuf = (char*)malloc(length);
            assert(readbuf);

            memset(readbuf, 0, length);
            ssize_t ret = pread(fd, readbuf, length, offset);
            assert (ret == length);
            close(fd);

            int cmp = memcmp(readbuf, iocb->u.c.buf, length);
            if (cmp != 0)
            {
                printf("tid=%ld data dismatch.cmp=%d file=%s
offset=%lu length=%lu!\n",
                        gettid(), cmp, path, offset, length);
                abort();
            }
            printf("tid=%ld check=success file=%s offset=%lu length=%lu\n",
                    gettid(), path, offset, length);

            free(iocb->u.c.buf);
            free(iocb);
            free(readbuf);
        }

        bool writeRequest(int fileseq, size_t offset, size_t length) {
            struct iocb *io = (struct iocb *)malloc(sizeof(struct iocb));
            assert (io);
            char path[1024];
            snprintf(path, sizeof(path), "%s%d", filename, fileseq);
            int fd  = open(path, O_RDWR|O_DIRECT|O_CREAT, S_IWUSR | S_IRUSR);
            assert (fd >= 0);
            void* buf=NULL;
            int ret = posix_memalign(&buf, getpagesize(), length);
            assert(ret == 0);
            memset(buf, 'a', length);
            io_prep_pwrite(io, fd, buf, length, offset);
            io->data = new cbParam(this, fileseq);

            int rc = io_submit(myctx, 1, &io);
            if (rc < 0){
                printf("tid=%ld io_submit fail.file=%s offset=%lu
length=%lu ret=%d errno=%s\n",
                        gettid(), path, offset, length, ret, strerror(errno));
                close(fd);
                free(buf);
                delete (cbParam*)(io->data);
                free(io);
                return false;
            }
            assert (rc != 0);
            printf("tid=%ld file=%s offset=%lu length=%lu\n",gettid(),
path, offset, length);
            return true;
        }

        void aioThread() {
            while (true)
            {
                struct io_event events[AIO_MAXIO];
                io_callback_t cb;
                int ret = io_getevents(myctx, 1, AIO_MAXIO, events, NULL);
                printf("tid=%ld %d io_request completed \n", gettid(), ret);

                for (int i = 0; i < ret; i++) {
                    struct iocb *io = events[i].obj;
                    printf("tid=%ld events[%d]res = %ld, res2 = %ld\n",
                            gettid(), i, events[i].res, events[i].res2);

                    cbParam* param = (cbParam*)io->data;
                    DioTest* p = param->p_;
                    p->wr_done(param->fileseq_, io, events[i].res,
events[i].res2);
                    delete param;
                }
            }
        }
    private:
        io_context_t myctx;
        const char* filename;
};

static void* aioThreadWrapper(void* arg)
{
    DioTest* p = (DioTest*)arg;
    p->aioThread();
    return NULL;
}

int main(int args, char *argv[])
{
    if (args < 2) {
        printf("./%s filename", argv[0]);
        exit(1);
    }
    const char* filename = argv[1];
    srand(time(NULL));
    vector<DioTest*> dioTests;
    const int threadNumber = 9;

    for (int i = 0; i < threadNumber; ++i) {
        dioTests.push_back(new DioTest(filename));
    }

    while (true) {
        size_t offset = (rand() % (64*1024*1024/AIO_BLKSIZE)) * AIO_BLKSIZE;
        size_t length = 0;
        while (length == 0) {
            length = abs(static_cast<int>(rand()*1.0/RAND_MAX*16))*AIO_BLKSIZE;
        }

        int seq = rand() % 100;
        DioTest* p = dioTests[rand() % threadNumber];
        for (int i = 0; i < 4; ++i){
            p->writeRequest(seq, offset, length);
            offset += (length + 4096);
        }
        usleep(rand() % 10000);
    }
    return 0;
}

-- 
Regards,

Zhi Yong Wu

^ permalink raw reply	[flat|nested] 4+ messages in thread