diff for duplicates of <56C05000.1040001@dev.mellanox.co.il> diff --git a/a/1.txt b/N1/1.txt index eca2b76..ecb145d 100644 --- a/a/1.txt +++ b/N1/1.txt @@ -44,415 +44,3 @@ wrt the virtual boundary so I think nvme will break as well. Attaching a small test program I used to force gappy I/O. $ ./scatter_data -l 64k -n 128 -d <dev> --------------- next part -------------- -/** - * Scattered IO test - * - * Author: Adir Lev - **/ -#define _GNU_SOURCE - -#include <ctype.h> -#include <stdio.h> -#include <stdlib.h> -#include <stdint.h> -#include <malloc.h> -#include <unistd.h> -#include <fcntl.h> -#include <sys/stat.h> -#include <sys/types.h> -#include <errno.h> -#include <string.h> -#include <getopt.h> -#include <sys/uio.h> -#include <sys/time.h> -#include <assert.h> - - -#define MAX_SGE 128 - -int do_write = 0; -int count = 1; -int num_sge = 0; -int bs = 0; -char *dev; -size_t page_size; -void *ibuf; -void *obuf; -long disk_sz = 0; - - -double time_diff(struct timeval x , struct timeval y) { - double x_ms , y_ms , diff; - x_ms = (double)x.tv_sec*1000000 + (double)x.tv_usec; - y_ms = (double)y.tv_sec*1000000 + (double)y.tv_usec; - diff = (double)y_ms - (double)x_ms; - return diff; -} - -void print_usage(char* cmd) { - printf("USAGE: %s -l 1024 -n 2 -d /dev/sdb [-C 1000]\n", cmd); - printf("\t-l bs in KBytes\n"); - printf("\t-n num of sges to use\n"); - printf("\t-d block device\n"); - printf("\t[-C] num of iterations\n"); -} - -int open_block_dev() { - FILE *fp; - int fd, rc; - long sz; - - printf("Device: %s\n", dev); - fd = open(dev, O_RDWR|O_DIRECT|O_SYNC, 777); - if (fd < 0) { - perror("Unable to open block device"); - return fd; - } - - fp = fdopen(fd, "w+"); - if (!fp) { - printf("failed to fdopen, errno=%d\n", errno); - return -1; - } - - rc = fseek(fp, 0, SEEK_END); - if (rc < 0) { - printf("failed to fseek, errno=%d\n", errno); - return -1; - } - - disk_sz = ftell(fp); - if (disk_sz < 0) { - printf("failed to ftell, errno=%d\n", errno); - return -1; - } - - rewind(fp); - - return fd; -} - -int my_rewind(fd) { - FILE *fp; - - fp = fdopen(fd, "w+"); - if (!fp) { - printf("failed to fdopen, errno=%d\n", errno); - return -1; - } - rewind(fp); - return 0; -} - -int parse_args(int argc, char **argv) { - int option = 0; - - while ((option = getopt(argc, argv,"wC:l:n:d:")) != -1) { - switch (option) { - case 'w': - do_write = 1; - break; - case 'C': - count = atoi(optarg); - break; - case 'd': - dev = optarg; - break; - case 'l': - bs = atoi(optarg); - break; - case 'n': - num_sge = atoi(optarg); - break; - default: - print_usage(argv[0]); - return -1; - } - } - - /* sanity check args */ - if (optind < 4) { - printf("Mandatory argument(s) missing\n"); - print_usage(argv[0]); - return -1; - } - - if (bs == 512) { - printf("ERROR: Block size must exceed 512Bytes \n"); - return -1; - } - bs = bs * 1024; - if (num_sge > MAX_SGE) { - printf("ERROR: num_sge (-n) cannot exceed 128\n"); - return -1; - } - - if (bs % 512 != 0) { - printf("ERROR: Block size must be multiple of 512\n"); - return -1; - } - - if ((bs / num_sge) % 512 != 0) { - printf("ERROR: Block size/num_sge must be multiple of 512\n"); - return -1; - } - - if (bs > (page_size * 128)) { - printf("ERROR: Block size cannot exceed 524288 Bytes (4096B * 128)\n"); - return -1; - } - - if (count < 1) { - printf("ERROR: count needs to be higher than 0\n"); - return -1; - } - - return 0; -} - -void* alloc_sges() -{ - void *buf; - int sge_size = bs / num_sge; - - if (sge_size > page_size) { - printf("ERROR: sge size cannot exceed page size\n"); - return NULL; - } - - buf = memalign(page_size, num_sge * page_size); - if (!buf) - perror( "ERROR: cannot allocate memory"); - - memset(buf, 0, num_sge * page_size); - - return buf; -} - -int sample_counter() { - FILE *fp; - int val; - - system("iscsiadm -m session -s | grep fmr_un | awk '{print $2}'" - " | awk '{ sum+=$1} END {print sum}' >> /tmp/indir_counter"); - - fp = fopen("/tmp/indir_counter", "rw"); - if (!fp) { - perror("Unable to open counter file"); - return -1; - } - - fscanf(fp, "%d", &val); - - if (val < 0) { - printf("Failed to get fmr_unaligned counter\n"); - return -1; - } - - fclose(fp); - unlink("/tmp/indir_counter"); - return val; -} - -void get_stats(struct timeval t_before, struct timeval t_after) { - double t_diff; - float iops; - long bw; - - t_diff = time_diff(t_before, t_after); - iops = (float)count / t_diff * 1000; - bw = iops * bs; - - printf("time elapsed in sec %f\n", t_diff/1000000); - printf("iops: %.2fkiops\n", iops); - printf("BW: %ldKB\n", bw); -} - -int calc_counter(int before, int after) { - int total = 0; - - total = after - before; - if (total != count * 2) { - printf("count: %d, fmr_unaligned_cntr: %d\n", count, total); - return -1; - } else { - return 0; - } -} - -static void dump_bufs(void *s1, void *s2, int len) -{ - int i; - - for (i = 0; i < len; i += 8) { - uint64_t idword = *(uint64_t *)&(((char *)s2)[i]); - uint64_t odword = *(uint64_t *)&(((char *)s1)[i]); - - printf("obuf[%x]: %x, ibuf[%x]: %x\n", - i, odword, i, idword); - } -} - -static int run_rw(int is_write, int fd, void *buf) -{ - struct iovec iov[num_sge]; - int sge_size = bs / num_sge; - int max = page_size - sge_size; - int i = 0, j = 0, offset = 0, rc = 0; - ssize_t bytes_read; - long bytes_left = disk_sz; - - /* for every iteration */ - for (i = 0; i < count; i++) { - if (max > 0) - offset = (512 * i) % max; - if (bytes_left < bs) { - rc = my_rewind(fd); - if (rc < 0) - return rc; - printf("count: %d, no space left on block " - "device, rewinding\n", i); - bytes_left = disk_sz; - } - /* for every sge */ - for (j = 0; j < num_sge; j++) { - /* change offset in page */ - iov[j].iov_base = buf + (page_size * j) + offset; - iov[j].iov_len = sge_size; - if (is_write) - memset(iov[j].iov_base, i+j, iov[j].iov_len); - } - - if (is_write) { - bytes_read = writev(fd, iov, num_sge); - if (bytes_read < bs) { - if (bytes_read < 0) { - printf("failed to writev, bytes=%d, " - "errno=%d\n", bytes_read, errno); - perror("failed to writev"); - } else - printf("writev less than expected. " - "Bytes=%d, expected %d\n", bytes_read, bs); - return -1; - } - } else { - bytes_read = readv(fd, iov, num_sge); - if (bytes_read < bs) { - if (bytes_read < 0) { - printf("failed to readv, bytes=%d, " - "errno=%d\n", bytes_read, errno); - perror("failed to readv"); - } else - printf("readv less than expected. " - "Bytes=%d, expected %d\n", bytes_read, bs); - return -1; - } - } - bytes_left -= bs; - } - - return 0; -} - -int run_iovec_traffic(int fd) -{ - int rc; - - rc = my_rewind(fd); - if (rc) { - printf("rewind failed\n"); - return -1; - } - - rc = run_rw(1, fd, obuf); - if (rc) { - printf("write failed\n"); - return -1; - } - - rc = my_rewind(fd); - if (rc) { - printf("rewind failed\n"); - return -1; - } - - rc = run_rw(0, fd, ibuf); - if (rc) { - printf("read failed\n"); - return -1; - } - - rc = memcmp(ibuf, obuf, bs); - if (rc) { - printf("memcmp failed\n"); - dump_bufs(obuf, ibuf, bs); - return -1; - } - - return rc; -} - -int main(int argc, char **argv) { - struct timeval t_before, t_after; - void **page_list = NULL; - int fd, before_counter = 0, after_counter = 0, rc = 0; - - page_size = sysconf(_SC_PAGESIZE); - rc = parse_args(argc, argv); - if (rc) - return -1; - - fd = open_block_dev(); - if (fd < 0) - return -1; - - ibuf = alloc_sges(); - if (!ibuf) { - rc = -ENOMEM; - goto out; - } - - obuf = alloc_sges(); - if (!obuf) { - rc = -ENOMEM; - goto out; - } - - before_counter = sample_counter(); - if (before_counter < 0) { - rc = -1; - goto out; - } - - gettimeofday(&t_before, NULL); - rc = run_iovec_traffic(fd); - gettimeofday(&t_after, NULL); - - if (rc) { - printf("Exiting with rc=%d\n", rc); - goto out; - } - - get_stats(t_before, t_after); - - after_counter = sample_counter(); - if (after_counter < 0) { - rc = -1; - goto out; - } - - rc = calc_counter(before_counter, after_counter); - if (rc) { - printf("Test Failed unaligned count\n"); - goto out; - } - - printf("Test Passes\n"); -out: - close(fd); - free (ibuf); - free (obuf); - - return rc; -} diff --git a/N1/2.hdr b/N1/2.hdr new file mode 100644 index 0000000..77828cc --- /dev/null +++ b/N1/2.hdr @@ -0,0 +1,5 @@ +Content-Type: text/plain; charset=UTF-8; + name="scattered_data.c" +Content-Transfer-Encoding: 7bit +Content-Disposition: attachment; + filename="scattered_data.c" diff --git a/N1/2.txt b/N1/2.txt new file mode 100644 index 0000000..a900cc6 --- /dev/null +++ b/N1/2.txt @@ -0,0 +1,411 @@ +/** + * Scattered IO test + * + * Author: Adir Lev + **/ +#define _GNU_SOURCE + +#include <ctype.h> +#include <stdio.h> +#include <stdlib.h> +#include <stdint.h> +#include <malloc.h> +#include <unistd.h> +#include <fcntl.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <errno.h> +#include <string.h> +#include <getopt.h> +#include <sys/uio.h> +#include <sys/time.h> +#include <assert.h> + + +#define MAX_SGE 128 + +int do_write = 0; +int count = 1; +int num_sge = 0; +int bs = 0; +char *dev; +size_t page_size; +void *ibuf; +void *obuf; +long disk_sz = 0; + + +double time_diff(struct timeval x , struct timeval y) { + double x_ms , y_ms , diff; + x_ms = (double)x.tv_sec*1000000 + (double)x.tv_usec; + y_ms = (double)y.tv_sec*1000000 + (double)y.tv_usec; + diff = (double)y_ms - (double)x_ms; + return diff; +} + +void print_usage(char* cmd) { + printf("USAGE: %s -l 1024 -n 2 -d /dev/sdb [-C 1000]\n", cmd); + printf("\t-l bs in KBytes\n"); + printf("\t-n num of sges to use\n"); + printf("\t-d block device\n"); + printf("\t[-C] num of iterations\n"); +} + +int open_block_dev() { + FILE *fp; + int fd, rc; + long sz; + + printf("Device: %s\n", dev); + fd = open(dev, O_RDWR|O_DIRECT|O_SYNC, 777); + if (fd < 0) { + perror("Unable to open block device"); + return fd; + } + + fp = fdopen(fd, "w+"); + if (!fp) { + printf("failed to fdopen, errno=%d\n", errno); + return -1; + } + + rc = fseek(fp, 0, SEEK_END); + if (rc < 0) { + printf("failed to fseek, errno=%d\n", errno); + return -1; + } + + disk_sz = ftell(fp); + if (disk_sz < 0) { + printf("failed to ftell, errno=%d\n", errno); + return -1; + } + + rewind(fp); + + return fd; +} + +int my_rewind(fd) { + FILE *fp; + + fp = fdopen(fd, "w+"); + if (!fp) { + printf("failed to fdopen, errno=%d\n", errno); + return -1; + } + rewind(fp); + return 0; +} + +int parse_args(int argc, char **argv) { + int option = 0; + + while ((option = getopt(argc, argv,"wC:l:n:d:")) != -1) { + switch (option) { + case 'w': + do_write = 1; + break; + case 'C': + count = atoi(optarg); + break; + case 'd': + dev = optarg; + break; + case 'l': + bs = atoi(optarg); + break; + case 'n': + num_sge = atoi(optarg); + break; + default: + print_usage(argv[0]); + return -1; + } + } + + /* sanity check args */ + if (optind < 4) { + printf("Mandatory argument(s) missing\n"); + print_usage(argv[0]); + return -1; + } + + if (bs == 512) { + printf("ERROR: Block size must exceed 512Bytes \n"); + return -1; + } + bs = bs * 1024; + if (num_sge > MAX_SGE) { + printf("ERROR: num_sge (-n) cannot exceed 128\n"); + return -1; + } + + if (bs % 512 != 0) { + printf("ERROR: Block size must be multiple of 512\n"); + return -1; + } + + if ((bs / num_sge) % 512 != 0) { + printf("ERROR: Block size/num_sge must be multiple of 512\n"); + return -1; + } + + if (bs > (page_size * 128)) { + printf("ERROR: Block size cannot exceed 524288 Bytes (4096B * 128)\n"); + return -1; + } + + if (count < 1) { + printf("ERROR: count needs to be higher than 0\n"); + return -1; + } + + return 0; +} + +void* alloc_sges() +{ + void *buf; + int sge_size = bs / num_sge; + + if (sge_size > page_size) { + printf("ERROR: sge size cannot exceed page size\n"); + return NULL; + } + + buf = memalign(page_size, num_sge * page_size); + if (!buf) + perror( "ERROR: cannot allocate memory"); + + memset(buf, 0, num_sge * page_size); + + return buf; +} + +int sample_counter() { + FILE *fp; + int val; + + system("iscsiadm -m session -s | grep fmr_un | awk '{print $2}'" + " | awk '{ sum+=$1} END {print sum}' >> /tmp/indir_counter"); + + fp = fopen("/tmp/indir_counter", "rw"); + if (!fp) { + perror("Unable to open counter file"); + return -1; + } + + fscanf(fp, "%d", &val); + + if (val < 0) { + printf("Failed to get fmr_unaligned counter\n"); + return -1; + } + + fclose(fp); + unlink("/tmp/indir_counter"); + return val; +} + +void get_stats(struct timeval t_before, struct timeval t_after) { + double t_diff; + float iops; + long bw; + + t_diff = time_diff(t_before, t_after); + iops = (float)count / t_diff * 1000; + bw = iops * bs; + + printf("time elapsed in sec %f\n", t_diff/1000000); + printf("iops: %.2fkiops\n", iops); + printf("BW: %ldKB\n", bw); +} + +int calc_counter(int before, int after) { + int total = 0; + + total = after - before; + if (total != count * 2) { + printf("count: %d, fmr_unaligned_cntr: %d\n", count, total); + return -1; + } else { + return 0; + } +} + +static void dump_bufs(void *s1, void *s2, int len) +{ + int i; + + for (i = 0; i < len; i += 8) { + uint64_t idword = *(uint64_t *)&(((char *)s2)[i]); + uint64_t odword = *(uint64_t *)&(((char *)s1)[i]); + + printf("obuf[%x]: %x, ibuf[%x]: %x\n", + i, odword, i, idword); + } +} + +static int run_rw(int is_write, int fd, void *buf) +{ + struct iovec iov[num_sge]; + int sge_size = bs / num_sge; + int max = page_size - sge_size; + int i = 0, j = 0, offset = 0, rc = 0; + ssize_t bytes_read; + long bytes_left = disk_sz; + + /* for every iteration */ + for (i = 0; i < count; i++) { + if (max > 0) + offset = (512 * i) % max; + if (bytes_left < bs) { + rc = my_rewind(fd); + if (rc < 0) + return rc; + printf("count: %d, no space left on block " + "device, rewinding\n", i); + bytes_left = disk_sz; + } + /* for every sge */ + for (j = 0; j < num_sge; j++) { + /* change offset in page */ + iov[j].iov_base = buf + (page_size * j) + offset; + iov[j].iov_len = sge_size; + if (is_write) + memset(iov[j].iov_base, i+j, iov[j].iov_len); + } + + if (is_write) { + bytes_read = writev(fd, iov, num_sge); + if (bytes_read < bs) { + if (bytes_read < 0) { + printf("failed to writev, bytes=%d, " + "errno=%d\n", bytes_read, errno); + perror("failed to writev"); + } else + printf("writev less than expected. " + "Bytes=%d, expected %d\n", bytes_read, bs); + return -1; + } + } else { + bytes_read = readv(fd, iov, num_sge); + if (bytes_read < bs) { + if (bytes_read < 0) { + printf("failed to readv, bytes=%d, " + "errno=%d\n", bytes_read, errno); + perror("failed to readv"); + } else + printf("readv less than expected. " + "Bytes=%d, expected %d\n", bytes_read, bs); + return -1; + } + } + bytes_left -= bs; + } + + return 0; +} + +int run_iovec_traffic(int fd) +{ + int rc; + + rc = my_rewind(fd); + if (rc) { + printf("rewind failed\n"); + return -1; + } + + rc = run_rw(1, fd, obuf); + if (rc) { + printf("write failed\n"); + return -1; + } + + rc = my_rewind(fd); + if (rc) { + printf("rewind failed\n"); + return -1; + } + + rc = run_rw(0, fd, ibuf); + if (rc) { + printf("read failed\n"); + return -1; + } + + rc = memcmp(ibuf, obuf, bs); + if (rc) { + printf("memcmp failed\n"); + dump_bufs(obuf, ibuf, bs); + return -1; + } + + return rc; +} + +int main(int argc, char **argv) { + struct timeval t_before, t_after; + void **page_list = NULL; + int fd, before_counter = 0, after_counter = 0, rc = 0; + + page_size = sysconf(_SC_PAGESIZE); + rc = parse_args(argc, argv); + if (rc) + return -1; + + fd = open_block_dev(); + if (fd < 0) + return -1; + + ibuf = alloc_sges(); + if (!ibuf) { + rc = -ENOMEM; + goto out; + } + + obuf = alloc_sges(); + if (!obuf) { + rc = -ENOMEM; + goto out; + } + + before_counter = sample_counter(); + if (before_counter < 0) { + rc = -1; + goto out; + } + + gettimeofday(&t_before, NULL); + rc = run_iovec_traffic(fd); + gettimeofday(&t_after, NULL); + + if (rc) { + printf("Exiting with rc=%d\n", rc); + goto out; + } + + get_stats(t_before, t_after); + + after_counter = sample_counter(); + if (after_counter < 0) { + rc = -1; + goto out; + } + + rc = calc_counter(before_counter, after_counter); + if (rc) { + printf("Test Failed unaligned count\n"); + goto out; + } + + printf("Test Passes\n"); +out: + close(fd); + free (ibuf); + free (obuf); + + return rc; +} diff --git a/a/content_digest b/N1/content_digest index fd08238..3f33973 100644 --- a/a/content_digest +++ b/N1/content_digest @@ -1,9 +1,15 @@ "ref\020160214074119.GA24558@infradead.org\0" "ref\056C04294.3090701@dev.mellanox.co.il\0" - "From\0sagig@dev.mellanox.co.il (Sagi Grimberg)\0" - "Subject\04.5-rc iser issues\0" + "ref\056C04294.3090701-LDSdmyG8hGV8YrgS2mwiifqBs+8SCbDb@public.gmane.org\0" + "From\0Sagi Grimberg <sagig-LDSdmyG8hGV8YrgS2mwiifqBs+8SCbDb@public.gmane.org>\0" + "Subject\0Re: 4.5-rc iser issues\0" "Date\0Sun, 14 Feb 2016 11:59:28 +0200\0" - "\00:1\0" + "To\0Christoph Hellwig <hch-wEGCiKHe2LqWVfeAwA7xHQ@public.gmane.org>\0" + "Cc\0linux-rdma-u79uwXL29TY76Z2rM5mHXA@public.gmane.org" + Ming Lin-SSI <ming.l-Vzezgt5dB6uUEJcrhfAQsw@public.gmane.org> + linux-nvme-IAPFreCvJWM7uuMidbF8XUB+6BGkLq7r@public.gmane.org <linux-nvme-IAPFreCvJWM7uuMidbF8XUB+6BGkLq7r@public.gmane.org> + " linux-block-u79uwXL29TY76Z2rM5mHXA@public.gmane.org <linux-block-u79uwXL29TY76Z2rM5mHXA@public.gmane.org>\0" + "\01:1\0" "b\0" "\n" ">> The only other kernel version I had available quickly is 3.16 from Debian\n" @@ -50,8 +56,10 @@ "\n" "Attaching a small test program I used to force gappy I/O.\n" "\n" - "$ ./scatter_data -l 64k -n 128 -d <dev>\n" - "-------------- next part --------------\n" + $ ./scatter_data -l 64k -n 128 -d <dev> + "\01:2\0" + "fn\0scattered_data.c\0" + "b\0" "/**\n" " * Scattered IO test\n" " *\n" @@ -464,4 +472,4 @@ "\treturn rc;\n" } -062517ef2e79505903e1c1594e6275b27ab2cd3af04eae0f7f5ffc08209f7c17 +ef1be0d38f4e408503904c76e7c0cbd183d91e23db833c4d276bc1ce72777fc3
This is an external index of several public inboxes, see mirroring instructions on how to clone and mirror all data and code used by this external index.