[f2fs-dev] [PATCH] f2fs_io: add fragread command to evaluate fragmented buffer for reads

[f2fs-dev] [PATCH] f2fs_io: add fragread command to evaluate fragmented buffer for reads

[Jaegeuk Kim via Linux-f2fs-devel]

From: Jaegeuk Kim <jaegeuk@google.com>

This adds a fragread command in f2fs_io, which is able to measure the
read performance on fragmented data buffer.

Signed-off-by: Jaegeuk Kim <jaegeuk@google.com>
---
 tools/f2fs_io/f2fs_io.c | 110 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 110 insertions(+)

diff --git a/tools/f2fs_io/f2fs_io.c b/tools/f2fs_io/f2fs_io.c
index b72c26648f56..8431262575e0 100644
--- a/tools/f2fs_io/f2fs_io.c
+++ b/tools/f2fs_io/f2fs_io.c
@@ -1018,6 +1018,115 @@ static void do_read(int argc, char **argv, const struct cmd_desc *cmd)
 	exit(0);
 }
 
+#define fragread_desc "read data with a fragmented buffer from file"
+#define fragread_help					\
+"f2fs_io fragread [chunk_size in 4kb] [offset in chunk_size] [count] [advice] [file_path]\n\n"	\
+"Read data in file_path and print nbytes\n"		\
+"advice can be\n"					\
+" 1 : set sequential|willneed\n"			\
+" 0 : none\n"						\
+
+#ifndef PAGE_SIZE
+#define PAGE_SIZE sysconf(_SC_PAGESIZE)
+#endif
+#define ALLOC_SIZE (2 * 1024 * 1024 - 4 * 1024) // 2MB - 4KB
+
+static void do_fragread(int argc, char **argv, const struct cmd_desc *cmd)
+{
+	u64 buf_size = 0, ret = 0, read_cnt = 0;
+	u64 offset;
+	char *buf = NULL;
+	uintptr_t idx, ptr;
+	unsigned bs, count, i;
+	u64 total_time = 0;
+	int flags = 0, alloc_count = 0;
+	void *mem_hole, **mem_holes;
+	int fd, advice;
+
+	if (argc != 6) {
+		fputs("Excess arguments\n\n", stderr);
+		fputs(cmd->cmd_help, stderr);
+		exit(1);
+	}
+
+	bs = atoi(argv[1]);
+	if (bs > 256 * 1024)
+		die("Too big chunk size - limit: 1GB");
+	buf_size = bs * F2FS_DEFAULT_BLKSIZE;
+
+	offset = atoi(argv[2]) * buf_size;
+	count = atoi(argv[3]);
+	advice = atoi(argv[4]);
+	mem_holes = xmalloc(sizeof(void *) * (buf_size / PAGE_SIZE));
+
+	/* 1. Allocate the buffer using mmap. */
+	buf = mmap(NULL, buf_size, PROT_READ | PROT_WRITE,
+				MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+
+	/* 2. Loop and touch each page. */
+	for (idx = (uintptr_t)buf; idx < (uintptr_t)buf + buf_size;
+						idx += PAGE_SIZE)
+	{
+		/* Touch the current page. */
+		volatile char *page = (volatile char *)idx;
+		*page;
+
+		/* 3. Allocate (2M - 4K) memory using mmap and touch all of it. */
+		mem_hole = mmap(NULL, ALLOC_SIZE, PROT_READ | PROT_WRITE,
+					MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+		if (mem_hole == MAP_FAILED)
+			die_errno("map failed");
+
+		/* Store the allocated memory pointer. */
+		mem_holes[alloc_count++] = mem_hole;
+
+		/* Touch all allocated memory. */
+		for (ptr = (uintptr_t)mem_hole;
+			ptr < (uintptr_t)mem_hole + ALLOC_SIZE;
+						ptr += PAGE_SIZE) {
+			volatile char *alloc_page = (volatile char *)ptr;
+			*alloc_page;
+		}
+	}
+	printf("Touched allocated memory: count = %u\n", alloc_count);
+	printf(" - allocated memory: = ");
+	for (idx = 0; idx < 5; idx++)
+		printf(" %p", mem_holes[idx]);
+	printf("\n");
+
+	fd = xopen(argv[5], O_RDONLY | flags, 0);
+
+	if (advice) {
+		if (posix_fadvise(fd, 0, F2FS_DEFAULT_BLKSIZE,
+				POSIX_FADV_SEQUENTIAL) != 0)
+			die_errno("fadvise failed");
+		if (posix_fadvise(fd, 0, F2FS_DEFAULT_BLKSIZE,
+				POSIX_FADV_WILLNEED) != 0)
+			die_errno("fadvise failed");
+		printf("fadvise SEQUENTIAL|WILLNEED to a file: %s\n", argv[5]);
+	}
+
+	total_time = get_current_us();
+
+	for (i = 0; i < count; i++) {
+		ret = pread(fd, buf, buf_size, offset + buf_size * i);
+		if (ret != buf_size) {
+			printf("pread expected: %"PRIu64", readed: %"PRIu64"\n",
+					buf_size, ret);
+			if (ret > 0)
+				read_cnt += ret;
+			break;
+		}
+
+		read_cnt += ret;
+	}
+	printf("Fragmented_Read %"PRIu64" bytes total_time = %"PRIu64" us, BW = %.Lf MB/s\n",
+		read_cnt, get_current_us() - total_time,
+		((long double)read_cnt / (get_current_us() - total_time)));
+	printf("\n");
+	exit(0);
+}
+
 #define randread_desc "random read data from file"
 #define randread_help					\
 "f2fs_io randread [chunk_size in 4kb] [count] [IO] [advise] [file_path]\n\n"	\
@@ -2002,6 +2111,7 @@ const struct cmd_desc cmd_list[] = {
 	CMD(write_advice),
 	CMD(read),
 	CMD(randread),
+	CMD(fragread),
 	CMD(fiemap),
 	CMD(gc_urgent),
 	CMD(defrag_file),
-- 
2.48.1.711.g2feabab25a-goog



_______________________________________________
Linux-f2fs-devel mailing list
Linux-f2fs-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel

Re: [f2fs-dev] [PATCH] f2fs_io: add fragread command to evaluate fragmented buffer for reads

[Chao Yu via Linux-f2fs-devel]

On 3/4/25 09:10, Jaegeuk Kim via Linux-f2fs-devel wrote:
> From: Jaegeuk Kim <jaegeuk@google.com>
> 
> This adds a fragread command in f2fs_io, which is able to measure the
> read performance on fragmented data buffer.
> 
> Signed-off-by: Jaegeuk Kim <jaegeuk@google.com>
> ---
>  tools/f2fs_io/f2fs_io.c | 110 ++++++++++++++++++++++++++++++++++++++++
>  1 file changed, 110 insertions(+)
> 
> diff --git a/tools/f2fs_io/f2fs_io.c b/tools/f2fs_io/f2fs_io.c
> index b72c26648f56..8431262575e0 100644
> --- a/tools/f2fs_io/f2fs_io.c
> +++ b/tools/f2fs_io/f2fs_io.c
> @@ -1018,6 +1018,115 @@ static void do_read(int argc, char **argv, const struct cmd_desc *cmd)
>  	exit(0);
>  }
>  
> +#define fragread_desc "read data with a fragmented buffer from file"
> +#define fragread_help					\
> +"f2fs_io fragread [chunk_size in 4kb] [offset in chunk_size] [count] [advice] [file_path]\n\n"	\
> +"Read data in file_path and print nbytes\n"		\
> +"advice can be\n"					\
> +" 1 : set sequential|willneed\n"			\
> +" 0 : none\n"						\
> +
> +#ifndef PAGE_SIZE
> +#define PAGE_SIZE sysconf(_SC_PAGESIZE)
> +#endif
> +#define ALLOC_SIZE (2 * 1024 * 1024 - 4 * 1024) // 2MB - 4KB
> +
> +static void do_fragread(int argc, char **argv, const struct cmd_desc *cmd)
> +{
> +	u64 buf_size = 0, ret = 0, read_cnt = 0;
> +	u64 offset;
> +	char *buf = NULL;
> +	uintptr_t idx, ptr;
> +	unsigned bs, count, i;
> +	u64 total_time = 0;
> +	int flags = 0, alloc_count = 0;
> +	void *mem_hole, **mem_holes;
> +	int fd, advice;
> +
> +	if (argc != 6) {
> +		fputs("Excess arguments\n\n", stderr);
> +		fputs(cmd->cmd_help, stderr);
> +		exit(1);
> +	}
> +
> +	bs = atoi(argv[1]);
> +	if (bs > 256 * 1024)
> +		die("Too big chunk size - limit: 1GB");
> +	buf_size = bs * F2FS_DEFAULT_BLKSIZE;
> +
> +	offset = atoi(argv[2]) * buf_size;
> +	count = atoi(argv[3]);
> +	advice = atoi(argv[4]);
> +	mem_holes = xmalloc(sizeof(void *) * (buf_size / PAGE_SIZE));
> +
> +	/* 1. Allocate the buffer using mmap. */
> +	buf = mmap(NULL, buf_size, PROT_READ | PROT_WRITE,
> +				MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
> +
> +	/* 2. Loop and touch each page. */
> +	for (idx = (uintptr_t)buf; idx < (uintptr_t)buf + buf_size;
> +						idx += PAGE_SIZE)
> +	{
> +		/* Touch the current page. */
> +		volatile char *page = (volatile char *)idx;
> +		*page;
> +
> +		/* 3. Allocate (2M - 4K) memory using mmap and touch all of it. */
> +		mem_hole = mmap(NULL, ALLOC_SIZE, PROT_READ | PROT_WRITE,
> +					MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
> +		if (mem_hole == MAP_FAILED)
> +			die_errno("map failed");
> +
> +		/* Store the allocated memory pointer. */
> +		mem_holes[alloc_count++] = mem_hole;

Do we need to call mlock to hold allocated cache in the memory, if
administrator enables swap, memory fragment will be gone once anonymous
pages are swapped to device?

Thanks,

> +
> +		/* Touch all allocated memory. */
> +		for (ptr = (uintptr_t)mem_hole;
> +			ptr < (uintptr_t)mem_hole + ALLOC_SIZE;
> +						ptr += PAGE_SIZE) {
> +			volatile char *alloc_page = (volatile char *)ptr;
> +			*alloc_page;
> +		}
> +	}
> +	printf("Touched allocated memory: count = %u\n", alloc_count);
> +	printf(" - allocated memory: = ");
> +	for (idx = 0; idx < 5; idx++)
> +		printf(" %p", mem_holes[idx]);
> +	printf("\n");
> +
> +	fd = xopen(argv[5], O_RDONLY | flags, 0);
> +
> +	if (advice) {
> +		if (posix_fadvise(fd, 0, F2FS_DEFAULT_BLKSIZE,
> +				POSIX_FADV_SEQUENTIAL) != 0)
> +			die_errno("fadvise failed");
> +		if (posix_fadvise(fd, 0, F2FS_DEFAULT_BLKSIZE,
> +				POSIX_FADV_WILLNEED) != 0)
> +			die_errno("fadvise failed");
> +		printf("fadvise SEQUENTIAL|WILLNEED to a file: %s\n", argv[5]);
> +	}
> +
> +	total_time = get_current_us();
> +
> +	for (i = 0; i < count; i++) {
> +		ret = pread(fd, buf, buf_size, offset + buf_size * i);
> +		if (ret != buf_size) {
> +			printf("pread expected: %"PRIu64", readed: %"PRIu64"\n",
> +					buf_size, ret);
> +			if (ret > 0)
> +				read_cnt += ret;
> +			break;
> +		}
> +
> +		read_cnt += ret;
> +	}
> +	printf("Fragmented_Read %"PRIu64" bytes total_time = %"PRIu64" us, BW = %.Lf MB/s\n",
> +		read_cnt, get_current_us() - total_time,
> +		((long double)read_cnt / (get_current_us() - total_time)));
> +	printf("\n");
> +	exit(0);
> +}
> +
>  #define randread_desc "random read data from file"
>  #define randread_help					\
>  "f2fs_io randread [chunk_size in 4kb] [count] [IO] [advise] [file_path]\n\n"	\
> @@ -2002,6 +2111,7 @@ const struct cmd_desc cmd_list[] = {
>  	CMD(write_advice),
>  	CMD(read),
>  	CMD(randread),
> +	CMD(fragread),
>  	CMD(fiemap),
>  	CMD(gc_urgent),
>  	CMD(defrag_file),



_______________________________________________
Linux-f2fs-devel mailing list
Linux-f2fs-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel

Re: [f2fs-dev] [PATCH] f2fs_io: add fragread command to evaluate fragmented buffer for reads

[Jaegeuk Kim via Linux-f2fs-devel]

On 03/11, Chao Yu wrote:
> On 3/4/25 09:10, Jaegeuk Kim via Linux-f2fs-devel wrote:
> > From: Jaegeuk Kim <jaegeuk@google.com>
> > 
> > This adds a fragread command in f2fs_io, which is able to measure the
> > read performance on fragmented data buffer.
> > 
> > Signed-off-by: Jaegeuk Kim <jaegeuk@google.com>
> > ---
> >  tools/f2fs_io/f2fs_io.c | 110 ++++++++++++++++++++++++++++++++++++++++
> >  1 file changed, 110 insertions(+)
> > 
> > diff --git a/tools/f2fs_io/f2fs_io.c b/tools/f2fs_io/f2fs_io.c
> > index b72c26648f56..8431262575e0 100644
> > --- a/tools/f2fs_io/f2fs_io.c
> > +++ b/tools/f2fs_io/f2fs_io.c
> > @@ -1018,6 +1018,115 @@ static void do_read(int argc, char **argv, const struct cmd_desc *cmd)
> >  	exit(0);
> >  }
> >  
> > +#define fragread_desc "read data with a fragmented buffer from file"
> > +#define fragread_help					\
> > +"f2fs_io fragread [chunk_size in 4kb] [offset in chunk_size] [count] [advice] [file_path]\n\n"	\
> > +"Read data in file_path and print nbytes\n"		\
> > +"advice can be\n"					\
> > +" 1 : set sequential|willneed\n"			\
> > +" 0 : none\n"						\
> > +
> > +#ifndef PAGE_SIZE
> > +#define PAGE_SIZE sysconf(_SC_PAGESIZE)
> > +#endif
> > +#define ALLOC_SIZE (2 * 1024 * 1024 - 4 * 1024) // 2MB - 4KB
> > +
> > +static void do_fragread(int argc, char **argv, const struct cmd_desc *cmd)
> > +{
> > +	u64 buf_size = 0, ret = 0, read_cnt = 0;
> > +	u64 offset;
> > +	char *buf = NULL;
> > +	uintptr_t idx, ptr;
> > +	unsigned bs, count, i;
> > +	u64 total_time = 0;
> > +	int flags = 0, alloc_count = 0;
> > +	void *mem_hole, **mem_holes;
> > +	int fd, advice;
> > +
> > +	if (argc != 6) {
> > +		fputs("Excess arguments\n\n", stderr);
> > +		fputs(cmd->cmd_help, stderr);
> > +		exit(1);
> > +	}
> > +
> > +	bs = atoi(argv[1]);
> > +	if (bs > 256 * 1024)
> > +		die("Too big chunk size - limit: 1GB");
> > +	buf_size = bs * F2FS_DEFAULT_BLKSIZE;
> > +
> > +	offset = atoi(argv[2]) * buf_size;
> > +	count = atoi(argv[3]);
> > +	advice = atoi(argv[4]);
> > +	mem_holes = xmalloc(sizeof(void *) * (buf_size / PAGE_SIZE));
> > +
> > +	/* 1. Allocate the buffer using mmap. */
> > +	buf = mmap(NULL, buf_size, PROT_READ | PROT_WRITE,
> > +				MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
> > +
> > +	/* 2. Loop and touch each page. */
> > +	for (idx = (uintptr_t)buf; idx < (uintptr_t)buf + buf_size;
> > +						idx += PAGE_SIZE)
> > +	{
> > +		/* Touch the current page. */
> > +		volatile char *page = (volatile char *)idx;
> > +		*page;
> > +
> > +		/* 3. Allocate (2M - 4K) memory using mmap and touch all of it. */
> > +		mem_hole = mmap(NULL, ALLOC_SIZE, PROT_READ | PROT_WRITE,
> > +					MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
> > +		if (mem_hole == MAP_FAILED)
> > +			die_errno("map failed");
> > +
> > +		/* Store the allocated memory pointer. */
> > +		mem_holes[alloc_count++] = mem_hole;
> 
> Do we need to call mlock to hold allocated cache in the memory, if
> administrator enables swap, memory fragment will be gone once anonymous
> pages are swapped to device?

Hmm, the test is going to use this very intensively, so is it possible?

> 
> Thanks,
> 
> > +
> > +		/* Touch all allocated memory. */
> > +		for (ptr = (uintptr_t)mem_hole;
> > +			ptr < (uintptr_t)mem_hole + ALLOC_SIZE;
> > +						ptr += PAGE_SIZE) {
> > +			volatile char *alloc_page = (volatile char *)ptr;
> > +			*alloc_page;
> > +		}
> > +	}
> > +	printf("Touched allocated memory: count = %u\n", alloc_count);
> > +	printf(" - allocated memory: = ");
> > +	for (idx = 0; idx < 5; idx++)
> > +		printf(" %p", mem_holes[idx]);
> > +	printf("\n");
> > +
> > +	fd = xopen(argv[5], O_RDONLY | flags, 0);
> > +
> > +	if (advice) {
> > +		if (posix_fadvise(fd, 0, F2FS_DEFAULT_BLKSIZE,
> > +				POSIX_FADV_SEQUENTIAL) != 0)
> > +			die_errno("fadvise failed");
> > +		if (posix_fadvise(fd, 0, F2FS_DEFAULT_BLKSIZE,
> > +				POSIX_FADV_WILLNEED) != 0)
> > +			die_errno("fadvise failed");
> > +		printf("fadvise SEQUENTIAL|WILLNEED to a file: %s\n", argv[5]);
> > +	}
> > +
> > +	total_time = get_current_us();
> > +
> > +	for (i = 0; i < count; i++) {
> > +		ret = pread(fd, buf, buf_size, offset + buf_size * i);
> > +		if (ret != buf_size) {
> > +			printf("pread expected: %"PRIu64", readed: %"PRIu64"\n",
> > +					buf_size, ret);
> > +			if (ret > 0)
> > +				read_cnt += ret;
> > +			break;
> > +		}
> > +
> > +		read_cnt += ret;
> > +	}
> > +	printf("Fragmented_Read %"PRIu64" bytes total_time = %"PRIu64" us, BW = %.Lf MB/s\n",
> > +		read_cnt, get_current_us() - total_time,
> > +		((long double)read_cnt / (get_current_us() - total_time)));
> > +	printf("\n");
> > +	exit(0);
> > +}
> > +
> >  #define randread_desc "random read data from file"
> >  #define randread_help					\
> >  "f2fs_io randread [chunk_size in 4kb] [count] [IO] [advise] [file_path]\n\n"	\
> > @@ -2002,6 +2111,7 @@ const struct cmd_desc cmd_list[] = {
> >  	CMD(write_advice),
> >  	CMD(read),
> >  	CMD(randread),
> > +	CMD(fragread),
> >  	CMD(fiemap),
> >  	CMD(gc_urgent),
> >  	CMD(defrag_file),


_______________________________________________
Linux-f2fs-devel mailing list
Linux-f2fs-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel

Re: [f2fs-dev] [PATCH] f2fs_io: add fragread command to evaluate fragmented buffer for reads

[Chao Yu via Linux-f2fs-devel]

On 3/12/25 03:36, Jaegeuk Kim wrote:
> On 03/11, Chao Yu wrote:
>> On 3/4/25 09:10, Jaegeuk Kim via Linux-f2fs-devel wrote:
>>> From: Jaegeuk Kim <jaegeuk@google.com>
>>>
>>> This adds a fragread command in f2fs_io, which is able to measure the
>>> read performance on fragmented data buffer.
>>>
>>> Signed-off-by: Jaegeuk Kim <jaegeuk@google.com>
>>> ---
>>>  tools/f2fs_io/f2fs_io.c | 110 ++++++++++++++++++++++++++++++++++++++++
>>>  1 file changed, 110 insertions(+)
>>>
>>> diff --git a/tools/f2fs_io/f2fs_io.c b/tools/f2fs_io/f2fs_io.c
>>> index b72c26648f56..8431262575e0 100644
>>> --- a/tools/f2fs_io/f2fs_io.c
>>> +++ b/tools/f2fs_io/f2fs_io.c
>>> @@ -1018,6 +1018,115 @@ static void do_read(int argc, char **argv, const struct cmd_desc *cmd)
>>>  	exit(0);
>>>  }
>>>  
>>> +#define fragread_desc "read data with a fragmented buffer from file"
>>> +#define fragread_help					\
>>> +"f2fs_io fragread [chunk_size in 4kb] [offset in chunk_size] [count] [advice] [file_path]\n\n"	\
>>> +"Read data in file_path and print nbytes\n"		\
>>> +"advice can be\n"					\
>>> +" 1 : set sequential|willneed\n"			\
>>> +" 0 : none\n"						\
>>> +
>>> +#ifndef PAGE_SIZE
>>> +#define PAGE_SIZE sysconf(_SC_PAGESIZE)
>>> +#endif
>>> +#define ALLOC_SIZE (2 * 1024 * 1024 - 4 * 1024) // 2MB - 4KB
>>> +
>>> +static void do_fragread(int argc, char **argv, const struct cmd_desc *cmd)
>>> +{
>>> +	u64 buf_size = 0, ret = 0, read_cnt = 0;
>>> +	u64 offset;
>>> +	char *buf = NULL;
>>> +	uintptr_t idx, ptr;
>>> +	unsigned bs, count, i;
>>> +	u64 total_time = 0;
>>> +	int flags = 0, alloc_count = 0;
>>> +	void *mem_hole, **mem_holes;
>>> +	int fd, advice;
>>> +
>>> +	if (argc != 6) {
>>> +		fputs("Excess arguments\n\n", stderr);
>>> +		fputs(cmd->cmd_help, stderr);
>>> +		exit(1);
>>> +	}
>>> +
>>> +	bs = atoi(argv[1]);
>>> +	if (bs > 256 * 1024)
>>> +		die("Too big chunk size - limit: 1GB");
>>> +	buf_size = bs * F2FS_DEFAULT_BLKSIZE;
>>> +
>>> +	offset = atoi(argv[2]) * buf_size;
>>> +	count = atoi(argv[3]);
>>> +	advice = atoi(argv[4]);
>>> +	mem_holes = xmalloc(sizeof(void *) * (buf_size / PAGE_SIZE));
>>> +
>>> +	/* 1. Allocate the buffer using mmap. */
>>> +	buf = mmap(NULL, buf_size, PROT_READ | PROT_WRITE,
>>> +				MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
>>> +
>>> +	/* 2. Loop and touch each page. */
>>> +	for (idx = (uintptr_t)buf; idx < (uintptr_t)buf + buf_size;
>>> +						idx += PAGE_SIZE)
>>> +	{
>>> +		/* Touch the current page. */
>>> +		volatile char *page = (volatile char *)idx;
>>> +		*page;
>>> +
>>> +		/* 3. Allocate (2M - 4K) memory using mmap and touch all of it. */
>>> +		mem_hole = mmap(NULL, ALLOC_SIZE, PROT_READ | PROT_WRITE,
>>> +					MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
>>> +		if (mem_hole == MAP_FAILED)
>>> +			die_errno("map failed");
>>> +
>>> +		/* Store the allocated memory pointer. */
>>> +		mem_holes[alloc_count++] = mem_hole;
>>
>> Do we need to call mlock to hold allocated cache in the memory, if
>> administrator enables swap, memory fragment will be gone once anonymous
>> pages are swapped to device?
> 
> Hmm, the test is going to use this very intensively, so is it possible?

Not sure, but I doubt it is possible to happen in low-end device? due to it has
very less memory and system may has already token the most of memory.

Thanks,

> 
>>
>> Thanks,
>>
>>> +
>>> +		/* Touch all allocated memory. */
>>> +		for (ptr = (uintptr_t)mem_hole;
>>> +			ptr < (uintptr_t)mem_hole + ALLOC_SIZE;
>>> +						ptr += PAGE_SIZE) {
>>> +			volatile char *alloc_page = (volatile char *)ptr;
>>> +			*alloc_page;
>>> +		}
>>> +	}
>>> +	printf("Touched allocated memory: count = %u\n", alloc_count);
>>> +	printf(" - allocated memory: = ");
>>> +	for (idx = 0; idx < 5; idx++)
>>> +		printf(" %p", mem_holes[idx]);
>>> +	printf("\n");
>>> +
>>> +	fd = xopen(argv[5], O_RDONLY | flags, 0);
>>> +
>>> +	if (advice) {
>>> +		if (posix_fadvise(fd, 0, F2FS_DEFAULT_BLKSIZE,
>>> +				POSIX_FADV_SEQUENTIAL) != 0)
>>> +			die_errno("fadvise failed");
>>> +		if (posix_fadvise(fd, 0, F2FS_DEFAULT_BLKSIZE,
>>> +				POSIX_FADV_WILLNEED) != 0)
>>> +			die_errno("fadvise failed");
>>> +		printf("fadvise SEQUENTIAL|WILLNEED to a file: %s\n", argv[5]);
>>> +	}
>>> +
>>> +	total_time = get_current_us();
>>> +
>>> +	for (i = 0; i < count; i++) {
>>> +		ret = pread(fd, buf, buf_size, offset + buf_size * i);
>>> +		if (ret != buf_size) {
>>> +			printf("pread expected: %"PRIu64", readed: %"PRIu64"\n",
>>> +					buf_size, ret);
>>> +			if (ret > 0)
>>> +				read_cnt += ret;
>>> +			break;
>>> +		}
>>> +
>>> +		read_cnt += ret;
>>> +	}
>>> +	printf("Fragmented_Read %"PRIu64" bytes total_time = %"PRIu64" us, BW = %.Lf MB/s\n",
>>> +		read_cnt, get_current_us() - total_time,
>>> +		((long double)read_cnt / (get_current_us() - total_time)));
>>> +	printf("\n");
>>> +	exit(0);
>>> +}
>>> +
>>>  #define randread_desc "random read data from file"
>>>  #define randread_help					\
>>>  "f2fs_io randread [chunk_size in 4kb] [count] [IO] [advise] [file_path]\n\n"	\
>>> @@ -2002,6 +2111,7 @@ const struct cmd_desc cmd_list[] = {
>>>  	CMD(write_advice),
>>>  	CMD(read),
>>>  	CMD(randread),
>>> +	CMD(fragread),
>>>  	CMD(fiemap),
>>>  	CMD(gc_urgent),
>>>  	CMD(defrag_file),



_______________________________________________
Linux-f2fs-devel mailing list
Linux-f2fs-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel

Re: [f2fs-dev] [PATCH] f2fs_io: add fragread command to evaluate fragmented buffer for reads

[Chao Yu via Linux-f2fs-devel]

On 3/12/25 11:02, Chao Yu wrote:
> On 3/12/25 03:36, Jaegeuk Kim wrote:
>> On 03/11, Chao Yu wrote:
>>> On 3/4/25 09:10, Jaegeuk Kim via Linux-f2fs-devel wrote:
>>>> From: Jaegeuk Kim <jaegeuk@google.com>
>>>>
>>>> This adds a fragread command in f2fs_io, which is able to measure the
>>>> read performance on fragmented data buffer.
>>>>
>>>> Signed-off-by: Jaegeuk Kim <jaegeuk@google.com>
>>>> ---
>>>>  tools/f2fs_io/f2fs_io.c | 110 ++++++++++++++++++++++++++++++++++++++++
>>>>  1 file changed, 110 insertions(+)
>>>>
>>>> diff --git a/tools/f2fs_io/f2fs_io.c b/tools/f2fs_io/f2fs_io.c
>>>> index b72c26648f56..8431262575e0 100644
>>>> --- a/tools/f2fs_io/f2fs_io.c
>>>> +++ b/tools/f2fs_io/f2fs_io.c
>>>> @@ -1018,6 +1018,115 @@ static void do_read(int argc, char **argv, const struct cmd_desc *cmd)
>>>>  	exit(0);
>>>>  }
>>>>  
>>>> +#define fragread_desc "read data with a fragmented buffer from file"
>>>> +#define fragread_help					\
>>>> +"f2fs_io fragread [chunk_size in 4kb] [offset in chunk_size] [count] [advice] [file_path]\n\n"	\
>>>> +"Read data in file_path and print nbytes\n"		\
>>>> +"advice can be\n"					\
>>>> +" 1 : set sequential|willneed\n"			\
>>>> +" 0 : none\n"						\
>>>> +
>>>> +#ifndef PAGE_SIZE
>>>> +#define PAGE_SIZE sysconf(_SC_PAGESIZE)
>>>> +#endif
>>>> +#define ALLOC_SIZE (2 * 1024 * 1024 - 4 * 1024) // 2MB - 4KB
>>>> +
>>>> +static void do_fragread(int argc, char **argv, const struct cmd_desc *cmd)
>>>> +{
>>>> +	u64 buf_size = 0, ret = 0, read_cnt = 0;
>>>> +	u64 offset;
>>>> +	char *buf = NULL;
>>>> +	uintptr_t idx, ptr;
>>>> +	unsigned bs, count, i;
>>>> +	u64 total_time = 0;
>>>> +	int flags = 0, alloc_count = 0;
>>>> +	void *mem_hole, **mem_holes;
>>>> +	int fd, advice;
>>>> +
>>>> +	if (argc != 6) {
>>>> +		fputs("Excess arguments\n\n", stderr);
>>>> +		fputs(cmd->cmd_help, stderr);
>>>> +		exit(1);
>>>> +	}
>>>> +
>>>> +	bs = atoi(argv[1]);
>>>> +	if (bs > 256 * 1024)
>>>> +		die("Too big chunk size - limit: 1GB");
>>>> +	buf_size = bs * F2FS_DEFAULT_BLKSIZE;
>>>> +
>>>> +	offset = atoi(argv[2]) * buf_size;
>>>> +	count = atoi(argv[3]);
>>>> +	advice = atoi(argv[4]);
>>>> +	mem_holes = xmalloc(sizeof(void *) * (buf_size / PAGE_SIZE));
>>>> +
>>>> +	/* 1. Allocate the buffer using mmap. */
>>>> +	buf = mmap(NULL, buf_size, PROT_READ | PROT_WRITE,
>>>> +				MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
>>>> +
>>>> +	/* 2. Loop and touch each page. */
>>>> +	for (idx = (uintptr_t)buf; idx < (uintptr_t)buf + buf_size;
>>>> +						idx += PAGE_SIZE)
>>>> +	{
>>>> +		/* Touch the current page. */
>>>> +		volatile char *page = (volatile char *)idx;
>>>> +		*page;
>>>> +
>>>> +		/* 3. Allocate (2M - 4K) memory using mmap and touch all of it. */
>>>> +		mem_hole = mmap(NULL, ALLOC_SIZE, PROT_READ | PROT_WRITE,
>>>> +					MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
>>>> +		if (mem_hole == MAP_FAILED)
>>>> +			die_errno("map failed");
>>>> +
>>>> +		/* Store the allocated memory pointer. */
>>>> +		mem_holes[alloc_count++] = mem_hole;
>>>
>>> Do we need to call mlock to hold allocated cache in the memory, if
>>> administrator enables swap, memory fragment will be gone once anonymous
>>> pages are swapped to device?
>>
>> Hmm, the test is going to use this very intensively, so is it possible?
> 
> Not sure, but I doubt it is possible to happen in low-end device? due to it has
> very less memory and system may has already token the most of memory.

Anyway, code looks clean, I think we can take a look at this later.

Reviewed-by: Chao Yu <chao@kernel.org>

Thanks,

> 
> Thanks,
> 
>>
>>>
>>> Thanks,
>>>
>>>> +
>>>> +		/* Touch all allocated memory. */
>>>> +		for (ptr = (uintptr_t)mem_hole;
>>>> +			ptr < (uintptr_t)mem_hole + ALLOC_SIZE;
>>>> +						ptr += PAGE_SIZE) {
>>>> +			volatile char *alloc_page = (volatile char *)ptr;
>>>> +			*alloc_page;
>>>> +		}
>>>> +	}
>>>> +	printf("Touched allocated memory: count = %u\n", alloc_count);
>>>> +	printf(" - allocated memory: = ");
>>>> +	for (idx = 0; idx < 5; idx++)
>>>> +		printf(" %p", mem_holes[idx]);
>>>> +	printf("\n");
>>>> +
>>>> +	fd = xopen(argv[5], O_RDONLY | flags, 0);
>>>> +
>>>> +	if (advice) {
>>>> +		if (posix_fadvise(fd, 0, F2FS_DEFAULT_BLKSIZE,
>>>> +				POSIX_FADV_SEQUENTIAL) != 0)
>>>> +			die_errno("fadvise failed");
>>>> +		if (posix_fadvise(fd, 0, F2FS_DEFAULT_BLKSIZE,
>>>> +				POSIX_FADV_WILLNEED) != 0)
>>>> +			die_errno("fadvise failed");
>>>> +		printf("fadvise SEQUENTIAL|WILLNEED to a file: %s\n", argv[5]);
>>>> +	}
>>>> +
>>>> +	total_time = get_current_us();
>>>> +
>>>> +	for (i = 0; i < count; i++) {
>>>> +		ret = pread(fd, buf, buf_size, offset + buf_size * i);
>>>> +		if (ret != buf_size) {
>>>> +			printf("pread expected: %"PRIu64", readed: %"PRIu64"\n",
>>>> +					buf_size, ret);
>>>> +			if (ret > 0)
>>>> +				read_cnt += ret;
>>>> +			break;
>>>> +		}
>>>> +
>>>> +		read_cnt += ret;
>>>> +	}
>>>> +	printf("Fragmented_Read %"PRIu64" bytes total_time = %"PRIu64" us, BW = %.Lf MB/s\n",
>>>> +		read_cnt, get_current_us() - total_time,
>>>> +		((long double)read_cnt / (get_current_us() - total_time)));
>>>> +	printf("\n");
>>>> +	exit(0);
>>>> +}
>>>> +
>>>>  #define randread_desc "random read data from file"
>>>>  #define randread_help					\
>>>>  "f2fs_io randread [chunk_size in 4kb] [count] [IO] [advise] [file_path]\n\n"	\
>>>> @@ -2002,6 +2111,7 @@ const struct cmd_desc cmd_list[] = {
>>>>  	CMD(write_advice),
>>>>  	CMD(read),
>>>>  	CMD(randread),
>>>> +	CMD(fragread),
>>>>  	CMD(fiemap),
>>>>  	CMD(gc_urgent),
>>>>  	CMD(defrag_file),
> 



_______________________________________________
Linux-f2fs-devel mailing list
Linux-f2fs-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel

Re: [f2fs-dev] [f2fs-dev v2] [PATCH] f2fs_io: add fragread command to evaluate fragmented buffer for reads

[Jaegeuk Kim via Linux-f2fs-devel]

This adds a fragread command in f2fs_io, which is able to measure the
read performance on fragmented data buffer.

Signed-off-by: Jaegeuk Kim <jaegeuk@google.com>
---

 Change log from v1:
  - add mlock

 tools/f2fs_io/f2fs_io.c | 114 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 114 insertions(+)

diff --git a/tools/f2fs_io/f2fs_io.c b/tools/f2fs_io/f2fs_io.c
index 44d389dbd771..57a931d5769c 100644
--- a/tools/f2fs_io/f2fs_io.c
+++ b/tools/f2fs_io/f2fs_io.c
@@ -1005,6 +1005,119 @@ static void do_read(int argc, char **argv, const struct cmd_desc *cmd)
 	exit(0);
 }
 
+#define fragread_desc "read data with a fragmented buffer from file"
+#define fragread_help					\
+"f2fs_io fragread [chunk_size in 4kb] [offset in chunk_size] [count] [advice] [file_path]\n\n"	\
+"Read data in file_path and print nbytes\n"		\
+"advice can be\n"					\
+" 1 : set sequential|willneed\n"			\
+" 0 : none\n"						\
+
+#ifndef PAGE_SIZE
+#define PAGE_SIZE sysconf(_SC_PAGESIZE)
+#endif
+#define ALLOC_SIZE (2 * 1024 * 1024 - 4 * 1024) // 2MB - 4KB
+
+static void do_fragread(int argc, char **argv, const struct cmd_desc *cmd)
+{
+	u64 buf_size = 0, ret = 0, read_cnt = 0;
+	u64 offset;
+	char *buf = NULL;
+	uintptr_t idx, ptr;
+	unsigned bs, count, i;
+	u64 total_time = 0;
+	int flags = 0, alloc_count = 0;
+	void *mem_hole, **mem_holes;
+	int fd, advice;
+
+	if (argc != 6) {
+		fputs("Excess arguments\n\n", stderr);
+		fputs(cmd->cmd_help, stderr);
+		exit(1);
+	}
+
+	bs = atoi(argv[1]);
+	if (bs > 256 * 1024)
+		die("Too big chunk size - limit: 1GB");
+	buf_size = bs * F2FS_DEFAULT_BLKSIZE;
+
+	offset = atoi(argv[2]) * buf_size;
+	count = atoi(argv[3]);
+	advice = atoi(argv[4]);
+	mem_holes = xmalloc(sizeof(void *) * (buf_size / PAGE_SIZE));
+
+	/* 1. Allocate the buffer using mmap. */
+	buf = mmap(NULL, buf_size, PROT_READ | PROT_WRITE,
+				MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+
+	/* 2. Loop and touch each page. */
+	for (idx = (uintptr_t)buf; idx < (uintptr_t)buf + buf_size;
+						idx += PAGE_SIZE)
+	{
+		/* Touch the current page. */
+		volatile char *page = (volatile char *)idx;
+		*page;
+
+		/* 3. Allocate (2M - 4K) memory using mmap and touch all of it. */
+		mem_hole = mmap(NULL, ALLOC_SIZE, PROT_READ | PROT_WRITE,
+					MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+		if (mem_hole == MAP_FAILED)
+			die_errno("map failed");
+
+		/* Store the allocated memory pointer. */
+		mem_holes[alloc_count++] = mem_hole;
+
+		/* Touch all allocated memory. */
+		for (ptr = (uintptr_t)mem_hole;
+			ptr < (uintptr_t)mem_hole + ALLOC_SIZE;
+						ptr += PAGE_SIZE) {
+			volatile char *alloc_page = (volatile char *)ptr;
+			*alloc_page;
+		}
+	}
+	printf("Touched allocated memory: count = %u\n", alloc_count);
+	printf(" - allocated memory: = ");
+	for (idx = 0; idx < 5; idx++)
+		printf(" %p", mem_holes[idx]);
+	printf("\n");
+
+	/* Pin the pages. */
+	if (mlock(buf, buf_size))
+		die_errno("mlock failed");
+
+	fd = xopen(argv[5], O_RDONLY | flags, 0);
+
+	if (advice) {
+		if (posix_fadvise(fd, 0, F2FS_DEFAULT_BLKSIZE,
+				POSIX_FADV_SEQUENTIAL) != 0)
+			die_errno("fadvise failed");
+		if (posix_fadvise(fd, 0, F2FS_DEFAULT_BLKSIZE,
+				POSIX_FADV_WILLNEED) != 0)
+			die_errno("fadvise failed");
+		printf("fadvise SEQUENTIAL|WILLNEED to a file: %s\n", argv[5]);
+	}
+
+	total_time = get_current_us();
+
+	for (i = 0; i < count; i++) {
+		ret = pread(fd, buf, buf_size, offset + buf_size * i);
+		if (ret != buf_size) {
+			printf("pread expected: %"PRIu64", readed: %"PRIu64"\n",
+					buf_size, ret);
+			if (ret > 0)
+				read_cnt += ret;
+			break;
+		}
+
+		read_cnt += ret;
+	}
+	printf("Fragmented_Read %"PRIu64" bytes total_time = %"PRIu64" us, BW = %.Lf MB/s\n",
+		read_cnt, get_current_us() - total_time,
+		((long double)read_cnt / (get_current_us() - total_time)));
+	printf("\n");
+	exit(0);
+}
+
 #define randread_desc "random read data from file"
 #define randread_help					\
 "f2fs_io randread [chunk_size in 4kb] [count] [IO] [advise] [file_path]\n\n"	\
@@ -1989,6 +2102,7 @@ const struct cmd_desc cmd_list[] = {
 	CMD(write_advice),
 	CMD(read),
 	CMD(randread),
+	CMD(fragread),
 	CMD(fiemap),
 	CMD(gc_urgent),
 	CMD(defrag_file),
-- 
2.49.0.rc1.451.g8f38331e32-goog



_______________________________________________
Linux-f2fs-devel mailing list
Linux-f2fs-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel

Re: [f2fs-dev] [f2fs-dev v2] [PATCH] f2fs_io: add fragread command to evaluate fragmented buffer for reads

[Jaegeuk Kim via Linux-f2fs-devel]

Ping.

On 03/17, Jaegeuk Kim via Linux-f2fs-devel wrote:
> This adds a fragread command in f2fs_io, which is able to measure the
> read performance on fragmented data buffer.
> 
> Signed-off-by: Jaegeuk Kim <jaegeuk@google.com>
> ---
> 
>  Change log from v1:
>   - add mlock
> 
>  tools/f2fs_io/f2fs_io.c | 114 ++++++++++++++++++++++++++++++++++++++++
>  1 file changed, 114 insertions(+)
> 
> diff --git a/tools/f2fs_io/f2fs_io.c b/tools/f2fs_io/f2fs_io.c
> index 44d389dbd771..57a931d5769c 100644
> --- a/tools/f2fs_io/f2fs_io.c
> +++ b/tools/f2fs_io/f2fs_io.c
> @@ -1005,6 +1005,119 @@ static void do_read(int argc, char **argv, const struct cmd_desc *cmd)
>  	exit(0);
>  }
>  
> +#define fragread_desc "read data with a fragmented buffer from file"
> +#define fragread_help					\
> +"f2fs_io fragread [chunk_size in 4kb] [offset in chunk_size] [count] [advice] [file_path]\n\n"	\
> +"Read data in file_path and print nbytes\n"		\
> +"advice can be\n"					\
> +" 1 : set sequential|willneed\n"			\
> +" 0 : none\n"						\
> +
> +#ifndef PAGE_SIZE
> +#define PAGE_SIZE sysconf(_SC_PAGESIZE)
> +#endif
> +#define ALLOC_SIZE (2 * 1024 * 1024 - 4 * 1024) // 2MB - 4KB
> +
> +static void do_fragread(int argc, char **argv, const struct cmd_desc *cmd)
> +{
> +	u64 buf_size = 0, ret = 0, read_cnt = 0;
> +	u64 offset;
> +	char *buf = NULL;
> +	uintptr_t idx, ptr;
> +	unsigned bs, count, i;
> +	u64 total_time = 0;
> +	int flags = 0, alloc_count = 0;
> +	void *mem_hole, **mem_holes;
> +	int fd, advice;
> +
> +	if (argc != 6) {
> +		fputs("Excess arguments\n\n", stderr);
> +		fputs(cmd->cmd_help, stderr);
> +		exit(1);
> +	}
> +
> +	bs = atoi(argv[1]);
> +	if (bs > 256 * 1024)
> +		die("Too big chunk size - limit: 1GB");
> +	buf_size = bs * F2FS_DEFAULT_BLKSIZE;
> +
> +	offset = atoi(argv[2]) * buf_size;
> +	count = atoi(argv[3]);
> +	advice = atoi(argv[4]);
> +	mem_holes = xmalloc(sizeof(void *) * (buf_size / PAGE_SIZE));
> +
> +	/* 1. Allocate the buffer using mmap. */
> +	buf = mmap(NULL, buf_size, PROT_READ | PROT_WRITE,
> +				MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
> +
> +	/* 2. Loop and touch each page. */
> +	for (idx = (uintptr_t)buf; idx < (uintptr_t)buf + buf_size;
> +						idx += PAGE_SIZE)
> +	{
> +		/* Touch the current page. */
> +		volatile char *page = (volatile char *)idx;
> +		*page;
> +
> +		/* 3. Allocate (2M - 4K) memory using mmap and touch all of it. */
> +		mem_hole = mmap(NULL, ALLOC_SIZE, PROT_READ | PROT_WRITE,
> +					MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
> +		if (mem_hole == MAP_FAILED)
> +			die_errno("map failed");
> +
> +		/* Store the allocated memory pointer. */
> +		mem_holes[alloc_count++] = mem_hole;
> +
> +		/* Touch all allocated memory. */
> +		for (ptr = (uintptr_t)mem_hole;
> +			ptr < (uintptr_t)mem_hole + ALLOC_SIZE;
> +						ptr += PAGE_SIZE) {
> +			volatile char *alloc_page = (volatile char *)ptr;
> +			*alloc_page;
> +		}
> +	}
> +	printf("Touched allocated memory: count = %u\n", alloc_count);
> +	printf(" - allocated memory: = ");
> +	for (idx = 0; idx < 5; idx++)
> +		printf(" %p", mem_holes[idx]);
> +	printf("\n");
> +
> +	/* Pin the pages. */
> +	if (mlock(buf, buf_size))
> +		die_errno("mlock failed");
> +
> +	fd = xopen(argv[5], O_RDONLY | flags, 0);
> +
> +	if (advice) {
> +		if (posix_fadvise(fd, 0, F2FS_DEFAULT_BLKSIZE,
> +				POSIX_FADV_SEQUENTIAL) != 0)
> +			die_errno("fadvise failed");
> +		if (posix_fadvise(fd, 0, F2FS_DEFAULT_BLKSIZE,
> +				POSIX_FADV_WILLNEED) != 0)
> +			die_errno("fadvise failed");
> +		printf("fadvise SEQUENTIAL|WILLNEED to a file: %s\n", argv[5]);
> +	}
> +
> +	total_time = get_current_us();
> +
> +	for (i = 0; i < count; i++) {
> +		ret = pread(fd, buf, buf_size, offset + buf_size * i);
> +		if (ret != buf_size) {
> +			printf("pread expected: %"PRIu64", readed: %"PRIu64"\n",
> +					buf_size, ret);
> +			if (ret > 0)
> +				read_cnt += ret;
> +			break;
> +		}
> +
> +		read_cnt += ret;
> +	}
> +	printf("Fragmented_Read %"PRIu64" bytes total_time = %"PRIu64" us, BW = %.Lf MB/s\n",
> +		read_cnt, get_current_us() - total_time,
> +		((long double)read_cnt / (get_current_us() - total_time)));
> +	printf("\n");
> +	exit(0);
> +}
> +
>  #define randread_desc "random read data from file"
>  #define randread_help					\
>  "f2fs_io randread [chunk_size in 4kb] [count] [IO] [advise] [file_path]\n\n"	\
> @@ -1989,6 +2102,7 @@ const struct cmd_desc cmd_list[] = {
>  	CMD(write_advice),
>  	CMD(read),
>  	CMD(randread),
> +	CMD(fragread),
>  	CMD(fiemap),
>  	CMD(gc_urgent),
>  	CMD(defrag_file),
> -- 
> 2.49.0.rc1.451.g8f38331e32-goog
> 
> 
> 
> _______________________________________________
> Linux-f2fs-devel mailing list
> Linux-f2fs-devel@lists.sourceforge.net
> https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel


_______________________________________________
Linux-f2fs-devel mailing list
Linux-f2fs-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel