Re: [PATCH] libibverbs: Add huge page support to ibv_madvise_range()

[Alex Vainman <alexonlists-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org>]

Hi Roland,

I would like to ask you if you have an estimation for when you will review the patches below?

Thanks,
AlexV

Alex Vainman wrote:
> Hi Roland,
> 
> Have you had a chance to look at this patch and at the patch 
> I've sent with this email:
> "Subject: [PATCH] libibverbs: Add huge page support to ibv_madvise_range()
> Sent: Sun, 29 Nov 2009 19:08:08 +0200"?
> 
> Thanks,
> Alexv
> 
> 
> Alex Vainman wrote:
>> ibv_reg_mr() fails to register a memory region allocated on huge page and not
>> the default page size. This happens because ibv_madvise_range() aligns memory
>> region to the default system page size before calling to madvise() which fails
>> with EINVAL error. madvise() fails because it expects that the start and end
>> pointer of the memory range be huge page aligned.
>> Patch handles the issue by:
>> 1. ibv_fork_init() gets kernel's default huge page size in addition
>>    to the default page size.
>> 2. ibv_madvise_range() first tries aligning users memory range to default
>>    page size and if madvise() fails with EINVAL error then it tries to align
>>    users memory range by huge page size and tries madvise() again.
>>
>> Signed-off-by: Alex Vaynman <alexv-smomgflXvOZWk0Htik3J/w@public.gmane.org>
>> ---
>>  src/memory.c |   69 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
>>  1 files changed, 68 insertions(+), 1 deletions(-)
>>
>> diff --git a/src/memory.c b/src/memory.c
>> index 550015a..73db083 100644
>> --- a/src/memory.c
>> +++ b/src/memory.c
>> @@ -40,6 +40,9 @@
>>  #include <unistd.h>
>>  #include <stdlib.h>
>>  #include <stdint.h>
>> +#include <ctype.h>
>> +#include <fcntl.h>
>> +#include <string.h>
>>  
>>  #include "ibverbs.h"
>>  
>> @@ -54,6 +57,8 @@
>>  #define MADV_DOFORK	11
>>  #endif
>>  
>> +#define MEMINFO_SIZE	2048
>> +
>>  struct ibv_mem_node {
>>  	enum {
>>  		IBV_RED,
>> @@ -68,8 +73,51 @@ struct ibv_mem_node {
>>  static struct ibv_mem_node *mm_root;
>>  static pthread_mutex_t mm_mutex = PTHREAD_MUTEX_INITIALIZER;
>>  static int page_size;
>> +static int huge_page_size;
>>  static int too_late;
>>  
>> +/*
>> + * Get the kernel default huge page size.
>> + */
>> +static int get_huge_page_size()
>> +{
>> +	int fd;
>> +	char buf[MEMINFO_SIZE];
>> +	int mem_file_len;
>> +	char *p_hpage_val = NULL;
>> +	char *end_pointer = NULL;
>> +	char file_name[] = "/proc/meminfo";
>> +	const char label[] = "Hugepagesize:";
>> +	int ret_val = 0;
>> +
>> +	fd = open(file_name, O_RDONLY);
>> +	if (fd < 0)
>> +		return fd;
>> +
>> +	mem_file_len = read(fd, buf, sizeof(buf) - 1);
>> +
>> +	close(fd);
>> +	if (mem_file_len < 0)
>> +		return mem_file_len;
>> +
>> +	buf[mem_file_len] = '\0';
>> +
>> +	p_hpage_val = strstr(buf, label);
>> +	if (!p_hpage_val) {
>> +		errno = EINVAL;
>> +		return -1;
>> +	}
>> +	p_hpage_val += strlen(label);
>> +
>> +	errno = 0;
>> +	ret_val = strtol(p_hpage_val, &end_pointer, 0);
>> +
>> +	if (errno != 0)
>> +		return -1;
>> +
>> +	return ret_val * 1024;
>> +}
>> +
>>  int ibv_fork_init(void)
>>  {
>>  	void *tmp;
>> @@ -85,6 +133,8 @@ int ibv_fork_init(void)
>>  	if (page_size < 0)
>>  		return errno;
>>  
>> +	huge_page_size = get_huge_page_size();
>> +
>>  	if (posix_memalign(&tmp, page_size, page_size))
>>  		return ENOMEM;
>>  
>> @@ -554,7 +604,8 @@ static struct ibv_mem_node *prepare_to_roll_back(struct ibv_mem_node *node,
>>  	return node;
>>  }
>>  
>> -static int ibv_madvise_range(void *base, size_t size, int advice)
>> +static int ibv_madvise_range_helper(void *base, size_t size, int advice,
>> +				    int page_size)
>>  {
>>  	uintptr_t start, end;
>>  	struct ibv_mem_node *node, *tmp;
>> @@ -646,6 +697,22 @@ out:
>>  	return ret;
>>  }
>>  
>> +static int ibv_madvise_range(void *base, size_t size, int advice)
>> +{
>> +	int ret_val = 0;
>> +
>> +	ret_val = ibv_madvise_range_helper(base, size, advice, page_size);
>> +
>> +	/*
>> +	 * if memory is backed by huge pages we need to align it
>> +	 * to huge page boundary in order madvise() will succeed.
>> +	 */
>> +	if (ret_val == -1 && errno == EINVAL && huge_page_size > 0)
>> +		ret_val = ibv_madvise_range_helper(base, size, advice, huge_page_size);
>> +
>> +	return ret_val;
>> +}
>> +
>>  int ibv_dontfork_range(void *base, size_t size)
>>  {
>>  	if (mm_root)
> 
> 
> --
> To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
> the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> 


--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html