[PATCH 1/2] powerpc: Allow memory that has been hot-removed to be hot-added

[PATCH 1/2] powerpc: Allow memory that has been hot-removed to be hot-added

[Rashmica Gupta]

This patch allows the memory removed by memtrace to be readded to the
kernel. So now you don't have to reboot your system to add the memory
back to the kernel or to have a different amount of memory removed.

Signed-off-by: Rashmica Gupta <rashmica.g@gmail.com>
---
To remove 1GB from each node:
echo 1073741824  >  /sys/kernel/debug/powerpc/memtrace/enable

To add this memory back and remove 2GB:
echo 2147483648 >  /sys/kernel/debug/powerpc/memtrace/enable 

To just re-add memory:
echo 0  >  /sys/kernel/debug/powerpc/memtrace/enable 


 arch/powerpc/platforms/powernv/memtrace.c | 93 ++++++++++++++++++++++++++++---
 1 file changed, 86 insertions(+), 7 deletions(-)

diff --git a/arch/powerpc/platforms/powernv/memtrace.c b/arch/powerpc/platforms/powernv/memtrace.c
index b99283df8584..51fe0862dcab 100644
--- a/arch/powerpc/platforms/powernv/memtrace.c
+++ b/arch/powerpc/platforms/powernv/memtrace.c
@@ -206,8 +206,11 @@ static int memtrace_init_debugfs(void)
 
 		snprintf(ent->name, 16, "%08x", ent->nid);
 		dir = debugfs_create_dir(ent->name, memtrace_debugfs_dir);
-		if (!dir)
+		if (!dir) {
+			pr_err("Failed to create debugfs directory for node %d\n",
+				ent->nid);
 			return -1;
+		}
 
 		ent->dir = dir;
 		debugfs_create_file("trace", 0400, dir, ent, &memtrace_fops);
@@ -218,18 +221,94 @@ static int memtrace_init_debugfs(void)
 	return ret;
 }
 
+static int online_mem_block(struct memory_block *mem, void *arg)
+{
+	return device_online(&mem->dev);
+}
+
+/*
+ * Iterate through the chunks of memory we have removed from the kernel
+ * and attempt to add them back to the kernel.
+ */
+static int memtrace_online(void)
+{
+	int i, ret = 0;
+	struct memtrace_entry *ent;
+
+	for (i = memtrace_array_nr - 1; i >= 0; i--) {
+		ent = &memtrace_array[i];
+
+		/* We have onlined this chunk previously */
+		if (ent->nid == -1)
+			continue;
+
+		/* Remove from io mappings */
+		if (ent->mem) {
+			iounmap(ent->mem);
+			ent->mem = 0;
+		}
+
+		if (add_memory(ent->nid, ent->start, ent->size)) {
+			pr_err("Failed to add trace memory to node %d\n",
+					 ent->nid);
+			ret += 1;
+			continue;
+		}
+
+		/*
+		 * If kernel isn't compiled with the auto online option
+		 * we need to online the memory ourselves.
+		 */
+		if (!memhp_auto_online) {
+			walk_memory_range(PFN_DOWN(ent->start),
+				PFN_UP(ent->start + ent->size - 1),
+				NULL, online_mem_block);
+		}
+
+		/*
+		 * Memory was added successfully so clean up references to it
+		 * so on reentry we can tell that this chunk was added.
+		 */
+		debugfs_remove_recursive(ent->dir);
+		pr_info("Added trace memory back to node %d\n", ent->nid);
+		ent->size = ent->start = ent->nid = -1;
+	}
+	if (ret)
+		return ret;
+
+	/* If all chunks of memory were added successfully, reset globals */
+	kfree(memtrace_array);
+	memtrace_array = NULL;
+	memtrace_size = 0;
+	memtrace_array_nr = 0;
+	return 0;
+
+}
+
 static int memtrace_enable_set(void *data, u64 val)
 {
-	if (memtrace_size)
+	uint64_t bytes;
+
+	/*
+	 * Don't attempt to do anything if size isn't aligned to a memory
+	 * block or equal to zero.
+	 */
+	bytes = memory_block_size_bytes();
+	if (val & (bytes - 1)) {
+		pr_err("Value must be aligned with 0x%llx\n", bytes);
 		return -EINVAL;
+	}
 
-	if (!val)
-		return -EINVAL;
+	/* Re-add/online previously removed/offlined memory */
+	if (memtrace_size) {
+		if (memtrace_online())
+			return -EAGAIN;
+	}
 
-	/* Make sure size is aligned to a memory block */
-	if (val & (memory_block_size_bytes() - 1))
-		return -EINVAL;
+	if (!val)
+		return 0;
 
+	/* Offline and remove memory */
 	if (memtrace_init_regions_runtime(val))
 		return -EINVAL;
 
-- 
2.14.4

[PATCH 2/2] Update documentation on ppc-memtrace

[Rashmica Gupta]

Signed-off-by: Rashmica Gupta <rashmica.g@gmail.com>
---
 Documentation/ABI/testing/ppc-memtrace | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/Documentation/ABI/testing/ppc-memtrace b/Documentation/ABI/testing/ppc-memtrace
index 2e8b93741270..9606aed33137 100644
--- a/Documentation/ABI/testing/ppc-memtrace
+++ b/Documentation/ABI/testing/ppc-memtrace
@@ -13,10 +13,11 @@ Contact:	linuxppc-dev@lists.ozlabs.org
 Description:	Write an integer containing the size in bytes of the memory
 		you want removed from each NUMA node to this file - it must be
 		aligned to the memblock size. This amount of RAM will be removed
-		from the kernel mappings and the following debugfs files will be
-		created. This can only be successfully done once per boot. Once
-		memory is successfully removed from each node, the following
-		files are created.
+		from each NUMA node in the kernel mappings and the following
+		debugfs files will be created. Once memory is successfully
+		removed from each node, the following files are created. To
+		re-add memory to the kernel, echo 0 into this file (it will be
+		automatically onlined).
 
 What:		/sys/kernel/debug/powerpc/memtrace/<node-id>
 Date:		Aug 2017
-- 
2.14.4

Re: [PATCH 1/2] powerpc: Allow memory that has been hot-removed to be hot-added

[Michael Neuling]

On Fri, 2018-08-03 at 16:06 +1000, Rashmica Gupta wrote:
> This patch allows the memory removed by memtrace to be readded to the
> kernel. So now you don't have to reboot your system to add the memory
> back to the kernel or to have a different amount of memory removed.
>=20
> Signed-off-by: Rashmica Gupta <rashmica.g@gmail.com>

Tested-by: Michael Neuling <mikey@neuling.org>

> ---
> To remove 1GB from each node:
> echo 1073741824  >  /sys/kernel/debug/powerpc/memtrace/enable
>=20
> To add this memory back and remove 2GB:
> echo 2147483648 >  /sys/kernel/debug/powerpc/memtrace/enable=20
>=20
> To just re-add memory:
> echo 0  >  /sys/kernel/debug/powerpc/memtrace/enable=20

Nice... thanks!

Mikey

>=20
>=20
>  arch/powerpc/platforms/powernv/memtrace.c | 93 +++++++++++++++++++++++++=
+++
> ---
>  1 file changed, 86 insertions(+), 7 deletions(-)
>=20
> diff --git a/arch/powerpc/platforms/powernv/memtrace.c
> b/arch/powerpc/platforms/powernv/memtrace.c
> index b99283df8584..51fe0862dcab 100644
> --- a/arch/powerpc/platforms/powernv/memtrace.c
> +++ b/arch/powerpc/platforms/powernv/memtrace.c
> @@ -206,8 +206,11 @@ static int memtrace_init_debugfs(void)
> =20
>  		snprintf(ent->name, 16, "%08x", ent->nid);
>  		dir =3D debugfs_create_dir(ent->name, memtrace_debugfs_dir);
> -		if (!dir)
> +		if (!dir) {
> +			pr_err("Failed to create debugfs directory for node
> %d\n",
> +				ent->nid);
>  			return -1;
> +		}
> =20
>  		ent->dir =3D dir;
>  		debugfs_create_file("trace", 0400, dir, ent, &memtrace_fops);
> @@ -218,18 +221,94 @@ static int memtrace_init_debugfs(void)
>  	return ret;
>  }
> =20
> +static int online_mem_block(struct memory_block *mem, void *arg)
> +{
> +	return device_online(&mem->dev);
> +}
> +
> +/*
> + * Iterate through the chunks of memory we have removed from the kernel
> + * and attempt to add them back to the kernel.
> + */
> +static int memtrace_online(void)
> +{
> +	int i, ret =3D 0;
> +	struct memtrace_entry *ent;
> +
> +	for (i =3D memtrace_array_nr - 1; i >=3D 0; i--) {
> +		ent =3D &memtrace_array[i];
> +
> +		/* We have onlined this chunk previously */
> +		if (ent->nid =3D=3D -1)
> +			continue;
> +
> +		/* Remove from io mappings */
> +		if (ent->mem) {
> +			iounmap(ent->mem);
> +			ent->mem =3D 0;
> +		}
> +
> +		if (add_memory(ent->nid, ent->start, ent->size)) {
> +			pr_err("Failed to add trace memory to node %d\n",
> +					 ent->nid);
> +			ret +=3D 1;
> +			continue;
> +		}
> +
> +		/*
> +		 * If kernel isn't compiled with the auto online option
> +		 * we need to online the memory ourselves.
> +		 */
> +		if (!memhp_auto_online) {
> +			walk_memory_range(PFN_DOWN(ent->start),
> +				PFN_UP(ent->start + ent->size - 1),
> +				NULL, online_mem_block);
> +		}
> +
> +		/*
> +		 * Memory was added successfully so clean up references to it
> +		 * so on reentry we can tell that this chunk was added.
> +		 */
> +		debugfs_remove_recursive(ent->dir);
> +		pr_info("Added trace memory back to node %d\n", ent->nid);
> +		ent->size =3D ent->start =3D ent->nid =3D -1;
> +	}
> +	if (ret)
> +		return ret;
> +
> +	/* If all chunks of memory were added successfully, reset globals */
> +	kfree(memtrace_array);
> +	memtrace_array =3D NULL;
> +	memtrace_size =3D 0;
> +	memtrace_array_nr =3D 0;
> +	return 0;
> +
> +}
> +
>  static int memtrace_enable_set(void *data, u64 val)
>  {
> -	if (memtrace_size)
> +	uint64_t bytes;
> +
> +	/*
> +	 * Don't attempt to do anything if size isn't aligned to a memory
> +	 * block or equal to zero.
> +	 */
> +	bytes =3D memory_block_size_bytes();
> +	if (val & (bytes - 1)) {
> +		pr_err("Value must be aligned with 0x%llx\n", bytes);
>  		return -EINVAL;
> +	}
> =20
> -	if (!val)
> -		return -EINVAL;
> +	/* Re-add/online previously removed/offlined memory */
> +	if (memtrace_size) {
> +		if (memtrace_online())
> +			return -EAGAIN;
> +	}
> =20
> -	/* Make sure size is aligned to a memory block */
> -	if (val & (memory_block_size_bytes() - 1))
> -		return -EINVAL;
> +	if (!val)
> +		return 0;
> =20
> +	/* Offline and remove memory */
>  	if (memtrace_init_regions_runtime(val))
>  		return -EINVAL;
> =20

Re: [1/2] powerpc: Allow memory that has been hot-removed to be hot-added

[Michael Ellerman]

On Fri, 2018-08-03 at 06:06:00 UTC, Rashmica Gupta wrote:
> This patch allows the memory removed by memtrace to be readded to the
> kernel. So now you don't have to reboot your system to add the memory
> back to the kernel or to have a different amount of memory removed.
> 
> Signed-off-by: Rashmica Gupta <rashmica.g@gmail.com>
> Tested-by: Michael Neuling <mikey@neuling.org>

Series applied to powerpc next, thanks.

https://git.kernel.org/powerpc/c/d3da701d3308ce1fa457f32c6c9e21

cheers