[PATCH v2 0/3] powerpc/pseries: Perform PRRN topology updates in kernel

[PATCH v2 0/3] powerpc/pseries: Perform PRRN topology updates in kernel

[John Allen]

Formerly, when we received a PRRN rtas event, device tree updating was
performed in the kernel and the actual topology updating was performed in
userspace. This was necessary as in order to update the topology for memory,
we must perform a hot remove and a subsequent hot add and until recently,
memory hotplug was not included in the kernel. Since memory hotplug is now
available, this patchset moves the PRRN topology updating into the kernel.

Changes from v1:
-Introduce patch to separate the acquire and release drc from existing
 memory hotplug
-Create new function "dlpar_memory_readd_by_index" that consolidates the
 necessary steps of memory hot remove and hot add into a single function
-Remove conversion of phandle to BE
-Since error messages are already generated in the memory hotplug code,
 remove redundant error messages in pseries_prrn_update_node. Since we no
 longer use the return code from the hotplug event, remove the
 wait_for_completion infrastructure.

John Allen (3):
      powerpc/pseries: Make the acquire/release of the drc for memory a seperate step
      powerpc/pseries: Introduce memory hotplug READD operation
      powerpc/pseries: Update affinity for memory and cpus specified in a PRRN event

 arch/powerpc/include/asm/rtas.h                 |    1
 arch/powerpc/kernel/rtasd.c                     |    7 ++
 arch/powerpc/platforms/pseries/hotplug-memory.c |   75 +++++++++++++++++++----
 arch/powerpc/platforms/pseries/mobility.c       |   32 ++++++++++
 4 files changed, 102 insertions(+), 13 deletions(-)

[PATCH v2 1/3] powerpc/pseries: Make the acquire/release of the drc for memory a seperate step

[John Allen]

When adding and removing LMBs we should make the acquire/release of
the DRC a separate step to allow for a few improvements. First
this will ensure that LMBs removed during a remove by count operation
are all available if a error occurs and we need to add them back. By
first removeing all the LMBs from the kernel before releasing their
DRCs the LMBs are available to add back should an error occur.

Also, this will allow for faster re-add operations of memory for
PRRN event handling since we can skip the unneeded step of having
to release the DRC and the acquire it back.

Signed-off-by: Nathan Fontenot <nfont@linux.vnet.ibm.com>
Signed-off-by: John Allen <jallen@linux.vnet.ibm.com>
---
 arch/powerpc/platforms/pseries/hotplug-memory.c |   34 +++++++++++++++--------
 1 file changed, 22 insertions(+), 12 deletions(-)

diff --git a/arch/powerpc/platforms/pseries/hotplug-memory.c b/arch/powerpc/platforms/pseries/hotplug-memory.c
index 2617f9f..be11fc3 100644
--- a/arch/powerpc/platforms/pseries/hotplug-memory.c
+++ b/arch/powerpc/platforms/pseries/hotplug-memory.c
@@ -446,9 +446,7 @@ static int dlpar_remove_lmb(struct of_drconf_cell *lmb)
 	/* Update memory regions for memory remove */
 	memblock_remove(lmb->base_addr, block_sz);

-	dlpar_release_drc(lmb->drc_index);
 	dlpar_remove_device_tree_lmb(lmb);
-
 	return 0;
 }

@@ -516,6 +514,7 @@ static int dlpar_memory_remove_by_count(u32 lmbs_to_remove,
 			if (!lmbs[i].reserved)
 				continue;

+			dlpar_release_drc(lmbs[i].drc_index);
 			pr_info("Memory at %llx was hot-removed\n",
 				lmbs[i].base_addr);

@@ -545,6 +544,9 @@ static int dlpar_memory_remove_by_index(u32 drc_index, struct property *prop)
 		if (lmbs[i].drc_index == drc_index) {
 			lmb_found = 1;
 			rc = dlpar_remove_lmb(&lmbs[i]);
+			if (!rc)
+				dlpar_release_drc(lmbs[i].drc_index);
+
 			break;
 		}
 	}
@@ -599,10 +601,6 @@ static int dlpar_add_lmb(struct of_drconf_cell *lmb)
 	if (lmb->flags & DRCONF_MEM_ASSIGNED)
 		return -EINVAL;

-	rc = dlpar_acquire_drc(lmb->drc_index);
-	if (rc)
-		return rc;
-
 	rc = dlpar_add_device_tree_lmb(lmb);
 	if (rc) {
 		pr_err("Couldn't update device tree for drc index %x\n",
@@ -618,12 +616,10 @@ static int dlpar_add_lmb(struct of_drconf_cell *lmb)

 	/* Add the memory */
 	rc = add_memory(nid, lmb->base_addr, block_sz);
-	if (rc) {
+	if (rc)
 		dlpar_remove_device_tree_lmb(lmb);
-		dlpar_release_drc(lmb->drc_index);
-	} else {
+	else
 		lmb->flags |= DRCONF_MEM_ASSIGNED;
-	}

 	return rc;
 }
@@ -655,10 +651,16 @@ static int dlpar_memory_add_by_count(u32 lmbs_to_add, struct property *prop)
 		return -EINVAL;

 	for (i = 0; i < num_lmbs && lmbs_to_add != lmbs_added; i++) {
-		rc = dlpar_add_lmb(&lmbs[i]);
+		rc = dlpar_acquire_drc(lmbs[i].drc_index);
 		if (rc)
 			continue;

+		rc = dlpar_add_lmb(&lmbs[i]);
+		if (rc) {
+			dlpar_release_drc(lmbs[i].drc_index);
+			continue;
+		}
+
 		lmbs_added++;

 		/* Mark this lmb so we can remove it later if all of the
@@ -678,6 +680,8 @@ static int dlpar_memory_add_by_count(u32 lmbs_to_add, struct property *prop)
 			if (rc)
 				pr_err("Failed to remove LMB, drc index %x\n",
 				       be32_to_cpu(lmbs[i].drc_index));
+			else
+				dlpar_release_drc(lmbs[i].drc_index);
 		}
 		rc = -EINVAL;
 	} else {
@@ -711,7 +715,13 @@ static int dlpar_memory_add_by_index(u32 drc_index, struct property *prop)
 	for (i = 0; i < num_lmbs; i++) {
 		if (lmbs[i].drc_index == drc_index) {
 			lmb_found = 1;
-			rc = dlpar_add_lmb(&lmbs[i]);
+			rc = dlpar_acquire_drc(lmbs[i].drc_index);
+			if (!rc) {
+				rc = dlpar_add_lmb(&lmbs[i]);
+				if (rc)
+					dlpar_release_drc(lmbs[i].drc_index);
+			}
+
 			break;
 		}
 	}

[PATCH v2 2/3] powerpc/pseries: Introduce memory hotplug READD operation

[John Allen]

Currently, memory must be hot removed and subsequently re-added in order
to dynamically update the affinity of LMBs specified by a PRRN event.
Earlier implementations of the PRRN event handler ran into issues in which
the hot remove would occur successfully, but a hotplug event would be
initiated from another source and grab the hotplug lock preventing the hot
add from occurring. To prevent this situation, this patch introduces the
notion of a hot "readd" action for memory which atomizes a hot remove and
a hot add into a single, serialized operation on the hotplug queue.

Signed-off-by: John Allen <jallen@linux.vnet.ibm.com>
---
diff --git a/arch/powerpc/include/asm/rtas.h b/arch/powerpc/include/asm/rtas.h
index 9c23baa..076b892 100644
--- a/arch/powerpc/include/asm/rtas.h
+++ b/arch/powerpc/include/asm/rtas.h
@@ -318,6 +318,7 @@ struct pseries_hp_errorlog {

 #define PSERIES_HP_ELOG_ACTION_ADD	1
 #define PSERIES_HP_ELOG_ACTION_REMOVE	2
+#define PSERIES_HP_ELOG_ACTION_READD	3

 #define PSERIES_HP_ELOG_ID_DRC_NAME	1
 #define PSERIES_HP_ELOG_ID_DRC_INDEX	2
diff --git a/arch/powerpc/platforms/pseries/hotplug-memory.c b/arch/powerpc/platforms/pseries/hotplug-memory.c
index 0eb4b1d..06f10a8 100644
--- a/arch/powerpc/platforms/pseries/hotplug-memory.c
+++ b/arch/powerpc/platforms/pseries/hotplug-memory.c
@@ -560,6 +560,44 @@ static int dlpar_memory_remove_by_index(u32 drc_index, struct property *prop)
 	return rc;
 }

+static int dlpar_memory_readd_by_index(u32 drc_index, struct property *prop)
+{
+	struct of_drconf_cell *lmbs;
+	u32 num_lmbs, *p;
+	int lmb_found;
+	int i, rc;
+
+	pr_info("Attempting to update LMB, drc index %x\n", drc_index);
+
+	p = prop->value;
+	num_lmbs = *p++;
+	lmbs = (struct of_drconf_cell *)p;
+
+	lmb_found = 0;
+	for (i = 0; i < num_lmbs; i++) {
+		if (lmbs[i].drc_index == drc_index) {
+			lmb_found = 1;
+			rc = dlpar_remove_lmb(&lmbs[i]);
+			if (!rc) {
+				rc = dlpar_add_lmb(&lmbs[i]);
+				if (rc)
+					dlpar_release_drc(lmbs[i].drc_index);
+			}
+			break;
+		}
+	}
+
+	if (!lmb_found)
+		rc = -EINVAL;
+
+	if (rc)
+		pr_info("Failed to update memory at %llx\n",
+			lmbs[i].base_addr);
+	else
+		pr_info("Memory at %llx was updated\n", lmbs[i].base_addr);
+
+	return rc;
+}
 #else
 static inline int pseries_remove_memblock(unsigned long base,
 					  unsigned int memblock_size)
@@ -776,6 +814,9 @@ int dlpar_memory(struct pseries_hp_errorlog *hp_elog)
 		else
 			rc = -EINVAL;
 		break;
+	case PSERIES_HP_ELOG_ACTION_READD:
+		rc = dlpar_memory_readd_by_index(drc_index, prop);
+		break;
 	default:
 		pr_err("Invalid action (%d) specified\n", hp_elog->action);
 		rc = -EINVAL;

[PATCH v2 3/3] powerpc/pseries: Update affinity for memory and cpus specified in a PRRN event

[John Allen]

Extend the existing PRRN infrastructure to perform the actual affinity
updating for cpus and memory in addition to the device tree updating. For
cpus, dynamic affinity updating already appears to exist in the kernel in
the form of arch_update_cpu_topology. For memory, we must place a READD
operation on the hotplug queue for any phandle included in the PRRN event
that is determined to be an LMB.

Signed-off-by: John Allen <jallen@linux.vnet.ibm.com>
---
diff --git a/arch/powerpc/kernel/rtasd.c b/arch/powerpc/kernel/rtasd.c
index a26a020..8836130 100644
--- a/arch/powerpc/kernel/rtasd.c
+++ b/arch/powerpc/kernel/rtasd.c
@@ -21,6 +21,7 @@
 #include <linux/cpu.h>
 #include <linux/workqueue.h>
 #include <linux/slab.h>
+#include <linux/topology.h>

 #include <asm/uaccess.h>
 #include <asm/io.h>
@@ -282,6 +283,7 @@ static void prrn_work_fn(struct work_struct *work)
 	 * the RTAS event.
 	 */
 	pseries_devicetree_update(-prrn_update_scope);
+	arch_update_cpu_topology();
 }

 static DECLARE_WORK(prrn_work, prrn_work_fn);
@@ -434,7 +436,10 @@ static void do_event_scan(void)
 		}

 		if (error == 0) {
-			pSeries_log_error(logdata, ERR_TYPE_RTAS_LOG, 0);
+			if (rtas_error_type((struct rtas_error_log *)logdata) !=
+			    RTAS_TYPE_PRRN)
+				pSeries_log_error(logdata, ERR_TYPE_RTAS_LOG,
+						  0);
 			handle_rtas_event((struct rtas_error_log *)logdata);
 		}

diff --git a/arch/powerpc/platforms/pseries/mobility.c b/arch/powerpc/platforms/pseries/mobility.c
index a560a98..d62d48a 100644
--- a/arch/powerpc/platforms/pseries/mobility.c
+++ b/arch/powerpc/platforms/pseries/mobility.c
@@ -39,6 +39,7 @@ struct update_props_workarea {
 #define ADD_DT_NODE	0x03000000

 #define MIGRATION_SCOPE	(1)
+#define PRRN_SCOPE -2

 static int mobility_rtas_call(int token, char *buf, s32 scope)
 {
@@ -236,6 +237,33 @@ static int add_dt_node(__be32 parent_phandle, __be32 drc_index)
 	return rc;
 }

+void pseries_prrn_update_node(__be32 phandle)
+{
+	struct pseries_hp_errorlog *hp_elog;
+	struct device_node *dn;
+
+	hp_elog = kzalloc(sizeof(*hp_elog), GFP_KERNEL);
+	if (!hp_elog)
+		return;
+
+	dn = of_find_node_by_phandle(be32_to_cpu(phandle));
+
+	/*
+	 * If the phandle was not found, assume phandle is the drc index of
+	 * an LMB.
+	 */
+	if (!dn) {
+		hp_elog->resource = PSERIES_HP_ELOG_RESOURCE_MEM;
+		hp_elog->action = PSERIES_HP_ELOG_ACTION_READD;
+		hp_elog->id_type = PSERIES_HP_ELOG_ID_DRC_INDEX;
+		hp_elog->_drc_u.drc_index = phandle;
+
+		queue_hotplug_event(hp_elog, NULL, NULL);
+	}
+
+	kfree(hp_elog);
+}
+
 int pseries_devicetree_update(s32 scope)
 {
 	char *rtas_buf;
@@ -274,6 +302,10 @@ int pseries_devicetree_update(s32 scope)
 					break;
 				case UPDATE_DT_NODE:
 					update_dt_node(phandle, scope);
+
+					if (scope == PRRN_SCOPE)
+						pseries_prrn_update_node(phandle);
+
 					break;
 				case ADD_DT_NODE:
 					drc_index = *data++;

Re: [PATCH v2 3/3] powerpc/pseries: Update affinity for memory and cpus specified in a PRRN event

[Nathan Fontenot]

On 12/15/2016 04:22 PM, John Allen wrote:
> Extend the existing PRRN infrastructure to perform the actual affinity
> updating for cpus and memory in addition to the device tree updating. For
> cpus, dynamic affinity updating already appears to exist in the kernel in
> the form of arch_update_cpu_topology. For memory, we must place a READD
> operation on the hotplug queue for any phandle included in the PRRN event
> that is determined to be an LMB.
> 
> Signed-off-by: John Allen <jallen@linux.vnet.ibm.com>
> ---
> diff --git a/arch/powerpc/kernel/rtasd.c b/arch/powerpc/kernel/rtasd.c
> index a26a020..8836130 100644
> --- a/arch/powerpc/kernel/rtasd.c
> +++ b/arch/powerpc/kernel/rtasd.c
> @@ -21,6 +21,7 @@
>  #include <linux/cpu.h>
>  #include <linux/workqueue.h>
>  #include <linux/slab.h>
> +#include <linux/topology.h>
> 
>  #include <asm/uaccess.h>
>  #include <asm/io.h>
> @@ -282,6 +283,7 @@ static void prrn_work_fn(struct work_struct *work)
>  	 * the RTAS event.
>  	 */
>  	pseries_devicetree_update(-prrn_update_scope);
> +	arch_update_cpu_topology();
>  }
> 
>  static DECLARE_WORK(prrn_work, prrn_work_fn);
> @@ -434,7 +436,10 @@ static void do_event_scan(void)
>  		}
> 
>  		if (error == 0) {
> -			pSeries_log_error(logdata, ERR_TYPE_RTAS_LOG, 0);
> +			if (rtas_error_type((struct rtas_error_log *)logdata) !=
> +			    RTAS_TYPE_PRRN)
> +				pSeries_log_error(logdata, ERR_TYPE_RTAS_LOG,
> +						  0);
>  			handle_rtas_event((struct rtas_error_log *)logdata);
>  		}
> 
> diff --git a/arch/powerpc/platforms/pseries/mobility.c b/arch/powerpc/platforms/pseries/mobility.c
> index a560a98..d62d48a 100644
> --- a/arch/powerpc/platforms/pseries/mobility.c
> +++ b/arch/powerpc/platforms/pseries/mobility.c
> @@ -39,6 +39,7 @@ struct update_props_workarea {
>  #define ADD_DT_NODE	0x03000000
> 
>  #define MIGRATION_SCOPE	(1)
> +#define PRRN_SCOPE -2
> 
>  static int mobility_rtas_call(int token, char *buf, s32 scope)
>  {
> @@ -236,6 +237,33 @@ static int add_dt_node(__be32 parent_phandle, __be32 drc_index)
>  	return rc;
>  }
> 
> +void pseries_prrn_update_node(__be32 phandle)
> +{
> +	struct pseries_hp_errorlog *hp_elog;
> +	struct device_node *dn;
> +
> +	hp_elog = kzalloc(sizeof(*hp_elog), GFP_KERNEL);
> +	if (!hp_elog)
> +		return;
> +
> +	dn = of_find_node_by_phandle(be32_to_cpu(phandle));

You should check for the presence of the device node prior to doing the
kzalloc, this way you can avoid the alloc/free calls in cases where it
is not needed.

Also, you do need to do a of_node_put() on the device node if it is found.

-Nathan
 
> +
> +	/*
> +	 * If the phandle was not found, assume phandle is the drc index of
> +	 * an LMB.
> +	 */
> +	if (!dn) {
> +		hp_elog->resource = PSERIES_HP_ELOG_RESOURCE_MEM;
> +		hp_elog->action = PSERIES_HP_ELOG_ACTION_READD;
> +		hp_elog->id_type = PSERIES_HP_ELOG_ID_DRC_INDEX;
> +		hp_elog->_drc_u.drc_index = phandle;
> +
> +		queue_hotplug_event(hp_elog, NULL, NULL);
> +	}
> +
> +	kfree(hp_elog);
> +}
> +
>  int pseries_devicetree_update(s32 scope)
>  {
>  	char *rtas_buf;
> @@ -274,6 +302,10 @@ int pseries_devicetree_update(s32 scope)
>  					break;
>  				case UPDATE_DT_NODE:
>  					update_dt_node(phandle, scope);
> +
> +					if (scope == PRRN_SCOPE)
> +						pseries_prrn_update_node(phandle);
> +
>  					break;
>  				case ADD_DT_NODE:
>  					drc_index = *data++;
>