[PATCH] kni: add module parameter 'bind_to_core'

[PATCH] kni: add module parameter 'bind_to_core'

[Vladyslav Buslov]

Allow binding KNI thread to specific core in single threaded mode.

Signed-off-by: Vladyslav Buslov <vladyslav.buslov@harmonicinc.com>
---
 lib/librte_eal/linuxapp/kni/kni_misc.c | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/lib/librte_eal/linuxapp/kni/kni_misc.c b/lib/librte_eal/linuxapp/kni/kni_misc.c
index 59d15ca..e98f4a9 100644
--- a/lib/librte_eal/linuxapp/kni/kni_misc.c
+++ b/lib/librte_eal/linuxapp/kni/kni_misc.c
@@ -93,6 +93,7 @@ static char *lo_mode = NULL;
 /* Kernel thread mode */
 static char *kthread_mode = NULL;
 static unsigned multiple_kthread_on = 0;
+static int bind_to_core = -1;
 
 #define KNI_DEV_IN_USE_BIT_NUM 0 /* Bit number for device in use */
 
@@ -239,12 +240,17 @@ kni_open(struct inode *inode, struct file *file)
 	if (multiple_kthread_on == 0) {
 		KNI_PRINT("Single kernel thread for all KNI devices\n");
 		/* Create kernel thread for RX */
-		knet->kni_kthread = kthread_run(kni_thread_single, (void *)knet,
+		knet->kni_kthread = kthread_create(kni_thread_single, (void *)knet,
 						"kni_single");
 		if (IS_ERR(knet->kni_kthread)) {
 			KNI_ERR("Unable to create kernel threaed\n");
 			return PTR_ERR(knet->kni_kthread);
 		}
+		if (bind_to_core >= 0) {
+			KNI_PRINT("Bind main thread to core %d\n", bind_to_core);
+			kthread_bind(knet->kni_kthread, bind_to_core);
+		}
+		wake_up_process(knet->kni_kthread);
 	} else
 		KNI_PRINT("Multiple kernel thread mode enabled\n");
 
@@ -698,3 +704,8 @@ MODULE_PARM_DESC(kthread_mode,
 "    multiple  Multiple kernel thread mode enabled.\n"
 "\n"
 );
+
+module_param(bind_to_core, int, S_IRUGO);
+MODULE_PARM_DESC(bind_to_core,
+"Bind KNI main kernel thread to specific core (default=-1(disabled)):\n"
+);
\ No newline at end of file
-- 
2.8.3

Re: [PATCH] kni: add module parameter 'bind_to_core'

[Ferruh Yigit]

Hi Vladyslav,

On 8/16/2016 7:24 PM, Vladyslav Buslov wrote:
> Allow binding KNI thread to specific core in single threaded mode.

I think this is good idea.
But I am not sure about making this a module parameter, setting core can
be more dynamic.

There is already a kni->core_id field, which is only used for
multiple_kthread mode.
What do you think moving single thread creation into kni_ioctl_create()
and use first kni->core_id to bind the thread? If there is no
kni->core_id set, it will behave as it is now.

> 
> Signed-off-by: Vladyslav Buslov <vladyslav.buslov@harmonicinc.com>
> ---
<...>

Re: [PATCH] kni: add module parameter 'bind_to_core'

[Vladyslav Buslov]

Hi Ferruh,

I agree that your solution is more flexible.
I'll work on moving single thread creation to kni_ioctl_create next week and get back to you with results.

Regards,
Vladyslav

-----Original Message-----
From: Ferruh Yigit [mailto:ferruh.yigit@intel.com] 
Sent: Thursday, August 25, 2016 5:47 PM
To: Vladyslav Buslov
Cc: dev@dpdk.org
Subject: Re: [PATCH] kni: add module parameter 'bind_to_core'

Hi Vladyslav,

On 8/16/2016 7:24 PM, Vladyslav Buslov wrote:
> Allow binding KNI thread to specific core in single threaded mode.

I think this is good idea.
But I am not sure about making this a module parameter, setting core can be more dynamic.

There is already a kni->core_id field, which is only used for multiple_kthread mode.
What do you think moving single thread creation into kni_ioctl_create() and use first kni->core_id to bind the thread? If there is no
kni->core_id set, it will behave as it is now.

> 
> Signed-off-by: Vladyslav Buslov <vladyslav.buslov@harmonicinc.com>
> ---
<...>

[PATCH] kni: add support for core_id param in single threaded mode

[Vladyslav Buslov]

Allow binding KNI thread to specific core in single threaded mode
by setting core_id and force_bind config parameters.

Signed-off-by: Vladyslav Buslov <vladyslav.buslov@harmonicinc.com>
---
 lib/librte_eal/linuxapp/kni/kni_misc.c | 48 ++++++++++++++++++++++------------
 1 file changed, 32 insertions(+), 16 deletions(-)

diff --git a/lib/librte_eal/linuxapp/kni/kni_misc.c b/lib/librte_eal/linuxapp/kni/kni_misc.c
index 59d15ca..5e7cf21 100644
--- a/lib/librte_eal/linuxapp/kni/kni_misc.c
+++ b/lib/librte_eal/linuxapp/kni/kni_misc.c
@@ -30,6 +30,7 @@
 #include <linux/pci.h>
 #include <linux/kthread.h>
 #include <linux/rwsem.h>
+#include <linux/mutex.h>
 #include <linux/nsproxy.h>
 #include <net/net_namespace.h>
 #include <net/netns/generic.h>
@@ -100,6 +101,7 @@ static int kni_net_id;
 
 struct kni_net {
 	unsigned long device_in_use; /* device in use flag */
+	struct mutex kni_kthread_lock;
 	struct task_struct *kni_kthread;
 	struct rw_semaphore kni_list_lock;
 	struct list_head kni_list_head;
@@ -123,6 +125,9 @@ static int __net_init kni_init_net(struct net *net)
 	/* Clear the bit of device in use */
 	clear_bit(KNI_DEV_IN_USE_BIT_NUM, &knet->device_in_use);
 
+	mutex_init(&knet->kni_kthread_lock);
+	knet->kni_kthread = NULL;
+
 	init_rwsem(&knet->kni_list_lock);
 	INIT_LIST_HEAD(&knet->kni_list_head);
 
@@ -139,9 +144,9 @@ static int __net_init kni_init_net(struct net *net)
 
 static void __net_exit kni_exit_net(struct net *net)
 {
-#ifndef HAVE_SIMPLIFIED_PERNET_OPERATIONS
 	struct kni_net *knet = net_generic(net, kni_net_id);
-
+	mutex_destroy(&knet->kni_kthread_lock);
+#ifndef HAVE_SIMPLIFIED_PERNET_OPERATIONS
 	kfree(knet);
 #endif
 }
@@ -236,16 +241,9 @@ kni_open(struct inode *inode, struct file *file)
 		return -EBUSY;
 
 	/* Create kernel thread for single mode */
-	if (multiple_kthread_on == 0) {
+	if (multiple_kthread_on == 0)
 		KNI_PRINT("Single kernel thread for all KNI devices\n");
-		/* Create kernel thread for RX */
-		knet->kni_kthread = kthread_run(kni_thread_single, (void *)knet,
-						"kni_single");
-		if (IS_ERR(knet->kni_kthread)) {
-			KNI_ERR("Unable to create kernel threaed\n");
-			return PTR_ERR(knet->kni_kthread);
-		}
-	} else
+	else
 		KNI_PRINT("Multiple kernel thread mode enabled\n");
 
 	file->private_data = get_net(net);
@@ -263,9 +261,13 @@ kni_release(struct inode *inode, struct file *file)
 
 	/* Stop kernel thread for single mode */
 	if (multiple_kthread_on == 0) {
+		mutex_lock(&knet->kni_kthread_lock);
 		/* Stop kernel thread */
-		kthread_stop(knet->kni_kthread);
-		knet->kni_kthread = NULL;
+		if (knet->kni_kthread != NULL) {
+			kthread_stop(knet->kni_kthread);
+			knet->kni_kthread = NULL;
+		}
+		mutex_unlock(&knet->kni_kthread_lock);
 	}
 
 	down_write(&knet->kni_list_lock);
@@ -415,10 +417,9 @@ kni_ioctl_create(struct net *net,
 	}
 
 	/**
-	 * Check if the cpu core id is valid for binding,
-	 * for multiple kernel thread mode.
+	 * Check if the cpu core id is valid for binding.
 	 */
-	if (multiple_kthread_on && dev_info.force_bind &&
+	if (dev_info.force_bind &&
 				!cpu_online(dev_info.core_id)) {
 		KNI_ERR("cpu %u is not online\n", dev_info.core_id);
 		return -EINVAL;
@@ -581,6 +582,21 @@ kni_ioctl_create(struct net *net,
 		if (dev_info.force_bind)
 			kthread_bind(kni->pthread, kni->core_id);
 		wake_up_process(kni->pthread);
+	} else {
+		mutex_lock(&knet->kni_kthread_lock);
+		if (knet->kni_kthread == NULL) {
+			knet->kni_kthread = kthread_create(kni_thread_single,
+									(void *)knet,
+									"kni_single");
+			if (IS_ERR(knet->kni_kthread)) {
+				kni_dev_remove(kni);
+				return -ECANCELED;
+			}
+			if (dev_info.force_bind)
+				kthread_bind(knet->kni_kthread, kni->core_id);
+			wake_up_process(knet->kni_kthread);
+		}
+		mutex_unlock(&knet->kni_kthread_lock);
 	}
 
 	down_write(&knet->kni_list_lock);
-- 
2.8.3