* [RFC] [PATCH 1/5] padata: generic interface for parallel processing
2009-06-08 7:07 [RFC] [PATCH 0/5] Parallel IPsec v4 Steffen Klassert
@ 2009-06-08 7:08 ` Steffen Klassert
2009-06-08 7:10 ` [RFC] [PATCH 2/5] pcrypt: Add pcrypt crypto parallelization wrapper Steffen Klassert
` (3 subsequent siblings)
4 siblings, 0 replies; 25+ messages in thread
From: Steffen Klassert @ 2009-06-08 7:08 UTC (permalink / raw)
To: Herbert Xu; +Cc: David Miller, linux-crypto
This patch introduces an interface to process data objects
in parallel. On request it is possible to serialize again.
The parallelized objects return after serialization in the
same order as they were before the parallelization.
Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
---
include/linux/interrupt.h | 3 +-
include/linux/padata.h | 116 +++++++++++
kernel/Makefile | 2 +-
kernel/padata.c | 490 +++++++++++++++++++++++++++++++++++++++++++++
kernel/softirq.c | 2 +-
5 files changed, 610 insertions(+), 3 deletions(-)
create mode 100644 include/linux/padata.h
create mode 100644 kernel/padata.c
diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
index 91bb76f..a17679c 100644
--- a/include/linux/interrupt.h
+++ b/include/linux/interrupt.h
@@ -338,7 +338,8 @@ enum
TASKLET_SOFTIRQ,
SCHED_SOFTIRQ,
HRTIMER_SOFTIRQ,
- RCU_SOFTIRQ, /* Preferable RCU should always be the last softirq */
+ PADATA_SOFTIRQ,
+ RCU_SOFTIRQ, /* Preferable RCU should always be the last softirq */
NR_SOFTIRQS
};
diff --git a/include/linux/padata.h b/include/linux/padata.h
new file mode 100644
index 0000000..469359f
--- /dev/null
+++ b/include/linux/padata.h
@@ -0,0 +1,116 @@
+/*
+ * padata.h - header for the padata parallelization interface
+ *
+ * Copyright (C) 2008, 2009 secunet Security Networks AG
+ * Copyright (C) 2008, 2009 Steffen Klassert <steffen.klassert@secunet.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#ifndef PADATA_H
+#define PADATA_H
+
+#include <linux/interrupt.h>
+#include <linux/smp.h>
+#include <linux/list.h>
+
+enum
+{
+ NO_PADATA=0,
+ AEAD_ENC_PADATA,
+ AEAD_DEC_PADATA,
+ NR_PADATA
+};
+
+struct padata_priv {
+ struct list_head list;
+ struct call_single_data csd;
+ int cb_cpu;
+ int seq_nr;
+ unsigned int nr;
+ int info;
+ void (*parallel)(struct padata_priv *padata);
+ void (*serial)(struct padata_priv *padata);
+};
+
+struct padata_queue {
+ struct list_head list;
+ atomic_t num_obj;
+ int cpu_index;
+ spinlock_t lock;
+};
+
+struct parallel_data {
+ struct work_struct work;
+ struct padata_queue *queue;
+ atomic_t seq_nr;
+ atomic_t queued_objects;
+ cpumask_t cpu_map;
+ cpumask_t new_cpu_map;
+ u8 flags;
+#define PADATA_INIT 1
+#define PADATA_FLUSH_HARD 2
+#define PADATA_RESET_IN_PROGRESS 4
+ spinlock_t lock;
+};
+
+#ifdef CONFIG_USE_GENERIC_SMP_HELPERS
+extern void __init padata_init(unsigned int nr, cpumask_t cpu_map);
+extern void padata_dont_wait(unsigned int nr, struct padata_priv *padata);
+extern int padata_do_parallel(unsigned int softirq_nr, unsigned int nr,
+ struct padata_priv *padata, int cb_cpu);
+extern int padata_do_serial(unsigned int nr, struct padata_priv *padata);
+extern cpumask_t padata_get_cpumap(unsigned int nr);
+extern void padata_set_cpumap(unsigned int nr, cpumask_t cpu_map);
+extern void padata_add_cpu(unsigned int nr, int cpu);
+extern void padata_remove_cpu(unsigned int nr, int cpu);
+extern void padata_start(unsigned int nr);
+extern void padata_stop(unsigned int nr);
+#else
+static inline void padata_init(unsigned int nr,cpumask_t cpu_map)
+{
+}
+static inline void padata_dont_wait(unsigned int nr, struct padata_priv *padata)
+{
+}
+static inline int padata_do_parallel(unsigned int softirq_nr, unsigned int nr,
+ struct padata_priv *padata, int cb_cpu)
+{
+ return 0;
+}
+static inline int padata_do_serial(unsigned int nr, struct padata_priv *padata)
+{
+ return 0;
+}
+static inline cpumask_t padata_get_cpumap(unsigned int nr)
+{
+ return cpu_online_map;
+}
+static inline void padata_set_cpumap(unsigned int nr, cpumask_t cpu_map)
+{
+}
+static inline padata_add_cpu(unsigned int nr, int cpu)
+{
+}
+static inline padata_remove_cpu(unsigned int nr, int cpu)
+{
+}
+static inline padata_start(unsigned int nr)
+{
+}
+static inline padata_stop(unsigned int nr)
+{
+}
+#endif
+#endif
diff --git a/kernel/Makefile b/kernel/Makefile
index 4242366..f2d7ea7 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -40,7 +40,7 @@ obj-$(CONFIG_RT_MUTEXES) += rtmutex.o
obj-$(CONFIG_DEBUG_RT_MUTEXES) += rtmutex-debug.o
obj-$(CONFIG_RT_MUTEX_TESTER) += rtmutex-tester.o
obj-$(CONFIG_GENERIC_ISA_DMA) += dma.o
-obj-$(CONFIG_USE_GENERIC_SMP_HELPERS) += smp.o
+obj-$(CONFIG_USE_GENERIC_SMP_HELPERS) += smp.o padata.o
ifneq ($(CONFIG_SMP),y)
obj-y += up.o
endif
diff --git a/kernel/padata.c b/kernel/padata.c
new file mode 100644
index 0000000..192c9a6
--- /dev/null
+++ b/kernel/padata.c
@@ -0,0 +1,490 @@
+/*
+ * padata.c - generic interface to process data streams in parallel
+ *
+ * Copyright (C) 2008, 2009 secunet Security Networks AG
+ * Copyright (C) 2008, 2009 Steffen Klassert <steffen.klassert@secunet.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include <linux/module.h>
+#include <linux/cpumask.h>
+#include <linux/err.h>
+#include <linux/padata.h>
+
+#define MAX_SEQ_NR 1000000000
+
+static struct parallel_data padata_vec[NR_PADATA];
+static struct padata_priv *padata_get_next(struct parallel_data *par_data);
+
+static void padata_flush_hard(struct parallel_data *par_data)
+{
+ int cpu;
+ struct padata_priv *padata;
+ struct padata_queue *queue;
+
+ for_each_cpu_mask(cpu, par_data->cpu_map) {
+ queue = per_cpu_ptr(par_data->queue, cpu);
+
+ while(!list_empty(&queue->list)) {
+ padata = list_entry(queue->list.next, struct padata_priv, list);
+
+ spin_lock(&queue->lock);
+ list_del_init(&padata->list);
+ spin_unlock(&queue->lock);
+
+ atomic_dec(&par_data->queued_objects);
+ padata->serial(padata);
+ }
+ }
+}
+
+static void padata_flush_order(struct parallel_data *par_data)
+{
+ struct padata_priv *padata;
+
+ while (1) {
+ padata = padata_get_next(par_data);
+
+ if (padata && !IS_ERR(padata))
+ padata->serial(padata);
+ else
+ break;
+ }
+
+ padata_flush_hard(par_data);
+}
+
+static void padata_reset_work(struct work_struct *work)
+{
+ int cpu, cpu_index;
+ struct padata_queue *queue;
+ struct parallel_data *par_data;
+
+ par_data = container_of(work, struct parallel_data, work);
+
+ if (par_data->flags & (PADATA_INIT|PADATA_RESET_IN_PROGRESS))
+ return;
+
+ spin_lock_bh(&par_data->lock);
+ par_data->flags |= PADATA_RESET_IN_PROGRESS;
+
+ if (!(par_data->flags & PADATA_FLUSH_HARD))
+ padata_flush_order(par_data);
+ else
+ padata_flush_hard(par_data);
+
+ cpu_index = 0;
+
+ par_data->cpu_map = par_data->new_cpu_map;
+
+ for_each_cpu_mask(cpu, par_data->cpu_map) {
+ queue = per_cpu_ptr(par_data->queue, cpu);
+
+ atomic_set(&queue->num_obj, 0);
+ queue->cpu_index = cpu_index;
+ cpu_index++;
+ }
+ spin_unlock_bh(&par_data->lock);
+
+ atomic_set(&par_data->seq_nr, -1);
+ par_data->flags &= ~PADATA_RESET_IN_PROGRESS;
+ par_data->flags |= PADATA_INIT;
+}
+
+static struct padata_priv *padata_get_next(struct parallel_data *par_data)
+{
+ int cpu, num_cpus, empty;
+ int seq_nr, calc_seq_nr, next_nr;
+ struct padata_queue *queue, *next_queue;
+ struct padata_priv *padata;
+
+ empty = 0;
+ next_nr = -1;
+ next_queue = NULL;
+
+ num_cpus = cpus_weight(par_data->cpu_map);
+
+ for_each_cpu_mask(cpu, par_data->cpu_map) {
+ queue = per_cpu_ptr(par_data->queue, cpu);
+
+ /*
+ * Calculate the seq_nr of the object that should be
+ * next in this queue.
+ */
+ calc_seq_nr = (atomic_read(&queue->num_obj) * num_cpus)
+ + queue->cpu_index;
+
+ if (!list_empty(&queue->list)) {
+ padata = list_entry(queue->list.next,
+ struct padata_priv, list);
+
+ seq_nr = padata->seq_nr;
+
+ if (unlikely(calc_seq_nr != seq_nr)) {
+ par_data->flags &= ~PADATA_INIT;
+ par_data->flags |= PADATA_FLUSH_HARD;
+ padata = NULL;
+ goto out;
+ }
+ } else {
+ seq_nr = calc_seq_nr;
+ empty++;
+ }
+
+ if (next_nr < 0 || seq_nr < next_nr) {
+ next_nr = seq_nr;
+ next_queue = queue;
+ }
+ }
+
+ padata = NULL;
+
+ if (empty == num_cpus)
+ goto out;
+
+ if (!list_empty(&next_queue->list)) {
+ padata = list_entry(next_queue->list.next,
+ struct padata_priv, list);
+
+ spin_lock(&next_queue->lock);
+ list_del_init(&padata->list);
+ spin_unlock(&next_queue->lock);
+
+ atomic_dec(&par_data->queued_objects);
+ atomic_inc(&next_queue->num_obj);
+
+ goto out;
+ }
+
+ if (next_nr % num_cpus == next_queue->cpu_index) {
+ padata = ERR_PTR(-ENODATA);
+ goto out;
+ }
+
+ padata = ERR_PTR(-EINPROGRESS);
+out:
+ return padata;
+}
+
+static void padata_action(struct softirq_action *h)
+{
+ struct list_head *cpu_list, local_list;
+
+ cpu_list = &__get_cpu_var(softirq_work_list[PADATA_SOFTIRQ]);
+
+ local_irq_disable();
+ list_replace_init(cpu_list, &local_list);
+ local_irq_enable();
+
+ while (!list_empty(&local_list)) {
+ struct padata_priv *padata;
+
+ padata = list_entry(local_list.next,
+ struct padata_priv, csd.list);
+
+ list_del_init(&padata->csd.list);
+
+ padata->serial(padata);
+ }
+}
+
+static int padata_cpu_hash(unsigned int nr, struct padata_priv *padata)
+{
+ int cpu, target_cpu, this_cpu, cpu_index;
+
+ this_cpu = smp_processor_id();
+
+ if (padata->nr != 0)
+ return this_cpu;
+
+ if (!(padata_vec[nr].flags & PADATA_INIT))
+ return this_cpu;
+
+ padata->seq_nr = atomic_inc_return(&padata_vec[nr].seq_nr);
+
+ if (padata->seq_nr > MAX_SEQ_NR) {
+ padata_vec[nr].flags &= ~PADATA_INIT;
+ padata->seq_nr = 0;
+ schedule_work(&padata_vec[nr].work);
+ return this_cpu;
+ }
+
+ padata->nr = nr;
+
+ /*
+ * Hash the sequence numbers to the cpus by taking
+ * seq_nr mod. number of cpus in use.
+ */
+ cpu_index = padata->seq_nr % cpus_weight(padata_vec[nr].cpu_map);
+
+ target_cpu = first_cpu(padata_vec[nr].cpu_map);
+ for (cpu = 0; cpu < cpu_index; cpu++)
+ target_cpu = next_cpu(target_cpu, padata_vec[nr].cpu_map);
+
+ return target_cpu;
+}
+
+/*
+ * padata_dont_wait - must be called if an object that runs in parallel will
+ * not be serialized with padata_do_serial
+ *
+ * @nr: number of the padata instance
+ * @padata: object that will not be seen by padata_do_serial
+ */
+void padata_dont_wait(unsigned int nr, struct padata_priv *padata)
+{
+ struct padata_queue *queue;
+
+ if (!(padata_vec[nr].flags & PADATA_INIT))
+ return;
+
+ if (padata->nr == 0 || padata->nr != nr)
+ return;
+
+ queue = per_cpu_ptr(padata_vec[nr].queue, smp_processor_id());
+ atomic_inc(&queue->num_obj);
+
+ padata->nr = 0;
+ padata->seq_nr = 0;
+}
+EXPORT_SYMBOL(padata_dont_wait);
+
+/*
+ * padata_do_parallel - padata parallelization function
+ *
+ * @softirq_nr: number of the softirq that will do the parallelization
+ * @nr: number of the padata instance
+ * @padata: object to be parallelized
+ * @cb_cpu: cpu number on which the serialization callback function will run
+ */
+int padata_do_parallel(unsigned int softirq_nr, unsigned int nr,
+ struct padata_priv *padata, int cb_cpu)
+{
+ int target_cpu;
+
+ padata->cb_cpu = cb_cpu;
+
+ local_bh_disable();
+ target_cpu = padata_cpu_hash(nr, padata);
+ local_bh_enable();
+
+ send_remote_softirq(&padata->csd, target_cpu, softirq_nr);
+
+ return 1;
+}
+EXPORT_SYMBOL(padata_do_parallel);
+
+/*
+ * padata_do_serial - padata serialization function
+ *
+ * @nr: number of the padata instance
+ * @padata: object to be serialized
+ *
+ * returns 1 if the serialization callback function will be called
+ * from padata, 0 else
+ */
+int padata_do_serial(unsigned int nr, struct padata_priv *padata)
+{
+ int cpu;
+ struct padata_queue *reorder_queue;
+
+ if (!(padata_vec[nr].flags & PADATA_INIT))
+ return 0;
+
+ if (padata->nr != nr || padata->nr == 0) {
+ padata->serial(padata);
+ return 1;
+ }
+
+ cpu = smp_processor_id();
+
+ reorder_queue = per_cpu_ptr(padata_vec[nr].queue, cpu);
+
+ spin_lock(&reorder_queue->lock);
+ list_add_tail(&padata->list, &reorder_queue->list);
+ spin_unlock(&reorder_queue->lock);
+
+ atomic_inc(&padata_vec[nr].queued_objects);
+
+try_again:
+ if (!spin_trylock(&padata_vec[nr].lock))
+ goto out;
+
+ while(1) {
+ padata = padata_get_next(&padata_vec[nr]);
+
+ if (!padata || PTR_ERR(padata) == -EINPROGRESS)
+ break;
+ if (PTR_ERR(padata) == -ENODATA) {
+ spin_unlock(&padata_vec[nr].lock);
+ goto out;
+ }
+
+ send_remote_softirq(&padata->csd, padata->cb_cpu,
+ PADATA_SOFTIRQ);
+ }
+
+ if (unlikely(!(padata_vec[nr].flags & PADATA_INIT))) {
+ spin_unlock(&padata_vec[nr].lock);
+ goto reset_out;
+ }
+
+ spin_unlock(&padata_vec[nr].lock);
+
+ if (atomic_read(&padata_vec[nr].queued_objects))
+ goto try_again;
+
+out:
+ return 1;
+reset_out:
+ schedule_work(&padata_vec[nr].work);
+ return 1;
+}
+EXPORT_SYMBOL(padata_do_serial);
+
+/*
+ * padata_get_cpumap - get the cpu map that is actually in use
+ *
+ * @nr: number of the padata instance
+ */
+cpumask_t padata_get_cpumap(unsigned int nr)
+{
+ return padata_vec[nr].cpu_map;
+}
+EXPORT_SYMBOL(padata_get_cpumap);
+
+/*
+ * padata_set_cpumap - set the cpu map that padata uses
+ *
+ * @nr: number of the padata instance
+ * @cpu_map: the cpu map to use
+ */
+void padata_set_cpumap(unsigned int nr, cpumask_t cpu_map)
+{
+ padata_vec[nr].new_cpu_map = cpu_map;
+ padata_vec[nr].flags &= ~PADATA_INIT;
+ padata_vec[nr].flags |= PADATA_FLUSH_HARD;
+
+ schedule_work(&padata_vec[nr].work);
+}
+EXPORT_SYMBOL(padata_set_cpumap);
+
+/*
+ * padata_add_cpu - add a cpu to the padata cpu map
+ *
+ * @nr: number of the padata instance
+ * @cpu: cpu to remove
+ */
+void padata_add_cpu(unsigned int nr, int cpu)
+{
+ cpumask_t cpu_map = padata_vec[nr].cpu_map;
+
+ cpu_set(cpu, cpu_map);
+ padata_set_cpumap(nr, cpu_map);
+}
+EXPORT_SYMBOL(padata_add_cpu);
+
+/*
+ * padata_remove_cpu - remove a cpu from the padata cpu map
+ *
+ * @nr: number of the padata instance
+ * @cpu: cpu to remove
+ */
+void padata_remove_cpu(unsigned int nr, int cpu)
+{
+ cpumask_t cpu_map = padata_vec[nr].cpu_map;
+
+ cpu_clear(cpu, cpu_map);
+ padata_set_cpumap(nr, cpu_map);
+}
+EXPORT_SYMBOL(padata_remove_cpu);
+
+/*
+ * padata_start - start the parallel processing
+ *
+ * @nr: number of the padata instance
+ */
+void padata_start(unsigned int nr)
+{
+ if (padata_vec[nr].flags & PADATA_INIT)
+ return;
+
+ schedule_work(&padata_vec[nr].work);
+}
+EXPORT_SYMBOL(padata_start);
+
+/*
+ * padata_stop - stop the parallel processing
+ *
+ * @nr: number of the padata instance
+ */
+void padata_stop(unsigned int nr)
+{
+ padata_vec[nr].flags &= ~PADATA_INIT;
+}
+EXPORT_SYMBOL(padata_stop);
+
+/*
+ * padata_init - initialize a padata instance
+ *
+ * @nr: number of the padata instance
+ * @cpu_map: map of the cpu set that padata uses for parallelization
+ */
+void __init padata_init(unsigned int nr, cpumask_t cpu_map)
+{
+ int cpu, cpu_index;
+ struct padata_queue *percpu_queue, *queue;
+
+ percpu_queue = alloc_percpu(struct padata_queue);
+
+ if (!percpu_queue) {
+ printk("padata_init: Failed to alloc the serialization"
+ "queues for padata nr %d, exiting!\n", nr);
+ return;
+ }
+
+ cpu_index = 0;
+
+ for_each_possible_cpu(cpu) {
+ queue = per_cpu_ptr(percpu_queue, cpu);
+
+ if (cpu_isset(cpu, cpu_map)) {
+ queue->cpu_index = cpu_index;
+ cpu_index++;
+ }
+
+ INIT_LIST_HEAD(&queue->list);
+ spin_lock_init(&queue->lock);
+ atomic_set(&queue->num_obj, 0);
+ }
+
+ INIT_WORK(&padata_vec[nr].work, padata_reset_work);
+
+ atomic_set(&padata_vec[nr].seq_nr, -1);
+ atomic_set(&padata_vec[nr].queued_objects, 0);
+ padata_vec[nr].cpu_map = cpu_map;
+ padata_vec[nr].new_cpu_map = cpu_map;
+ padata_vec[nr].queue = percpu_queue;
+ padata_vec[nr].flags = 0;
+ spin_lock_init(&padata_vec[nr].lock);
+}
+
+static int __init padata_initcall(void)
+{
+ open_softirq(PADATA_SOFTIRQ, padata_action);
+
+ return 0;
+}
+subsys_initcall(padata_initcall);
diff --git a/kernel/softirq.c b/kernel/softirq.c
index b525dd3..12c9b64 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -56,7 +56,7 @@ static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
char *softirq_to_name[NR_SOFTIRQS] = {
"HI", "TIMER", "NET_TX", "NET_RX", "BLOCK",
- "TASKLET", "SCHED", "HRTIMER", "RCU"
+ "TASKLET", "SCHED", "HRTIMER", "PADATA", "RCU"
};
/*
--
1.5.4.2
^ permalink raw reply related [flat|nested] 25+ messages in thread* [RFC] [PATCH 2/5] pcrypt: Add pcrypt crypto parallelization wrapper
2009-06-08 7:07 [RFC] [PATCH 0/5] Parallel IPsec v4 Steffen Klassert
2009-06-08 7:08 ` [RFC] [PATCH 1/5] padata: generic interface for parallel processing Steffen Klassert
@ 2009-06-08 7:10 ` Steffen Klassert
2009-06-19 12:21 ` Herbert Xu
2009-06-19 12:29 ` Herbert Xu
2009-06-08 7:10 ` [RFC] [PATCH 3/5] eseqiv: Add support for aead algorithms Steffen Klassert
` (2 subsequent siblings)
4 siblings, 2 replies; 25+ messages in thread
From: Steffen Klassert @ 2009-06-08 7:10 UTC (permalink / raw)
To: Herbert Xu; +Cc: David Miller, linux-crypto
This patch adds a parallel crypto template that takes a crypto
algorithm and converts it to process the crypto transforms in
parallel. For the moment only aead is supported.
Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
---
crypto/Kconfig | 13 ++
crypto/Makefile | 2 +
crypto/pcrypt.c | 413 +++++++++++++++++++++++++++++++++++++++++++++
crypto/pcrypt_core.c | 125 ++++++++++++++
include/crypto/pcrypt.h | 85 +++++++++
include/linux/interrupt.h | 2 +
6 files changed, 640 insertions(+), 0 deletions(-)
create mode 100644 crypto/pcrypt.c
create mode 100644 crypto/pcrypt_core.c
create mode 100644 include/crypto/pcrypt.h
diff --git a/crypto/Kconfig b/crypto/Kconfig
index 74d0e62..b05fc95 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -112,6 +112,19 @@ config CRYPTO_NULL
help
These are 'Null' algorithms, used by IPsec, which do nothing.
+config CRYPTO_PCRYPT_CORE
+ bool
+ select CRYPTO_AEAD
+
+config CRYPTO_PCRYPT
+ tristate "Parallel crypto engine (EXPERIMENTAL)"
+ depends on USE_GENERIC_SMP_HELPERS && EXPERIMENTAL
+ select CRYPTO_MANAGER
+ select CRYPTO_PCRYPT_CORE
+ help
+ This converts an arbitrary crypto algorithm into a parallel
+ algorithm that is executed in a softirq.
+
config CRYPTO_WORKQUEUE
tristate
diff --git a/crypto/Makefile b/crypto/Makefile
index 673d9f7..84b9d17 100644
--- a/crypto/Makefile
+++ b/crypto/Makefile
@@ -56,6 +56,8 @@ obj-$(CONFIG_CRYPTO_XTS) += xts.o
obj-$(CONFIG_CRYPTO_CTR) += ctr.o
obj-$(CONFIG_CRYPTO_GCM) += gcm.o
obj-$(CONFIG_CRYPTO_CCM) += ccm.o
+obj-$(CONFIG_CRYPTO_PCRYPT_CORE) += pcrypt_core.o
+obj-$(CONFIG_CRYPTO_PCRYPT) += pcrypt.o
obj-$(CONFIG_CRYPTO_CRYPTD) += cryptd.o
obj-$(CONFIG_CRYPTO_DES) += des_generic.o
obj-$(CONFIG_CRYPTO_FCRYPT) += fcrypt.o
diff --git a/crypto/pcrypt.c b/crypto/pcrypt.c
new file mode 100644
index 0000000..cc960df
--- /dev/null
+++ b/crypto/pcrypt.c
@@ -0,0 +1,413 @@
+/*
+ * pcrypt - Parallel crypto wrapper.
+ *
+ * Copyright (C) 2009 secunet Security Networks AG
+ * Copyright (C) 2009 Steffen Klassert <steffen.klassert@secunet.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include <crypto/algapi.h>
+#include <crypto/internal/aead.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <crypto/pcrypt.h>
+
+struct pcrypt_instance_ctx {
+ struct crypto_spawn spawn;
+ unsigned int tfm_count;
+};
+
+struct pcrypt_aead_ctx {
+ struct crypto_aead *child;
+ unsigned int tfm_nr;
+};
+
+static int pcrypt_do_parallel(struct padata_priv *padata, unsigned int tfm_nr,
+ unsigned int softirq, unsigned int padata_nr)
+{
+ unsigned int cpu, cpu_index, num_cpus, cb_cpu;
+ cpumask_t cpu_map;
+
+ cpu_map = padata_get_cpumap(padata_nr);
+ num_cpus = cpus_weight(cpu_map);
+
+ cpu_index = tfm_nr % num_cpus;
+
+ cb_cpu = first_cpu(cpu_map);
+ for (cpu = 0; cpu < cpu_index; cpu++)
+ cb_cpu = next_cpu(cb_cpu, cpu_map);
+
+ return padata_do_parallel(softirq, padata_nr, padata, cb_cpu);
+}
+
+static int pcrypt_aead_setkey(struct crypto_aead *parent,
+ const u8 *key, unsigned int keylen)
+{
+ struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(parent);
+
+ return crypto_aead_setkey(ctx->child, key, keylen);
+}
+
+static int pcrypt_aead_setauthsize(struct crypto_aead *parent,
+ unsigned int authsize)
+{
+ struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(parent);
+
+ return crypto_aead_setauthsize(ctx->child, authsize);
+}
+
+static void pcrypt_aead_serial(struct padata_priv *padata)
+{
+ struct pcrypt_request *preq = pcrypt_padata_request(padata);
+ struct aead_request *req = pcrypt_request_ctx(preq);
+
+ aead_request_complete(req->base.data, padata->info);
+}
+
+static void pcrypt_aead_giv_serial(struct padata_priv *padata)
+{
+ struct pcrypt_request *preq = pcrypt_padata_request(padata);
+ struct aead_givcrypt_request *req = pcrypt_request_ctx(preq);
+
+ aead_request_complete(req->areq.base.data, padata->info);
+}
+
+static void pcrypt_aead_done(struct crypto_async_request *areq, int err)
+{
+ struct aead_request *req = areq->data;
+ struct pcrypt_request *preq = aead_request_ctx(req);
+ struct padata_priv *padata = pcrypt_request_padata(preq);
+
+ padata->info = err;
+ req->base.flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
+
+ local_bh_disable();
+ if (padata_do_serial(padata->nr, padata))
+ goto out;
+
+ aead_request_complete(req, padata->info);
+
+out:
+ local_bh_enable();
+}
+
+static void pcrypt_aead_enc(struct padata_priv *padata)
+{
+ struct pcrypt_request *preq = pcrypt_padata_request(padata);
+ struct aead_request *req = pcrypt_request_ctx(preq);
+
+ padata->info = crypto_aead_encrypt(req);
+
+ if (padata->info)
+ return;
+
+ if (padata_do_serial(AEAD_ENC_PADATA, padata))
+ return;
+
+ aead_request_complete(req->base.data, padata->info);
+}
+
+static int pcrypt_aead_encrypt(struct aead_request *req)
+{
+ int err;
+ struct pcrypt_request *preq = aead_request_ctx(req);
+ struct aead_request *creq = pcrypt_request_ctx(preq);
+ struct padata_priv *padata = pcrypt_request_padata(preq);
+ struct crypto_aead *aead = crypto_aead_reqtfm(req);
+ struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead);
+ u32 flags = aead_request_flags(req);
+
+ memset(padata, 0, sizeof(struct padata_priv));
+
+ padata->parallel = pcrypt_aead_enc;
+ padata->serial = pcrypt_aead_serial;
+
+ aead_request_set_tfm(creq, ctx->child);
+ aead_request_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP,
+ pcrypt_aead_done, req);
+ aead_request_set_crypt(creq, req->src, req->dst,
+ req->cryptlen, req->iv);
+ aead_request_set_assoc(creq, req->assoc, req->assoclen);
+
+ if (pcrypt_do_parallel(padata, ctx->tfm_nr, AEAD_ENC_SOFTIRQ,
+ AEAD_ENC_PADATA))
+ err = -EINPROGRESS;
+ else
+ err = crypto_aead_encrypt(creq);
+
+ return err;
+}
+
+static void pcrypt_aead_dec(struct padata_priv *padata)
+{
+ struct pcrypt_request *preq = pcrypt_padata_request(padata);
+ struct aead_request *req = pcrypt_request_ctx(preq);
+
+ padata->info = crypto_aead_decrypt(req);
+
+ if (padata->info)
+ return;
+
+ if (padata_do_serial(AEAD_DEC_PADATA, padata))
+ return;
+
+ aead_request_complete(req->base.data, padata->info);
+}
+
+static int pcrypt_aead_decrypt(struct aead_request *req)
+{
+ int err;
+ struct pcrypt_request *preq = aead_request_ctx(req);
+ struct aead_request *creq = pcrypt_request_ctx(preq);
+ struct padata_priv *padata = pcrypt_request_padata(preq);
+ struct crypto_aead *aead = crypto_aead_reqtfm(req);
+ struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead);
+ u32 flags = aead_request_flags(req);
+
+ memset(padata, 0, sizeof(struct padata_priv));
+
+ padata->parallel = pcrypt_aead_dec;
+ padata->serial = pcrypt_aead_serial;
+
+ aead_request_set_tfm(creq, ctx->child);
+ aead_request_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP,
+ pcrypt_aead_done, req);
+ aead_request_set_crypt(creq, req->src, req->dst,
+ req->cryptlen, req->iv);
+ aead_request_set_assoc(creq, req->assoc, req->assoclen);
+
+ if (pcrypt_do_parallel(padata, ctx->tfm_nr, AEAD_DEC_SOFTIRQ,
+ AEAD_DEC_PADATA))
+ err = -EINPROGRESS;
+ else
+ err = crypto_aead_decrypt(creq);
+
+ return err;
+}
+
+static void pcrypt_aead_givenc(struct padata_priv *padata)
+{
+ struct pcrypt_request *preq = pcrypt_padata_request(padata);
+ struct aead_givcrypt_request *req = pcrypt_request_ctx(preq);
+
+ padata->info = crypto_aead_givencrypt(req);
+
+ if (padata->info)
+ return;
+
+ if (padata_do_serial(AEAD_ENC_PADATA, padata))
+ return;
+
+ aead_request_complete(req->areq.base.data, padata->info);
+}
+
+static int pcrypt_aead_givencrypt(struct aead_givcrypt_request *req)
+{
+ int err;
+ struct aead_request *areq = &req->areq;
+ struct pcrypt_request *preq = aead_request_ctx(areq);
+ struct aead_givcrypt_request *creq = pcrypt_request_ctx(preq);
+ struct padata_priv *padata = pcrypt_request_padata(preq);
+ struct crypto_aead *aead = aead_givcrypt_reqtfm(req);
+ struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead);
+ u32 flags = aead_request_flags(areq);
+
+ memset(padata, 0, sizeof(struct padata_priv));
+
+ padata->parallel = pcrypt_aead_givenc;
+ padata->serial = pcrypt_aead_giv_serial;
+
+ aead_givcrypt_set_tfm(creq, ctx->child);
+ aead_givcrypt_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP,
+ pcrypt_aead_done, areq);
+ aead_givcrypt_set_crypt(creq, areq->src, areq->dst,
+ areq->cryptlen, areq->iv);
+ aead_givcrypt_set_assoc(creq, areq->assoc, areq->assoclen);
+ aead_givcrypt_set_giv(creq, req->giv, req->seq);
+
+
+ if (pcrypt_do_parallel(padata, ctx->tfm_nr, AEAD_ENC_SOFTIRQ,
+ AEAD_ENC_PADATA))
+ err = -EINPROGRESS;
+ else
+ err = crypto_aead_givencrypt(creq);
+
+ return err;
+}
+
+static int pcrypt_aead_init_tfm(struct crypto_tfm *tfm)
+{
+ struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
+ struct pcrypt_instance_ctx *ictx = crypto_instance_ctx(inst);
+ struct pcrypt_aead_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct crypto_aead *cipher;
+
+ ictx->tfm_count++;
+ ctx->tfm_nr = ictx->tfm_count;
+
+ cipher = crypto_spawn_aead(crypto_instance_ctx(inst));
+
+ if (IS_ERR(cipher))
+ return PTR_ERR(cipher);
+
+ ctx->child = cipher;
+ tfm->crt_aead.reqsize = sizeof(struct pcrypt_request)
+ + sizeof(struct aead_givcrypt_request)
+ + crypto_aead_reqsize(cipher);
+
+ return 0;
+}
+
+static void pcrypt_aead_exit_tfm(struct crypto_tfm *tfm)
+{
+ struct pcrypt_aead_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ crypto_free_aead(ctx->child);
+}
+
+static struct crypto_instance *pcrypt_alloc_instance(struct crypto_alg *alg)
+{
+ struct crypto_instance *inst;
+ struct pcrypt_instance_ctx *ctx;
+ int err;
+
+ inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
+ if (!inst) {
+ inst = ERR_PTR(-ENOMEM);
+ goto out;
+ }
+
+ err = -ENAMETOOLONG;
+ if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
+ "pcrypt(%s)", alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
+ goto out_free_inst;
+
+ memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME);
+
+ ctx = crypto_instance_ctx(inst);
+ err = crypto_init_spawn(&ctx->spawn, alg, inst,
+ CRYPTO_ALG_TYPE_MASK);
+ if (err)
+ goto out_free_inst;
+
+ inst->alg.cra_priority = alg->cra_priority + 100;
+ inst->alg.cra_blocksize = alg->cra_blocksize;
+ inst->alg.cra_alignmask = alg->cra_alignmask;
+
+out:
+ return inst;
+
+out_free_inst:
+ kfree(inst);
+ inst = ERR_PTR(err);
+ goto out;
+}
+
+static struct crypto_instance *pcrypt_alloc_aead(struct rtattr **tb)
+{
+ struct crypto_instance *inst;
+ struct crypto_alg *alg;
+ struct crypto_attr_type *algt;
+
+ algt = crypto_get_attr_type(tb);
+
+ alg = crypto_get_attr_alg(tb, algt->type,
+ (algt->mask & CRYPTO_ALG_TYPE_MASK));
+ if (IS_ERR(alg))
+ return ERR_CAST(alg);
+
+ inst = pcrypt_alloc_instance(alg);
+ if (IS_ERR(inst))
+ goto out_put_alg;
+
+ inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC;
+ inst->alg.cra_type = &crypto_nivaead_type;
+
+ inst->alg.cra_aead.ivsize = alg->cra_aead.ivsize;
+ inst->alg.cra_aead.geniv = alg->cra_aead.geniv;
+ inst->alg.cra_aead.maxauthsize = alg->cra_aead.maxauthsize;
+
+ inst->alg.cra_ctxsize = sizeof(struct pcrypt_aead_ctx);
+
+ inst->alg.cra_init = pcrypt_aead_init_tfm;
+ inst->alg.cra_exit = pcrypt_aead_exit_tfm;
+
+ inst->alg.cra_aead.setkey = pcrypt_aead_setkey;
+ inst->alg.cra_aead.setauthsize = pcrypt_aead_setauthsize;
+ inst->alg.cra_aead.encrypt = pcrypt_aead_encrypt;
+ inst->alg.cra_aead.decrypt = pcrypt_aead_decrypt;
+ inst->alg.cra_aead.givencrypt = pcrypt_aead_givencrypt;
+
+ inst->alg.cra_aead.geniv = "eseqiv";
+
+out_put_alg:
+ crypto_mod_put(alg);
+ return inst;
+}
+
+static struct crypto_instance *pcrypt_alloc(struct rtattr **tb)
+{
+ struct crypto_attr_type *algt;
+
+ algt = crypto_get_attr_type(tb);
+ if (IS_ERR(algt))
+ return ERR_CAST(algt);
+
+ switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) {
+ case CRYPTO_ALG_TYPE_AEAD:
+ return pcrypt_alloc_aead(tb);
+ }
+
+ return ERR_PTR(-EINVAL);
+}
+
+static void pcrypt_free(struct crypto_instance *inst)
+{
+ struct pcrypt_instance_ctx *ctx = crypto_instance_ctx(inst);
+
+ crypto_drop_spawn(&ctx->spawn);
+ kfree(inst);
+}
+
+static struct crypto_template pcrypt_tmpl = {
+ .name = "pcrypt",
+ .alloc = pcrypt_alloc,
+ .free = pcrypt_free,
+ .module = THIS_MODULE,
+};
+
+static int __init pcrypt_init(void)
+{
+ padata_start(AEAD_ENC_PADATA);
+ padata_start(AEAD_DEC_PADATA);
+
+ return crypto_register_template(&pcrypt_tmpl);
+}
+
+static void __exit pcrypt_exit(void)
+{
+ padata_stop(AEAD_ENC_PADATA);
+ padata_stop(AEAD_DEC_PADATA);
+
+ crypto_unregister_template(&pcrypt_tmpl);
+}
+
+module_init(pcrypt_init);
+module_exit(pcrypt_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Parallel crypto engine");
diff --git a/crypto/pcrypt_core.c b/crypto/pcrypt_core.c
new file mode 100644
index 0000000..7c807e0
--- /dev/null
+++ b/crypto/pcrypt_core.c
@@ -0,0 +1,125 @@
+/*
+ * pcrypt_core.c - Core functions for the pcrypt crypto parallelization
+ *
+ * Copyright (C) 2009 secunet Security Networks AG
+ * Copyright (C) 2009 Steffen Klassert <steffen.klassert@secunet.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include <linux/interrupt.h>
+#include <linux/cpu.h>
+#include <linux/err.h>
+#include <linux/module.h>
+#include <crypto/pcrypt.h>
+
+struct crypto_aead *pcrypt_alloc_aead_tfm(const char *alg_name,
+ u32 type, u32 mask)
+{
+ char pcrypt_alg_name[CRYPTO_MAX_ALG_NAME];
+ struct crypto_aead *tfm;
+
+ if (snprintf(pcrypt_alg_name, CRYPTO_MAX_ALG_NAME,
+ "pcrypt(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
+ return ERR_PTR(-EINVAL);
+
+ tfm = crypto_alloc_aead(pcrypt_alg_name, type, mask);
+
+ if (IS_ERR(tfm))
+ return ERR_CAST(tfm);
+
+ return tfm;
+}
+EXPORT_SYMBOL_GPL(pcrypt_alloc_aead_tfm);
+
+static void aead_enc_action(struct softirq_action *h)
+{
+ struct list_head *cpu_list, local_list;
+
+ cpu_list = &__get_cpu_var(softirq_work_list[AEAD_ENC_SOFTIRQ]);
+
+ local_irq_disable();
+ list_replace_init(cpu_list, &local_list);
+ local_irq_enable();
+
+ while (!list_empty(&local_list)) {
+ struct padata_priv *padata;
+
+ padata = list_entry(local_list.next, struct padata_priv,
+ csd.list);
+
+ list_del_init(&padata->csd.list);
+
+ padata->parallel(padata);
+ }
+}
+
+static void aead_dec_action(struct softirq_action *h)
+{
+ struct list_head *cpu_list, local_list;
+
+ cpu_list = &__get_cpu_var(softirq_work_list[AEAD_DEC_SOFTIRQ]);
+
+ local_irq_disable();
+ list_replace_init(cpu_list, &local_list);
+ local_irq_enable();
+
+ while (!list_empty(&local_list)) {
+ struct padata_priv *padata;
+
+ padata = list_entry(local_list.next, struct padata_priv,
+ csd.list);
+
+ list_del_init(&padata->csd.list);
+
+ padata->parallel(padata);
+ }
+}
+
+static int __devinit pcrypt_cpu_callback(struct notifier_block *nfb,
+ unsigned long action, void *hcpu)
+{
+ int cpu = (unsigned long)hcpu;
+
+ switch (action) {
+ case CPU_ONLINE:
+ case CPU_ONLINE_FROZEN:
+ padata_add_cpu(AEAD_ENC_PADATA, cpu);
+ padata_add_cpu(AEAD_DEC_PADATA, cpu);
+ break;
+
+ case CPU_DEAD:
+ case CPU_DEAD_FROZEN:
+ padata_remove_cpu(AEAD_ENC_PADATA, cpu);
+ padata_remove_cpu(AEAD_DEC_PADATA, cpu);
+
+ break;
+ }
+
+ return NOTIFY_OK;
+}
+
+static int __init pcrypt_init_padata(void)
+{
+ open_softirq(AEAD_ENC_SOFTIRQ, aead_enc_action);
+ open_softirq(AEAD_DEC_SOFTIRQ, aead_dec_action);
+
+ padata_init(AEAD_ENC_PADATA, cpu_online_map);
+ padata_init(AEAD_DEC_PADATA, cpu_online_map);
+
+ hotcpu_notifier(pcrypt_cpu_callback, 0);
+
+ return 0;
+}
+subsys_initcall(pcrypt_init_padata);
diff --git a/include/crypto/pcrypt.h b/include/crypto/pcrypt.h
new file mode 100644
index 0000000..9cea12b
--- /dev/null
+++ b/include/crypto/pcrypt.h
@@ -0,0 +1,85 @@
+/*
+ * pcrypt - Parallel crypto engine.
+ *
+ * Copyright (C) 2009 secunet Security Networks AG
+ * Copyright (C) 2009 Steffen Klassert <steffen.klassert@secunet.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#ifndef _CRYPTO_PCRYPT_H
+#define _CRYPTO_PCRYPT_H
+
+#include <linux/crypto.h>
+#include <linux/kernel.h>
+#include <linux/padata.h>
+
+struct pcrypt_request {
+ struct padata_priv padata;
+ void *data;
+ void *__ctx[] CRYPTO_MINALIGN_ATTR;
+};
+
+static inline void *pcrypt_request_ctx(struct pcrypt_request *req)
+{
+ return req->__ctx;
+}
+
+static inline
+struct padata_priv *pcrypt_request_padata(struct pcrypt_request *req)
+{
+ return &req->padata;
+}
+
+static inline
+struct pcrypt_request *pcrypt_padata_request(struct padata_priv *padata)
+{
+ return container_of(padata, struct pcrypt_request, padata);
+}
+
+struct crypto_aead *pcrypt_alloc_aead_tfm(const char *alg_name, u32 type,
+ u32 mask);
+
+struct crypto_ablkcipher *pcrypt_alloc_ablkcipher_tfm(const char *alg_name,
+ u32 type, u32 mask);
+
+#ifdef CONFIG_CRYPTO_PCRYPT_CORE
+static inline struct crypto_aead *crypto_alloc_aead_tfm(const char *alg_name,
+ u32 type, u32 mask)
+{
+ return pcrypt_alloc_aead_tfm(alg_name, type, mask);
+}
+
+static inline
+struct crypto_ablkcipher *crypto_alloc_ablkcipher_tfm(const char *alg_name,
+ u32 type, u32 mask)
+{
+ return pcrypt_alloc_ablkcipher_tfm(alg_name, type, mask);
+}
+#else
+static inline struct crypto_aead *crypto_alloc_aead_tfm(const char *alg_name,
+ u32 type, u32 mask)
+{
+ return crypto_alloc_aead(alg_name, type, mask);
+}
+
+static inline
+struct crypto_ablkcipher *crypto_alloc_ablkcipher_tfm(const char *alg_name,
+ u32 type, u32 mask)
+{
+ return crypto_alloc_ablkcipher(alg_name, type, mask);
+}
+#endif
+
+#endif
diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
index a17679c..93e0c9f 100644
--- a/include/linux/interrupt.h
+++ b/include/linux/interrupt.h
@@ -338,6 +338,8 @@ enum
TASKLET_SOFTIRQ,
SCHED_SOFTIRQ,
HRTIMER_SOFTIRQ,
+ AEAD_ENC_SOFTIRQ,
+ AEAD_DEC_SOFTIRQ,
PADATA_SOFTIRQ,
RCU_SOFTIRQ, /* Preferable RCU should always be the last softirq */
--
1.5.4.2
^ permalink raw reply related [flat|nested] 25+ messages in thread* Re: [RFC] [PATCH 2/5] pcrypt: Add pcrypt crypto parallelization wrapper
2009-06-08 7:10 ` [RFC] [PATCH 2/5] pcrypt: Add pcrypt crypto parallelization wrapper Steffen Klassert
@ 2009-06-19 12:21 ` Herbert Xu
2009-06-23 6:21 ` Steffen Klassert
2009-06-19 12:29 ` Herbert Xu
1 sibling, 1 reply; 25+ messages in thread
From: Herbert Xu @ 2009-06-19 12:21 UTC (permalink / raw)
To: Steffen Klassert; +Cc: David Miller, linux-crypto
On Mon, Jun 08, 2009 at 09:10:00AM +0200, Steffen Klassert wrote:
>
> +config CRYPTO_PCRYPT_CORE
> + bool
> + select CRYPTO_AEAD
BTW just noticed that this forces AEAD to be built-in even when
pcrypt is a module. We should fix this up.
Thanks,
--
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
^ permalink raw reply [flat|nested] 25+ messages in thread* Re: [RFC] [PATCH 2/5] pcrypt: Add pcrypt crypto parallelization wrapper
2009-06-19 12:21 ` Herbert Xu
@ 2009-06-23 6:21 ` Steffen Klassert
0 siblings, 0 replies; 25+ messages in thread
From: Steffen Klassert @ 2009-06-23 6:21 UTC (permalink / raw)
To: Herbert Xu; +Cc: David Miller, linux-crypto
On Fri, Jun 19, 2009 at 08:21:13PM +0800, Herbert Xu wrote:
> On Mon, Jun 08, 2009 at 09:10:00AM +0200, Steffen Klassert wrote:
> >
> > +config CRYPTO_PCRYPT_CORE
> > + bool
> > + select CRYPTO_AEAD
>
> BTW just noticed that this forces AEAD to be built-in even when
> pcrypt is a module. We should fix this up.
>
Yes, I'll change this with the next version.
^ permalink raw reply [flat|nested] 25+ messages in thread
* Re: [RFC] [PATCH 2/5] pcrypt: Add pcrypt crypto parallelization wrapper
2009-06-08 7:10 ` [RFC] [PATCH 2/5] pcrypt: Add pcrypt crypto parallelization wrapper Steffen Klassert
2009-06-19 12:21 ` Herbert Xu
@ 2009-06-19 12:29 ` Herbert Xu
2009-06-23 8:29 ` Steffen Klassert
1 sibling, 1 reply; 25+ messages in thread
From: Herbert Xu @ 2009-06-19 12:29 UTC (permalink / raw)
To: Steffen Klassert; +Cc: David Miller, linux-crypto
On Mon, Jun 08, 2009 at 09:10:00AM +0200, Steffen Klassert wrote:
>
> +config CRYPTO_PCRYPT
> + tristate "Parallel crypto engine (EXPERIMENTAL)"
> + depends on USE_GENERIC_SMP_HELPERS && EXPERIMENTAL
> + select CRYPTO_MANAGER
> + select CRYPTO_PCRYPT_CORE
> + help
> + This converts an arbitrary crypto algorithm into a parallel
> + algorithm that is executed in a softirq.
BTW, do we really need softirqs for these things? Cryptographic
operations are typically slow (especially when you have to move
them to other CPUs to handle a single NIC:) so if we're going to
the trouble of making it async then we should put them in work
queues instead.
Cheers,
--
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
^ permalink raw reply [flat|nested] 25+ messages in thread* Re: [RFC] [PATCH 2/5] pcrypt: Add pcrypt crypto parallelization wrapper
2009-06-19 12:29 ` Herbert Xu
@ 2009-06-23 8:29 ` Steffen Klassert
2009-06-23 8:34 ` Herbert Xu
0 siblings, 1 reply; 25+ messages in thread
From: Steffen Klassert @ 2009-06-23 8:29 UTC (permalink / raw)
To: Herbert Xu; +Cc: David Miller, linux-crypto
On Fri, Jun 19, 2009 at 08:29:33PM +0800, Herbert Xu wrote:
> On Mon, Jun 08, 2009 at 09:10:00AM +0200, Steffen Klassert wrote:
> >
> > +config CRYPTO_PCRYPT
> > + tristate "Parallel crypto engine (EXPERIMENTAL)"
> > + depends on USE_GENERIC_SMP_HELPERS && EXPERIMENTAL
> > + select CRYPTO_MANAGER
> > + select CRYPTO_PCRYPT_CORE
> > + help
> > + This converts an arbitrary crypto algorithm into a parallel
> > + algorithm that is executed in a softirq.
>
> BTW, do we really need softirqs for these things? Cryptographic
> operations are typically slow (especially when you have to move
> them to other CPUs to handle a single NIC:) so if we're going to
> the trouble of making it async then we should put them in work
> queues instead.
I did already tests with a work queue based parallelization, but the
throughput and the latency on networking tests were much better with the
remote softirq based version. The serialization engine must always wait
for the cpu that processes the 'next' packet to keep the packets in the
right order. And softirqs are a good way to ensure that the packets are
processed as soon as possible.
Anyway, we can always use pcrypt(cryptd(..)) combinations if we need to
make the request async.
^ permalink raw reply [flat|nested] 25+ messages in thread
* Re: [RFC] [PATCH 2/5] pcrypt: Add pcrypt crypto parallelization wrapper
2009-06-23 8:29 ` Steffen Klassert
@ 2009-06-23 8:34 ` Herbert Xu
2009-06-23 9:14 ` Steffen Klassert
0 siblings, 1 reply; 25+ messages in thread
From: Herbert Xu @ 2009-06-23 8:34 UTC (permalink / raw)
To: Steffen Klassert; +Cc: David Miller, linux-crypto
On Tue, Jun 23, 2009 at 10:29:21AM +0200, Steffen Klassert wrote:
>
> I did already tests with a work queue based parallelization, but the
> throughput and the latency on networking tests were much better with the
> remote softirq based version. The serialization engine must always wait
> for the cpu that processes the 'next' packet to keep the packets in the
> right order. And softirqs are a good way to ensure that the packets are
> processed as soon as possible.
OK, do you have some numbers that we can add to the patches?
Also note that the real time folks are turning softirq into threads
so this complexity might be moot in the end.
Cheers,
--
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
^ permalink raw reply [flat|nested] 25+ messages in thread* Re: [RFC] [PATCH 2/5] pcrypt: Add pcrypt crypto parallelization wrapper
2009-06-23 8:34 ` Herbert Xu
@ 2009-06-23 9:14 ` Steffen Klassert
2009-06-23 9:18 ` Herbert Xu
0 siblings, 1 reply; 25+ messages in thread
From: Steffen Klassert @ 2009-06-23 9:14 UTC (permalink / raw)
To: Herbert Xu; +Cc: David Miller, linux-crypto
On Tue, Jun 23, 2009 at 04:34:38PM +0800, Herbert Xu wrote:
> On Tue, Jun 23, 2009 at 10:29:21AM +0200, Steffen Klassert wrote:
> >
> > I did already tests with a work queue based parallelization, but the
> > throughput and the latency on networking tests were much better with the
> > remote softirq based version. The serialization engine must always wait
> > for the cpu that processes the 'next' packet to keep the packets in the
> > right order. And softirqs are a good way to ensure that the packets are
> > processed as soon as possible.
>
> OK, do you have some numbers that we can add to the patches?
I posted some numbers for the actual softirq based version with the
first patchset, see:
http://www.mail-archive.com/linux-crypto@vger.kernel.org/msg03035.html
But as I pointed out there, these numbers are not that meaningful
as long as we have the lock in crypto_authenc_hash()
Actually I'm about to rewrite authenc/hmac to use ahash. As soon as I
finished this I will redo the tests without the lock.
>
> Also note that the real time folks are turning softirq into threads
> so this complexity might be moot in the end.
>
Yes, I know. The one of the main problems of the thread based version
was that it had to wait for the cpu that serves the interrupt for the
NICs. As soon as other softirqs (like the NET_RX_SOFTIRQ) are thread
based too this should not be a problem any more.
^ permalink raw reply [flat|nested] 25+ messages in thread
* Re: [RFC] [PATCH 2/5] pcrypt: Add pcrypt crypto parallelization wrapper
2009-06-23 9:14 ` Steffen Klassert
@ 2009-06-23 9:18 ` Herbert Xu
2009-06-23 10:19 ` Steffen Klassert
0 siblings, 1 reply; 25+ messages in thread
From: Herbert Xu @ 2009-06-23 9:18 UTC (permalink / raw)
To: Steffen Klassert; +Cc: David Miller, linux-crypto
On Tue, Jun 23, 2009 at 11:14:29AM +0200, Steffen Klassert wrote:
>
> I posted some numbers for the actual softirq based version with the
> first patchset, see:
> http://www.mail-archive.com/linux-crypto@vger.kernel.org/msg03035.html
Thanks, but I was thinking of softirq numbers vs. workqueue numbers.
Cheers,
--
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
^ permalink raw reply [flat|nested] 25+ messages in thread* Re: [RFC] [PATCH 2/5] pcrypt: Add pcrypt crypto parallelization wrapper
2009-06-23 9:18 ` Herbert Xu
@ 2009-06-23 10:19 ` Steffen Klassert
2009-06-23 10:19 ` Herbert Xu
0 siblings, 1 reply; 25+ messages in thread
From: Steffen Klassert @ 2009-06-23 10:19 UTC (permalink / raw)
To: Herbert Xu; +Cc: David Miller, linux-crypto
On Tue, Jun 23, 2009 at 05:18:52PM +0800, Herbert Xu wrote:
> On Tue, Jun 23, 2009 at 11:14:29AM +0200, Steffen Klassert wrote:
> >
> > I posted some numbers for the actual softirq based version with the
> > first patchset, see:
> > http://www.mail-archive.com/linux-crypto@vger.kernel.org/msg03035.html
>
> Thanks, but I was thinking of softirq numbers vs. workqueue numbers.
>
I have just some numbers from a network based parallelization
that uses threads. So it's probaply not really comparable to
the pcrypt version. To compare the pcrypt version, we would need a
thread based padata version. If you think that it is needed I could
care about this as soon as I finished the ahash work.
Anyway, here are numbers for the network based versions:
I did forwarding tests with the thread version and two quad core machines
(Intel Core 2 Quad Q6600):
linux-2.6.25.2 + thread based (network) parallelization
Packetsize: 1420 byte
Encryption: aes192-sha1
Unidirectional throughput: 660 Mbit/s (tcp)
After rewriting this to use remote softirqs I've got with the same
environment:
linux-2.6-git (Dec 01, 2008) + softirq based (network) parallelization
Packetsize: 1420 byte
Encryption: aes192-sha1
Unidirectional throughput: 910 Mbit/s (tcp)
^ permalink raw reply [flat|nested] 25+ messages in thread
* Re: [RFC] [PATCH 2/5] pcrypt: Add pcrypt crypto parallelization wrapper
2009-06-23 10:19 ` Steffen Klassert
@ 2009-06-23 10:19 ` Herbert Xu
0 siblings, 0 replies; 25+ messages in thread
From: Herbert Xu @ 2009-06-23 10:19 UTC (permalink / raw)
To: Steffen Klassert; +Cc: David Miller, linux-crypto
On Tue, Jun 23, 2009 at 12:19:58PM +0200, Steffen Klassert wrote:
>
> Anyway, here are numbers for the network based versions:
>
> I did forwarding tests with the thread version and two quad core machines
> (Intel Core 2 Quad Q6600):
>
> linux-2.6.25.2 + thread based (network) parallelization
> Packetsize: 1420 byte
> Encryption: aes192-sha1
> Unidirectional throughput: 660 Mbit/s (tcp)
>
> After rewriting this to use remote softirqs I've got with the same
> environment:
>
> linux-2.6-git (Dec 01, 2008) + softirq based (network) parallelization
> Packetsize: 1420 byte
> Encryption: aes192-sha1
> Unidirectional throughput: 910 Mbit/s (tcp)
OK, as long as there something to justify the added complexity
it's fine by me. We could always remove the softirqs later if
necessary.
Cheers,
--
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
^ permalink raw reply [flat|nested] 25+ messages in thread
* [RFC] [PATCH 3/5] eseqiv: Add support for aead algorithms
2009-06-08 7:07 [RFC] [PATCH 0/5] Parallel IPsec v4 Steffen Klassert
2009-06-08 7:08 ` [RFC] [PATCH 1/5] padata: generic interface for parallel processing Steffen Klassert
2009-06-08 7:10 ` [RFC] [PATCH 2/5] pcrypt: Add pcrypt crypto parallelization wrapper Steffen Klassert
@ 2009-06-08 7:10 ` Steffen Klassert
2009-06-25 10:46 ` Herbert Xu
2009-06-08 7:11 ` [RFC] [PATCH 4/5] authenc: Check if the IV is already added to the scatterlist Steffen Klassert
2009-06-08 7:12 ` [RFC] [PATCH 5/5] tcrypt: Test algorithms by name Steffen Klassert
4 siblings, 1 reply; 25+ messages in thread
From: Steffen Klassert @ 2009-06-08 7:10 UTC (permalink / raw)
To: Herbert Xu; +Cc: David Miller, linux-crypto
This adds eseqiv support for aead algorithms, this is usefull
for aead algorithms that need eseqiv as it's default IV generator.
Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
---
crypto/eseqiv.c | 244 +++++++++++++++++++++++++++++++++++++++++++++---
include/linux/crypto.h | 1 +
2 files changed, 232 insertions(+), 13 deletions(-)
diff --git a/crypto/eseqiv.c b/crypto/eseqiv.c
index 3ca3b66..04a8bba 100644
--- a/crypto/eseqiv.c
+++ b/crypto/eseqiv.c
@@ -15,6 +15,7 @@
*
*/
+#include <crypto/internal/aead.h>
#include <crypto/internal/skcipher.h>
#include <crypto/rng.h>
#include <crypto/scatterwalk.h>
@@ -62,6 +63,29 @@ out:
skcipher_givcrypt_complete(req, err);
}
+static void eseqiv_aead_complete2(struct aead_givcrypt_request *req)
+{
+ struct crypto_aead *geniv = aead_givcrypt_reqtfm(req);
+ struct eseqiv_request_ctx *reqctx = aead_givcrypt_reqctx(req);
+
+ memcpy(req->giv, PTR_ALIGN((u8 *)reqctx->tail,
+ crypto_aead_alignmask(geniv) + 1),
+ crypto_aead_ivsize(geniv));
+}
+
+static void eseqiv_aead_complete(struct crypto_async_request *base, int err)
+{
+ struct aead_givcrypt_request *req = base->data;
+
+ if (err)
+ goto out;
+
+ eseqiv_aead_complete2(req);
+
+out:
+ aead_givcrypt_complete(req, err);
+}
+
static void eseqiv_chain(struct scatterlist *head, struct scatterlist *sg,
int chain)
{
@@ -155,6 +179,90 @@ static int eseqiv_givencrypt(struct skcipher_givcrypt_request *req)
if (giv != req->giv)
eseqiv_complete2(req);
+out:
+ return err;
+}
+
+static int eseqiv_aead_givencrypt(struct aead_givcrypt_request *req)
+{
+ struct crypto_aead *geniv = aead_givcrypt_reqtfm(req);
+ struct eseqiv_ctx *ctx = crypto_aead_ctx(geniv);
+ struct eseqiv_request_ctx *reqctx = aead_givcrypt_reqctx(req);
+ struct aead_request *subreq;
+ crypto_completion_t complete;
+ void *data;
+ struct scatterlist *osrc, *odst;
+ struct scatterlist *dst;
+ struct page *srcp;
+ struct page *dstp;
+ u8 *giv;
+ u8 *vsrc;
+ u8 *vdst;
+ __be64 seq;
+ unsigned int ivsize;
+ unsigned int len;
+ unsigned int flags;
+ int err;
+
+ subreq = (void *)(reqctx->tail + ctx->reqoff);
+ aead_request_set_tfm(subreq, aead_geniv_base(geniv));
+
+ giv = req->giv;
+ complete = req->areq.base.complete;
+ data = req->areq.base.data;
+
+ osrc = req->areq.src;
+ odst = req->areq.dst;
+ srcp = sg_page(osrc);
+ dstp = sg_page(odst);
+ vsrc = PageHighMem(srcp) ? NULL : page_address(srcp) + osrc->offset;
+ vdst = PageHighMem(dstp) ? NULL : page_address(dstp) + odst->offset;
+
+ ivsize = crypto_aead_ivsize(geniv);
+ flags = req->areq.base.flags | CRYPTO_TFM_REQ_SG_HAS_IV;
+
+ if (vsrc != giv + ivsize && vdst != giv + ivsize) {
+ giv = PTR_ALIGN((u8 *)reqctx->tail,
+ crypto_aead_alignmask(geniv) + 1);
+ complete = eseqiv_aead_complete;
+ data = req;
+ }
+
+ aead_request_set_callback(subreq, flags, complete, data);
+
+ sg_init_table(reqctx->src, 2);
+ sg_set_buf(reqctx->src, giv, ivsize);
+ eseqiv_chain(reqctx->src, osrc, vsrc == giv + ivsize);
+
+ dst = reqctx->src;
+ if (osrc != odst) {
+ sg_init_table(reqctx->dst, 2);
+ sg_set_buf(reqctx->dst, giv, ivsize);
+ eseqiv_chain(reqctx->dst, odst, vdst == giv + ivsize);
+
+ dst = reqctx->dst;
+ }
+
+ aead_request_set_crypt(subreq, reqctx->src, dst,
+ req->areq.cryptlen + ivsize, req->areq.iv);
+ aead_request_set_assoc(subreq, req->areq.assoc, req->areq.assoclen);
+
+ memcpy(req->areq.iv, ctx->salt, ivsize);
+
+ len = ivsize;
+ if (ivsize > sizeof(u64)) {
+ memset(req->giv, 0, ivsize - sizeof(u64));
+ len = sizeof(u64);
+ }
+ seq = cpu_to_be64(req->seq);
+ memcpy(req->giv + ivsize - len, &seq, len);
+
+ err = crypto_aead_encrypt(subreq);
+ if (err)
+ goto out;
+
+ if (giv != req->giv)
+ eseqiv_aead_complete2(req);
out:
return err;
@@ -183,6 +291,29 @@ unlock:
return eseqiv_givencrypt(req);
}
+static int eseqiv_aead_givencrypt_first(struct aead_givcrypt_request *req)
+{
+ struct crypto_aead *geniv = aead_givcrypt_reqtfm(req);
+ struct eseqiv_ctx *ctx = crypto_aead_ctx(geniv);
+ int err = 0;
+
+ spin_lock_bh(&ctx->lock);
+ if (crypto_aead_crt(geniv)->givencrypt != eseqiv_aead_givencrypt_first)
+ goto unlock;
+
+ crypto_aead_crt(geniv)->givencrypt = eseqiv_aead_givencrypt;
+ err = crypto_rng_get_bytes(crypto_default_rng, ctx->salt,
+ crypto_aead_ivsize(geniv));
+
+unlock:
+ spin_unlock_bh(&ctx->lock);
+
+ if (err)
+ return err;
+
+ return eseqiv_aead_givencrypt(req);
+}
+
static int eseqiv_init(struct crypto_tfm *tfm)
{
struct crypto_ablkcipher *geniv = __crypto_ablkcipher_cast(tfm);
@@ -215,20 +346,47 @@ static int eseqiv_init(struct crypto_tfm *tfm)
return skcipher_geniv_init(tfm);
}
+static int eseqiv_aead_init(struct crypto_tfm *tfm)
+{
+ struct crypto_aead *geniv = __crypto_aead_cast(tfm);
+ struct eseqiv_ctx *ctx = crypto_aead_ctx(geniv);
+ unsigned long alignmask;
+ unsigned int reqsize;
+
+ spin_lock_init(&ctx->lock);
+
+ alignmask = crypto_tfm_ctx_alignment() - 1;
+ reqsize = sizeof(struct eseqiv_request_ctx);
+
+ if (alignmask & reqsize) {
+ alignmask &= reqsize;
+ alignmask--;
+ }
+
+ alignmask = ~alignmask;
+ alignmask &= crypto_aead_alignmask(geniv);
+
+ reqsize += alignmask;
+ reqsize += crypto_aead_ivsize(geniv);
+ reqsize = ALIGN(reqsize, crypto_tfm_ctx_alignment());
+
+ ctx->reqoff = reqsize - sizeof(struct eseqiv_request_ctx);
+
+ tfm->crt_aead.reqsize = reqsize + sizeof(struct aead_request);
+
+ return aead_geniv_init(tfm);
+}
+
static struct crypto_template eseqiv_tmpl;
-static struct crypto_instance *eseqiv_alloc(struct rtattr **tb)
+static struct crypto_instance *eseqiv_ablkcipher_alloc(struct rtattr **tb)
{
- struct crypto_instance *inst;
int err;
-
- err = crypto_get_default_rng();
- if (err)
- return ERR_PTR(err);
+ struct crypto_instance *inst;
inst = skcipher_geniv_alloc(&eseqiv_tmpl, tb, 0, 0);
if (IS_ERR(inst))
- goto put_rng;
+ goto out;
err = -EINVAL;
if (inst->alg.cra_ablkcipher.ivsize != inst->alg.cra_blocksize)
@@ -236,18 +394,75 @@ static struct crypto_instance *eseqiv_alloc(struct rtattr **tb)
inst->alg.cra_ablkcipher.givencrypt = eseqiv_givencrypt_first;
- inst->alg.cra_init = eseqiv_init;
- inst->alg.cra_exit = skcipher_geniv_exit;
+ inst->alg.cra_init = eseqiv_init;
+ inst->alg.cra_exit = skcipher_geniv_exit;
- inst->alg.cra_ctxsize = sizeof(struct eseqiv_ctx);
- inst->alg.cra_ctxsize += inst->alg.cra_ablkcipher.ivsize;
+ inst->alg.cra_ctxsize = inst->alg.cra_ablkcipher.ivsize;
out:
return inst;
free_inst:
skcipher_geniv_free(inst);
- inst = ERR_PTR(err);
+ return ERR_PTR(err);
+}
+
+static struct crypto_instance *eseqiv_aead_alloc(struct rtattr **tb)
+{
+ int err;
+ struct crypto_instance *inst;
+
+ inst = aead_geniv_alloc(&eseqiv_tmpl, tb, 0, 0);
+ if (IS_ERR(inst))
+ goto out;
+
+ err = -EINVAL;
+ if (inst->alg.cra_aead.ivsize != inst->alg.cra_blocksize)
+ goto free_inst;
+
+ inst->alg.cra_aead.givencrypt = eseqiv_aead_givencrypt_first;
+
+ inst->alg.cra_init = eseqiv_aead_init;
+ inst->alg.cra_exit = aead_geniv_exit;
+
+ inst->alg.cra_ctxsize = inst->alg.cra_aead.ivsize;
+
+out:
+ return inst;
+
+free_inst:
+ aead_geniv_free(inst);
+ return ERR_PTR(err);
+}
+
+static struct crypto_instance *eseqiv_alloc(struct rtattr **tb)
+{
+ struct crypto_attr_type *algt;
+ struct crypto_instance *inst;
+ int err;
+
+ algt = crypto_get_attr_type(tb);
+ err = PTR_ERR(algt);
+ if (IS_ERR(algt))
+ return ERR_PTR(err);
+
+ err = crypto_get_default_rng();
+ if (err)
+ return ERR_PTR(err);
+
+ if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & CRYPTO_ALG_TYPE_MASK)
+ inst = eseqiv_ablkcipher_alloc(tb);
+ else
+ inst = eseqiv_aead_alloc(tb);
+
+ if (IS_ERR(inst))
+ goto put_rng;
+
+ inst->alg.cra_ctxsize += sizeof(struct eseqiv_ctx);
+
+out:
+ return inst;
+
put_rng:
crypto_put_default_rng();
goto out;
@@ -255,7 +470,10 @@ put_rng:
static void eseqiv_free(struct crypto_instance *inst)
{
- skcipher_geniv_free(inst);
+ if ((inst->alg.cra_flags ^ CRYPTO_ALG_TYPE_AEAD) & CRYPTO_ALG_TYPE_MASK)
+ skcipher_geniv_free(inst);
+ else
+ aead_geniv_free(inst);
crypto_put_default_rng();
}
diff --git a/include/linux/crypto.h b/include/linux/crypto.h
index ec29fa2..1af50e6 100644
--- a/include/linux/crypto.h
+++ b/include/linux/crypto.h
@@ -80,6 +80,7 @@
#define CRYPTO_TFM_REQ_WEAK_KEY 0x00000100
#define CRYPTO_TFM_REQ_MAY_SLEEP 0x00000200
#define CRYPTO_TFM_REQ_MAY_BACKLOG 0x00000400
+#define CRYPTO_TFM_REQ_SG_HAS_IV 0x00000800
#define CRYPTO_TFM_RES_WEAK_KEY 0x00100000
#define CRYPTO_TFM_RES_BAD_KEY_LEN 0x00200000
#define CRYPTO_TFM_RES_BAD_KEY_SCHED 0x00400000
--
1.5.4.2
^ permalink raw reply related [flat|nested] 25+ messages in thread* Re: [RFC] [PATCH 3/5] eseqiv: Add support for aead algorithms
2009-06-08 7:10 ` [RFC] [PATCH 3/5] eseqiv: Add support for aead algorithms Steffen Klassert
@ 2009-06-25 10:46 ` Herbert Xu
2009-06-29 11:12 ` Steffen Klassert
0 siblings, 1 reply; 25+ messages in thread
From: Herbert Xu @ 2009-06-25 10:46 UTC (permalink / raw)
To: Steffen Klassert; +Cc: David Miller, linux-crypto
On Mon, Jun 08, 2009 at 09:10:46AM +0200, Steffen Klassert wrote:
> This adds eseqiv support for aead algorithms, this is usefull
> for aead algorithms that need eseqiv as it's default IV generator.
>
> Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
I'm going to change the default sync geniv to eseqiv for SMP.
That should render this patch unnecessary, right?
Here's the change.
commit 0b67fb65d1b2ba1396de69112b8b9bc95d8d5feb
Author: Herbert Xu <herbert@gondor.apana.org.au>
Date: Thu Jun 25 18:43:48 2009 +0800
crypto: skcipher - Change default sync geniv on SMP to eseqiv
As it stands we use chainiv for sync algorithms and eseqiv for
async algorithms. However, when there is more than one CPU
chainiv forces all processing to be serialised which is usually
not what you want. Also, the added overhead of eseqiv isn't that
great.
Therefore this patch changes the default sync geniv on SMP machines
to eseqiv. For the odd situation where the overhead is unacceptable
then chainiv is still available as an option.
Note that on UP machines chainiv is still preferred over eseqiv
for sync algorithms.
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
diff --git a/crypto/ablkcipher.c b/crypto/ablkcipher.c
index 43fc8fb..03fb5fa 100644
--- a/crypto/ablkcipher.c
+++ b/crypto/ablkcipher.c
@@ -14,6 +14,7 @@
*/
#include <crypto/internal/skcipher.h>
+#include <linux/cpumask.h>
#include <linux/err.h>
#include <linux/init.h>
#include <linux/kernel.h>
@@ -25,6 +26,8 @@
#include "internal.h"
+static const char *skcipher_default_geniv __read_mostly;
+
static int setkey_unaligned(struct crypto_ablkcipher *tfm, const u8 *key,
unsigned int keylen)
{
@@ -180,7 +183,8 @@ EXPORT_SYMBOL_GPL(crypto_givcipher_type);
const char *crypto_default_geniv(const struct crypto_alg *alg)
{
- return alg->cra_flags & CRYPTO_ALG_ASYNC ? "eseqiv" : "chainiv";
+ return alg->cra_flags & CRYPTO_ALG_ASYNC ?
+ "eseqiv" : skcipher_default_geniv;
}
static int crypto_givcipher_default(struct crypto_alg *alg, u32 type, u32 mask)
@@ -361,3 +365,17 @@ err:
return ERR_PTR(err);
}
EXPORT_SYMBOL_GPL(crypto_alloc_ablkcipher);
+
+static int __init skcipher_module_init(void)
+{
+ skcipher_default_geniv = num_possible_cpus() > 1 ?
+ "eseqiv" : "chainiv";
+ return 0;
+}
+
+static void skcipher_module_exit(void)
+{
+}
+
+module_init(skcipher_module_init);
+module_exit(skcipher_module_exit);
Cheers,
--
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
^ permalink raw reply related [flat|nested] 25+ messages in thread* Re: [RFC] [PATCH 3/5] eseqiv: Add support for aead algorithms
2009-06-25 10:46 ` Herbert Xu
@ 2009-06-29 11:12 ` Steffen Klassert
0 siblings, 0 replies; 25+ messages in thread
From: Steffen Klassert @ 2009-06-29 11:12 UTC (permalink / raw)
To: Herbert Xu; +Cc: David Miller, linux-crypto
On Thu, Jun 25, 2009 at 06:46:02PM +0800, Herbert Xu wrote:
> On Mon, Jun 08, 2009 at 09:10:46AM +0200, Steffen Klassert wrote:
> > This adds eseqiv support for aead algorithms, this is usefull
> > for aead algorithms that need eseqiv as it's default IV generator.
> >
> > Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
>
> I'm going to change the default sync geniv to eseqiv for SMP.
> That should render this patch unnecessary, right?
>
Right, this would reduce the patchset to the padata and the pcrypt
patch. I'll strip the patchset to these two patches as soon as I see
your patch applied to cryptodev-2.6.
^ permalink raw reply [flat|nested] 25+ messages in thread
* [RFC] [PATCH 4/5] authenc: Check if the IV is already added to the scatterlist
2009-06-08 7:07 [RFC] [PATCH 0/5] Parallel IPsec v4 Steffen Klassert
` (2 preceding siblings ...)
2009-06-08 7:10 ` [RFC] [PATCH 3/5] eseqiv: Add support for aead algorithms Steffen Klassert
@ 2009-06-08 7:11 ` Steffen Klassert
2009-06-08 7:12 ` [RFC] [PATCH 5/5] tcrypt: Test algorithms by name Steffen Klassert
4 siblings, 0 replies; 25+ messages in thread
From: Steffen Klassert @ 2009-06-08 7:11 UTC (permalink / raw)
To: Herbert Xu; +Cc: David Miller, linux-crypto
aead eseqiv needs to add the IV to the scatterlist itself.
So check if the IV is already contained in the scatterlist
and add it just if it is not already there.
Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
---
crypto/authenc.c | 5 +++--
1 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/crypto/authenc.c b/crypto/authenc.c
index 5793b64..e11029a 100644
--- a/crypto/authenc.c
+++ b/crypto/authenc.c
@@ -157,15 +157,16 @@ static int crypto_authenc_genicv(struct aead_request *req, u8 *iv,
dstp = sg_page(dst);
vdst = PageHighMem(dstp) ? NULL : page_address(dstp) + dst->offset;
+ cryptlen = req->cryptlen;
- if (ivsize) {
+ if (ivsize && !(aead_request_flags(req) & CRYPTO_TFM_REQ_SG_HAS_IV)) {
sg_init_table(cipher, 2);
sg_set_buf(cipher, iv, ivsize);
authenc_chain(cipher, dst, vdst == iv + ivsize);
dst = cipher;
+ cryptlen += ivsize;
}
- cryptlen = req->cryptlen + ivsize;
hash = crypto_authenc_hash(req, flags, dst, cryptlen);
if (IS_ERR(hash))
return PTR_ERR(hash);
--
1.5.4.2
^ permalink raw reply related [flat|nested] 25+ messages in thread* [RFC] [PATCH 5/5] tcrypt: Test algorithms by name
2009-06-08 7:07 [RFC] [PATCH 0/5] Parallel IPsec v4 Steffen Klassert
` (3 preceding siblings ...)
2009-06-08 7:11 ` [RFC] [PATCH 4/5] authenc: Check if the IV is already added to the scatterlist Steffen Klassert
@ 2009-06-08 7:12 ` Steffen Klassert
2009-06-19 10:53 ` Herbert Xu
4 siblings, 1 reply; 25+ messages in thread
From: Steffen Klassert @ 2009-06-08 7:12 UTC (permalink / raw)
To: Herbert Xu; +Cc: David Miller, linux-crypto
This adds the 'alg' module parameter to be able to test an
algorithm by name. If the algorithm type is not ad-hoc
clear for a algorithm (e.g. pcrypt, cryptd) it is possilbe
to set the algorithm type with the 'type' module parameter.
Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
---
crypto/tcrypt.c | 14 +++++++++++++-
1 files changed, 13 insertions(+), 1 deletions(-)
diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c
index c3c9124..9d5b33c 100644
--- a/crypto/tcrypt.c
+++ b/crypto/tcrypt.c
@@ -44,6 +44,8 @@
*/
static unsigned int sec;
+static char *alg = NULL;
+static u32 type;
static int mode;
static char *tvmem[TVMEMSIZE];
@@ -864,6 +866,11 @@ static void do_test(int m)
}
}
+static void do_alg_test(const char *alg, u32 type)
+{
+ crypto_has_alg(alg, type, CRYPTO_ALG_TYPE_MASK);
+}
+
static int __init tcrypt_mod_init(void)
{
int err = -ENOMEM;
@@ -875,7 +882,10 @@ static int __init tcrypt_mod_init(void)
goto err_free_tv;
}
- do_test(mode);
+ if (alg)
+ do_alg_test(alg, type);
+ else
+ do_test(mode);
/* We intentionaly return -EAGAIN to prevent keeping
* the module. It does all its work from init()
@@ -901,6 +911,8 @@ static void __exit tcrypt_mod_fini(void) { }
module_init(tcrypt_mod_init);
module_exit(tcrypt_mod_fini);
+module_param(alg, charp, 0);
+module_param(type, uint, 0);
module_param(mode, int, 0);
module_param(sec, uint, 0);
MODULE_PARM_DESC(sec, "Length in seconds of speed tests "
--
1.5.4.2
^ permalink raw reply related [flat|nested] 25+ messages in thread* Re: [RFC] [PATCH 5/5] tcrypt: Test algorithms by name
2009-06-08 7:12 ` [RFC] [PATCH 5/5] tcrypt: Test algorithms by name Steffen Klassert
@ 2009-06-19 10:53 ` Herbert Xu
2009-06-19 10:56 ` Herbert Xu
0 siblings, 1 reply; 25+ messages in thread
From: Herbert Xu @ 2009-06-19 10:53 UTC (permalink / raw)
To: Steffen Klassert; +Cc: David Miller, linux-crypto
On Mon, Jun 08, 2009 at 09:12:46AM +0200, Steffen Klassert wrote:
> This adds the 'alg' module parameter to be able to test an
> algorithm by name. If the algorithm type is not ad-hoc
> clear for a algorithm (e.g. pcrypt, cryptd) it is possilbe
> to set the algorithm type with the 'type' module parameter.
>
> Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
I'll apply this one right now. Thanks!
--
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
^ permalink raw reply [flat|nested] 25+ messages in thread* Re: [RFC] [PATCH 5/5] tcrypt: Test algorithms by name
2009-06-19 10:53 ` Herbert Xu
@ 2009-06-19 10:56 ` Herbert Xu
2009-06-19 11:40 ` [PATCH] " Steffen Klassert
0 siblings, 1 reply; 25+ messages in thread
From: Herbert Xu @ 2009-06-19 10:56 UTC (permalink / raw)
To: Steffen Klassert; +Cc: David Miller, linux-crypto
On Fri, Jun 19, 2009 at 06:53:25PM +0800, Herbert Xu wrote:
> On Mon, Jun 08, 2009 at 09:12:46AM +0200, Steffen Klassert wrote:
> > This adds the 'alg' module parameter to be able to test an
> > algorithm by name. If the algorithm type is not ad-hoc
> > clear for a algorithm (e.g. pcrypt, cryptd) it is possilbe
> > to set the algorithm type with the 'type' module parameter.
> >
> > Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
>
> I'll apply this one right now. Thanks!
Oops it doesn't apply anymore. Could you rebase this one for me?
Thanks!
--
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
^ permalink raw reply [flat|nested] 25+ messages in thread* [PATCH] tcrypt: Test algorithms by name
2009-06-19 10:56 ` Herbert Xu
@ 2009-06-19 11:40 ` Steffen Klassert
2009-06-19 11:47 ` Herbert Xu
0 siblings, 1 reply; 25+ messages in thread
From: Steffen Klassert @ 2009-06-19 11:40 UTC (permalink / raw)
To: Herbert Xu; +Cc: David Miller, linux-crypto
This adds the 'alg' module parameter to be able to test an
algorithm by name. If the algorithm type is not ad-hoc
clear for a algorithm (e.g. pcrypt, cryptd) it is possilbe
to set the algorithm type with the 'type' module parameter.
Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
---
crypto/tcrypt.c | 15 ++++++++++++++-
1 files changed, 14 insertions(+), 1 deletions(-)
diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c
index d59ba50..dfeec0c 100644
--- a/crypto/tcrypt.c
+++ b/crypto/tcrypt.c
@@ -45,6 +45,8 @@
*/
static unsigned int sec;
+static char *alg = NULL;
+static u32 type;
static int mode;
static char *tvmem[TVMEMSIZE];
@@ -885,6 +887,11 @@ static int do_test(int m)
return ret;
}
+static int do_alg_test(const char *alg, u32 type)
+{
+ return crypto_has_alg(alg, type, CRYPTO_ALG_TYPE_MASK);
+}
+
static int __init tcrypt_mod_init(void)
{
int err = -ENOMEM;
@@ -896,7 +903,11 @@ static int __init tcrypt_mod_init(void)
goto err_free_tv;
}
- err = do_test(mode);
+ if (alg)
+ err = do_alg_test(alg, type);
+ else
+ err = do_test(mode);
+
if (err) {
printk(KERN_ERR "tcrypt: one or more tests failed!\n");
goto err_free_tv;
@@ -928,6 +939,8 @@ static void __exit tcrypt_mod_fini(void) { }
module_init(tcrypt_mod_init);
module_exit(tcrypt_mod_fini);
+module_param(alg, charp, 0);
+module_param(type, uint, 0);
module_param(mode, int, 0);
module_param(sec, uint, 0);
MODULE_PARM_DESC(sec, "Length in seconds of speed tests "
--
1.5.4.2
^ permalink raw reply related [flat|nested] 25+ messages in thread* Re: [PATCH] tcrypt: Test algorithms by name
2009-06-19 11:40 ` [PATCH] " Steffen Klassert
@ 2009-06-19 11:47 ` Herbert Xu
2009-06-19 12:38 ` Herbert Xu
0 siblings, 1 reply; 25+ messages in thread
From: Herbert Xu @ 2009-06-19 11:47 UTC (permalink / raw)
To: Steffen Klassert; +Cc: David Miller, linux-crypto
On Fri, Jun 19, 2009 at 01:40:58PM +0200, Steffen Klassert wrote:
> This adds the 'alg' module parameter to be able to test an
> algorithm by name. If the algorithm type is not ad-hoc
> clear for a algorithm (e.g. pcrypt, cryptd) it is possilbe
> to set the algorithm type with the 'type' module parameter.
>
> Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
That was quick :)
Patch applied. Thanks a lot Steffen!
--
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
^ permalink raw reply [flat|nested] 25+ messages in thread* Re: [PATCH] tcrypt: Test algorithms by name
2009-06-19 11:47 ` Herbert Xu
@ 2009-06-19 12:38 ` Herbert Xu
2009-06-23 6:04 ` Steffen Klassert
0 siblings, 1 reply; 25+ messages in thread
From: Herbert Xu @ 2009-06-19 12:38 UTC (permalink / raw)
To: Steffen Klassert; +Cc: David Miller, linux-crypto
On Fri, Jun 19, 2009 at 07:47:37PM +0800, Herbert Xu wrote:
> On Fri, Jun 19, 2009 at 01:40:58PM +0200, Steffen Klassert wrote:
> > This adds the 'alg' module parameter to be able to test an
> > algorithm by name. If the algorithm type is not ad-hoc
> > clear for a algorithm (e.g. pcrypt, cryptd) it is possilbe
> > to set the algorithm type with the 'type' module parameter.
> >
> > Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
>
> That was quick :)
>
> Patch applied. Thanks a lot Steffen!
BTW, I added this on top so that modprobe doesn't complain about
the return code:
tcrypt: one or more tests failed!
sys_init_module: 'tcrypt'->init suspiciously returned 1, it should follow 0/-E convention
sys_init_module: loading module anyway...
Pid: 3064, comm: modprobe Not tainted 2.6.30 #2
Call Trace:
[<ffffffff81076826>] level2_spare_pgt+0x6e826/0x2d1760
[<ffffffff8100b4ab>] level2_spare_pgt+0x34ab/0x2d1760
commit ea4006576945195ed35824acfb4007ca7cb78b70
Author: Herbert Xu <herbert@gondor.apana.org.au>
Date: Fri Jun 19 20:37:00 2009 +0800
crypto: tcrypt - Fix module return code when testing by name
We should return 0/-ENOENT instead of 1/0 when testing by name.
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c
index dfeec0c..a890a67 100644
--- a/crypto/tcrypt.c
+++ b/crypto/tcrypt.c
@@ -889,7 +889,7 @@ static int do_test(int m)
static int do_alg_test(const char *alg, u32 type)
{
- return crypto_has_alg(alg, type, CRYPTO_ALG_TYPE_MASK);
+ return crypto_has_alg(alg, type, CRYPTO_ALG_TYPE_MASK) ? 0 : -ENOENT;
}
static int __init tcrypt_mod_init(void)
Cheers,
--
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
^ permalink raw reply related [flat|nested] 25+ messages in thread
* [RFC] [PATCH 1/5] padata: generic interface for parallel processing
2009-05-13 13:06 [RFC] [PATCH 0/5] Parallel IPsec v3 Steffen Klassert
@ 2009-05-13 13:07 ` Steffen Klassert
0 siblings, 0 replies; 25+ messages in thread
From: Steffen Klassert @ 2009-05-13 13:07 UTC (permalink / raw)
To: Herbert Xu; +Cc: David Miller, linux-crypto
This patch introduces an interface to process data objects
in parallel. On request it is possible to serialize again.
The parallelized objects return after serialization in the
same order as they were before the parallelization.
Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
---
include/linux/interrupt.h | 3 +-
include/linux/padata.h | 116 +++++++++++
kernel/Makefile | 2 +-
kernel/padata.c | 490 +++++++++++++++++++++++++++++++++++++++++++++
kernel/softirq.c | 2 +-
5 files changed, 610 insertions(+), 3 deletions(-)
create mode 100644 include/linux/padata.h
create mode 100644 kernel/padata.c
diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
index 91bb76f..a17679c 100644
--- a/include/linux/interrupt.h
+++ b/include/linux/interrupt.h
@@ -338,7 +338,8 @@ enum
TASKLET_SOFTIRQ,
SCHED_SOFTIRQ,
HRTIMER_SOFTIRQ,
- RCU_SOFTIRQ, /* Preferable RCU should always be the last softirq */
+ PADATA_SOFTIRQ,
+ RCU_SOFTIRQ, /* Preferable RCU should always be the last softirq */
NR_SOFTIRQS
};
diff --git a/include/linux/padata.h b/include/linux/padata.h
new file mode 100644
index 0000000..469359f
--- /dev/null
+++ b/include/linux/padata.h
@@ -0,0 +1,116 @@
+/*
+ * padata.h - header for the padata parallelization interface
+ *
+ * Copyright (C) 2008, 2009 secunet Security Networks AG
+ * Copyright (C) 2008, 2009 Steffen Klassert <steffen.klassert@secunet.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#ifndef PADATA_H
+#define PADATA_H
+
+#include <linux/interrupt.h>
+#include <linux/smp.h>
+#include <linux/list.h>
+
+enum
+{
+ NO_PADATA=0,
+ AEAD_ENC_PADATA,
+ AEAD_DEC_PADATA,
+ NR_PADATA
+};
+
+struct padata_priv {
+ struct list_head list;
+ struct call_single_data csd;
+ int cb_cpu;
+ int seq_nr;
+ unsigned int nr;
+ int info;
+ void (*parallel)(struct padata_priv *padata);
+ void (*serial)(struct padata_priv *padata);
+};
+
+struct padata_queue {
+ struct list_head list;
+ atomic_t num_obj;
+ int cpu_index;
+ spinlock_t lock;
+};
+
+struct parallel_data {
+ struct work_struct work;
+ struct padata_queue *queue;
+ atomic_t seq_nr;
+ atomic_t queued_objects;
+ cpumask_t cpu_map;
+ cpumask_t new_cpu_map;
+ u8 flags;
+#define PADATA_INIT 1
+#define PADATA_FLUSH_HARD 2
+#define PADATA_RESET_IN_PROGRESS 4
+ spinlock_t lock;
+};
+
+#ifdef CONFIG_USE_GENERIC_SMP_HELPERS
+extern void __init padata_init(unsigned int nr, cpumask_t cpu_map);
+extern void padata_dont_wait(unsigned int nr, struct padata_priv *padata);
+extern int padata_do_parallel(unsigned int softirq_nr, unsigned int nr,
+ struct padata_priv *padata, int cb_cpu);
+extern int padata_do_serial(unsigned int nr, struct padata_priv *padata);
+extern cpumask_t padata_get_cpumap(unsigned int nr);
+extern void padata_set_cpumap(unsigned int nr, cpumask_t cpu_map);
+extern void padata_add_cpu(unsigned int nr, int cpu);
+extern void padata_remove_cpu(unsigned int nr, int cpu);
+extern void padata_start(unsigned int nr);
+extern void padata_stop(unsigned int nr);
+#else
+static inline void padata_init(unsigned int nr,cpumask_t cpu_map)
+{
+}
+static inline void padata_dont_wait(unsigned int nr, struct padata_priv *padata)
+{
+}
+static inline int padata_do_parallel(unsigned int softirq_nr, unsigned int nr,
+ struct padata_priv *padata, int cb_cpu)
+{
+ return 0;
+}
+static inline int padata_do_serial(unsigned int nr, struct padata_priv *padata)
+{
+ return 0;
+}
+static inline cpumask_t padata_get_cpumap(unsigned int nr)
+{
+ return cpu_online_map;
+}
+static inline void padata_set_cpumap(unsigned int nr, cpumask_t cpu_map)
+{
+}
+static inline padata_add_cpu(unsigned int nr, int cpu)
+{
+}
+static inline padata_remove_cpu(unsigned int nr, int cpu)
+{
+}
+static inline padata_start(unsigned int nr)
+{
+}
+static inline padata_stop(unsigned int nr)
+{
+}
+#endif
+#endif
diff --git a/kernel/Makefile b/kernel/Makefile
index 4242366..f2d7ea7 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -40,7 +40,7 @@ obj-$(CONFIG_RT_MUTEXES) += rtmutex.o
obj-$(CONFIG_DEBUG_RT_MUTEXES) += rtmutex-debug.o
obj-$(CONFIG_RT_MUTEX_TESTER) += rtmutex-tester.o
obj-$(CONFIG_GENERIC_ISA_DMA) += dma.o
-obj-$(CONFIG_USE_GENERIC_SMP_HELPERS) += smp.o
+obj-$(CONFIG_USE_GENERIC_SMP_HELPERS) += smp.o padata.o
ifneq ($(CONFIG_SMP),y)
obj-y += up.o
endif
diff --git a/kernel/padata.c b/kernel/padata.c
new file mode 100644
index 0000000..192c9a6
--- /dev/null
+++ b/kernel/padata.c
@@ -0,0 +1,490 @@
+/*
+ * padata.c - generic interface to process data streams in parallel
+ *
+ * Copyright (C) 2008, 2009 secunet Security Networks AG
+ * Copyright (C) 2008, 2009 Steffen Klassert <steffen.klassert@secunet.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include <linux/module.h>
+#include <linux/cpumask.h>
+#include <linux/err.h>
+#include <linux/padata.h>
+
+#define MAX_SEQ_NR 1000000000
+
+static struct parallel_data padata_vec[NR_PADATA];
+static struct padata_priv *padata_get_next(struct parallel_data *par_data);
+
+static void padata_flush_hard(struct parallel_data *par_data)
+{
+ int cpu;
+ struct padata_priv *padata;
+ struct padata_queue *queue;
+
+ for_each_cpu_mask(cpu, par_data->cpu_map) {
+ queue = per_cpu_ptr(par_data->queue, cpu);
+
+ while(!list_empty(&queue->list)) {
+ padata = list_entry(queue->list.next, struct padata_priv, list);
+
+ spin_lock(&queue->lock);
+ list_del_init(&padata->list);
+ spin_unlock(&queue->lock);
+
+ atomic_dec(&par_data->queued_objects);
+ padata->serial(padata);
+ }
+ }
+}
+
+static void padata_flush_order(struct parallel_data *par_data)
+{
+ struct padata_priv *padata;
+
+ while (1) {
+ padata = padata_get_next(par_data);
+
+ if (padata && !IS_ERR(padata))
+ padata->serial(padata);
+ else
+ break;
+ }
+
+ padata_flush_hard(par_data);
+}
+
+static void padata_reset_work(struct work_struct *work)
+{
+ int cpu, cpu_index;
+ struct padata_queue *queue;
+ struct parallel_data *par_data;
+
+ par_data = container_of(work, struct parallel_data, work);
+
+ if (par_data->flags & (PADATA_INIT|PADATA_RESET_IN_PROGRESS))
+ return;
+
+ spin_lock_bh(&par_data->lock);
+ par_data->flags |= PADATA_RESET_IN_PROGRESS;
+
+ if (!(par_data->flags & PADATA_FLUSH_HARD))
+ padata_flush_order(par_data);
+ else
+ padata_flush_hard(par_data);
+
+ cpu_index = 0;
+
+ par_data->cpu_map = par_data->new_cpu_map;
+
+ for_each_cpu_mask(cpu, par_data->cpu_map) {
+ queue = per_cpu_ptr(par_data->queue, cpu);
+
+ atomic_set(&queue->num_obj, 0);
+ queue->cpu_index = cpu_index;
+ cpu_index++;
+ }
+ spin_unlock_bh(&par_data->lock);
+
+ atomic_set(&par_data->seq_nr, -1);
+ par_data->flags &= ~PADATA_RESET_IN_PROGRESS;
+ par_data->flags |= PADATA_INIT;
+}
+
+static struct padata_priv *padata_get_next(struct parallel_data *par_data)
+{
+ int cpu, num_cpus, empty;
+ int seq_nr, calc_seq_nr, next_nr;
+ struct padata_queue *queue, *next_queue;
+ struct padata_priv *padata;
+
+ empty = 0;
+ next_nr = -1;
+ next_queue = NULL;
+
+ num_cpus = cpus_weight(par_data->cpu_map);
+
+ for_each_cpu_mask(cpu, par_data->cpu_map) {
+ queue = per_cpu_ptr(par_data->queue, cpu);
+
+ /*
+ * Calculate the seq_nr of the object that should be
+ * next in this queue.
+ */
+ calc_seq_nr = (atomic_read(&queue->num_obj) * num_cpus)
+ + queue->cpu_index;
+
+ if (!list_empty(&queue->list)) {
+ padata = list_entry(queue->list.next,
+ struct padata_priv, list);
+
+ seq_nr = padata->seq_nr;
+
+ if (unlikely(calc_seq_nr != seq_nr)) {
+ par_data->flags &= ~PADATA_INIT;
+ par_data->flags |= PADATA_FLUSH_HARD;
+ padata = NULL;
+ goto out;
+ }
+ } else {
+ seq_nr = calc_seq_nr;
+ empty++;
+ }
+
+ if (next_nr < 0 || seq_nr < next_nr) {
+ next_nr = seq_nr;
+ next_queue = queue;
+ }
+ }
+
+ padata = NULL;
+
+ if (empty == num_cpus)
+ goto out;
+
+ if (!list_empty(&next_queue->list)) {
+ padata = list_entry(next_queue->list.next,
+ struct padata_priv, list);
+
+ spin_lock(&next_queue->lock);
+ list_del_init(&padata->list);
+ spin_unlock(&next_queue->lock);
+
+ atomic_dec(&par_data->queued_objects);
+ atomic_inc(&next_queue->num_obj);
+
+ goto out;
+ }
+
+ if (next_nr % num_cpus == next_queue->cpu_index) {
+ padata = ERR_PTR(-ENODATA);
+ goto out;
+ }
+
+ padata = ERR_PTR(-EINPROGRESS);
+out:
+ return padata;
+}
+
+static void padata_action(struct softirq_action *h)
+{
+ struct list_head *cpu_list, local_list;
+
+ cpu_list = &__get_cpu_var(softirq_work_list[PADATA_SOFTIRQ]);
+
+ local_irq_disable();
+ list_replace_init(cpu_list, &local_list);
+ local_irq_enable();
+
+ while (!list_empty(&local_list)) {
+ struct padata_priv *padata;
+
+ padata = list_entry(local_list.next,
+ struct padata_priv, csd.list);
+
+ list_del_init(&padata->csd.list);
+
+ padata->serial(padata);
+ }
+}
+
+static int padata_cpu_hash(unsigned int nr, struct padata_priv *padata)
+{
+ int cpu, target_cpu, this_cpu, cpu_index;
+
+ this_cpu = smp_processor_id();
+
+ if (padata->nr != 0)
+ return this_cpu;
+
+ if (!(padata_vec[nr].flags & PADATA_INIT))
+ return this_cpu;
+
+ padata->seq_nr = atomic_inc_return(&padata_vec[nr].seq_nr);
+
+ if (padata->seq_nr > MAX_SEQ_NR) {
+ padata_vec[nr].flags &= ~PADATA_INIT;
+ padata->seq_nr = 0;
+ schedule_work(&padata_vec[nr].work);
+ return this_cpu;
+ }
+
+ padata->nr = nr;
+
+ /*
+ * Hash the sequence numbers to the cpus by taking
+ * seq_nr mod. number of cpus in use.
+ */
+ cpu_index = padata->seq_nr % cpus_weight(padata_vec[nr].cpu_map);
+
+ target_cpu = first_cpu(padata_vec[nr].cpu_map);
+ for (cpu = 0; cpu < cpu_index; cpu++)
+ target_cpu = next_cpu(target_cpu, padata_vec[nr].cpu_map);
+
+ return target_cpu;
+}
+
+/*
+ * padata_dont_wait - must be called if an object that runs in parallel will
+ * not be serialized with padata_do_serial
+ *
+ * @nr: number of the padata instance
+ * @padata: object that will not be seen by padata_do_serial
+ */
+void padata_dont_wait(unsigned int nr, struct padata_priv *padata)
+{
+ struct padata_queue *queue;
+
+ if (!(padata_vec[nr].flags & PADATA_INIT))
+ return;
+
+ if (padata->nr == 0 || padata->nr != nr)
+ return;
+
+ queue = per_cpu_ptr(padata_vec[nr].queue, smp_processor_id());
+ atomic_inc(&queue->num_obj);
+
+ padata->nr = 0;
+ padata->seq_nr = 0;
+}
+EXPORT_SYMBOL(padata_dont_wait);
+
+/*
+ * padata_do_parallel - padata parallelization function
+ *
+ * @softirq_nr: number of the softirq that will do the parallelization
+ * @nr: number of the padata instance
+ * @padata: object to be parallelized
+ * @cb_cpu: cpu number on which the serialization callback function will run
+ */
+int padata_do_parallel(unsigned int softirq_nr, unsigned int nr,
+ struct padata_priv *padata, int cb_cpu)
+{
+ int target_cpu;
+
+ padata->cb_cpu = cb_cpu;
+
+ local_bh_disable();
+ target_cpu = padata_cpu_hash(nr, padata);
+ local_bh_enable();
+
+ send_remote_softirq(&padata->csd, target_cpu, softirq_nr);
+
+ return 1;
+}
+EXPORT_SYMBOL(padata_do_parallel);
+
+/*
+ * padata_do_serial - padata serialization function
+ *
+ * @nr: number of the padata instance
+ * @padata: object to be serialized
+ *
+ * returns 1 if the serialization callback function will be called
+ * from padata, 0 else
+ */
+int padata_do_serial(unsigned int nr, struct padata_priv *padata)
+{
+ int cpu;
+ struct padata_queue *reorder_queue;
+
+ if (!(padata_vec[nr].flags & PADATA_INIT))
+ return 0;
+
+ if (padata->nr != nr || padata->nr == 0) {
+ padata->serial(padata);
+ return 1;
+ }
+
+ cpu = smp_processor_id();
+
+ reorder_queue = per_cpu_ptr(padata_vec[nr].queue, cpu);
+
+ spin_lock(&reorder_queue->lock);
+ list_add_tail(&padata->list, &reorder_queue->list);
+ spin_unlock(&reorder_queue->lock);
+
+ atomic_inc(&padata_vec[nr].queued_objects);
+
+try_again:
+ if (!spin_trylock(&padata_vec[nr].lock))
+ goto out;
+
+ while(1) {
+ padata = padata_get_next(&padata_vec[nr]);
+
+ if (!padata || PTR_ERR(padata) == -EINPROGRESS)
+ break;
+ if (PTR_ERR(padata) == -ENODATA) {
+ spin_unlock(&padata_vec[nr].lock);
+ goto out;
+ }
+
+ send_remote_softirq(&padata->csd, padata->cb_cpu,
+ PADATA_SOFTIRQ);
+ }
+
+ if (unlikely(!(padata_vec[nr].flags & PADATA_INIT))) {
+ spin_unlock(&padata_vec[nr].lock);
+ goto reset_out;
+ }
+
+ spin_unlock(&padata_vec[nr].lock);
+
+ if (atomic_read(&padata_vec[nr].queued_objects))
+ goto try_again;
+
+out:
+ return 1;
+reset_out:
+ schedule_work(&padata_vec[nr].work);
+ return 1;
+}
+EXPORT_SYMBOL(padata_do_serial);
+
+/*
+ * padata_get_cpumap - get the cpu map that is actually in use
+ *
+ * @nr: number of the padata instance
+ */
+cpumask_t padata_get_cpumap(unsigned int nr)
+{
+ return padata_vec[nr].cpu_map;
+}
+EXPORT_SYMBOL(padata_get_cpumap);
+
+/*
+ * padata_set_cpumap - set the cpu map that padata uses
+ *
+ * @nr: number of the padata instance
+ * @cpu_map: the cpu map to use
+ */
+void padata_set_cpumap(unsigned int nr, cpumask_t cpu_map)
+{
+ padata_vec[nr].new_cpu_map = cpu_map;
+ padata_vec[nr].flags &= ~PADATA_INIT;
+ padata_vec[nr].flags |= PADATA_FLUSH_HARD;
+
+ schedule_work(&padata_vec[nr].work);
+}
+EXPORT_SYMBOL(padata_set_cpumap);
+
+/*
+ * padata_add_cpu - add a cpu to the padata cpu map
+ *
+ * @nr: number of the padata instance
+ * @cpu: cpu to remove
+ */
+void padata_add_cpu(unsigned int nr, int cpu)
+{
+ cpumask_t cpu_map = padata_vec[nr].cpu_map;
+
+ cpu_set(cpu, cpu_map);
+ padata_set_cpumap(nr, cpu_map);
+}
+EXPORT_SYMBOL(padata_add_cpu);
+
+/*
+ * padata_remove_cpu - remove a cpu from the padata cpu map
+ *
+ * @nr: number of the padata instance
+ * @cpu: cpu to remove
+ */
+void padata_remove_cpu(unsigned int nr, int cpu)
+{
+ cpumask_t cpu_map = padata_vec[nr].cpu_map;
+
+ cpu_clear(cpu, cpu_map);
+ padata_set_cpumap(nr, cpu_map);
+}
+EXPORT_SYMBOL(padata_remove_cpu);
+
+/*
+ * padata_start - start the parallel processing
+ *
+ * @nr: number of the padata instance
+ */
+void padata_start(unsigned int nr)
+{
+ if (padata_vec[nr].flags & PADATA_INIT)
+ return;
+
+ schedule_work(&padata_vec[nr].work);
+}
+EXPORT_SYMBOL(padata_start);
+
+/*
+ * padata_stop - stop the parallel processing
+ *
+ * @nr: number of the padata instance
+ */
+void padata_stop(unsigned int nr)
+{
+ padata_vec[nr].flags &= ~PADATA_INIT;
+}
+EXPORT_SYMBOL(padata_stop);
+
+/*
+ * padata_init - initialize a padata instance
+ *
+ * @nr: number of the padata instance
+ * @cpu_map: map of the cpu set that padata uses for parallelization
+ */
+void __init padata_init(unsigned int nr, cpumask_t cpu_map)
+{
+ int cpu, cpu_index;
+ struct padata_queue *percpu_queue, *queue;
+
+ percpu_queue = alloc_percpu(struct padata_queue);
+
+ if (!percpu_queue) {
+ printk("padata_init: Failed to alloc the serialization"
+ "queues for padata nr %d, exiting!\n", nr);
+ return;
+ }
+
+ cpu_index = 0;
+
+ for_each_possible_cpu(cpu) {
+ queue = per_cpu_ptr(percpu_queue, cpu);
+
+ if (cpu_isset(cpu, cpu_map)) {
+ queue->cpu_index = cpu_index;
+ cpu_index++;
+ }
+
+ INIT_LIST_HEAD(&queue->list);
+ spin_lock_init(&queue->lock);
+ atomic_set(&queue->num_obj, 0);
+ }
+
+ INIT_WORK(&padata_vec[nr].work, padata_reset_work);
+
+ atomic_set(&padata_vec[nr].seq_nr, -1);
+ atomic_set(&padata_vec[nr].queued_objects, 0);
+ padata_vec[nr].cpu_map = cpu_map;
+ padata_vec[nr].new_cpu_map = cpu_map;
+ padata_vec[nr].queue = percpu_queue;
+ padata_vec[nr].flags = 0;
+ spin_lock_init(&padata_vec[nr].lock);
+}
+
+static int __init padata_initcall(void)
+{
+ open_softirq(PADATA_SOFTIRQ, padata_action);
+
+ return 0;
+}
+subsys_initcall(padata_initcall);
diff --git a/kernel/softirq.c b/kernel/softirq.c
index b525dd3..12c9b64 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -56,7 +56,7 @@ static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
char *softirq_to_name[NR_SOFTIRQS] = {
"HI", "TIMER", "NET_TX", "NET_RX", "BLOCK",
- "TASKLET", "SCHED", "HRTIMER", "RCU"
+ "TASKLET", "SCHED", "HRTIMER", "PADATA", "RCU"
};
/*
--
1.5.4.2
^ permalink raw reply related [flat|nested] 25+ messages in thread
* [RFC PATCH 1/5] padata: generic interface for parallel processing
2008-12-01 7:16 [RFC PATCH 0/5] IPsec parallelization Steffen Klassert
@ 2008-12-01 7:17 ` Steffen Klassert
0 siblings, 0 replies; 25+ messages in thread
From: Steffen Klassert @ 2008-12-01 7:17 UTC (permalink / raw)
To: netdev; +Cc: davem, herbert, klassert
From: Steffen Klassert <steffen.klassert@secunet.com>
This patch introduces an interface to process data objects
in parallel. On request it is possible to serialize again.
The parallelized objects return in the same order as they
were before the parallelization.
Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
---
include/linux/interrupt.h | 1 +
include/linux/padata.h | 115 +++++++++++
kernel/Makefile | 2 +-
kernel/padata.c | 489 +++++++++++++++++++++++++++++++++++++++++++++
4 files changed, 606 insertions(+), 1 deletions(-)
create mode 100644 include/linux/padata.h
create mode 100644 kernel/padata.c
diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
index f58a0cf..4d2f4bb 100644
--- a/include/linux/interrupt.h
+++ b/include/linux/interrupt.h
@@ -254,6 +254,7 @@ enum
#ifdef CONFIG_HIGH_RES_TIMERS
HRTIMER_SOFTIRQ,
#endif
+ PADATA_SOFTIRQ,
RCU_SOFTIRQ, /* Preferable RCU should always be the last softirq */
NR_SOFTIRQS
diff --git a/include/linux/padata.h b/include/linux/padata.h
new file mode 100644
index 0000000..6447c93
--- /dev/null
+++ b/include/linux/padata.h
@@ -0,0 +1,115 @@
+/*
+ * padata.h - header for the padata parallelization interface
+ *
+ * Copyright (C) 2008 secunet Security Networks AG
+ * Copyright (C) 2008 Steffen Klassert <steffen.klassert@secunet.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#ifndef PADATA_H
+#define PADATA_H
+
+#include <linux/interrupt.h>
+#include <linux/smp.h>
+#include <linux/list.h>
+
+enum
+{
+ NO_PADATA=0,
+ NR_PADATA
+};
+
+struct padata_priv {
+ struct list_head list;
+ struct call_single_data csd;
+ int cb_cpu;
+ int seq_nr;
+ unsigned int nr;
+ int info;
+};
+
+struct padata_queue {
+ struct list_head list;
+ atomic_t num_obj;
+ int cpu_index;
+ spinlock_t lock;
+};
+
+struct parallel_data {
+ struct work_struct work;
+ struct padata_queue *queue;
+ atomic_t seq_nr;
+ atomic_t queued_objects;
+ cpumask_t cpu_map;
+ cpumask_t new_cpu_map;
+ u8 flags;
+#define PADATA_INIT 1
+#define PADATA_FLUSH_HARD 2
+#define PADATA_RESET_IN_PROGRESS 4
+ void (*serial)(unsigned long data);
+ spinlock_t lock;
+};
+
+#ifdef CONFIG_USE_GENERIC_SMP_HELPERS
+extern void __init padata_init(unsigned int nr, cpumask_t cpu_map,
+ void (*serial)(unsigned long data));
+extern void padata_dont_wait(unsigned int nr, struct padata_priv *padata);
+extern int padata_do_parallel(unsigned int softirq_nr, unsigned int nr,
+ struct padata_priv *padata, int cb_cpu);
+extern int padata_do_serial(unsigned int nr, struct padata_priv *padata);
+extern cpumask_t padata_get_cpumap(unsigned int nr);
+extern void padata_set_cpumap(unsigned int nr, cpumask_t cpu_map);
+extern void padata_add_cpu(unsigned int nr, int cpu);
+extern void padata_remove_cpu(unsigned int nr, int cpu);
+extern void padata_start(unsigned int nr);
+extern void padata_stop(unsigned int nr);
+#else
+void padata_init(unsigned int nr,cpumask_t cpu_map,
+ void (*serial)(unsigned long data));
+{
+}
+void padata_dont_wait(unsigned int nr, struct padata_priv *padata)
+{
+}
+int padata_do_parallel(unsigned int softirq_nr, unsigned int nr,
+ struct padata_priv *padata, int cb_cpu)
+{
+ return 0;
+}
+int padata_do_serial(unsigned int nr, struct padata_priv *padata)
+{
+ return 0;
+}
+cpumask_t padata_get_cpumap(unsigned int nr)
+{
+ return cpu_online_map;
+}
+void padata_set_cpumap(unsigned int nr, cpumask_t cpu_map)
+{
+}
+padata_add_cpu(unsigned int nr, int cpu)
+{
+}
+padata_remove_cpu(unsigned int nr, int cpu)
+{
+}
+padata_start(unsigned int nr)
+{
+}
+padata_stop(unsigned int nr)
+{
+}
+#endif
+#endif
diff --git a/kernel/Makefile b/kernel/Makefile
index 19fad00..730a401 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -40,7 +40,7 @@ obj-$(CONFIG_RT_MUTEXES) += rtmutex.o
obj-$(CONFIG_DEBUG_RT_MUTEXES) += rtmutex-debug.o
obj-$(CONFIG_RT_MUTEX_TESTER) += rtmutex-tester.o
obj-$(CONFIG_GENERIC_ISA_DMA) += dma.o
-obj-$(CONFIG_USE_GENERIC_SMP_HELPERS) += smp.o
+obj-$(CONFIG_USE_GENERIC_SMP_HELPERS) += smp.o padata.o
obj-$(CONFIG_SMP) += spinlock.o
obj-$(CONFIG_DEBUG_SPINLOCK) += spinlock.o
obj-$(CONFIG_PROVE_LOCKING) += spinlock.o
diff --git a/kernel/padata.c b/kernel/padata.c
new file mode 100644
index 0000000..20eccbd
--- /dev/null
+++ b/kernel/padata.c
@@ -0,0 +1,489 @@
+/*
+ * padata.c - generic interface to process data streams in parallel
+ *
+ * Copyright (C) 2008 secunet Security Networks AG
+ * Copyright (C) 2008 Steffen Klassert <steffen.klassert@secunet.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include <linux/module.h>
+#include <linux/cpumask.h>
+#include <linux/err.h>
+#include <linux/padata.h>
+
+#define MAX_SEQ_NR 1000000000
+
+static struct parallel_data padata_vec[NR_PADATA];
+static struct padata_priv *padata_get_next(struct parallel_data *par_data);
+
+static void padata_flush_hard(struct parallel_data *par_data)
+{
+ int cpu;
+ struct padata_priv *padata;
+ struct padata_queue *queue;
+
+ for_each_cpu_mask(cpu, par_data->cpu_map) {
+ queue = per_cpu_ptr(par_data->queue, cpu);
+
+ while(!list_empty(&queue->list)) {
+ padata = list_entry(queue->list.next, struct padata_priv, list);
+
+ spin_lock(&queue->lock);
+ list_del_init(&padata->list);
+ spin_unlock(&queue->lock);
+
+ atomic_dec(&par_data->queued_objects);
+ par_data->serial((unsigned long ) padata);
+ }
+ }
+}
+
+static void padata_flush_order(struct parallel_data *par_data)
+{
+ struct padata_priv *padata;
+
+ while (1) {
+ padata = padata_get_next(par_data);
+
+ if (padata && !IS_ERR(padata))
+ par_data->serial((unsigned long ) padata);
+ else
+ break;
+ }
+
+ padata_flush_hard(par_data);
+}
+
+static void padata_reset_work(struct work_struct *work)
+{
+ int cpu, cpu_index;
+ struct padata_queue *queue;
+ struct parallel_data *par_data;
+
+ par_data = container_of(work, struct parallel_data, work);
+
+ if (par_data->flags & (PADATA_INIT|PADATA_RESET_IN_PROGRESS))
+ return;
+
+ spin_lock_bh(&par_data->lock);
+ par_data->flags |= PADATA_RESET_IN_PROGRESS;
+
+ if (!(par_data->flags & PADATA_FLUSH_HARD))
+ padata_flush_order(par_data);
+ else
+ padata_flush_hard(par_data);
+
+ cpu_index = 0;
+
+ par_data->cpu_map = par_data->new_cpu_map;
+
+ for_each_cpu_mask(cpu, par_data->cpu_map) {
+ queue = per_cpu_ptr(par_data->queue, cpu);
+
+ atomic_set(&queue->num_obj, 0);
+ queue->cpu_index = cpu_index;
+ cpu_index++;
+ }
+ spin_unlock_bh(&par_data->lock);
+
+ atomic_set(&par_data->seq_nr, -1);
+ par_data->flags &= ~PADATA_RESET_IN_PROGRESS;
+ par_data->flags |= PADATA_INIT;
+}
+
+static struct padata_priv *padata_get_next(struct parallel_data *par_data)
+{
+ int cpu, num_cpus, empty;
+ int seq_nr, calc_seq_nr, next_nr;
+ struct padata_queue *queue, *next_queue;
+ struct padata_priv *padata;
+
+ empty = 0;
+ next_nr = -1;
+ next_queue = NULL;
+
+ num_cpus = cpus_weight(par_data->cpu_map);
+
+ for_each_cpu_mask(cpu, par_data->cpu_map) {
+ queue = per_cpu_ptr(par_data->queue, cpu);
+
+ /*
+ * Calculate the seq_nr of the object that should be
+ * next in this queue.
+ */
+ calc_seq_nr = (atomic_read(&queue->num_obj) * num_cpus)
+ + queue->cpu_index;
+
+ if (!list_empty(&queue->list)) {
+ padata = list_entry(queue->list.next,
+ struct padata_priv, list);
+
+ seq_nr = padata->seq_nr;
+
+ if (unlikely(calc_seq_nr != seq_nr)) {
+ par_data->flags &= ~PADATA_INIT;
+ par_data->flags |= PADATA_FLUSH_HARD;
+ padata = NULL;
+ goto out;
+ }
+ } else {
+ seq_nr = calc_seq_nr;
+ empty++;
+ }
+
+ if (next_nr < 0 || seq_nr < next_nr) {
+ next_nr = seq_nr;
+ next_queue = queue;
+ }
+ }
+
+ padata = NULL;
+
+ if (empty == num_cpus)
+ goto out;
+
+ if (!list_empty(&next_queue->list)) {
+ padata = list_entry(next_queue->list.next,
+ struct padata_priv, list);
+
+ spin_lock(&next_queue->lock);
+ list_del_init(&padata->list);
+ spin_unlock(&next_queue->lock);
+
+ atomic_dec(&par_data->queued_objects);
+ atomic_inc(&next_queue->num_obj);
+
+ goto out;
+ }
+
+ if (next_nr % num_cpus == next_queue->cpu_index) {
+ padata = ERR_PTR(-ENODATA);
+ goto out;
+ }
+
+ padata = ERR_PTR(-EINPROGRESS);
+out:
+ return padata;
+}
+
+static void padata_action(struct softirq_action *h)
+{
+ struct list_head *cpu_list, local_list;
+
+ cpu_list = &__get_cpu_var(softirq_work_list[PADATA_SOFTIRQ]);
+
+ local_irq_disable();
+ list_replace_init(cpu_list, &local_list);
+ local_irq_enable();
+
+ while (!list_empty(&local_list)) {
+ struct padata_priv *padata;
+
+ padata = list_entry(local_list.next,
+ struct padata_priv, csd.list);
+
+ list_del_init(&padata->csd.list);
+
+ padata_vec[padata->nr].serial((unsigned long ) padata);
+ }
+}
+
+static int padata_cpu_hash(unsigned int nr, struct padata_priv *padata)
+{
+ int cpu, target_cpu, this_cpu, cpu_index;
+
+ this_cpu = smp_processor_id();
+
+ if (padata->nr != 0)
+ return this_cpu;
+
+ if (!(padata_vec[nr].flags & PADATA_INIT))
+ return this_cpu;
+
+ padata->seq_nr = atomic_inc_return(&padata_vec[nr].seq_nr);
+
+ if (padata->seq_nr > MAX_SEQ_NR) {
+ padata_vec[nr].flags &= ~PADATA_INIT;
+ padata->seq_nr = 0;
+ schedule_work(&padata_vec[nr].work);
+ return this_cpu;
+ }
+
+ padata->nr = nr;
+
+ /*
+ * Hash the sequence numbers to the cpus by taking
+ * seq_nr mod. number of cpus in use.
+ */
+ cpu_index = padata->seq_nr % cpus_weight(padata_vec[nr].cpu_map);
+
+ target_cpu = first_cpu(padata_vec[nr].cpu_map);
+ for (cpu = 0; cpu < cpu_index; cpu++)
+ target_cpu = next_cpu(target_cpu, padata_vec[nr].cpu_map);
+
+ return target_cpu;
+}
+
+/*
+ * padata_dont_wait - must be called if an object that runs in parallel will
+ * not be serialized with padata_do_serial
+ *
+ * @nr: number of the padata istance
+ * @padata: object that will not be seen by padata_do_serial
+ */
+void padata_dont_wait(unsigned int nr, struct padata_priv *padata)
+{
+ struct padata_queue *queue;
+
+ if (!(padata_vec[nr].flags & PADATA_INIT))
+ return;
+
+ if (padata->nr == 0 || padata->nr != nr)
+ return;
+
+ queue = per_cpu_ptr(padata_vec[nr].queue, smp_processor_id());
+ atomic_inc(&queue->num_obj);
+
+ padata->nr = 0;
+ padata->seq_nr = 0;
+}
+EXPORT_SYMBOL(padata_dont_wait);
+
+/*
+ * padata_do_parallel - padata parallelization function
+ *
+ * @softirq_nr: number of the softirq that will do the parallelization
+ * @nr: number of the padata istance
+ * @padata: object to be paralellized
+ * @cb_cpu: cpu number on which the serialization callback function will run
+ */
+int padata_do_parallel(unsigned int softirq_nr, unsigned int nr,
+ struct padata_priv *padata, int cb_cpu)
+{
+ int target_cpu;
+
+ padata->cb_cpu = cb_cpu;
+
+ local_bh_disable();
+ target_cpu = padata_cpu_hash(nr, padata);
+ local_bh_enable();
+
+ send_remote_softirq(&padata->csd, target_cpu, softirq_nr);
+
+ return 1;
+}
+EXPORT_SYMBOL(padata_do_parallel);
+
+/*
+ * padata_do_serial - padata serialization function
+ *
+ * @nr: number of the padata istance
+ * @padata: object to be serialized
+ *
+ * returns 1 if the serialization callback function will be called
+ * from padata, 0 else
+ */
+int padata_do_serial(unsigned int nr, struct padata_priv *padata)
+{
+ int cpu;
+ struct padata_queue *reorder_queue;
+
+ if (!(padata_vec[nr].flags & PADATA_INIT))
+ return 0;
+
+ if (padata->nr != nr || padata->nr == 0) {
+ padata_vec[nr].serial((unsigned long ) padata);
+ return 1;
+ }
+
+ cpu = smp_processor_id();
+
+ reorder_queue = per_cpu_ptr(padata_vec[nr].queue, cpu);
+
+ spin_lock(&reorder_queue->lock);
+ list_add_tail(&padata->list, &reorder_queue->list);
+ spin_unlock(&reorder_queue->lock);
+
+ atomic_inc(&padata_vec[nr].queued_objects);
+
+try_again:
+ if (!spin_trylock(&padata_vec[nr].lock))
+ goto out;
+
+ while(1) {
+ padata = padata_get_next(&padata_vec[nr]);
+
+ if (!padata || PTR_ERR(padata) == -EINPROGRESS)
+ break;
+ if (PTR_ERR(padata) == -ENODATA) {
+ spin_unlock(&padata_vec[nr].lock);
+ goto out;
+ }
+
+ send_remote_softirq(&padata->csd, padata->cb_cpu,
+ PADATA_SOFTIRQ);
+ }
+
+ if (unlikely(!(padata_vec[nr].flags & PADATA_INIT))) {
+ spin_unlock(&padata_vec[nr].lock);
+ goto reset_out;
+ }
+
+ spin_unlock(&padata_vec[nr].lock);
+
+ if (atomic_read(&padata_vec[nr].queued_objects))
+ goto try_again;
+
+out:
+ return 1;
+reset_out:
+ schedule_work(&padata_vec[nr].work);
+ return 1;
+}
+EXPORT_SYMBOL(padata_do_serial);
+
+/*
+ * padata_get_cpumap - get the cpu map that is actually in use
+ *
+ * @nr: number of the padata istance
+ */
+cpumask_t padata_get_cpumap(unsigned int nr)
+{
+ return padata_vec[nr].cpu_map;
+}
+EXPORT_SYMBOL(padata_get_cpumap);
+
+/*
+ * padata_set_cpumap - set the cpu map that padata uses
+ *
+ * @nr: number of the padata istance
+ * @cpu_map: the cpu map to use
+ */
+void padata_set_cpumap(unsigned int nr, cpumask_t cpu_map)
+{
+ padata_vec[nr].new_cpu_map = cpu_map;
+ padata_vec[nr].flags &= ~PADATA_INIT;
+ padata_vec[nr].flags |= PADATA_FLUSH_HARD;
+
+ schedule_work(&padata_vec[nr].work);
+}
+EXPORT_SYMBOL(padata_set_cpumap);
+
+/*
+ * padata_add_cpu - add a cpu to the padata cpu map
+ *
+ * @nr: number of the padata istance
+ * @cpu: cpu to remove
+ */
+void padata_add_cpu(unsigned int nr, int cpu)
+{
+ cpumask_t cpu_map = padata_vec[nr].cpu_map;
+
+ cpu_set(cpu, cpu_map);
+ padata_set_cpumap(nr, cpu_map);
+}
+EXPORT_SYMBOL(padata_add_cpu);
+
+/*
+ * padata_remove_cpu - remove a cpu from the padata cpu map
+ *
+ * @nr: number of the padata istance
+ * @cpu: cpu to remove
+ */
+void padata_remove_cpu(unsigned int nr, int cpu)
+{
+ cpumask_t cpu_map = padata_vec[nr].cpu_map;
+
+ cpu_clear(cpu, cpu_map);
+ padata_set_cpumap(nr, cpu_map);
+}
+EXPORT_SYMBOL(padata_remove_cpu);
+
+/*
+ * padata_start - start the parallel processing
+ *
+ * @nr: number of the padata istance
+ */
+void padata_start(unsigned int nr)
+{
+ if (padata_vec[nr].flags & PADATA_INIT)
+ return;
+
+ schedule_work(&padata_vec[nr].work);
+}
+EXPORT_SYMBOL(padata_start);
+
+/*
+ * padata_stop - stop the parallel processing
+ *
+ * @nr: number of the padata istance
+ */
+void padata_stop(unsigned int nr)
+{
+ padata_vec[nr].flags &= ~PADATA_INIT;
+}
+EXPORT_SYMBOL(padata_stop);
+
+/*
+ * padata_init - initialize a padata instance
+ *
+ * @nr: number of the padata istance
+ * @cpu_map: map of the cpu set that padata uses for parallelization
+ * @serial: the serialization callback function
+ *
+ * The serialization callback function must be able to run in softirq context.
+ */
+void __init padata_init(unsigned int nr, cpumask_t cpu_map,
+ void (*serial)(unsigned long data))
+{
+ int cpu, cpu_index;
+ struct padata_queue *percpu_queue, *queue;
+
+ percpu_queue = alloc_percpu(struct padata_queue);
+
+ if (!percpu_queue) {
+ printk("padata_init: Failed to alloc the serialization"
+ "queues for padata nr %d, exiting!\n", nr);
+ return;
+ }
+
+ open_softirq(PADATA_SOFTIRQ, padata_action);
+
+ cpu_index = 0;
+
+ for_each_possible_cpu(cpu) {
+ queue = per_cpu_ptr(percpu_queue, cpu);
+
+ if (cpu_isset(cpu, cpu_map)) {
+ queue->cpu_index = cpu_index;
+ cpu_index++;
+ }
+
+ INIT_LIST_HEAD(&queue->list);
+ spin_lock_init(&queue->lock);
+ atomic_set(&queue->num_obj, 0);
+ }
+
+ INIT_WORK(&padata_vec[nr].work, padata_reset_work);
+
+ atomic_set(&padata_vec[nr].seq_nr, -1);
+ atomic_set(&padata_vec[nr].queued_objects, 0);
+ padata_vec[nr].cpu_map = cpu_map;
+ padata_vec[nr].new_cpu_map = cpu_map;
+ padata_vec[nr].queue = percpu_queue;
+ padata_vec[nr].serial = serial;
+ padata_vec[nr].flags = 0;
+}
+EXPORT_SYMBOL(padata_init);
--
1.5.4.2
^ permalink raw reply related [flat|nested] 25+ messages in thread