[RFC] [PATCH 0/5] Parallel IPsec v3

[RFC] [PATCH 0/5] Parallel IPsec v3

[Steffen Klassert]

This patchset adds the 'pcrypt' parallel crypto template. With this template it
is possible to process the crypto requests of a transform in parallel without
getting request reorder. This is in particular interesting for IPsec.

The parallel crypto template is based on a generic parallelization/serialization
method. This method uses the remote softirq invocation infrastructure for
parallelization and serialization. With this method data objects can be
processed in parallel, starting at some given point.
After doing some expensive operations in parallel, it is possible to serialize
again. The parallelized data objects return after serialization in the order as
they were before the parallelization. In the case of IPsec, this makes it
possible to run the expensive parts in parallel without getting packet
reordering.

Changes from v2:

- The xfrm netlink configuration code is dropped,
  this will be an extra patchset.

- Add generic aead wrapper interface to be able to wrap an aead algorithm
  with an arbitrary crypto template. 

- Convert pcrypt to use the generic aead wrapper.

- Add support for aead algorithms to eseqiv.

- Add support for the pcrypt aead wrapper to authenc. It's now possible to
  choose for pcrypt as the default authenc wrapper with a module parameter.

- Patchset applies to linux-2.6 git current.

Changes from v1:

- cpu_chainiv is dropped, pcrypt uses eseqiv as it's IV generator now.

- Add a xfrm netlink message to be able to choose for pcrypt from userspace.

- Use pcrypt just if it is selected from userspace.

- Patchset applies to 2.6.30-rc3

Steffen

[RFC] [PATCH 1/5] padata: generic interface for parallel processing

[Steffen Klassert]

This patch introduces an interface to process data objects
in parallel. On request it is possible to serialize again.
The parallelized objects return after serialization in the
same order as they were before the parallelization.

Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
---
 include/linux/interrupt.h |    3 +-
 include/linux/padata.h    |  116 +++++++++++
 kernel/Makefile           |    2 +-
 kernel/padata.c           |  490 +++++++++++++++++++++++++++++++++++++++++++++
 kernel/softirq.c          |    2 +-
 5 files changed, 610 insertions(+), 3 deletions(-)
 create mode 100644 include/linux/padata.h
 create mode 100644 kernel/padata.c

diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
index 91bb76f..a17679c 100644
--- a/include/linux/interrupt.h
+++ b/include/linux/interrupt.h
@@ -338,7 +338,8 @@ enum
 	TASKLET_SOFTIRQ,
 	SCHED_SOFTIRQ,
 	HRTIMER_SOFTIRQ,
-	RCU_SOFTIRQ,	/* Preferable RCU should always be the last softirq */
+	PADATA_SOFTIRQ,
+	RCU_SOFTIRQ, 	/* Preferable RCU should always be the last softirq */
 
 	NR_SOFTIRQS
 };
diff --git a/include/linux/padata.h b/include/linux/padata.h
new file mode 100644
index 0000000..469359f
--- /dev/null
+++ b/include/linux/padata.h
@@ -0,0 +1,116 @@
+/*
+ * padata.h - header for the padata parallelization interface
+ *
+ * Copyright (C) 2008, 2009 secunet Security Networks AG
+ * Copyright (C) 2008, 2009 Steffen Klassert <steffen.klassert@secunet.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#ifndef PADATA_H
+#define PADATA_H
+
+#include <linux/interrupt.h>
+#include <linux/smp.h>
+#include <linux/list.h>
+
+enum
+{
+	NO_PADATA=0,
+	AEAD_ENC_PADATA,
+	AEAD_DEC_PADATA,
+	NR_PADATA
+};
+
+struct padata_priv {
+	struct list_head	list;
+	struct call_single_data	csd;
+	int			cb_cpu;
+	int			seq_nr;
+	unsigned int		nr;
+	int			info;
+	void                    (*parallel)(struct padata_priv *padata);
+	void                    (*serial)(struct padata_priv *padata);
+};
+
+struct padata_queue {
+	struct list_head        list;
+	atomic_t                num_obj;
+	int               	cpu_index;
+	spinlock_t              lock;
+};
+
+struct parallel_data {
+	struct work_struct	work;
+	struct padata_queue	*queue;
+	atomic_t		seq_nr;
+	atomic_t		queued_objects;
+	cpumask_t		cpu_map;
+	cpumask_t		new_cpu_map;
+	u8			flags;
+#define	PADATA_INIT			1
+#define	PADATA_FLUSH_HARD		2
+#define	PADATA_RESET_IN_PROGRESS	4
+	spinlock_t              lock;
+};
+
+#ifdef CONFIG_USE_GENERIC_SMP_HELPERS
+extern void __init padata_init(unsigned int nr, cpumask_t cpu_map);
+extern void padata_dont_wait(unsigned int nr, struct padata_priv *padata);
+extern int padata_do_parallel(unsigned int softirq_nr, unsigned int nr,
+			      struct padata_priv *padata, int cb_cpu);
+extern int padata_do_serial(unsigned int nr, struct padata_priv *padata);
+extern cpumask_t padata_get_cpumap(unsigned int nr);
+extern void padata_set_cpumap(unsigned int nr, cpumask_t cpu_map);
+extern void padata_add_cpu(unsigned int nr, int cpu);
+extern void padata_remove_cpu(unsigned int nr, int cpu);
+extern void padata_start(unsigned int nr);
+extern void padata_stop(unsigned int nr);
+#else
+static inline void padata_init(unsigned int nr,cpumask_t cpu_map)
+{
+}
+static inline void padata_dont_wait(unsigned int nr, struct padata_priv *padata)
+{
+}
+static inline int padata_do_parallel(unsigned int softirq_nr, unsigned int nr,
+				     struct padata_priv *padata, int cb_cpu)
+{
+	return 0;
+}
+static inline int padata_do_serial(unsigned int nr, struct padata_priv *padata)
+{
+	return 0;
+}
+static inline cpumask_t padata_get_cpumap(unsigned int nr)
+{
+	return cpu_online_map;
+}
+static inline void padata_set_cpumap(unsigned int nr, cpumask_t cpu_map)
+{
+}
+static inline padata_add_cpu(unsigned int nr, int cpu)
+{
+}
+static inline padata_remove_cpu(unsigned int nr, int cpu)
+{
+}
+static inline padata_start(unsigned int nr)
+{
+}
+static inline padata_stop(unsigned int nr)
+{
+}
+#endif
+#endif
diff --git a/kernel/Makefile b/kernel/Makefile
index 4242366..f2d7ea7 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -40,7 +40,7 @@ obj-$(CONFIG_RT_MUTEXES) += rtmutex.o
 obj-$(CONFIG_DEBUG_RT_MUTEXES) += rtmutex-debug.o
 obj-$(CONFIG_RT_MUTEX_TESTER) += rtmutex-tester.o
 obj-$(CONFIG_GENERIC_ISA_DMA) += dma.o
-obj-$(CONFIG_USE_GENERIC_SMP_HELPERS) += smp.o
+obj-$(CONFIG_USE_GENERIC_SMP_HELPERS) += smp.o padata.o
 ifneq ($(CONFIG_SMP),y)
 obj-y += up.o
 endif
diff --git a/kernel/padata.c b/kernel/padata.c
new file mode 100644
index 0000000..192c9a6
--- /dev/null
+++ b/kernel/padata.c
@@ -0,0 +1,490 @@
+/*
+ * padata.c - generic interface to process data streams in parallel
+ *
+ * Copyright (C) 2008, 2009 secunet Security Networks AG
+ * Copyright (C) 2008, 2009 Steffen Klassert <steffen.klassert@secunet.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include <linux/module.h>
+#include <linux/cpumask.h>
+#include <linux/err.h>
+#include <linux/padata.h>
+
+#define MAX_SEQ_NR  1000000000
+
+static struct parallel_data padata_vec[NR_PADATA];
+static struct padata_priv *padata_get_next(struct parallel_data *par_data);
+
+static void padata_flush_hard(struct parallel_data *par_data)
+{
+	int cpu;
+	struct padata_priv *padata;
+	struct padata_queue *queue;
+
+	for_each_cpu_mask(cpu, par_data->cpu_map) {
+		queue = per_cpu_ptr(par_data->queue, cpu);
+
+		while(!list_empty(&queue->list)) {
+			padata = list_entry(queue->list.next, struct padata_priv, list);
+
+			spin_lock(&queue->lock);
+			list_del_init(&padata->list);
+			spin_unlock(&queue->lock);
+
+			atomic_dec(&par_data->queued_objects);
+			padata->serial(padata);
+		}
+	}
+}
+
+static void padata_flush_order(struct parallel_data *par_data)
+{
+	struct padata_priv *padata;
+
+	while (1) {
+		padata = padata_get_next(par_data);
+
+		if (padata && !IS_ERR(padata))
+			padata->serial(padata);
+		else
+			break;
+	}
+
+	padata_flush_hard(par_data);
+}
+
+static void padata_reset_work(struct work_struct *work)
+{
+	int cpu, cpu_index;
+	struct padata_queue *queue;
+	struct parallel_data *par_data;
+
+	par_data = container_of(work, struct parallel_data, work);
+
+	if (par_data->flags & (PADATA_INIT|PADATA_RESET_IN_PROGRESS))
+		return;
+
+	spin_lock_bh(&par_data->lock);
+	par_data->flags |= PADATA_RESET_IN_PROGRESS;
+
+	if (!(par_data->flags & PADATA_FLUSH_HARD))
+		padata_flush_order(par_data);
+	else
+		padata_flush_hard(par_data);
+
+	cpu_index = 0;
+
+	par_data->cpu_map = par_data->new_cpu_map;
+
+	for_each_cpu_mask(cpu, par_data->cpu_map) {
+		queue = per_cpu_ptr(par_data->queue, cpu);
+
+		atomic_set(&queue->num_obj, 0);
+		queue->cpu_index = cpu_index;
+		cpu_index++;
+	}
+	spin_unlock_bh(&par_data->lock);
+
+	atomic_set(&par_data->seq_nr, -1);
+	par_data->flags &= ~PADATA_RESET_IN_PROGRESS;
+	par_data->flags |= PADATA_INIT;
+}
+
+static struct padata_priv *padata_get_next(struct parallel_data *par_data)
+{
+	int cpu, num_cpus, empty;
+	int seq_nr, calc_seq_nr, next_nr;
+	struct padata_queue *queue, *next_queue;
+	struct padata_priv *padata;
+
+	empty = 0;
+	next_nr = -1;
+	next_queue = NULL;
+
+	num_cpus = cpus_weight(par_data->cpu_map);
+
+	for_each_cpu_mask(cpu, par_data->cpu_map) {
+		queue = per_cpu_ptr(par_data->queue, cpu);
+
+		/*
+		 * Calculate the seq_nr of the object that should be
+		 * next in this queue.
+		 */
+		calc_seq_nr = (atomic_read(&queue->num_obj) * num_cpus)
+			+ queue->cpu_index;
+
+		if (!list_empty(&queue->list)) {
+			padata = list_entry(queue->list.next,
+					    struct padata_priv, list);
+
+			seq_nr  = padata->seq_nr;
+
+			if (unlikely(calc_seq_nr != seq_nr)) {
+				par_data->flags &= ~PADATA_INIT;
+				par_data->flags |= PADATA_FLUSH_HARD;
+				padata = NULL;
+				goto out;
+			}
+		} else {
+			seq_nr = calc_seq_nr;
+			empty++;
+		}
+
+		if (next_nr < 0 || seq_nr < next_nr) {
+			next_nr = seq_nr;
+			next_queue = queue;
+		}
+	}
+
+	padata = NULL;
+
+	if (empty == num_cpus)
+		goto out;
+
+	if (!list_empty(&next_queue->list)) {
+		padata = list_entry(next_queue->list.next,
+				    struct padata_priv, list);
+
+		spin_lock(&next_queue->lock);
+		list_del_init(&padata->list);
+		spin_unlock(&next_queue->lock);
+
+		atomic_dec(&par_data->queued_objects);
+		atomic_inc(&next_queue->num_obj);
+
+		goto out;
+	}
+
+	if (next_nr % num_cpus == next_queue->cpu_index) {
+		padata = ERR_PTR(-ENODATA);
+		goto out;
+	}
+
+	padata = ERR_PTR(-EINPROGRESS);
+out:
+	return padata;
+}
+
+static void padata_action(struct softirq_action *h)
+{
+	struct list_head *cpu_list, local_list;
+
+	cpu_list = &__get_cpu_var(softirq_work_list[PADATA_SOFTIRQ]);
+
+	local_irq_disable();
+	list_replace_init(cpu_list, &local_list);
+	local_irq_enable();
+
+	while (!list_empty(&local_list)) {
+		struct padata_priv *padata;
+
+		padata = list_entry(local_list.next,
+				    struct padata_priv, csd.list);
+
+		list_del_init(&padata->csd.list);
+
+		padata->serial(padata);
+	}
+}
+
+static int padata_cpu_hash(unsigned int nr, struct padata_priv *padata)
+{
+	int cpu, target_cpu, this_cpu, cpu_index;
+
+	this_cpu = smp_processor_id();
+
+	if (padata->nr != 0)
+		return this_cpu;
+
+	if (!(padata_vec[nr].flags & PADATA_INIT))
+		return this_cpu;
+
+	padata->seq_nr = atomic_inc_return(&padata_vec[nr].seq_nr);
+
+	if (padata->seq_nr > MAX_SEQ_NR) {
+		padata_vec[nr].flags &= ~PADATA_INIT;
+		padata->seq_nr = 0;
+		schedule_work(&padata_vec[nr].work);
+		return this_cpu;
+	}
+
+	padata->nr = nr;
+
+	/*
+	 * Hash the sequence numbers to the cpus by taking
+	 * seq_nr mod. number of cpus in use.
+	 */
+	cpu_index =  padata->seq_nr % cpus_weight(padata_vec[nr].cpu_map);
+
+	target_cpu = first_cpu(padata_vec[nr].cpu_map);
+	for (cpu = 0; cpu < cpu_index; cpu++)
+		target_cpu = next_cpu(target_cpu, padata_vec[nr].cpu_map);
+
+	return target_cpu;
+}
+
+/*
+ * padata_dont_wait - must be called if an object that runs in parallel will
+ * not be serialized with padata_do_serial
+ *
+ * @nr: number of the padata instance
+ * @padata: object that will not be seen by padata_do_serial
+ */
+void padata_dont_wait(unsigned int nr, struct padata_priv *padata)
+{
+	struct padata_queue *queue;
+
+	if (!(padata_vec[nr].flags & PADATA_INIT))
+		return;
+
+	if (padata->nr == 0 || padata->nr != nr)
+		return;
+
+	queue = per_cpu_ptr(padata_vec[nr].queue, smp_processor_id());
+	atomic_inc(&queue->num_obj);
+
+	padata->nr = 0;
+	padata->seq_nr = 0;
+}
+EXPORT_SYMBOL(padata_dont_wait);
+
+/*
+ * padata_do_parallel - padata parallelization function
+ *
+ * @softirq_nr: number of the softirq that will do the parallelization
+ * @nr: number of the padata instance
+ * @padata: object to be parallelized
+ * @cb_cpu: cpu number on which the serialization callback function will run
+ */
+int padata_do_parallel(unsigned int softirq_nr, unsigned int nr,
+		       struct padata_priv *padata, int cb_cpu)
+{
+	int target_cpu;
+
+	padata->cb_cpu = cb_cpu;
+
+	local_bh_disable();
+	target_cpu = padata_cpu_hash(nr, padata);
+	local_bh_enable();
+
+	send_remote_softirq(&padata->csd, target_cpu, softirq_nr);
+
+	return 1;
+}
+EXPORT_SYMBOL(padata_do_parallel);
+
+/*
+ * padata_do_serial - padata serialization function
+ *
+ * @nr: number of the padata instance
+ * @padata: object to be serialized
+ *
+ * returns 1 if the serialization callback function will be called
+ * from padata, 0 else
+ */
+int padata_do_serial(unsigned int nr, struct padata_priv *padata)
+{
+	int cpu;
+	struct padata_queue *reorder_queue;
+
+	if (!(padata_vec[nr].flags & PADATA_INIT))
+		return 0;
+
+	if (padata->nr != nr || padata->nr == 0) {
+		padata->serial(padata);
+		return 1;
+	}
+
+	cpu = smp_processor_id();
+
+	reorder_queue = per_cpu_ptr(padata_vec[nr].queue, cpu);
+
+	spin_lock(&reorder_queue->lock);
+	list_add_tail(&padata->list, &reorder_queue->list);
+	spin_unlock(&reorder_queue->lock);
+
+	atomic_inc(&padata_vec[nr].queued_objects);
+
+try_again:
+	if (!spin_trylock(&padata_vec[nr].lock))
+		goto out;
+
+	while(1) {
+		padata = padata_get_next(&padata_vec[nr]);
+
+		if (!padata || PTR_ERR(padata) == -EINPROGRESS)
+			break;
+		if (PTR_ERR(padata) == -ENODATA) {
+			spin_unlock(&padata_vec[nr].lock);
+			goto out;
+		}
+
+		send_remote_softirq(&padata->csd, padata->cb_cpu,
+				    PADATA_SOFTIRQ);
+	}
+
+	if (unlikely(!(padata_vec[nr].flags & PADATA_INIT))) {
+		spin_unlock(&padata_vec[nr].lock);
+		goto reset_out;
+	}
+
+	spin_unlock(&padata_vec[nr].lock);
+
+	if (atomic_read(&padata_vec[nr].queued_objects))
+		goto try_again;
+
+out:
+	return 1;
+reset_out:
+	schedule_work(&padata_vec[nr].work);
+	return 1;
+}
+EXPORT_SYMBOL(padata_do_serial);
+
+/*
+ * padata_get_cpumap - get the cpu map that is actually in use
+ *
+ * @nr: number of the padata instance
+ */
+cpumask_t padata_get_cpumap(unsigned int nr)
+{
+	return padata_vec[nr].cpu_map;
+}
+EXPORT_SYMBOL(padata_get_cpumap);
+
+/*
+ * padata_set_cpumap - set the cpu map that padata uses
+ *
+ * @nr: number of the padata instance
+ * @cpu_map: the cpu map to use
+ */
+void padata_set_cpumap(unsigned int nr, cpumask_t cpu_map)
+{
+	padata_vec[nr].new_cpu_map = cpu_map;
+	padata_vec[nr].flags &= ~PADATA_INIT;
+	padata_vec[nr].flags |= PADATA_FLUSH_HARD;
+
+	schedule_work(&padata_vec[nr].work);
+}
+EXPORT_SYMBOL(padata_set_cpumap);
+
+/*
+ * padata_add_cpu - add a cpu to the padata cpu map
+ *
+ * @nr: number of the padata instance
+ * @cpu: cpu to remove
+ */
+void padata_add_cpu(unsigned int nr, int cpu)
+{
+	cpumask_t cpu_map = padata_vec[nr].cpu_map;
+
+	cpu_set(cpu, cpu_map);
+	padata_set_cpumap(nr, cpu_map);
+}
+EXPORT_SYMBOL(padata_add_cpu);
+
+/*
+ * padata_remove_cpu - remove a cpu from the padata cpu map
+ *
+ * @nr: number of the padata instance
+ * @cpu: cpu to remove
+ */
+void padata_remove_cpu(unsigned int nr, int cpu)
+{
+	cpumask_t cpu_map = padata_vec[nr].cpu_map;
+
+	cpu_clear(cpu, cpu_map);
+	padata_set_cpumap(nr, cpu_map);
+}
+EXPORT_SYMBOL(padata_remove_cpu);
+
+/*
+ * padata_start - start the parallel processing
+ *
+ * @nr: number of the padata instance
+ */
+void padata_start(unsigned int nr)
+{
+	if (padata_vec[nr].flags & PADATA_INIT)
+		return;
+
+	schedule_work(&padata_vec[nr].work);
+}
+EXPORT_SYMBOL(padata_start);
+
+/*
+ * padata_stop - stop the parallel processing
+ *
+ * @nr: number of the padata instance
+ */
+void padata_stop(unsigned int nr)
+{
+	padata_vec[nr].flags &= ~PADATA_INIT;
+}
+EXPORT_SYMBOL(padata_stop);
+
+/*
+ * padata_init - initialize a padata instance
+ *
+ * @nr: number of the padata instance
+ * @cpu_map: map of the cpu set that padata uses for parallelization
+ */
+void __init padata_init(unsigned int nr, cpumask_t cpu_map)
+{
+	int cpu, cpu_index;
+	struct padata_queue *percpu_queue, *queue;
+
+	percpu_queue = alloc_percpu(struct padata_queue);
+
+	if (!percpu_queue) {
+		printk("padata_init: Failed to alloc the serialization"
+		       "queues for padata nr %d, exiting!\n", nr);
+		return;
+	}
+
+	cpu_index = 0;
+
+	for_each_possible_cpu(cpu) {
+		queue = per_cpu_ptr(percpu_queue, cpu);
+
+		if (cpu_isset(cpu, cpu_map)) {
+			queue->cpu_index = cpu_index;
+			cpu_index++;
+		}
+
+		INIT_LIST_HEAD(&queue->list);
+		spin_lock_init(&queue->lock);
+		atomic_set(&queue->num_obj, 0);
+	}
+
+	INIT_WORK(&padata_vec[nr].work, padata_reset_work);
+
+	atomic_set(&padata_vec[nr].seq_nr, -1);
+	atomic_set(&padata_vec[nr].queued_objects, 0);
+	padata_vec[nr].cpu_map = cpu_map;
+	padata_vec[nr].new_cpu_map = cpu_map;
+	padata_vec[nr].queue = percpu_queue;
+	padata_vec[nr].flags = 0;
+	spin_lock_init(&padata_vec[nr].lock);
+}
+
+static int __init padata_initcall(void)
+{
+	open_softirq(PADATA_SOFTIRQ, padata_action);
+
+	return 0;
+}
+subsys_initcall(padata_initcall);
diff --git a/kernel/softirq.c b/kernel/softirq.c
index b525dd3..12c9b64 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -56,7 +56,7 @@ static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
 
 char *softirq_to_name[NR_SOFTIRQS] = {
 	"HI", "TIMER", "NET_TX", "NET_RX", "BLOCK",
-	"TASKLET", "SCHED", "HRTIMER",	"RCU"
+	"TASKLET", "SCHED", "HRTIMER", "PADATA", "RCU"
 };
 
 /*
-- 
1.5.4.2

[RFC] [PATCH 2/5] aead: Add generic aead wrapper interface

[Steffen Klassert]

This patch adds an interface to wrap aead algorithms with an
arbitrary crypto template. This is useful to make a certain
version of an aead algorithm the system default of that algorithm.

Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
---
 crypto/aead.c                  |  286 +++++++++++++++++++++++++++++++++++++++-
 include/crypto/internal/aead.h |    8 +
 include/linux/crypto.h         |   10 +-
 3 files changed, 295 insertions(+), 9 deletions(-)

diff --git a/crypto/aead.c b/crypto/aead.c
index d9aa733..f45e27a 100644
--- a/crypto/aead.c
+++ b/crypto/aead.c
@@ -286,6 +286,7 @@ struct crypto_instance *aead_geniv_alloc(struct crypto_template *tmpl,
 	inst->alg.cra_aead.ivsize = alg->cra_aead.ivsize;
 	inst->alg.cra_aead.maxauthsize = alg->cra_aead.maxauthsize;
 	inst->alg.cra_aead.geniv = alg->cra_aead.geniv;
+	inst->alg.cra_aead.wrapper = alg->cra_aead.wrapper;
 
 	inst->alg.cra_aead.setkey = alg->cra_aead.setkey;
 	inst->alg.cra_aead.setauthsize = alg->cra_aead.setauthsize;
@@ -407,6 +408,262 @@ out:
 	return err;
 }
 
+static int crypto_init_wrapaead_ops(struct crypto_tfm *tfm, u32 type, u32 mask)
+{
+	struct aead_alg *alg = &tfm->__crt_alg->cra_aead;
+	struct aead_tfm *crt = &tfm->crt_aead;
+
+	if (max(alg->maxauthsize, alg->ivsize) > PAGE_SIZE / 8)
+		return -EINVAL;
+
+	crt->setkey = alg->setkey;
+	crt->encrypt = alg->encrypt;
+	crt->decrypt = alg->decrypt;
+	crt->givencrypt = alg->givencrypt ?: no_givcrypt;
+	crt->givdecrypt = alg->givdecrypt ?: no_givcrypt;
+	crt->base = __crypto_aead_cast(tfm);
+	crt->ivsize = alg->ivsize;
+	crt->authsize = alg->maxauthsize;
+
+	return 0;
+}
+
+static void crypto_wrapaead_show(struct seq_file *m, struct crypto_alg *alg)
+	__attribute__ ((unused));
+static void crypto_wrapaead_show(struct seq_file *m, struct crypto_alg *alg)
+{
+	struct aead_alg *aead = &alg->cra_aead;
+
+	seq_printf(m, "type         : wrapped aead\n");
+	seq_printf(m, "async        : %s\n", alg->cra_flags & CRYPTO_ALG_ASYNC ?
+					     "yes" : "no");
+	seq_printf(m, "blocksize    : %u\n", alg->cra_blocksize);
+	seq_printf(m, "ivsize       : %u\n", aead->ivsize);
+	seq_printf(m, "maxauthsize  : %u\n", aead->maxauthsize);
+	seq_printf(m, "geniv        : %s\n", aead->geniv ?: "<built-in>");
+	seq_printf(m, "wrapper      : %s\n", alg->cra_flags & CRYPTO_ALG_WRAP ?
+					     aead->wrapper : "<built-in>");
+}
+
+const struct crypto_type crypto_wrapaead_type = {
+	.ctxsize = crypto_aead_ctxsize,
+	.init = crypto_init_wrapaead_ops,
+#ifdef CONFIG_PROC_FS
+	.show = crypto_wrapaead_show,
+#endif
+};
+EXPORT_SYMBOL_GPL(crypto_wrapaead_type);
+
+static int crypto_grab_wrapaead(struct crypto_aead_spawn *spawn,
+			       const char *name, u32 type, u32 mask)
+{
+	struct crypto_alg *alg;
+	int err;
+
+	type &= ~(CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_WRAP);
+	type |= CRYPTO_ALG_TYPE_AEAD;
+	mask |= CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_WRAP;
+
+	alg = crypto_alg_mod_lookup(name, type, mask);
+	if (IS_ERR(alg))
+		return PTR_ERR(alg);
+
+	err = crypto_init_spawn(&spawn->base, alg, spawn->base.inst, mask);
+	crypto_mod_put(alg);
+	return err;
+}
+
+struct crypto_instance *aead_wrap_alloc(struct crypto_template *tmpl,
+					 struct rtattr **tb,
+					 unsigned int ctx_size, u32 type,
+					 u32 mask)
+{
+	const char *name;
+	struct crypto_aead_spawn *spawn;
+	struct crypto_attr_type *algt;
+	struct crypto_instance *inst;
+	struct crypto_alg *alg;
+	int err;
+
+	algt = crypto_get_attr_type(tb);
+	err = PTR_ERR(algt);
+	if (IS_ERR(algt))
+		return ERR_PTR(err);
+
+	if ((algt->type ^ (CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_WRAP)) &
+	    algt->mask)
+		return ERR_PTR(-EINVAL);
+
+	name = crypto_attr_alg_name(tb[1]);
+	err = PTR_ERR(name);
+	if (IS_ERR(name))
+		return ERR_PTR(err);
+
+	inst = kzalloc(sizeof(*inst) + ctx_size, GFP_KERNEL);
+	if (!inst)
+		return ERR_PTR(-ENOMEM);
+
+	spawn = crypto_instance_ctx(inst);
+
+	/* Ignore async algorithms if necessary. */
+	mask |= crypto_requires_sync(algt->type, algt->mask);
+
+	crypto_set_aead_spawn(spawn, inst);
+	err = crypto_grab_wrapaead(spawn, name, type, mask);
+	if (err)
+		goto err_free_inst;
+
+	alg = crypto_aead_spawn_alg(spawn);
+
+	err = -EINVAL;
+	if (alg->cra_type != &crypto_aead_type)
+		goto err_drop_alg;
+
+	if (algt->mask & CRYPTO_ALG_WRAP) {
+		if (strcmp(tmpl->name, alg->cra_aead.wrapper))
+			goto err_drop_alg;
+
+		memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME);
+		memcpy(inst->alg.cra_driver_name, alg->cra_driver_name,
+		       CRYPTO_MAX_ALG_NAME);
+	} else {
+		err = -ENAMETOOLONG;
+		if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME,
+			     "%s(%s)", tmpl->name, alg->cra_name) >=
+		    CRYPTO_MAX_ALG_NAME)
+			goto err_drop_alg;
+		if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
+			     "%s(%s)", tmpl->name, alg->cra_driver_name) >=
+		    CRYPTO_MAX_ALG_NAME)
+			goto err_drop_alg;
+	}
+
+	inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_WRAP;
+	inst->alg.cra_flags |= alg->cra_flags
+			     & (CRYPTO_ALG_GENIV | CRYPTO_ALG_ASYNC);
+	inst->alg.cra_blocksize = alg->cra_blocksize;
+	inst->alg.cra_alignmask = alg->cra_alignmask;
+	inst->alg.cra_aead.ivsize = alg->cra_aead.ivsize;
+	inst->alg.cra_aead.maxauthsize = alg->cra_aead.maxauthsize;
+	inst->alg.cra_type = &crypto_wrapaead_type;
+	inst->alg.cra_aead.wrapper = alg->cra_aead.wrapper;
+	inst->alg.cra_aead.geniv = alg->cra_aead.geniv;
+	inst->alg.cra_priority = alg->cra_priority + 100;
+
+out:
+	return inst;
+
+err_drop_alg:
+	crypto_drop_aead(spawn);
+err_free_inst:
+	kfree(inst);
+	inst = ERR_PTR(err);
+	goto out;
+}
+EXPORT_SYMBOL_GPL(aead_wrap_alloc);
+
+void aead_wrap_free(struct crypto_instance *inst)
+{
+	crypto_drop_aead(crypto_instance_ctx(inst));
+	kfree(inst);
+}
+EXPORT_SYMBOL_GPL(aead_wrap_free);
+
+int aead_wrap_init(struct crypto_tfm *tfm)
+{
+	struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
+	struct crypto_aead *aead;
+
+	aead = crypto_spawn_aead(crypto_instance_ctx(inst));
+	if (IS_ERR(aead))
+		return PTR_ERR(aead);
+
+	tfm->crt_aead.base = aead;
+	tfm->crt_aead.reqsize += crypto_aead_reqsize(aead);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(aead_wrap_init);
+
+void aead_wrap_exit(struct crypto_tfm *tfm)
+{
+	crypto_free_aead(tfm->crt_aead.base);
+}
+EXPORT_SYMBOL_GPL(aead_wrap_exit);
+
+static int crypto_wrapaead_default(struct crypto_alg *alg, u32 type, u32 mask)
+{
+	struct rtattr *tb[3];
+	struct {
+		struct rtattr attr;
+		struct crypto_attr_type data;
+	} ptype;
+	struct {
+		struct rtattr attr;
+		struct crypto_attr_alg data;
+	} palg;
+	struct crypto_template *tmpl;
+	struct crypto_instance *inst;
+	struct crypto_alg *larval;
+	const char *wrapper;
+	int err;
+
+	larval = crypto_larval_lookup(alg->cra_driver_name,
+				      CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_WRAP,
+				      CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_WRAP);
+	err = PTR_ERR(larval);
+	if (IS_ERR(larval))
+		goto out;
+
+	err = -EAGAIN;
+	if (!crypto_is_larval(larval))
+		goto drop_larval;
+
+	ptype.attr.rta_len = sizeof(ptype);
+	ptype.attr.rta_type = CRYPTOA_TYPE;
+	ptype.data.type = type | CRYPTO_ALG_WRAP;
+	ptype.data.mask = mask | CRYPTO_ALG_WRAP;
+	tb[0] = &ptype.attr;
+
+	palg.attr.rta_len = sizeof(palg);
+	palg.attr.rta_type = CRYPTOA_ALG;
+	/* Must use the exact name to locate ourselves. */
+	memcpy(palg.data.name, alg->cra_driver_name, CRYPTO_MAX_ALG_NAME);
+	tb[1] = &palg.attr;
+
+	tb[2] = NULL;
+
+	wrapper = alg->cra_aead.wrapper;
+
+	tmpl = crypto_lookup_template(wrapper);
+	err = -ENOENT;
+	if (!tmpl)
+		goto kill_larval;
+
+	inst = tmpl->alloc(tb);
+	err = PTR_ERR(inst);
+	if (IS_ERR(inst))
+		goto put_tmpl;
+
+	if ((err = crypto_register_instance(tmpl, inst))) {
+		tmpl->free(inst);
+		goto put_tmpl;
+	}
+
+	/* Redo the lookup to use the instance we just registered. */
+	err = -EAGAIN;
+
+put_tmpl:
+	crypto_tmpl_put(tmpl);
+kill_larval:
+	crypto_larval_kill(larval);
+drop_larval:
+	crypto_mod_put(larval);
+out:
+	crypto_mod_put(alg);
+	return err;
+}
+
 static struct crypto_alg *crypto_lookup_aead(const char *name, u32 type,
 					     u32 mask)
 {
@@ -416,10 +673,13 @@ static struct crypto_alg *crypto_lookup_aead(const char *name, u32 type,
 	if (IS_ERR(alg))
 		return alg;
 
-	if (alg->cra_type == &crypto_aead_type)
+	if (alg->cra_type == &crypto_wrapaead_type)
 		return alg;
 
-	if (!alg->cra_aead.ivsize)
+	if ((alg->cra_type == &crypto_aead_type) && !alg->cra_aead.wrapper)
+		return alg;
+
+	if (!alg->cra_aead.ivsize && !alg->cra_aead.wrapper)
 		return alg;
 
 	crypto_mod_put(alg);
@@ -428,7 +688,7 @@ static struct crypto_alg *crypto_lookup_aead(const char *name, u32 type,
 	if (IS_ERR(alg))
 		return alg;
 
-	if (alg->cra_type == &crypto_aead_type) {
+	if (alg->cra_type == &crypto_wrapaead_type) {
 		if ((alg->cra_flags ^ type ^ ~mask) & CRYPTO_ALG_TESTED) {
 			crypto_mod_put(alg);
 			alg = ERR_PTR(-ENOENT);
@@ -436,6 +696,18 @@ static struct crypto_alg *crypto_lookup_aead(const char *name, u32 type,
 		return alg;
 	}
 
+	if (alg->cra_type == &crypto_aead_type) {
+		if ((alg->cra_flags ^ type ^ ~mask) & CRYPTO_ALG_TESTED) {
+			crypto_mod_put(alg);
+			alg = ERR_PTR(-ENOENT);
+		}
+
+		if (!alg->cra_aead.wrapper)
+			return alg;
+
+		return ERR_PTR(crypto_wrapaead_default(alg, type, mask));
+	}
+
 	BUG_ON(!alg->cra_aead.ivsize);
 
 	return ERR_PTR(crypto_nivaead_default(alg, type, mask));
@@ -447,9 +719,9 @@ int crypto_grab_aead(struct crypto_aead_spawn *spawn, const char *name,
 	struct crypto_alg *alg;
 	int err;
 
-	type &= ~(CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_GENIV);
+	type &= ~(CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_GENIV | CRYPTO_ALG_WRAP);
 	type |= CRYPTO_ALG_TYPE_AEAD;
-	mask &= ~(CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_GENIV);
+	mask &= ~(CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_GENIV | CRYPTO_ALG_WRAP);
 	mask |= CRYPTO_ALG_TYPE_MASK;
 
 	alg = crypto_lookup_aead(name, type, mask);
@@ -467,9 +739,9 @@ struct crypto_aead *crypto_alloc_aead(const char *alg_name, u32 type, u32 mask)
 	struct crypto_tfm *tfm;
 	int err;
 
-	type &= ~(CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_GENIV);
+	type &= ~(CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_GENIV | CRYPTO_ALG_WRAP);
 	type |= CRYPTO_ALG_TYPE_AEAD;
-	mask &= ~(CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_GENIV);
+	mask &= ~(CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_GENIV | CRYPTO_ALG_WRAP);
 	mask |= CRYPTO_ALG_TYPE_MASK;
 
 	for (;;) {
diff --git a/include/crypto/internal/aead.h b/include/crypto/internal/aead.h
index d838c94..4bf8a50 100644
--- a/include/crypto/internal/aead.h
+++ b/include/crypto/internal/aead.h
@@ -65,6 +65,14 @@ static inline struct crypto_aead *aead_geniv_base(struct crypto_aead *geniv)
 	return crypto_aead_crt(geniv)->base;
 }
 
+struct crypto_instance *aead_wrap_alloc(struct crypto_template *tmpl,
+					 struct rtattr **tb,
+					 unsigned int ctx_size, u32 type,
+					 u32 mask);
+void aead_wrap_free(struct crypto_instance *inst);
+int aead_wrap_init(struct crypto_tfm *tfm);
+void aead_wrap_exit(struct crypto_tfm *tfm);
+
 static inline void *aead_givcrypt_reqctx(struct aead_givcrypt_request *req)
 {
 	return aead_request_ctx(&req->areq);
diff --git a/include/linux/crypto.h b/include/linux/crypto.h
index ec29fa2..2548bf0 100644
--- a/include/linux/crypto.h
+++ b/include/linux/crypto.h
@@ -71,6 +71,8 @@
 
 #define CRYPTO_ALG_TESTED		0x00000400
 
+#define CRYPTO_ALG_WRAP			0x00000800
+
 /*
  * Transform masks and values (for crt_flags).
  */
@@ -242,6 +244,7 @@ struct aead_alg {
 	int (*givdecrypt)(struct aead_givcrypt_request *req);
 
 	const char *geniv;
+	const char *wrapper;
 
 	unsigned int ivsize;
 	unsigned int maxauthsize;
@@ -881,9 +884,12 @@ static inline unsigned int crypto_aead_reqsize(struct crypto_aead *tfm)
 }
 
 static inline void aead_request_set_tfm(struct aead_request *req,
-					struct crypto_aead *tfm)
+					struct crypto_aead *aead)
 {
-	req->base.tfm = crypto_aead_tfm(crypto_aead_crt(tfm)->base);
+	struct crypto_tfm *tfm = crypto_aead_tfm(aead);
+
+	req->base.tfm = tfm->__crt_alg->cra_flags & CRYPTO_ALG_WRAP ?
+			tfm : crypto_aead_tfm(crypto_aead_crt(aead)->base);
 }
 
 static inline struct aead_request *aead_request_alloc(struct crypto_aead *tfm,
-- 
1.5.4.2

Re: [RFC] [PATCH 2/5] aead: Add generic aead wrapper interface

[Herbert Xu]

On Wed, May 13, 2009 at 03:08:18PM +0200, Steffen Klassert wrote:
> This patch adds an interface to wrap aead algorithms with an
> arbitrary crypto template. This is useful to make a certain
> version of an aead algorithm the system default of that algorithm.
> 
> Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>

Sorry for the late response, but rather than making ad-hoc user
interfaces like this, I'd like this to be part of the crypto user-
space API where you should be able to configure the priorities
of each implementation which determines the system default.

Do we really need this patch right now? If not I think it could
be removed from this set so the others can go in first.

Thanks,
-- 
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt

Re: [RFC] [PATCH 2/5] aead: Add generic aead wrapper interface

[Herbert Xu]

On Wed, May 13, 2009 at 03:08:18PM +0200, Steffen Klassert wrote:
> This patch adds an interface to wrap aead algorithms with an
> arbitrary crypto template. This is useful to make a certain
> version of an aead algorithm the system default of that algorithm.
> 
> Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>

Hmm, it seems that this patch is completely intertwined into the
rest of the patches so we can't just kill it.  Can you elaborate
on the rationale behind this wrap work? I'm curious to find out
what we gain by this over the much simpler patch you had in your
previous set.

Thanks,
-- 
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt

Re: [RFC] [PATCH 2/5] aead: Add generic aead wrapper interface

[Steffen Klassert]

On Tue, Jun 02, 2009 at 01:50:41PM +1000, Herbert Xu wrote:
> 
> Hmm, it seems that this patch is completely intertwined into the
> rest of the patches so we can't just kill it.  Can you elaborate
> on the rationale behind this wrap work? I'm curious to find out
> what we gain by this over the much simpler patch you had in your
> previous set.
> 

The reason for the wrap work is to have a possibility to choose a
certain version of an algorithm as the system default. The advantage
e.g. for pcrypt is that we can turn over the whole system to pcrypt,
or we can choose for pcrypt by the algorithm name if we want to use
it just for a subset of transforms. In particular we have a possibility
to use pcrypt without touching other subsystems (like networking) and
userspace tools for now.

Re: [RFC] [PATCH 2/5] aead: Add generic aead wrapper interface

[Herbert Xu]

On Tue, Jun 02, 2009 at 11:21:51AM +0200, Steffen Klassert wrote:
> 
> The reason for the wrap work is to have a possibility to choose a
> certain version of an algorithm as the system default. The advantage
> e.g. for pcrypt is that we can turn over the whole system to pcrypt,
> or we can choose for pcrypt by the algorithm name if we want to use
> it just for a subset of transforms. In particular we have a possibility
> to use pcrypt without touching other subsystems (like networking) and
> userspace tools for now.

Yes but what you're creating is a user-space API.  IMHO we don't
want to have ad-hoc APIs such as this scattered around the place.
pcrypt is certainly not the only algorithm that needs to be able
to decide whether it should serve as the system default.

So what I suggest is that you make pcrypt take a higher priority
for now, so that it always is the default once instantiated.
After all if you instantiate it then you probably want to use it
as the default.

We can then work on the generic user-space API to control crypto
API algorithms, in particular, allow the user to change their
priorities so as to determine the system default.

This has been on my todo list for years, but it has never risen
to the top.  What I have in mind is a netlink protocol not unlike
xfrm which controls crypto algorithm properties (and queries them
so we can obsolete /proc/crypto).

Thanks,
-- 
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt

Re: [RFC] [PATCH 2/5] aead: Add generic aead wrapper interface

[Steffen Klassert]

On Tue, Jun 02, 2009 at 07:28:15PM +1000, Herbert Xu wrote:
> On Tue, Jun 02, 2009 at 11:21:51AM +0200, Steffen Klassert wrote:
> > 
> > The reason for the wrap work is to have a possibility to choose a
> > certain version of an algorithm as the system default. The advantage
> > e.g. for pcrypt is that we can turn over the whole system to pcrypt,
> > or we can choose for pcrypt by the algorithm name if we want to use
> > it just for a subset of transforms. In particular we have a possibility
> > to use pcrypt without touching other subsystems (like networking) and
> > userspace tools for now.
> 
> Yes but what you're creating is a user-space API.  IMHO we don't
> want to have ad-hoc APIs such as this scattered around the place.
> pcrypt is certainly not the only algorithm that needs to be able
> to decide whether it should serve as the system default.

Hm, I have not considered this as an user-space API. It just adds the
possibility to wrap an arbitrary crypto template arround a given aead
type algorithm, similar than aead_geniv wraps a IV generator template
arround a nivaead type algorithm.

The thing that connects this to user-space is the authenc patch by
adding the possibility to set a wrapper name with a module parameter.
This is probaply such an ad-hoc API that you want to avoid, right?

> 
> So what I suggest is that you make pcrypt take a higher priority
> for now, so that it always is the default once instantiated.
> After all if you instantiate it then you probably want to use it
> as the default.

Yes, in fact the instantiating is my problem. E.g. esp asks for an
authenc(...,...) algorithm, so the crypto manager tries to instantiate
a template with name authenc. If I don't want to touch the network
subsystem I can't change the name to pcrypt(authenc(...,...)) easy.
So one solution was to add a default wrapper template arround authenc
that will be instantiated along with authenc.

I'm not insisting on that wap work. I just want to have a easy possibility
to instantiate pcrypt on the users choice for now, at best without the
need to touch other subsystems.
 

Re: [RFC] [PATCH 2/5] aead: Add generic aead wrapper interface

[Herbert Xu]

On Wed, Jun 03, 2009 at 11:32:16AM +0200, Steffen Klassert wrote:
>
> > So what I suggest is that you make pcrypt take a higher priority
> > for now, so that it always is the default once instantiated.
> > After all if you instantiate it then you probably want to use it
> > as the default.
> 
> Yes, in fact the instantiating is my problem. E.g. esp asks for an
> authenc(...,...) algorithm, so the crypto manager tries to instantiate
> a template with name authenc. If I don't want to touch the network
> subsystem I can't change the name to pcrypt(authenc(...,...)) easy.
> So one solution was to add a default wrapper template arround authenc
> that will be instantiated along with authenc.

I see.  How about if we let tcrypt test algorithms by name, e.g.,
something like

	modprobe tcrypt alg='pcrypt(authenc(hmac(sha1),cbc(aes))'

which would simply call crypto_has_alg on the string alg?

That should instantiate alg, and test it too if there are vectors.

This is a new interface too, but at least it pretends to be
generic :)

Cheers,
-- 
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt

Re: [RFC] [PATCH 2/5] aead: Add generic aead wrapper interface

[Steffen Klassert]

On Wed, Jun 03, 2009 at 07:40:50PM +1000, Herbert Xu wrote:
> 
> I see.  How about if we let tcrypt test algorithms by name, e.g.,
> something like
> 
> 	modprobe tcrypt alg='pcrypt(authenc(hmac(sha1),cbc(aes))'
> 

I'm not that sure whether this does what I want.

If pcrypt has cra_name = pcrypt(authenc(hmac(sha1),cbc(aes))) this
would instatiate this algorithm, but esp wants an algorithm with
cra_name = authenc(hmac(sha1),cbc(aes)). 
These names are not matching, so __crypto_alg_lookup() will not
choose for the pcrypt version regardless of the higher priority.

Setting cra_name = authenc(hmac(sha1),cbc(aes)) for pcrypt seems to
be not possible too, because pcrypt needs authenc as an underlain 
algorithm and the system will not load two aead algorithms with the
same name. This was one of the reasons why I added a wrapper to
authenc that is instantiated along with authenc.

Is this the case or do I miss something?

Re: [RFC] [PATCH 2/5] aead: Add generic aead wrapper interface

[Herbert Xu]

On Wed, Jun 03, 2009 at 01:23:53PM +0200, Steffen Klassert wrote:
> On Wed, Jun 03, 2009 at 07:40:50PM +1000, Herbert Xu wrote:
> > 
> > I see.  How about if we let tcrypt test algorithms by name, e.g.,
> > something like
> > 
> > 	modprobe tcrypt alg='pcrypt(authenc(hmac(sha1),cbc(aes))'
> > 
> 
> I'm not that sure whether this does what I want.
> 
> If pcrypt has cra_name = pcrypt(authenc(hmac(sha1),cbc(aes))) this
> would instatiate this algorithm, but esp wants an algorithm with
> cra_name = authenc(hmac(sha1),cbc(aes)). 
> These names are not matching, so __crypto_alg_lookup() will not
> choose for the pcrypt version regardless of the higher priority.

When pcrypt instantiates an algorithm, it should set cra_name to
%s and cra_driver_name to pcrypt(%s).  So as long as the pcrypt
priority is higher than the underlying algorithm, it should all
work.

See for instance how cryptd does it.

Cheers,
-- 
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt

Re: [RFC] [PATCH 2/5] aead: Add generic aead wrapper interface

[Steffen Klassert]

On Wed, Jun 03, 2009 at 09:59:31PM +1000, Herbert Xu wrote:
> 
> When pcrypt instantiates an algorithm, it should set cra_name to
> %s and cra_driver_name to pcrypt(%s).  So as long as the pcrypt
> priority is higher than the underlying algorithm, it should all
> work.
> 
> See for instance how cryptd does it.
> 

Aha, ok I see. So I will see whether I can instantiate pcrypt
with tcrypt as you suggested and drop the wrapper from the next
patchset.

Thanks!

Re: [RFC] [PATCH 2/5] aead: Add generic aead wrapper interface

[Herbert Xu]

On Wed, Jun 03, 2009 at 02:14:35PM +0200, Steffen Klassert wrote:
>
> Aha, ok I see. So I will see whether I can instantiate pcrypt
> with tcrypt as you suggested and drop the wrapper from the next
> patchset.

It should be straightforward as you just need to call crypto_has_alg.

Thanks,
-- 
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt

Re: [RFC] [PATCH 2/5] aead: Add generic aead wrapper interface

[Steffen Klassert]

On Wed, Jun 03, 2009 at 09:59:31PM +1000, Herbert Xu wrote:
> 
> When pcrypt instantiates an algorithm, it should set cra_name to
> %s and cra_driver_name to pcrypt(%s).  So as long as the pcrypt
> priority is higher than the underlying algorithm, it should all
> work.
> 

As it is, I can instantiate pcrypt if the priority of pcrypt is lower
than the priority of the underlying algorithm. If I do the priority
check in crypto_alg_tested() the other way arround, I get it to work if
pcrypt has a higher priority than the underlying algorithm. So I guess
we need the patch below, right? If so, I would send a signed-off patch.

 
diff --git a/crypto/algapi.c b/crypto/algapi.c
index 56c62e2..2492e6c 100644
--- a/crypto/algapi.c
+++ b/crypto/algapi.c
@@ -255,7 +255,7 @@ found:
 			continue;
 
 		if (strcmp(alg->cra_driver_name, q->cra_driver_name) &&
-		    q->cra_priority > alg->cra_priority)
+		    q->cra_priority < alg->cra_priority)
 			continue;
 
 		crypto_remove_spawns(&q->cra_users, &list, alg->cra_flags);

Re: [RFC] [PATCH 2/5] aead: Add generic aead wrapper interface

[Herbert Xu]

On Fri, Jun 05, 2009 at 11:20:30AM +0200, Steffen Klassert wrote:
> 
> As it is, I can instantiate pcrypt if the priority of pcrypt is lower
> than the priority of the underlying algorithm. If I do the priority
> check in crypto_alg_tested() the other way arround, I get it to work if
> pcrypt has a higher priority than the underlying algorithm. So I guess
> we need the patch below, right? If so, I would send a signed-off patch.

Can you give me an example with actual numbers and what you expect
to happen?

Thanks,
-- 
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt

Re: [RFC] [PATCH 2/5] aead: Add generic aead wrapper interface

[Steffen Klassert]

On Fri, Jun 05, 2009 at 07:20:21PM +1000, Herbert Xu wrote:
> On Fri, Jun 05, 2009 at 11:20:30AM +0200, Steffen Klassert wrote:
> > 
> > As it is, I can instantiate pcrypt if the priority of pcrypt is lower
> > than the priority of the underlying algorithm. If I do the priority
> > check in crypto_alg_tested() the other way arround, I get it to work if
> > pcrypt has a higher priority than the underlying algorithm. So I guess
> > we need the patch below, right? If so, I would send a signed-off patch.
> 
> Can you give me an example with actual numbers and what you expect
> to happen?
> 

In pcrypt_alloc_instance() I do
inst->alg.cra_priority = alg->cra_priority + 100;

So, in my case authenc has priority 2000 and pcrypt has priority 2100.
In this case pcrypt is not instantiated if I use %s for pcrypt as
cra_name. If I do 
inst->alg.cra_priority = alg->cra_priority - 100
it will be instantiated with priority 1900 but it will not be used
because the priority of authenc is higher.

So I did the priority check in crypto_alg_tested() the other way around.
Then I can instantiate pcrypt with priority 2100 and I can use it.

Re: [RFC] [PATCH 2/5] aead: Add generic aead wrapper interface

[Herbert Xu]

On Fri, Jun 05, 2009 at 11:34:30AM +0200, Steffen Klassert wrote:
>
> In pcrypt_alloc_instance() I do
> inst->alg.cra_priority = alg->cra_priority + 100;
> 
> So, in my case authenc has priority 2000 and pcrypt has priority 2100.
> In this case pcrypt is not instantiated if I use %s for pcrypt as
> cra_name. If I do 
> inst->alg.cra_priority = alg->cra_priority - 100
> it will be instantiated with priority 1900 but it will not be used
> because the priority of authenc is higher.
> 
> So I did the priority check in crypto_alg_tested() the other way around.
> Then I can instantiate pcrypt with priority 2100 and I can use it.

Can you send me a pcrypt patch that I can use to reproduce this?

The check modified is meant to replace instances of the same
implementation (i.e., you're replaceing aes-x86-64 with a newer
version of aes-x86-64).  It should never do anything when you add
a different implementation of the same algorithm.

So I'm surprised that you're seeing a difference when changing
that check.  Because unless you're creating two pcrypt objects
with the same driver name, or your pcrypt object has the wrong
driver name, then this change should make no difference whatsoever.

Cheers,
-- 
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt

Re: [RFC] [PATCH 2/5] aead: Add generic aead wrapper interface

[Steffen Klassert]

On Mon, Jun 08, 2009 at 03:28:08PM +1000, Herbert Xu wrote:
> On Fri, Jun 05, 2009 at 11:34:30AM +0200, Steffen Klassert wrote:
> >
> > In pcrypt_alloc_instance() I do
> > inst->alg.cra_priority = alg->cra_priority + 100;
> > 
> > So, in my case authenc has priority 2000 and pcrypt has priority 2100.
> > In this case pcrypt is not instantiated if I use %s for pcrypt as
> > cra_name. If I do 
> > inst->alg.cra_priority = alg->cra_priority - 100
> > it will be instantiated with priority 1900 but it will not be used
> > because the priority of authenc is higher.
> > 
> > So I did the priority check in crypto_alg_tested() the other way around.
> > Then I can instantiate pcrypt with priority 2100 and I can use it.
> 
> Can you send me a pcrypt patch that I can use to reproduce this?

Yes, I will send the full patchset including the tcrypt changes to
instantiate pcrypt.

As the patchset is, I'm not able to instantiate pcrypt here. I need to
either change the priority check in crypto_alg_tested() or to make
pcrypt using a lower priority than authenc.

> 
> The check modified is meant to replace instances of the same
> implementation (i.e., you're replaceing aes-x86-64 with a newer
> version of aes-x86-64).  It should never do anything when you add
> a different implementation of the same algorithm.
> 
> So I'm surprised that you're seeing a difference when changing
> that check.  Because unless you're creating two pcrypt objects
> with the same driver name, or your pcrypt object has the wrong
> driver name, then this change should make no difference whatsoever.
> 

I was just surprised that I was able to instantiate pcrypt if it has a
lower priority than the underlying authenc algorithm. So I searched for
priority checks like the one in crypto_alg_tested() and in fact changing
this check got it to work as I described above.

Re: [RFC] [PATCH 2/5] aead: Add generic aead wrapper interface

[Herbert Xu]

On Mon, Jun 08, 2009 at 08:45:18AM +0200, Steffen Klassert wrote:
>
> Yes, I will send the full patchset including the tcrypt changes to
> instantiate pcrypt.

OK, the patch I just posted to the list should fix the problem.

I was able to test it suing

modprobe tcrypt alg='pcrypt(authenc(hmac(sha1-generic),cbc(aes-generic)))' type=3

Cheers,
-- 
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt

Re: [RFC] [PATCH 2/5] aead: Add generic aead wrapper interface

[Steffen Klassert]

On Thu, Jun 25, 2009 at 02:51:12PM +0800, Herbert Xu wrote:
> 
> OK, the patch I just posted to the list should fix the problem.
> 
> I was able to test it suing
> 
> modprobe tcrypt alg='pcrypt(authenc(hmac(sha1-generic),cbc(aes-generic)))' type=3
> 

I applied your patch on top of the last pcrypt patchset, but
unfortunately it does not change anything here.

If I do
modprobe tcrypt alg='pcrypt(authenc(hmac(sha1-generic),cbc(aes-generic)))' type=3

it instantiates

authenc(hmac(sha1-generic),cbc(aes-generic))

but not 

pcrypt(authenc(hmac(sha1-generic),cbc(aes-generic)))

regardless your patch applied or not.

Re: [RFC] [PATCH 2/5] aead: Add generic aead wrapper interface

[Herbert Xu]

On Mon, Jun 29, 2009 at 01:04:10PM +0200, Steffen Klassert wrote:
>
> I applied your patch on top of the last pcrypt patchset, but
> unfortunately it does not change anything here.
> 
> If I do
> modprobe tcrypt alg='pcrypt(authenc(hmac(sha1-generic),cbc(aes-generic)))' type=3
> 
> it instantiates
> 
> authenc(hmac(sha1-generic),cbc(aes-generic))
> 
> but not 
> 
> pcrypt(authenc(hmac(sha1-generic),cbc(aes-generic)))
> 
> regardless your patch applied or not.

OK.  Can you send me the patches you used against the current
cryptodev tree (I only just pushed so give it an hour or so)?
I'll see if I can reproduce it here.

Thanks,
-- 
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt

Re: [RFC] [PATCH 2/5] aead: Add generic aead wrapper interface

[Steffen Klassert]

On Mon, Jun 29, 2009 at 07:59:50PM +0800, Herbert Xu wrote:
> 
> OK.  Can you send me the patches you used against the current
> cryptodev tree (I only just pushed so give it an hour or so)?
> I'll see if I can reproduce it here.
> 

I'll send the remaining two patches in reply to this mail.
The strange thing is that I can instantiate pcrypt if the priority is
lower than the priority of authenc, but not if it's equal or higher.

[PATCH 1/2] padata: generic interface for parallel processing

[Steffen Klassert]

This patch introduces an interface to process data objects
in parallel. On request it is possible to serialize again.
The parallelized objects return after serialization in the
same order as they were before the parallelization.

Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
---
 include/linux/interrupt.h |    3 +-
 include/linux/padata.h    |  116 +++++++++++
 kernel/Makefile           |    2 +-
 kernel/padata.c           |  490 +++++++++++++++++++++++++++++++++++++++++++++
 kernel/softirq.c          |    2 +-
 5 files changed, 610 insertions(+), 3 deletions(-)
 create mode 100644 include/linux/padata.h
 create mode 100644 kernel/padata.c

diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
index 2721f07..4aad58f 100644
--- a/include/linux/interrupt.h
+++ b/include/linux/interrupt.h
@@ -344,7 +344,8 @@ enum
 	TASKLET_SOFTIRQ,
 	SCHED_SOFTIRQ,
 	HRTIMER_SOFTIRQ,
-	RCU_SOFTIRQ,	/* Preferable RCU should always be the last softirq */
+	PADATA_SOFTIRQ,
+	RCU_SOFTIRQ, 	/* Preferable RCU should always be the last softirq */
 
 	NR_SOFTIRQS
 };
diff --git a/include/linux/padata.h b/include/linux/padata.h
new file mode 100644
index 0000000..469359f
--- /dev/null
+++ b/include/linux/padata.h
@@ -0,0 +1,116 @@
+/*
+ * padata.h - header for the padata parallelization interface
+ *
+ * Copyright (C) 2008, 2009 secunet Security Networks AG
+ * Copyright (C) 2008, 2009 Steffen Klassert <steffen.klassert@secunet.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#ifndef PADATA_H
+#define PADATA_H
+
+#include <linux/interrupt.h>
+#include <linux/smp.h>
+#include <linux/list.h>
+
+enum
+{
+	NO_PADATA=0,
+	AEAD_ENC_PADATA,
+	AEAD_DEC_PADATA,
+	NR_PADATA
+};
+
+struct padata_priv {
+	struct list_head	list;
+	struct call_single_data	csd;
+	int			cb_cpu;
+	int			seq_nr;
+	unsigned int		nr;
+	int			info;
+	void                    (*parallel)(struct padata_priv *padata);
+	void                    (*serial)(struct padata_priv *padata);
+};
+
+struct padata_queue {
+	struct list_head        list;
+	atomic_t                num_obj;
+	int               	cpu_index;
+	spinlock_t              lock;
+};
+
+struct parallel_data {
+	struct work_struct	work;
+	struct padata_queue	*queue;
+	atomic_t		seq_nr;
+	atomic_t		queued_objects;
+	cpumask_t		cpu_map;
+	cpumask_t		new_cpu_map;
+	u8			flags;
+#define	PADATA_INIT			1
+#define	PADATA_FLUSH_HARD		2
+#define	PADATA_RESET_IN_PROGRESS	4
+	spinlock_t              lock;
+};
+
+#ifdef CONFIG_USE_GENERIC_SMP_HELPERS
+extern void __init padata_init(unsigned int nr, cpumask_t cpu_map);
+extern void padata_dont_wait(unsigned int nr, struct padata_priv *padata);
+extern int padata_do_parallel(unsigned int softirq_nr, unsigned int nr,
+			      struct padata_priv *padata, int cb_cpu);
+extern int padata_do_serial(unsigned int nr, struct padata_priv *padata);
+extern cpumask_t padata_get_cpumap(unsigned int nr);
+extern void padata_set_cpumap(unsigned int nr, cpumask_t cpu_map);
+extern void padata_add_cpu(unsigned int nr, int cpu);
+extern void padata_remove_cpu(unsigned int nr, int cpu);
+extern void padata_start(unsigned int nr);
+extern void padata_stop(unsigned int nr);
+#else
+static inline void padata_init(unsigned int nr,cpumask_t cpu_map)
+{
+}
+static inline void padata_dont_wait(unsigned int nr, struct padata_priv *padata)
+{
+}
+static inline int padata_do_parallel(unsigned int softirq_nr, unsigned int nr,
+				     struct padata_priv *padata, int cb_cpu)
+{
+	return 0;
+}
+static inline int padata_do_serial(unsigned int nr, struct padata_priv *padata)
+{
+	return 0;
+}
+static inline cpumask_t padata_get_cpumap(unsigned int nr)
+{
+	return cpu_online_map;
+}
+static inline void padata_set_cpumap(unsigned int nr, cpumask_t cpu_map)
+{
+}
+static inline padata_add_cpu(unsigned int nr, int cpu)
+{
+}
+static inline padata_remove_cpu(unsigned int nr, int cpu)
+{
+}
+static inline padata_start(unsigned int nr)
+{
+}
+static inline padata_stop(unsigned int nr)
+{
+}
+#endif
+#endif
diff --git a/kernel/Makefile b/kernel/Makefile
index 9df4501..d029314 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -41,7 +41,7 @@ obj-$(CONFIG_RT_MUTEXES) += rtmutex.o
 obj-$(CONFIG_DEBUG_RT_MUTEXES) += rtmutex-debug.o
 obj-$(CONFIG_RT_MUTEX_TESTER) += rtmutex-tester.o
 obj-$(CONFIG_GENERIC_ISA_DMA) += dma.o
-obj-$(CONFIG_USE_GENERIC_SMP_HELPERS) += smp.o
+obj-$(CONFIG_USE_GENERIC_SMP_HELPERS) += smp.o padata.o
 ifneq ($(CONFIG_SMP),y)
 obj-y += up.o
 endif
diff --git a/kernel/padata.c b/kernel/padata.c
new file mode 100644
index 0000000..192c9a6
--- /dev/null
+++ b/kernel/padata.c
@@ -0,0 +1,490 @@
+/*
+ * padata.c - generic interface to process data streams in parallel
+ *
+ * Copyright (C) 2008, 2009 secunet Security Networks AG
+ * Copyright (C) 2008, 2009 Steffen Klassert <steffen.klassert@secunet.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include <linux/module.h>
+#include <linux/cpumask.h>
+#include <linux/err.h>
+#include <linux/padata.h>
+
+#define MAX_SEQ_NR  1000000000
+
+static struct parallel_data padata_vec[NR_PADATA];
+static struct padata_priv *padata_get_next(struct parallel_data *par_data);
+
+static void padata_flush_hard(struct parallel_data *par_data)
+{
+	int cpu;
+	struct padata_priv *padata;
+	struct padata_queue *queue;
+
+	for_each_cpu_mask(cpu, par_data->cpu_map) {
+		queue = per_cpu_ptr(par_data->queue, cpu);
+
+		while(!list_empty(&queue->list)) {
+			padata = list_entry(queue->list.next, struct padata_priv, list);
+
+			spin_lock(&queue->lock);
+			list_del_init(&padata->list);
+			spin_unlock(&queue->lock);
+
+			atomic_dec(&par_data->queued_objects);
+			padata->serial(padata);
+		}
+	}
+}
+
+static void padata_flush_order(struct parallel_data *par_data)
+{
+	struct padata_priv *padata;
+
+	while (1) {
+		padata = padata_get_next(par_data);
+
+		if (padata && !IS_ERR(padata))
+			padata->serial(padata);
+		else
+			break;
+	}
+
+	padata_flush_hard(par_data);
+}
+
+static void padata_reset_work(struct work_struct *work)
+{
+	int cpu, cpu_index;
+	struct padata_queue *queue;
+	struct parallel_data *par_data;
+
+	par_data = container_of(work, struct parallel_data, work);
+
+	if (par_data->flags & (PADATA_INIT|PADATA_RESET_IN_PROGRESS))
+		return;
+
+	spin_lock_bh(&par_data->lock);
+	par_data->flags |= PADATA_RESET_IN_PROGRESS;
+
+	if (!(par_data->flags & PADATA_FLUSH_HARD))
+		padata_flush_order(par_data);
+	else
+		padata_flush_hard(par_data);
+
+	cpu_index = 0;
+
+	par_data->cpu_map = par_data->new_cpu_map;
+
+	for_each_cpu_mask(cpu, par_data->cpu_map) {
+		queue = per_cpu_ptr(par_data->queue, cpu);
+
+		atomic_set(&queue->num_obj, 0);
+		queue->cpu_index = cpu_index;
+		cpu_index++;
+	}
+	spin_unlock_bh(&par_data->lock);
+
+	atomic_set(&par_data->seq_nr, -1);
+	par_data->flags &= ~PADATA_RESET_IN_PROGRESS;
+	par_data->flags |= PADATA_INIT;
+}
+
+static struct padata_priv *padata_get_next(struct parallel_data *par_data)
+{
+	int cpu, num_cpus, empty;
+	int seq_nr, calc_seq_nr, next_nr;
+	struct padata_queue *queue, *next_queue;
+	struct padata_priv *padata;
+
+	empty = 0;
+	next_nr = -1;
+	next_queue = NULL;
+
+	num_cpus = cpus_weight(par_data->cpu_map);
+
+	for_each_cpu_mask(cpu, par_data->cpu_map) {
+		queue = per_cpu_ptr(par_data->queue, cpu);
+
+		/*
+		 * Calculate the seq_nr of the object that should be
+		 * next in this queue.
+		 */
+		calc_seq_nr = (atomic_read(&queue->num_obj) * num_cpus)
+			+ queue->cpu_index;
+
+		if (!list_empty(&queue->list)) {
+			padata = list_entry(queue->list.next,
+					    struct padata_priv, list);
+
+			seq_nr  = padata->seq_nr;
+
+			if (unlikely(calc_seq_nr != seq_nr)) {
+				par_data->flags &= ~PADATA_INIT;
+				par_data->flags |= PADATA_FLUSH_HARD;
+				padata = NULL;
+				goto out;
+			}
+		} else {
+			seq_nr = calc_seq_nr;
+			empty++;
+		}
+
+		if (next_nr < 0 || seq_nr < next_nr) {
+			next_nr = seq_nr;
+			next_queue = queue;
+		}
+	}
+
+	padata = NULL;
+
+	if (empty == num_cpus)
+		goto out;
+
+	if (!list_empty(&next_queue->list)) {
+		padata = list_entry(next_queue->list.next,
+				    struct padata_priv, list);
+
+		spin_lock(&next_queue->lock);
+		list_del_init(&padata->list);
+		spin_unlock(&next_queue->lock);
+
+		atomic_dec(&par_data->queued_objects);
+		atomic_inc(&next_queue->num_obj);
+
+		goto out;
+	}
+
+	if (next_nr % num_cpus == next_queue->cpu_index) {
+		padata = ERR_PTR(-ENODATA);
+		goto out;
+	}
+
+	padata = ERR_PTR(-EINPROGRESS);
+out:
+	return padata;
+}
+
+static void padata_action(struct softirq_action *h)
+{
+	struct list_head *cpu_list, local_list;
+
+	cpu_list = &__get_cpu_var(softirq_work_list[PADATA_SOFTIRQ]);
+
+	local_irq_disable();
+	list_replace_init(cpu_list, &local_list);
+	local_irq_enable();
+
+	while (!list_empty(&local_list)) {
+		struct padata_priv *padata;
+
+		padata = list_entry(local_list.next,
+				    struct padata_priv, csd.list);
+
+		list_del_init(&padata->csd.list);
+
+		padata->serial(padata);
+	}
+}
+
+static int padata_cpu_hash(unsigned int nr, struct padata_priv *padata)
+{
+	int cpu, target_cpu, this_cpu, cpu_index;
+
+	this_cpu = smp_processor_id();
+
+	if (padata->nr != 0)
+		return this_cpu;
+
+	if (!(padata_vec[nr].flags & PADATA_INIT))
+		return this_cpu;
+
+	padata->seq_nr = atomic_inc_return(&padata_vec[nr].seq_nr);
+
+	if (padata->seq_nr > MAX_SEQ_NR) {
+		padata_vec[nr].flags &= ~PADATA_INIT;
+		padata->seq_nr = 0;
+		schedule_work(&padata_vec[nr].work);
+		return this_cpu;
+	}
+
+	padata->nr = nr;
+
+	/*
+	 * Hash the sequence numbers to the cpus by taking
+	 * seq_nr mod. number of cpus in use.
+	 */
+	cpu_index =  padata->seq_nr % cpus_weight(padata_vec[nr].cpu_map);
+
+	target_cpu = first_cpu(padata_vec[nr].cpu_map);
+	for (cpu = 0; cpu < cpu_index; cpu++)
+		target_cpu = next_cpu(target_cpu, padata_vec[nr].cpu_map);
+
+	return target_cpu;
+}
+
+/*
+ * padata_dont_wait - must be called if an object that runs in parallel will
+ * not be serialized with padata_do_serial
+ *
+ * @nr: number of the padata instance
+ * @padata: object that will not be seen by padata_do_serial
+ */
+void padata_dont_wait(unsigned int nr, struct padata_priv *padata)
+{
+	struct padata_queue *queue;
+
+	if (!(padata_vec[nr].flags & PADATA_INIT))
+		return;
+
+	if (padata->nr == 0 || padata->nr != nr)
+		return;
+
+	queue = per_cpu_ptr(padata_vec[nr].queue, smp_processor_id());
+	atomic_inc(&queue->num_obj);
+
+	padata->nr = 0;
+	padata->seq_nr = 0;
+}
+EXPORT_SYMBOL(padata_dont_wait);
+
+/*
+ * padata_do_parallel - padata parallelization function
+ *
+ * @softirq_nr: number of the softirq that will do the parallelization
+ * @nr: number of the padata instance
+ * @padata: object to be parallelized
+ * @cb_cpu: cpu number on which the serialization callback function will run
+ */
+int padata_do_parallel(unsigned int softirq_nr, unsigned int nr,
+		       struct padata_priv *padata, int cb_cpu)
+{
+	int target_cpu;
+
+	padata->cb_cpu = cb_cpu;
+
+	local_bh_disable();
+	target_cpu = padata_cpu_hash(nr, padata);
+	local_bh_enable();
+
+	send_remote_softirq(&padata->csd, target_cpu, softirq_nr);
+
+	return 1;
+}
+EXPORT_SYMBOL(padata_do_parallel);
+
+/*
+ * padata_do_serial - padata serialization function
+ *
+ * @nr: number of the padata instance
+ * @padata: object to be serialized
+ *
+ * returns 1 if the serialization callback function will be called
+ * from padata, 0 else
+ */
+int padata_do_serial(unsigned int nr, struct padata_priv *padata)
+{
+	int cpu;
+	struct padata_queue *reorder_queue;
+
+	if (!(padata_vec[nr].flags & PADATA_INIT))
+		return 0;
+
+	if (padata->nr != nr || padata->nr == 0) {
+		padata->serial(padata);
+		return 1;
+	}
+
+	cpu = smp_processor_id();
+
+	reorder_queue = per_cpu_ptr(padata_vec[nr].queue, cpu);
+
+	spin_lock(&reorder_queue->lock);
+	list_add_tail(&padata->list, &reorder_queue->list);
+	spin_unlock(&reorder_queue->lock);
+
+	atomic_inc(&padata_vec[nr].queued_objects);
+
+try_again:
+	if (!spin_trylock(&padata_vec[nr].lock))
+		goto out;
+
+	while(1) {
+		padata = padata_get_next(&padata_vec[nr]);
+
+		if (!padata || PTR_ERR(padata) == -EINPROGRESS)
+			break;
+		if (PTR_ERR(padata) == -ENODATA) {
+			spin_unlock(&padata_vec[nr].lock);
+			goto out;
+		}
+
+		send_remote_softirq(&padata->csd, padata->cb_cpu,
+				    PADATA_SOFTIRQ);
+	}
+
+	if (unlikely(!(padata_vec[nr].flags & PADATA_INIT))) {
+		spin_unlock(&padata_vec[nr].lock);
+		goto reset_out;
+	}
+
+	spin_unlock(&padata_vec[nr].lock);
+
+	if (atomic_read(&padata_vec[nr].queued_objects))
+		goto try_again;
+
+out:
+	return 1;
+reset_out:
+	schedule_work(&padata_vec[nr].work);
+	return 1;
+}
+EXPORT_SYMBOL(padata_do_serial);
+
+/*
+ * padata_get_cpumap - get the cpu map that is actually in use
+ *
+ * @nr: number of the padata instance
+ */
+cpumask_t padata_get_cpumap(unsigned int nr)
+{
+	return padata_vec[nr].cpu_map;
+}
+EXPORT_SYMBOL(padata_get_cpumap);
+
+/*
+ * padata_set_cpumap - set the cpu map that padata uses
+ *
+ * @nr: number of the padata instance
+ * @cpu_map: the cpu map to use
+ */
+void padata_set_cpumap(unsigned int nr, cpumask_t cpu_map)
+{
+	padata_vec[nr].new_cpu_map = cpu_map;
+	padata_vec[nr].flags &= ~PADATA_INIT;
+	padata_vec[nr].flags |= PADATA_FLUSH_HARD;
+
+	schedule_work(&padata_vec[nr].work);
+}
+EXPORT_SYMBOL(padata_set_cpumap);
+
+/*
+ * padata_add_cpu - add a cpu to the padata cpu map
+ *
+ * @nr: number of the padata instance
+ * @cpu: cpu to remove
+ */
+void padata_add_cpu(unsigned int nr, int cpu)
+{
+	cpumask_t cpu_map = padata_vec[nr].cpu_map;
+
+	cpu_set(cpu, cpu_map);
+	padata_set_cpumap(nr, cpu_map);
+}
+EXPORT_SYMBOL(padata_add_cpu);
+
+/*
+ * padata_remove_cpu - remove a cpu from the padata cpu map
+ *
+ * @nr: number of the padata instance
+ * @cpu: cpu to remove
+ */
+void padata_remove_cpu(unsigned int nr, int cpu)
+{
+	cpumask_t cpu_map = padata_vec[nr].cpu_map;
+
+	cpu_clear(cpu, cpu_map);
+	padata_set_cpumap(nr, cpu_map);
+}
+EXPORT_SYMBOL(padata_remove_cpu);
+
+/*
+ * padata_start - start the parallel processing
+ *
+ * @nr: number of the padata instance
+ */
+void padata_start(unsigned int nr)
+{
+	if (padata_vec[nr].flags & PADATA_INIT)
+		return;
+
+	schedule_work(&padata_vec[nr].work);
+}
+EXPORT_SYMBOL(padata_start);
+
+/*
+ * padata_stop - stop the parallel processing
+ *
+ * @nr: number of the padata instance
+ */
+void padata_stop(unsigned int nr)
+{
+	padata_vec[nr].flags &= ~PADATA_INIT;
+}
+EXPORT_SYMBOL(padata_stop);
+
+/*
+ * padata_init - initialize a padata instance
+ *
+ * @nr: number of the padata instance
+ * @cpu_map: map of the cpu set that padata uses for parallelization
+ */
+void __init padata_init(unsigned int nr, cpumask_t cpu_map)
+{
+	int cpu, cpu_index;
+	struct padata_queue *percpu_queue, *queue;
+
+	percpu_queue = alloc_percpu(struct padata_queue);
+
+	if (!percpu_queue) {
+		printk("padata_init: Failed to alloc the serialization"
+		       "queues for padata nr %d, exiting!\n", nr);
+		return;
+	}
+
+	cpu_index = 0;
+
+	for_each_possible_cpu(cpu) {
+		queue = per_cpu_ptr(percpu_queue, cpu);
+
+		if (cpu_isset(cpu, cpu_map)) {
+			queue->cpu_index = cpu_index;
+			cpu_index++;
+		}
+
+		INIT_LIST_HEAD(&queue->list);
+		spin_lock_init(&queue->lock);
+		atomic_set(&queue->num_obj, 0);
+	}
+
+	INIT_WORK(&padata_vec[nr].work, padata_reset_work);
+
+	atomic_set(&padata_vec[nr].seq_nr, -1);
+	atomic_set(&padata_vec[nr].queued_objects, 0);
+	padata_vec[nr].cpu_map = cpu_map;
+	padata_vec[nr].new_cpu_map = cpu_map;
+	padata_vec[nr].queue = percpu_queue;
+	padata_vec[nr].flags = 0;
+	spin_lock_init(&padata_vec[nr].lock);
+}
+
+static int __init padata_initcall(void)
+{
+	open_softirq(PADATA_SOFTIRQ, padata_action);
+
+	return 0;
+}
+subsys_initcall(padata_initcall);
diff --git a/kernel/softirq.c b/kernel/softirq.c
index b41fb71..e0faebf 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -58,7 +58,7 @@ static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
 
 char *softirq_to_name[NR_SOFTIRQS] = {
 	"HI", "TIMER", "NET_TX", "NET_RX", "BLOCK",
-	"TASKLET", "SCHED", "HRTIMER",	"RCU"
+	"TASKLET", "SCHED", "HRTIMER", "PADATA", "RCU"
 };
 
 /*
-- 
1.5.4.2

[PATCH 2/2] pcrypt: Add pcrypt crypto parallelization wrapper

[Steffen Klassert]

This patch adds a parallel crypto template that takes a crypto
algorithm and converts it to process the crypto transforms in
parallel. For the moment only aead is supported.

Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
---
 crypto/Kconfig            |   13 ++
 crypto/Makefile           |    2 +
 crypto/pcrypt.c           |  411 +++++++++++++++++++++++++++++++++++++++++++++
 crypto/pcrypt_core.c      |  106 ++++++++++++
 include/crypto/pcrypt.h   |   51 ++++++
 include/linux/interrupt.h |    2 +
 6 files changed, 585 insertions(+), 0 deletions(-)
 create mode 100644 crypto/pcrypt.c
 create mode 100644 crypto/pcrypt_core.c
 create mode 100644 include/crypto/pcrypt.h

diff --git a/crypto/Kconfig b/crypto/Kconfig
index f2002d8..1a3b277 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -113,6 +113,19 @@ config CRYPTO_NULL
 	help
 	  These are 'Null' algorithms, used by IPsec, which do nothing.
 
+config CRYPTO_PCRYPT_CORE
+	bool
+
+config CRYPTO_PCRYPT
+	tristate "Parallel crypto engine (EXPERIMENTAL)"
+	depends on USE_GENERIC_SMP_HELPERS && EXPERIMENTAL
+	select CRYPTO_MANAGER
+	select CRYPTO_PCRYPT_CORE
+	select CRYPTO_AEAD
+	help
+	  This converts an arbitrary crypto algorithm into a parallel
+	  algorithm that is executed in a softirq.
+
 config CRYPTO_WORKQUEUE
        tristate
 
diff --git a/crypto/Makefile b/crypto/Makefile
index 673d9f7..84b9d17 100644
--- a/crypto/Makefile
+++ b/crypto/Makefile
@@ -56,6 +56,8 @@ obj-$(CONFIG_CRYPTO_XTS) += xts.o
 obj-$(CONFIG_CRYPTO_CTR) += ctr.o
 obj-$(CONFIG_CRYPTO_GCM) += gcm.o
 obj-$(CONFIG_CRYPTO_CCM) += ccm.o
+obj-$(CONFIG_CRYPTO_PCRYPT_CORE) += pcrypt_core.o
+obj-$(CONFIG_CRYPTO_PCRYPT) += pcrypt.o
 obj-$(CONFIG_CRYPTO_CRYPTD) += cryptd.o
 obj-$(CONFIG_CRYPTO_DES) += des_generic.o
 obj-$(CONFIG_CRYPTO_FCRYPT) += fcrypt.o
diff --git a/crypto/pcrypt.c b/crypto/pcrypt.c
new file mode 100644
index 0000000..2debf8b
--- /dev/null
+++ b/crypto/pcrypt.c
@@ -0,0 +1,411 @@
+/*
+ * pcrypt - Parallel crypto wrapper.
+ *
+ * Copyright (C) 2009 secunet Security Networks AG
+ * Copyright (C) 2009 Steffen Klassert <steffen.klassert@secunet.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include <crypto/algapi.h>
+#include <crypto/internal/aead.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <crypto/pcrypt.h>
+
+struct pcrypt_instance_ctx {
+	struct crypto_spawn spawn;
+	unsigned int tfm_count;
+};
+
+struct pcrypt_aead_ctx {
+	struct crypto_aead *child;
+	unsigned int tfm_nr;
+};
+
+static int pcrypt_do_parallel(struct padata_priv *padata, unsigned int tfm_nr,
+			      unsigned int softirq, unsigned int padata_nr)
+{
+	unsigned int cpu, cpu_index, num_cpus, cb_cpu;
+	cpumask_t cpu_map;
+
+	cpu_map = padata_get_cpumap(padata_nr);
+	num_cpus = cpus_weight(cpu_map);
+
+	cpu_index = tfm_nr % num_cpus;
+
+	cb_cpu = first_cpu(cpu_map);
+	for (cpu = 0; cpu < cpu_index; cpu++)
+		cb_cpu = next_cpu(cb_cpu, cpu_map);
+
+	return padata_do_parallel(softirq, padata_nr, padata, cb_cpu);
+}
+
+static int pcrypt_aead_setkey(struct crypto_aead *parent,
+			      const u8 *key, unsigned int keylen)
+{
+	struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(parent);
+
+	return crypto_aead_setkey(ctx->child, key, keylen);
+}
+
+static int pcrypt_aead_setauthsize(struct crypto_aead *parent,
+				   unsigned int authsize)
+{
+	struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(parent);
+
+	return crypto_aead_setauthsize(ctx->child, authsize);
+}
+
+static void pcrypt_aead_serial(struct padata_priv *padata)
+{
+	struct pcrypt_request *preq = pcrypt_padata_request(padata);
+	struct aead_request *req = pcrypt_request_ctx(preq);
+
+	aead_request_complete(req->base.data, padata->info);
+}
+
+static void pcrypt_aead_giv_serial(struct padata_priv *padata)
+{
+	struct pcrypt_request *preq = pcrypt_padata_request(padata);
+	struct aead_givcrypt_request *req = pcrypt_request_ctx(preq);
+
+	aead_request_complete(req->areq.base.data, padata->info);
+}
+
+static void pcrypt_aead_done(struct crypto_async_request *areq, int err)
+{
+	struct aead_request *req = areq->data;
+	struct pcrypt_request *preq = aead_request_ctx(req);
+	struct padata_priv *padata = pcrypt_request_padata(preq);
+
+	padata->info = err;
+	req->base.flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
+
+	local_bh_disable();
+	if (padata_do_serial(padata->nr, padata))
+		goto out;
+
+	aead_request_complete(req, padata->info);
+
+out:
+	local_bh_enable();
+}
+
+static void pcrypt_aead_enc(struct padata_priv *padata)
+{
+	struct pcrypt_request *preq = pcrypt_padata_request(padata);
+	struct aead_request *req = pcrypt_request_ctx(preq);
+
+	padata->info = crypto_aead_encrypt(req);
+
+	if (padata->info)
+		return;
+
+	if (padata_do_serial(AEAD_ENC_PADATA, padata))
+		return;
+
+	aead_request_complete(req->base.data, padata->info);
+}
+
+static int pcrypt_aead_encrypt(struct aead_request *req)
+{
+	int err;
+	struct pcrypt_request *preq = aead_request_ctx(req);
+	struct aead_request *creq = pcrypt_request_ctx(preq);
+	struct padata_priv *padata = pcrypt_request_padata(preq);
+	struct crypto_aead *aead = crypto_aead_reqtfm(req);
+	struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead);
+	u32 flags = aead_request_flags(req);
+
+	memset(padata, 0, sizeof(struct padata_priv));
+
+	padata->parallel = pcrypt_aead_enc;
+	padata->serial = pcrypt_aead_serial;
+
+	aead_request_set_tfm(creq, ctx->child);
+	aead_request_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP,
+				  pcrypt_aead_done, req);
+	aead_request_set_crypt(creq, req->src, req->dst,
+			       req->cryptlen, req->iv);
+	aead_request_set_assoc(creq, req->assoc, req->assoclen);
+
+	if (pcrypt_do_parallel(padata, ctx->tfm_nr, AEAD_ENC_SOFTIRQ,
+			       AEAD_ENC_PADATA))
+		err = -EINPROGRESS;
+	else
+		err = crypto_aead_encrypt(creq);
+
+	return err;
+}
+
+static void pcrypt_aead_dec(struct padata_priv *padata)
+{
+	struct pcrypt_request *preq = pcrypt_padata_request(padata);
+	struct aead_request *req = pcrypt_request_ctx(preq);
+
+	padata->info = crypto_aead_decrypt(req);
+
+	if (padata->info)
+		return;
+
+	if (padata_do_serial(AEAD_DEC_PADATA, padata))
+		return;
+
+	aead_request_complete(req->base.data, padata->info);
+}
+
+static int pcrypt_aead_decrypt(struct aead_request *req)
+{
+	int err;
+	struct pcrypt_request *preq = aead_request_ctx(req);
+	struct aead_request *creq = pcrypt_request_ctx(preq);
+	struct padata_priv *padata = pcrypt_request_padata(preq);
+	struct crypto_aead *aead = crypto_aead_reqtfm(req);
+	struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead);
+	u32 flags = aead_request_flags(req);
+
+	memset(padata, 0, sizeof(struct padata_priv));
+
+	padata->parallel = pcrypt_aead_dec;
+	padata->serial = pcrypt_aead_serial;
+
+	aead_request_set_tfm(creq, ctx->child);
+	aead_request_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP,
+				  pcrypt_aead_done, req);
+	aead_request_set_crypt(creq, req->src, req->dst,
+			       req->cryptlen, req->iv);
+	aead_request_set_assoc(creq, req->assoc, req->assoclen);
+
+	if (pcrypt_do_parallel(padata, ctx->tfm_nr, AEAD_DEC_SOFTIRQ,
+			       AEAD_DEC_PADATA))
+		err = -EINPROGRESS;
+	else
+		err = crypto_aead_decrypt(creq);
+
+	return err;
+}
+
+static void pcrypt_aead_givenc(struct padata_priv *padata)
+{
+	struct pcrypt_request *preq = pcrypt_padata_request(padata);
+	struct aead_givcrypt_request *req = pcrypt_request_ctx(preq);
+
+	padata->info = crypto_aead_givencrypt(req);
+
+	if (padata->info)
+		return;
+
+	if (padata_do_serial(AEAD_ENC_PADATA, padata))
+		return;
+
+	aead_request_complete(req->areq.base.data, padata->info);
+}
+
+static int pcrypt_aead_givencrypt(struct aead_givcrypt_request *req)
+{
+	int err;
+	struct aead_request *areq = &req->areq;
+	struct pcrypt_request *preq = aead_request_ctx(areq);
+	struct aead_givcrypt_request *creq = pcrypt_request_ctx(preq);
+	struct padata_priv *padata = pcrypt_request_padata(preq);
+	struct crypto_aead *aead = aead_givcrypt_reqtfm(req);
+	struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead);
+	u32 flags = aead_request_flags(areq);
+
+	memset(padata, 0, sizeof(struct padata_priv));
+
+	padata->parallel = pcrypt_aead_givenc;
+	padata->serial = pcrypt_aead_giv_serial;
+
+	aead_givcrypt_set_tfm(creq, ctx->child);
+	aead_givcrypt_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP,
+				   pcrypt_aead_done, areq);
+	aead_givcrypt_set_crypt(creq, areq->src, areq->dst,
+				areq->cryptlen, areq->iv);
+	aead_givcrypt_set_assoc(creq, areq->assoc, areq->assoclen);
+	aead_givcrypt_set_giv(creq, req->giv, req->seq);
+
+
+	if (pcrypt_do_parallel(padata, ctx->tfm_nr, AEAD_ENC_SOFTIRQ,
+			       AEAD_ENC_PADATA))
+		err = -EINPROGRESS;
+	else
+		err = crypto_aead_givencrypt(creq);
+
+	return err;
+}
+
+static int pcrypt_aead_init_tfm(struct crypto_tfm *tfm)
+{
+	struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
+	struct pcrypt_instance_ctx *ictx = crypto_instance_ctx(inst);
+	struct pcrypt_aead_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct crypto_aead *cipher;
+	
+	ictx->tfm_count++;
+	ctx->tfm_nr = ictx->tfm_count;
+
+	cipher = crypto_spawn_aead(crypto_instance_ctx(inst));
+
+	if (IS_ERR(cipher))
+		return PTR_ERR(cipher);
+
+	ctx->child = cipher;
+	tfm->crt_aead.reqsize = sizeof(struct pcrypt_request)
+		+ sizeof(struct aead_givcrypt_request)
+		+ crypto_aead_reqsize(cipher);
+
+	return 0;
+}
+
+static void pcrypt_aead_exit_tfm(struct crypto_tfm *tfm)
+{
+	struct pcrypt_aead_ctx *ctx = crypto_tfm_ctx(tfm);
+
+	crypto_free_aead(ctx->child);
+}
+
+static struct crypto_instance *pcrypt_alloc_instance(struct crypto_alg *alg)
+{
+	struct crypto_instance *inst;
+	struct pcrypt_instance_ctx *ctx;
+	int err;
+
+	inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
+	if (!inst) {
+		inst = ERR_PTR(-ENOMEM);
+		goto out;
+	}
+
+	err = -ENAMETOOLONG;
+	if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
+		     "pcrypt(%s)", alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
+		goto out_free_inst;
+
+	memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME);
+
+	ctx = crypto_instance_ctx(inst);
+	err = crypto_init_spawn(&ctx->spawn, alg, inst,
+				CRYPTO_ALG_TYPE_MASK);
+	if (err)
+		goto out_free_inst;
+
+	inst->alg.cra_priority = alg->cra_priority + 100;
+	inst->alg.cra_blocksize = alg->cra_blocksize;
+	inst->alg.cra_alignmask = alg->cra_alignmask;
+
+out:
+	return inst;
+
+out_free_inst:
+	kfree(inst);
+	inst = ERR_PTR(err);
+	goto out;
+}
+
+static struct crypto_instance *pcrypt_alloc_aead(struct rtattr **tb)
+{
+	struct crypto_instance *inst;
+	struct crypto_alg *alg;
+	struct crypto_attr_type *algt;
+
+	algt = crypto_get_attr_type(tb);
+
+	alg = crypto_get_attr_alg(tb, algt->type,
+				  (algt->mask & CRYPTO_ALG_TYPE_MASK));
+	if (IS_ERR(alg))
+		return ERR_CAST(alg);
+
+	inst = pcrypt_alloc_instance(alg);
+	if (IS_ERR(inst))
+		goto out_put_alg;
+
+	inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC; 
+	inst->alg.cra_type = &crypto_aead_type;
+
+	inst->alg.cra_aead.ivsize = alg->cra_aead.ivsize;
+	inst->alg.cra_aead.geniv = alg->cra_aead.geniv;
+	inst->alg.cra_aead.maxauthsize = alg->cra_aead.maxauthsize;
+
+	inst->alg.cra_ctxsize = sizeof(struct pcrypt_aead_ctx);
+
+	inst->alg.cra_init = pcrypt_aead_init_tfm;
+	inst->alg.cra_exit = pcrypt_aead_exit_tfm;
+
+	inst->alg.cra_aead.setkey = pcrypt_aead_setkey;
+	inst->alg.cra_aead.setauthsize = pcrypt_aead_setauthsize;
+	inst->alg.cra_aead.encrypt = pcrypt_aead_encrypt;
+	inst->alg.cra_aead.decrypt = pcrypt_aead_decrypt;
+	inst->alg.cra_aead.givencrypt = pcrypt_aead_givencrypt;
+
+out_put_alg:
+	crypto_mod_put(alg);
+	return inst;
+}
+
+static struct crypto_instance *pcrypt_alloc(struct rtattr **tb)
+{
+	struct crypto_attr_type *algt;
+
+	algt = crypto_get_attr_type(tb);
+	if (IS_ERR(algt))
+		return ERR_CAST(algt);
+
+	switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) {
+	case CRYPTO_ALG_TYPE_AEAD:
+		return pcrypt_alloc_aead(tb);
+	}
+
+	return ERR_PTR(-EINVAL);
+}
+
+static void pcrypt_free(struct crypto_instance *inst)
+{
+	struct pcrypt_instance_ctx *ctx = crypto_instance_ctx(inst);
+
+	crypto_drop_spawn(&ctx->spawn);
+	kfree(inst);
+}
+
+static struct crypto_template pcrypt_tmpl = {
+	.name = "pcrypt",
+	.alloc = pcrypt_alloc,
+	.free = pcrypt_free,
+	.module = THIS_MODULE,
+};
+
+static int __init pcrypt_init(void)
+{
+	padata_start(AEAD_ENC_PADATA);
+	padata_start(AEAD_DEC_PADATA);
+
+	return crypto_register_template(&pcrypt_tmpl);
+}
+
+static void __exit pcrypt_exit(void)
+{
+	padata_stop(AEAD_ENC_PADATA);
+	padata_stop(AEAD_DEC_PADATA);
+
+	crypto_unregister_template(&pcrypt_tmpl);
+}
+
+module_init(pcrypt_init);
+module_exit(pcrypt_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Parallel crypto engine");
diff --git a/crypto/pcrypt_core.c b/crypto/pcrypt_core.c
new file mode 100644
index 0000000..61c0411
--- /dev/null
+++ b/crypto/pcrypt_core.c
@@ -0,0 +1,106 @@
+/*
+ * pcrypt_core.c - Core functions for the pcrypt crypto parallelization
+ *
+ * Copyright (C) 2009 secunet Security Networks AG
+ * Copyright (C) 2009 Steffen Klassert <steffen.klassert@secunet.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include <linux/interrupt.h>
+#include <linux/cpu.h>
+#include <linux/err.h>
+#include <linux/module.h>
+#include <crypto/pcrypt.h>
+
+static void aead_enc_action(struct softirq_action *h)
+{
+	struct list_head *cpu_list, local_list;
+
+	cpu_list = &__get_cpu_var(softirq_work_list[AEAD_ENC_SOFTIRQ]);
+
+	local_irq_disable();
+	list_replace_init(cpu_list, &local_list);
+	local_irq_enable();
+
+	while (!list_empty(&local_list)) {
+		struct padata_priv *padata;
+
+		padata = list_entry(local_list.next, struct padata_priv,
+				    csd.list);
+
+		list_del_init(&padata->csd.list);
+
+		padata->parallel(padata);
+	}
+}
+
+static void aead_dec_action(struct softirq_action *h)
+{
+	struct list_head *cpu_list, local_list;
+
+	cpu_list = &__get_cpu_var(softirq_work_list[AEAD_DEC_SOFTIRQ]);
+
+	local_irq_disable();
+	list_replace_init(cpu_list, &local_list);
+	local_irq_enable();
+
+	while (!list_empty(&local_list)) {
+		struct padata_priv *padata;
+
+		padata = list_entry(local_list.next, struct padata_priv,
+				    csd.list);
+
+		list_del_init(&padata->csd.list);
+
+		padata->parallel(padata);
+	}
+}
+
+static int __devinit pcrypt_cpu_callback(struct notifier_block *nfb,
+					 unsigned long action, void *hcpu)
+{
+	int cpu = (unsigned long)hcpu;
+
+	switch (action) {
+	case CPU_ONLINE:
+	case CPU_ONLINE_FROZEN:
+		padata_add_cpu(AEAD_ENC_PADATA, cpu);
+		padata_add_cpu(AEAD_DEC_PADATA, cpu);
+		break;
+
+	case CPU_DEAD:
+	case CPU_DEAD_FROZEN:
+		padata_remove_cpu(AEAD_ENC_PADATA, cpu);
+		padata_remove_cpu(AEAD_DEC_PADATA, cpu);
+
+		break;
+	}
+
+	return NOTIFY_OK;
+}
+
+static int __init pcrypt_init_padata(void)
+{
+	open_softirq(AEAD_ENC_SOFTIRQ, aead_enc_action);
+	open_softirq(AEAD_DEC_SOFTIRQ, aead_dec_action);
+
+	padata_init(AEAD_ENC_PADATA, cpu_online_map);
+	padata_init(AEAD_DEC_PADATA, cpu_online_map);
+
+	hotcpu_notifier(pcrypt_cpu_callback, 0);
+
+	return 0;
+}
+subsys_initcall(pcrypt_init_padata);
diff --git a/include/crypto/pcrypt.h b/include/crypto/pcrypt.h
new file mode 100644
index 0000000..d7d8bd8
--- /dev/null
+++ b/include/crypto/pcrypt.h
@@ -0,0 +1,51 @@
+/*
+ * pcrypt - Parallel crypto engine.
+ *
+ * Copyright (C) 2009 secunet Security Networks AG
+ * Copyright (C) 2009 Steffen Klassert <steffen.klassert@secunet.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#ifndef _CRYPTO_PCRYPT_H
+#define _CRYPTO_PCRYPT_H
+
+#include <linux/crypto.h>
+#include <linux/kernel.h>
+#include <linux/padata.h>
+
+struct pcrypt_request {
+	struct padata_priv	padata;
+	void			*data;
+	void			*__ctx[] CRYPTO_MINALIGN_ATTR;
+};
+
+static inline void *pcrypt_request_ctx(struct pcrypt_request *req)
+{
+	return req->__ctx;
+}
+
+static inline
+struct padata_priv *pcrypt_request_padata(struct pcrypt_request *req)
+{
+	return &req->padata;
+}
+
+static inline
+struct pcrypt_request *pcrypt_padata_request(struct padata_priv *padata)
+{
+	return container_of(padata, struct pcrypt_request, padata);
+}
+
+#endif
diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
index 4aad58f..67d46e9 100644
--- a/include/linux/interrupt.h
+++ b/include/linux/interrupt.h
@@ -344,6 +344,8 @@ enum
 	TASKLET_SOFTIRQ,
 	SCHED_SOFTIRQ,
 	HRTIMER_SOFTIRQ,
+	AEAD_ENC_SOFTIRQ,
+	AEAD_DEC_SOFTIRQ,
 	PADATA_SOFTIRQ,
 	RCU_SOFTIRQ, 	/* Preferable RCU should always be the last softirq */
 
-- 
1.5.4.2

Re: [RFC] [PATCH 2/5] aead: Add generic aead wrapper interface

[Steffen Klassert]

On Mon, Jun 29, 2009 at 03:52:57PM +0200, Steffen Klassert wrote:
> On Mon, Jun 29, 2009 at 07:59:50PM +0800, Herbert Xu wrote:
> > 
> > OK.  Can you send me the patches you used against the current
> > cryptodev tree (I only just pushed so give it an hour or so)?
> > I'll see if I can reproduce it here.
> > 
> 
> I'll send the remaining two patches in reply to this mail.
> The strange thing is that I can instantiate pcrypt if the priority is
> lower than the priority of authenc, but not if it's equal or higher.

Any ideas why it's not possible to instantiate pcrypt?

In between I updated padata to use the dynamic percpu allocator,
so the instantiating of pcrypt is the last issue I know of for the
moment.

Re: [RFC] [PATCH 2/5] aead: Add generic aead wrapper interface

[Herbert Xu]

On Wed, Aug 19, 2009 at 09:15:49AM +0200, Steffen Klassert wrote:
> 
> Any ideas why it's not possible to instantiate pcrypt?

Sorry for the late response.

In fact it was being instantiated, it was just getting killed
immediately after completing its self-test :)

Please let me know if this fixes it for you.

commit 2bf2901669a564b402cd0e40eb3f941c391e64c4
Author: Herbert Xu <herbert@gondor.apana.org.au>
Date:   Mon Aug 31 15:56:54 2009 +1000

    crypto: api - Do not displace newly registered algorithms
    
    We have a mechanism where newly registered algorithms of a higher
    priority can displace existing instances that use a different
    implementation of the same algorithm with a lower priority.
    
    Unfortunately the same mechanism can cause a newly registered
    algorithm to displace itself if it depends on an existing version
    of the same algorithm.
    
    This patch fixes this by keeping all algorithms that the newly
    reigstered algorithm depends on, thus protecting them from being
    removed.
    
    Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

diff --git a/crypto/algapi.c b/crypto/algapi.c
index 6a98076..feb77e4 100644
--- a/crypto/algapi.c
+++ b/crypto/algapi.c
@@ -81,16 +81,35 @@ static void crypto_destroy_instance(struct crypto_alg *alg)
 	crypto_tmpl_put(tmpl);
 }
 
+static struct list_head *crypto_more_spawns(struct crypto_alg *alg,
+					    struct list_head *stack,
+					    struct list_head *top,
+					    struct list_head *secondary_spawns)
+{
+	struct crypto_spawn *spawn, *n;
+
+	if (list_empty(stack))
+		return NULL;
+
+	spawn = list_first_entry(stack, struct crypto_spawn, list);
+	n = list_entry(spawn->list.next, struct crypto_spawn, list);
+
+	if (spawn->alg && &n->list != stack && !n->alg)
+		n->alg = (n->list.next == stack) ? alg :
+			 &list_entry(n->list.next, struct crypto_spawn,
+				     list)->inst->alg;
+
+	list_move(&spawn->list, secondary_spawns);
+
+	return &n->list == stack ? top : &n->inst->alg.cra_users;
+}
+
 static void crypto_remove_spawn(struct crypto_spawn *spawn,
-				struct list_head *list,
-				struct list_head *secondary_spawns)
+				struct list_head *list)
 {
 	struct crypto_instance *inst = spawn->inst;
 	struct crypto_template *tmpl = inst->tmpl;
 
-	list_del_init(&spawn->list);
-	spawn->alg = NULL;
-
 	if (crypto_is_dead(&inst->alg))
 		return;
 
@@ -106,25 +125,55 @@ static void crypto_remove_spawn(struct crypto_spawn *spawn,
 	hlist_del(&inst->list);
 	inst->alg.cra_destroy = crypto_destroy_instance;
 
-	list_splice(&inst->alg.cra_users, secondary_spawns);
+	BUG_ON(!list_empty(&inst->alg.cra_users));
 }
 
-static void crypto_remove_spawns(struct list_head *spawns,
-				 struct list_head *list, u32 new_type)
+static void crypto_remove_spawns(struct crypto_alg *alg,
+				 struct list_head *list,
+				 struct crypto_alg *nalg)
 {
+	u32 new_type = (nalg ?: alg)->cra_flags;
 	struct crypto_spawn *spawn, *n;
 	LIST_HEAD(secondary_spawns);
+	struct list_head *spawns;
+	LIST_HEAD(stack);
+	LIST_HEAD(top);
 
+	spawns = &alg->cra_users;
 	list_for_each_entry_safe(spawn, n, spawns, list) {
 		if ((spawn->alg->cra_flags ^ new_type) & spawn->mask)
 			continue;
 
-		crypto_remove_spawn(spawn, list, &secondary_spawns);
+		list_move(&spawn->list, &top);
 	}
 
-	while (!list_empty(&secondary_spawns)) {
-		list_for_each_entry_safe(spawn, n, &secondary_spawns, list)
-			crypto_remove_spawn(spawn, list, &secondary_spawns);
+	spawns = &top;
+	do {
+		while (!list_empty(spawns)) {
+			struct crypto_instance *inst;
+
+			spawn = list_first_entry(spawns, struct crypto_spawn,
+						 list);
+			inst = spawn->inst;
+
+			BUG_ON(&inst->alg == alg);
+
+			list_move(&spawn->list, &stack);
+
+			if (&inst->alg == nalg)
+				break;
+
+			spawn->alg = NULL;
+			spawns = &inst->alg.cra_users;
+		}
+	} while ((spawns = crypto_more_spawns(alg, &stack, &top,
+					      &secondary_spawns)));
+
+	list_for_each_entry_safe(spawn, n, &secondary_spawns, list) {
+		if (spawn->alg)
+			list_move(&spawn->list, &spawn->alg->cra_users);
+		else
+			crypto_remove_spawn(spawn, list);
 	}
 }
 
@@ -258,7 +307,7 @@ found:
 		    q->cra_priority > alg->cra_priority)
 			continue;
 
-		crypto_remove_spawns(&q->cra_users, &list, alg->cra_flags);
+		crypto_remove_spawns(q, &list, alg);
 	}
 
 complete:
@@ -330,7 +379,7 @@ static int crypto_remove_alg(struct crypto_alg *alg, struct list_head *list)
 
 	crypto_notify(CRYPTO_MSG_ALG_UNREGISTER, alg);
 	list_del_init(&alg->cra_list);
-	crypto_remove_spawns(&alg->cra_users, list, alg->cra_flags);
+	crypto_remove_spawns(alg, list, NULL);
 
 	return 0;
 }

Thanks,
-- 
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt

Re: [RFC] [PATCH 2/5] aead: Add generic aead wrapper interface

[Steffen Klassert]

On Mon, Aug 31, 2009 at 03:58:16PM +1000, Herbert Xu wrote:
> 
> In fact it was being instantiated, it was just getting killed
> immediately after completing its self-test :)
> 
> Please let me know if this fixes it for you.
> 

Yes, it fixes it. Thanks a lot!

So pcrypt would be ready to use from my point of view.
I'll send a next version of pcrypt and padata later.

[RFC] [PATCH 3/5] pcrypt: Add pcrypt crypto parallelization wrapper

[Steffen Klassert]

This patch adds a parallel crypto template that takes a crypto
algorithm and converts it to process the crypto transforms in
parallel. For the moment only aead is supported.

Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
---
 crypto/Kconfig            |   13 ++
 crypto/Makefile           |    2 +
 crypto/pcrypt.c           |  353 +++++++++++++++++++++++++++++++++++++++++++++
 crypto/pcrypt_core.c      |  106 ++++++++++++++
 include/crypto/pcrypt.h   |   56 +++++++
 include/linux/interrupt.h |    2 +
 kernel/softirq.c          |    3 +-
 7 files changed, 534 insertions(+), 1 deletions(-)
 create mode 100644 crypto/pcrypt.c
 create mode 100644 crypto/pcrypt_core.c
 create mode 100644 include/crypto/pcrypt.h

diff --git a/crypto/Kconfig b/crypto/Kconfig
index 74d0e62..36da74f 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -112,6 +112,19 @@ config CRYPTO_NULL
 	help
 	  These are 'Null' algorithms, used by IPsec, which do nothing.
 
+config CRYPTO_PCRYPT_CORE
+	bool
+	select CRYPTO_AEAD
+
+config CRYPTO_PCRYPT
+	tristate "Parallel crypto wrapper (EXPERIMENTAL)"
+	depends on USE_GENERIC_SMP_HELPERS && EXPERIMENTAL
+	select CRYPTO_MANAGER
+	select CRYPTO_PCRYPT_CORE
+	help
+	  This converts an arbitrary crypto algorithm into a parallel
+	  algorithm that is executed in a softirq.
+
 config CRYPTO_WORKQUEUE
        tristate
 
diff --git a/crypto/Makefile b/crypto/Makefile
index 673d9f7..84b9d17 100644
--- a/crypto/Makefile
+++ b/crypto/Makefile
@@ -56,6 +56,8 @@ obj-$(CONFIG_CRYPTO_XTS) += xts.o
 obj-$(CONFIG_CRYPTO_CTR) += ctr.o
 obj-$(CONFIG_CRYPTO_GCM) += gcm.o
 obj-$(CONFIG_CRYPTO_CCM) += ccm.o
+obj-$(CONFIG_CRYPTO_PCRYPT_CORE) += pcrypt_core.o
+obj-$(CONFIG_CRYPTO_PCRYPT) += pcrypt.o
 obj-$(CONFIG_CRYPTO_CRYPTD) += cryptd.o
 obj-$(CONFIG_CRYPTO_DES) += des_generic.o
 obj-$(CONFIG_CRYPTO_FCRYPT) += fcrypt.o
diff --git a/crypto/pcrypt.c b/crypto/pcrypt.c
new file mode 100644
index 0000000..32e88fb
--- /dev/null
+++ b/crypto/pcrypt.c
@@ -0,0 +1,353 @@
+/*
+ * pcrypt - Parallel crypto wrapper.
+ *
+ * Copyright (C) 2009 secunet Security Networks AG
+ * Copyright (C) 2009 Steffen Klassert <steffen.klassert@secunet.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include <crypto/algapi.h>
+#include <crypto/internal/aead.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <crypto/pcrypt.h>
+
+struct pcrypt_instance_ctx {
+	struct crypto_spawn spawn;
+	unsigned int tfm_count;
+};
+
+struct pcrypt_aead_ctx {
+	unsigned int tfm_nr;
+};
+
+static int pcrypt_do_parallel(struct padata_priv *padata, unsigned int tfm_nr,
+			      unsigned int softirq, unsigned int padata_nr)
+{
+	unsigned int cpu, cpu_index, num_cpus, cb_cpu;
+	cpumask_t cpu_map;
+
+	cpu_map = padata_get_cpumap(padata_nr);
+	num_cpus = cpus_weight(cpu_map);
+
+	cpu_index = tfm_nr % num_cpus;
+
+	cb_cpu = first_cpu(cpu_map);
+	for (cpu = 0; cpu < cpu_index; cpu++)
+		cb_cpu = next_cpu(cb_cpu, cpu_map);
+
+	return padata_do_parallel(softirq, padata_nr, padata, cb_cpu);
+}
+
+static int pcrypt_aead_setkey(struct crypto_aead *parent,
+			      const u8 *key, unsigned int keylen)
+{
+	struct crypto_aead *child = crypto_aead_crt(parent)->base;
+
+	return crypto_aead_setkey(child, key, keylen);
+}
+
+static int pcrypt_aead_setauthsize(struct crypto_aead *parent,
+				   unsigned int authsize)
+{
+	struct crypto_aead *child = crypto_aead_crt(parent)->base;
+
+	return crypto_aead_setauthsize(child, authsize);
+}
+
+static void pcrypt_aead_serial(struct padata_priv *padata)
+{
+	struct pcrypt_request *preq = pcrypt_padata_request(padata);
+	struct aead_request *req = pcrypt_request_ctx(preq);
+
+	aead_request_complete(req->base.data, padata->info);
+}
+
+static void pcrypt_aead_giv_serial(struct padata_priv *padata)
+{
+	struct pcrypt_request *preq = pcrypt_padata_request(padata);
+	struct aead_givcrypt_request *req = pcrypt_request_ctx(preq);
+
+	aead_request_complete(req->areq.base.data, padata->info);
+}
+
+static void pcrypt_aead_done(struct crypto_async_request *areq, int err)
+{
+	struct aead_request *req = areq->data;
+	struct pcrypt_request *preq = aead_request_ctx(req);
+	struct padata_priv *padata = pcrypt_request_padata(preq);
+
+	padata->info = err;
+	req->base.flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
+
+	local_bh_disable();
+	if (padata_do_serial(padata->nr, padata))
+		goto out;
+
+	aead_request_complete(req, padata->info);
+
+out:
+	local_bh_enable();
+}
+
+static void pcrypt_aead_enc(struct padata_priv *padata)
+{
+	struct pcrypt_request *preq = pcrypt_padata_request(padata);
+	struct aead_request *req = pcrypt_request_ctx(preq);
+
+	padata->info = crypto_aead_encrypt(req);
+
+	if (padata->info)
+		return;
+
+	if (padata_do_serial(AEAD_ENC_PADATA, padata))
+		return;
+
+	aead_request_complete(req->base.data, padata->info);
+}
+
+static int pcrypt_aead_encrypt(struct aead_request *req)
+{
+	int err;
+	struct pcrypt_request *preq = aead_request_ctx(req);
+	struct aead_request *creq = pcrypt_request_ctx(preq);
+	struct padata_priv *padata = pcrypt_request_padata(preq);
+	struct crypto_aead *aead = crypto_aead_reqtfm(req);
+	struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead);
+	u32 flags = aead_request_flags(req);
+
+	memset(padata, 0, sizeof(struct padata_priv));
+
+	padata->parallel = pcrypt_aead_enc;
+	padata->serial = pcrypt_aead_serial;
+
+	aead_request_set_tfm(creq, crypto_aead_crt(aead)->base);
+	aead_request_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP,
+				  pcrypt_aead_done, req);
+	aead_request_set_crypt(creq, req->src, req->dst,
+			       req->cryptlen, req->iv);
+	aead_request_set_assoc(creq, req->assoc, req->assoclen);
+
+	if (pcrypt_do_parallel(padata, ctx->tfm_nr, AEAD_ENC_SOFTIRQ,
+			       AEAD_ENC_PADATA))
+		err = -EINPROGRESS;
+	else
+		err = crypto_aead_encrypt(creq);
+
+	return err;
+}
+
+static void pcrypt_aead_dec(struct padata_priv *padata)
+{
+	struct pcrypt_request *preq = pcrypt_padata_request(padata);
+	struct aead_request *req = pcrypt_request_ctx(preq);
+
+	padata->info = crypto_aead_decrypt(req);
+
+	if (padata->info)
+		return;
+
+	if (padata_do_serial(AEAD_DEC_PADATA, padata))
+		return;
+
+	aead_request_complete(req->base.data, padata->info);
+}
+
+static int pcrypt_aead_decrypt(struct aead_request *req)
+{
+	int err;
+	struct pcrypt_request *preq = aead_request_ctx(req);
+	struct aead_request *creq = pcrypt_request_ctx(preq);
+	struct padata_priv *padata = pcrypt_request_padata(preq);
+	struct crypto_aead *aead = crypto_aead_reqtfm(req);
+	struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead);
+	u32 flags = aead_request_flags(req);
+
+	memset(padata, 0, sizeof(struct padata_priv));
+
+	padata->parallel = pcrypt_aead_dec;
+	padata->serial = pcrypt_aead_serial;
+
+	aead_request_set_tfm(creq, crypto_aead_crt(aead)->base);
+	aead_request_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP,
+				  pcrypt_aead_done, req);
+	aead_request_set_crypt(creq, req->src, req->dst,
+			       req->cryptlen, req->iv);
+	aead_request_set_assoc(creq, req->assoc, req->assoclen);
+
+	if (pcrypt_do_parallel(padata, ctx->tfm_nr, AEAD_DEC_SOFTIRQ,
+			       AEAD_DEC_PADATA))
+		err = -EINPROGRESS;
+	else
+		err = crypto_aead_decrypt(creq);
+
+	return err;
+}
+
+static void pcrypt_aead_givenc(struct padata_priv *padata)
+{
+	struct pcrypt_request *preq = pcrypt_padata_request(padata);
+	struct aead_givcrypt_request *req = pcrypt_request_ctx(preq);
+
+	padata->info = crypto_aead_givencrypt(req);
+
+	if (padata->info)
+		return;
+
+	if (padata_do_serial(AEAD_ENC_PADATA, padata))
+		return;
+
+	aead_request_complete(req->areq.base.data, padata->info);
+}
+
+static int pcrypt_aead_givencrypt(struct aead_givcrypt_request *req)
+{
+	int err;
+	struct aead_request *areq = &req->areq;
+	struct pcrypt_request *preq = aead_request_ctx(areq);
+	struct aead_givcrypt_request *creq = pcrypt_request_ctx(preq);
+	struct padata_priv *padata = pcrypt_request_padata(preq);
+	struct crypto_aead *aead = aead_givcrypt_reqtfm(req);
+	struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead);
+	u32 flags = aead_request_flags(areq);
+
+	memset(padata, 0, sizeof(struct padata_priv));
+
+	padata->parallel = pcrypt_aead_givenc;
+	padata->serial = pcrypt_aead_giv_serial;
+
+	aead_givcrypt_set_tfm(creq, crypto_aead_crt(aead)->base);
+	aead_givcrypt_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP,
+				   pcrypt_aead_done, areq);
+	aead_givcrypt_set_crypt(creq, areq->src, areq->dst,
+				areq->cryptlen, areq->iv);
+	aead_givcrypt_set_assoc(creq, areq->assoc, areq->assoclen);
+	aead_givcrypt_set_giv(creq, req->giv, req->seq);
+
+
+	if (pcrypt_do_parallel(padata, ctx->tfm_nr, AEAD_ENC_SOFTIRQ,
+			       AEAD_ENC_PADATA))
+		err = -EINPROGRESS;
+	else
+		err = crypto_aead_givencrypt(creq);
+
+	return err;
+}
+
+static int pcrypt_aead_init_tfm(struct crypto_tfm *tfm)
+{
+	struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
+	struct pcrypt_instance_ctx *ictx = crypto_instance_ctx(inst);
+	struct pcrypt_aead_ctx *ctx = crypto_tfm_ctx(tfm);
+	int err = 0;
+	
+	ictx->tfm_count++;
+	ctx->tfm_nr = ictx->tfm_count;
+
+	err = aead_wrap_init(tfm);
+	if (err)
+		return err;
+
+	tfm->crt_aead.reqsize += sizeof(struct pcrypt_request)
+			       + sizeof(struct aead_givcrypt_request);
+
+	return err;
+}
+
+static void pcrypt_aead_exit_tfm(struct crypto_tfm *tfm)
+{
+	aead_wrap_exit(tfm);
+}
+
+static struct crypto_template pcrypt_tmpl;
+
+static struct crypto_instance *pcrypt_alloc_aead(struct rtattr **tb)
+{
+	struct crypto_instance *inst;
+	unsigned int ctx_size = sizeof(struct pcrypt_instance_ctx);
+
+	inst = aead_wrap_alloc(&pcrypt_tmpl, tb, ctx_size, 0, 0);
+	if (IS_ERR(inst))
+		goto out;
+
+	inst->alg.cra_flags |= CRYPTO_ALG_ASYNC;
+
+	inst->alg.cra_ctxsize = sizeof(struct pcrypt_aead_ctx);
+
+	inst->alg.cra_init = pcrypt_aead_init_tfm;
+	inst->alg.cra_exit = pcrypt_aead_exit_tfm;
+
+	inst->alg.cra_aead.setkey = pcrypt_aead_setkey;
+	inst->alg.cra_aead.setauthsize = pcrypt_aead_setauthsize;
+	inst->alg.cra_aead.encrypt = pcrypt_aead_encrypt;
+	inst->alg.cra_aead.decrypt = pcrypt_aead_decrypt;
+	inst->alg.cra_aead.givencrypt = pcrypt_aead_givencrypt;
+
+out:
+	return inst;
+}
+
+static struct crypto_instance *pcrypt_alloc(struct rtattr **tb)
+{
+	struct crypto_attr_type *algt;
+
+	algt = crypto_get_attr_type(tb);
+	if (IS_ERR(algt))
+		return ERR_CAST(algt);
+
+	switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) {
+	case CRYPTO_ALG_TYPE_AEAD:
+		return pcrypt_alloc_aead(tb);
+	}
+
+	return ERR_PTR(-EINVAL);
+}
+
+static void pcrypt_free(struct crypto_instance *inst)
+{
+	aead_wrap_free(inst);
+}
+
+static struct crypto_template pcrypt_tmpl = {
+	.name = "pcrypt",
+	.alloc = pcrypt_alloc,
+	.free = pcrypt_free,
+	.module = THIS_MODULE,
+};
+
+static int __init pcrypt_init(void)
+{
+	padata_start(AEAD_ENC_PADATA);
+	padata_start(AEAD_DEC_PADATA);
+
+	return crypto_register_template(&pcrypt_tmpl);
+}
+
+static void __exit pcrypt_exit(void)
+{
+	padata_stop(AEAD_ENC_PADATA);
+	padata_stop(AEAD_DEC_PADATA);
+
+	crypto_unregister_template(&pcrypt_tmpl);
+}
+
+module_init(pcrypt_init);
+module_exit(pcrypt_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Steffen Klassert <steffen.klassert@secunet.com>");
+MODULE_DESCRIPTION("Parallel crypto wrapper");
diff --git a/crypto/pcrypt_core.c b/crypto/pcrypt_core.c
new file mode 100644
index 0000000..61c0411
--- /dev/null
+++ b/crypto/pcrypt_core.c
@@ -0,0 +1,106 @@
+/*
+ * pcrypt_core.c - Core functions for the pcrypt crypto parallelization
+ *
+ * Copyright (C) 2009 secunet Security Networks AG
+ * Copyright (C) 2009 Steffen Klassert <steffen.klassert@secunet.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include <linux/interrupt.h>
+#include <linux/cpu.h>
+#include <linux/err.h>
+#include <linux/module.h>
+#include <crypto/pcrypt.h>
+
+static void aead_enc_action(struct softirq_action *h)
+{
+	struct list_head *cpu_list, local_list;
+
+	cpu_list = &__get_cpu_var(softirq_work_list[AEAD_ENC_SOFTIRQ]);
+
+	local_irq_disable();
+	list_replace_init(cpu_list, &local_list);
+	local_irq_enable();
+
+	while (!list_empty(&local_list)) {
+		struct padata_priv *padata;
+
+		padata = list_entry(local_list.next, struct padata_priv,
+				    csd.list);
+
+		list_del_init(&padata->csd.list);
+
+		padata->parallel(padata);
+	}
+}
+
+static void aead_dec_action(struct softirq_action *h)
+{
+	struct list_head *cpu_list, local_list;
+
+	cpu_list = &__get_cpu_var(softirq_work_list[AEAD_DEC_SOFTIRQ]);
+
+	local_irq_disable();
+	list_replace_init(cpu_list, &local_list);
+	local_irq_enable();
+
+	while (!list_empty(&local_list)) {
+		struct padata_priv *padata;
+
+		padata = list_entry(local_list.next, struct padata_priv,
+				    csd.list);
+
+		list_del_init(&padata->csd.list);
+
+		padata->parallel(padata);
+	}
+}
+
+static int __devinit pcrypt_cpu_callback(struct notifier_block *nfb,
+					 unsigned long action, void *hcpu)
+{
+	int cpu = (unsigned long)hcpu;
+
+	switch (action) {
+	case CPU_ONLINE:
+	case CPU_ONLINE_FROZEN:
+		padata_add_cpu(AEAD_ENC_PADATA, cpu);
+		padata_add_cpu(AEAD_DEC_PADATA, cpu);
+		break;
+
+	case CPU_DEAD:
+	case CPU_DEAD_FROZEN:
+		padata_remove_cpu(AEAD_ENC_PADATA, cpu);
+		padata_remove_cpu(AEAD_DEC_PADATA, cpu);
+
+		break;
+	}
+
+	return NOTIFY_OK;
+}
+
+static int __init pcrypt_init_padata(void)
+{
+	open_softirq(AEAD_ENC_SOFTIRQ, aead_enc_action);
+	open_softirq(AEAD_DEC_SOFTIRQ, aead_dec_action);
+
+	padata_init(AEAD_ENC_PADATA, cpu_online_map);
+	padata_init(AEAD_DEC_PADATA, cpu_online_map);
+
+	hotcpu_notifier(pcrypt_cpu_callback, 0);
+
+	return 0;
+}
+subsys_initcall(pcrypt_init_padata);
diff --git a/include/crypto/pcrypt.h b/include/crypto/pcrypt.h
new file mode 100644
index 0000000..65c1f94
--- /dev/null
+++ b/include/crypto/pcrypt.h
@@ -0,0 +1,56 @@
+/*
+ * pcrypt - Parallel crypto engine.
+ *
+ * Copyright (C) 2009 secunet Security Networks AG
+ * Copyright (C) 2009 Steffen Klassert <steffen.klassert@secunet.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#ifndef _CRYPTO_PCRYPT_H
+#define _CRYPTO_PCRYPT_H
+
+#include <linux/crypto.h>
+#include <linux/kernel.h>
+#include <linux/padata.h>
+
+struct pcrypt_request {
+	struct padata_priv	padata;
+	void			*data;
+	void			*__ctx[] CRYPTO_MINALIGN_ATTR;
+};
+
+static inline void *pcrypt_request_ctx(struct pcrypt_request *req)
+{
+	return req->__ctx;
+}
+
+static inline
+struct padata_priv *pcrypt_request_padata(struct pcrypt_request *req)
+{
+	return &req->padata;
+}
+
+static inline
+struct pcrypt_request *pcrypt_padata_request(struct padata_priv *padata)
+{
+	return container_of(padata, struct pcrypt_request, padata);
+}
+
+struct crypto_aead *pcrypt_alloc_aead_tfm(const char *alg_name, u32 type,
+					  u32 mask);
+
+struct crypto_ablkcipher *pcrypt_alloc_ablkcipher_tfm(const char *alg_name,
+						      u32 type, u32 mask);
+#endif
diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
index a17679c..93e0c9f 100644
--- a/include/linux/interrupt.h
+++ b/include/linux/interrupt.h
@@ -338,6 +338,8 @@ enum
 	TASKLET_SOFTIRQ,
 	SCHED_SOFTIRQ,
 	HRTIMER_SOFTIRQ,
+	AEAD_ENC_SOFTIRQ,
+	AEAD_DEC_SOFTIRQ,
 	PADATA_SOFTIRQ,
 	RCU_SOFTIRQ, 	/* Preferable RCU should always be the last softirq */
 
diff --git a/kernel/softirq.c b/kernel/softirq.c
index 12c9b64..c76ace0 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -56,7 +56,8 @@ static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
 
 char *softirq_to_name[NR_SOFTIRQS] = {
 	"HI", "TIMER", "NET_TX", "NET_RX", "BLOCK",
-	"TASKLET", "SCHED", "HRTIMER", "PADATA", "RCU"
+	"TASKLET", "SCHED", "HRTIMER", "AEAD_ENC",
+	"AEAD_DEC", "PADATA", "RCU"
 };
 
 /*
-- 
1.5.4.2

[RFC] [PATCH 4/5] eseqiv: Add support for aead algorithms

[Steffen Klassert]

This adds eseqiv support for aead algorithms, this is usefull
for aead wrappers that need eseqiv as it's default IV generator.

Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
---
 crypto/eseqiv.c        |  248 +++++++++++++++++++++++++++++++++++++++++++++---
 include/linux/crypto.h |    1 +
 2 files changed, 235 insertions(+), 14 deletions(-)

diff --git a/crypto/eseqiv.c b/crypto/eseqiv.c
index 2a342c8..b6185f4 100644
--- a/crypto/eseqiv.c
+++ b/crypto/eseqiv.c
@@ -15,6 +15,7 @@
  *
  */
 
+#include <crypto/internal/aead.h>
 #include <crypto/internal/skcipher.h>
 #include <crypto/rng.h>
 #include <crypto/scatterwalk.h>
@@ -62,6 +63,29 @@ out:
 	skcipher_givcrypt_complete(req, err);
 }
 
+static void eseqiv_aead_complete2(struct aead_givcrypt_request *req)
+{
+	struct crypto_aead *geniv = aead_givcrypt_reqtfm(req);
+	struct eseqiv_request_ctx *reqctx = aead_givcrypt_reqctx(req);
+
+	memcpy(req->giv, PTR_ALIGN((u8 *)reqctx->tail,
+			 crypto_aead_alignmask(geniv) + 1),
+	       crypto_aead_ivsize(geniv));
+}
+
+static void eseqiv_aead_complete(struct crypto_async_request *base, int err)
+{
+	struct aead_givcrypt_request *req = base->data;
+
+	if (err)
+		goto out;
+
+	eseqiv_aead_complete2(req);
+
+out:
+	aead_givcrypt_complete(req, err);
+}
+
 static void eseqiv_chain(struct scatterlist *head, struct scatterlist *sg,
 			 int chain)
 {
@@ -153,7 +177,93 @@ static int eseqiv_givencrypt(struct skcipher_givcrypt_request *req)
 	if (err)
 		goto out;
 
-	eseqiv_complete2(req);
+	if (giv != req->giv)
+		eseqiv_complete2(req);
+
+out:
+	return err;
+}
+
+static int eseqiv_aead_givencrypt(struct aead_givcrypt_request *req)
+{
+	struct crypto_aead *geniv = aead_givcrypt_reqtfm(req);
+	struct eseqiv_ctx *ctx = crypto_aead_ctx(geniv);
+	struct eseqiv_request_ctx *reqctx = aead_givcrypt_reqctx(req);
+	struct aead_request *subreq;
+	crypto_completion_t complete;
+	void *data;
+	struct scatterlist *osrc, *odst;
+	struct scatterlist *dst;
+	struct page *srcp;
+	struct page *dstp;
+	u8 *giv;
+	u8 *vsrc;
+	u8 *vdst;
+	__be64 seq;
+	unsigned int ivsize;
+	unsigned int len;
+	unsigned int flags;
+	int err;
+
+	subreq = (void *)(reqctx->tail + ctx->reqoff);
+	aead_request_set_tfm(subreq, aead_geniv_base(geniv));
+
+	giv = req->giv;
+	complete = req->areq.base.complete;
+	data = req->areq.base.data;
+
+	osrc = req->areq.src;
+	odst = req->areq.dst;
+	srcp = sg_page(osrc);
+	dstp = sg_page(odst);
+	vsrc = PageHighMem(srcp) ? NULL : page_address(srcp) + osrc->offset;
+	vdst = PageHighMem(dstp) ? NULL : page_address(dstp) + odst->offset;
+
+	ivsize = crypto_aead_ivsize(geniv);
+	flags = req->areq.base.flags | CRYPTO_TFM_REQ_SG_HAS_IV;
+
+	if (vsrc != giv + ivsize && vdst != giv + ivsize) {
+		giv = PTR_ALIGN((u8 *)reqctx->tail,
+				crypto_aead_alignmask(geniv) + 1);
+		complete = eseqiv_aead_complete;
+		data = req;
+	}
+
+	aead_request_set_callback(subreq, flags, complete, data);
+
+	sg_init_table(reqctx->src, 2);
+	sg_set_buf(reqctx->src, giv, ivsize);
+	eseqiv_chain(reqctx->src, osrc, vsrc == giv + ivsize);
+
+	dst = reqctx->src;
+	if (osrc != odst) {
+		sg_init_table(reqctx->dst, 2);
+		sg_set_buf(reqctx->dst, giv, ivsize);
+		eseqiv_chain(reqctx->dst, odst, vdst == giv + ivsize);
+
+		dst = reqctx->dst;
+	}
+
+	aead_request_set_crypt(subreq, reqctx->src, dst,
+				     req->areq.cryptlen + ivsize, req->areq.iv);
+	aead_request_set_assoc(subreq, req->areq.assoc, req->areq.assoclen);
+
+	memcpy(req->areq.iv, ctx->salt, ivsize);
+
+	len = ivsize;
+	if (ivsize > sizeof(u64)) {
+		memset(req->giv, 0, ivsize - sizeof(u64));
+		len = sizeof(u64);
+	}
+	seq = cpu_to_be64(req->seq);
+	memcpy(req->giv + ivsize - len, &seq, len);
+
+	err = crypto_aead_encrypt(subreq);
+	if (err)
+		goto out;
+
+	if (giv != req->giv)
+		eseqiv_aead_complete2(req);
 
 out:
 	return err;
@@ -182,6 +292,29 @@ unlock:
 	return eseqiv_givencrypt(req);
 }
 
+static int eseqiv_aead_givencrypt_first(struct aead_givcrypt_request *req)
+{
+	struct crypto_aead *geniv = aead_givcrypt_reqtfm(req);
+	struct eseqiv_ctx *ctx = crypto_aead_ctx(geniv);
+	int err = 0;
+
+	spin_lock_bh(&ctx->lock);
+	if (crypto_aead_crt(geniv)->givencrypt != eseqiv_aead_givencrypt_first)
+		goto unlock;
+
+	crypto_aead_crt(geniv)->givencrypt = eseqiv_aead_givencrypt;
+	err = crypto_rng_get_bytes(crypto_default_rng, ctx->salt,
+				   crypto_aead_ivsize(geniv));
+
+unlock:
+	spin_unlock_bh(&ctx->lock);
+
+	if (err)
+		return err;
+
+	return eseqiv_aead_givencrypt(req);
+}
+
 static int eseqiv_init(struct crypto_tfm *tfm)
 {
 	struct crypto_ablkcipher *geniv = __crypto_ablkcipher_cast(tfm);
@@ -214,20 +347,47 @@ static int eseqiv_init(struct crypto_tfm *tfm)
 	return skcipher_geniv_init(tfm);
 }
 
+static int eseqiv_aead_init(struct crypto_tfm *tfm)
+{
+	struct crypto_aead *geniv = __crypto_aead_cast(tfm);
+	struct eseqiv_ctx *ctx = crypto_aead_ctx(geniv);
+	unsigned long alignmask;
+	unsigned int reqsize;
+
+	spin_lock_init(&ctx->lock);
+
+	alignmask = crypto_tfm_ctx_alignment() - 1;
+	reqsize = sizeof(struct eseqiv_request_ctx);
+
+	if (alignmask & reqsize) {
+		alignmask &= reqsize;
+		alignmask--;
+	}
+
+	alignmask = ~alignmask;
+	alignmask &= crypto_aead_alignmask(geniv);
+
+	reqsize += alignmask;
+	reqsize += crypto_aead_ivsize(geniv);
+	reqsize = ALIGN(reqsize, crypto_tfm_ctx_alignment());
+
+	ctx->reqoff = reqsize - sizeof(struct eseqiv_request_ctx);
+
+	tfm->crt_aead.reqsize = reqsize + sizeof(struct aead_request);
+
+	return aead_geniv_init(tfm);
+}
+
 static struct crypto_template eseqiv_tmpl;
 
-static struct crypto_instance *eseqiv_alloc(struct rtattr **tb)
+static struct crypto_instance *eseqiv_ablkcipher_alloc(struct rtattr **tb)
 {
-	struct crypto_instance *inst;
 	int err;
-
-	err = crypto_get_default_rng();
-	if (err)
-		return ERR_PTR(err);
+	struct crypto_instance *inst;
 
 	inst = skcipher_geniv_alloc(&eseqiv_tmpl, tb, 0, 0);
 	if (IS_ERR(inst))
-		goto put_rng;
+		goto out;
 
 	err = -EINVAL;
 	if (inst->alg.cra_ablkcipher.ivsize != inst->alg.cra_blocksize)
@@ -235,18 +395,75 @@ static struct crypto_instance *eseqiv_alloc(struct rtattr **tb)
 
 	inst->alg.cra_ablkcipher.givencrypt = eseqiv_givencrypt_first;
 
-	inst->alg.cra_init = eseqiv_init;
-	inst->alg.cra_exit = skcipher_geniv_exit;
+	 inst->alg.cra_init = eseqiv_init;
+	 inst->alg.cra_exit = skcipher_geniv_exit;
 
-	inst->alg.cra_ctxsize = sizeof(struct eseqiv_ctx);
-	inst->alg.cra_ctxsize += inst->alg.cra_ablkcipher.ivsize;
+	inst->alg.cra_ctxsize = inst->alg.cra_ablkcipher.ivsize;
 
 out:
 	return inst;
 
 free_inst:
 	skcipher_geniv_free(inst);
-	inst = ERR_PTR(err);
+	return ERR_PTR(err);
+}
+
+static struct crypto_instance *eseqiv_aead_alloc(struct rtattr **tb)
+{
+	int err;
+	struct crypto_instance *inst;
+
+	inst = aead_geniv_alloc(&eseqiv_tmpl, tb, 0, 0);
+	if (IS_ERR(inst))
+		goto out;
+
+	err = -EINVAL;
+	if (inst->alg.cra_aead.ivsize != inst->alg.cra_blocksize)
+		goto free_inst;
+
+	inst->alg.cra_aead.givencrypt = eseqiv_aead_givencrypt_first;
+
+	 inst->alg.cra_init = eseqiv_aead_init;
+	 inst->alg.cra_exit = aead_geniv_exit;
+
+	inst->alg.cra_ctxsize = inst->alg.cra_aead.ivsize;
+
+out:
+	return inst;
+
+free_inst:
+	aead_geniv_free(inst);
+	return ERR_PTR(err);
+}
+
+static struct crypto_instance *eseqiv_alloc(struct rtattr **tb)
+{
+	struct crypto_attr_type *algt;
+	struct crypto_instance *inst;
+	int err;
+
+	algt = crypto_get_attr_type(tb);
+	err = PTR_ERR(algt);
+	if (IS_ERR(algt))
+		return ERR_PTR(err);
+
+	err = crypto_get_default_rng();
+	if (err)
+		return ERR_PTR(err);
+
+	if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & CRYPTO_ALG_TYPE_MASK)
+		inst = eseqiv_ablkcipher_alloc(tb);
+	else
+		inst = eseqiv_aead_alloc(tb);
+
+	if (IS_ERR(inst))
+		goto put_rng;
+
+	inst->alg.cra_ctxsize += sizeof(struct eseqiv_ctx);
+
+out:
+	return inst;
+
 put_rng:
 	crypto_put_default_rng();
 	goto out;
@@ -254,7 +471,10 @@ put_rng:
 
 static void eseqiv_free(struct crypto_instance *inst)
 {
-	skcipher_geniv_free(inst);
+	if ((inst->alg.cra_flags ^ CRYPTO_ALG_TYPE_AEAD) & CRYPTO_ALG_TYPE_MASK)
+		skcipher_geniv_free(inst);
+	else
+		aead_geniv_free(inst);
 	crypto_put_default_rng();
 }
 
diff --git a/include/linux/crypto.h b/include/linux/crypto.h
index 2548bf0..5e336b2 100644
--- a/include/linux/crypto.h
+++ b/include/linux/crypto.h
@@ -82,6 +82,7 @@
 #define CRYPTO_TFM_REQ_WEAK_KEY		0x00000100
 #define CRYPTO_TFM_REQ_MAY_SLEEP	0x00000200
 #define CRYPTO_TFM_REQ_MAY_BACKLOG	0x00000400
+#define CRYPTO_TFM_REQ_SG_HAS_IV	0x00000800
 #define CRYPTO_TFM_RES_WEAK_KEY		0x00100000
 #define CRYPTO_TFM_RES_BAD_KEY_LEN   	0x00200000
 #define CRYPTO_TFM_RES_BAD_KEY_SCHED 	0x00400000
-- 
1.5.4.2

[RFC] [PATCH 5/5] authenc: Add support for the pcrypt aead wrapper

[Steffen Klassert]

This adds support for aead wrapper templates. An aead wrapper
can be coosen via a module parameter. For the moment only
pcrypt is supported.

Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
---
 crypto/authenc.c |   33 +++++++++++++++++++++++++++++++--
 1 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/crypto/authenc.c b/crypto/authenc.c
index 5793b64..aaa0fcc 100644
--- a/crypto/authenc.c
+++ b/crypto/authenc.c
@@ -13,6 +13,7 @@
 #include <crypto/aead.h>
 #include <crypto/internal/hash.h>
 #include <crypto/internal/skcipher.h>
+#include <crypto/internal/aead.h>
 #include <crypto/authenc.h>
 #include <crypto/scatterwalk.h>
 #include <linux/err.h>
@@ -23,6 +24,10 @@
 #include <linux/slab.h>
 #include <linux/spinlock.h>
 
+static char *wrapper = NULL;
+module_param(wrapper, charp, 0);
+MODULE_PARM_DESC(wrapper, "Wrapper template for AEAD algorithms");
+
 struct authenc_instance_ctx {
 	struct crypto_spawn auth;
 	struct crypto_skcipher_spawn enc;
@@ -157,15 +162,16 @@ static int crypto_authenc_genicv(struct aead_request *req, u8 *iv,
 
 	dstp = sg_page(dst);
 	vdst = PageHighMem(dstp) ? NULL : page_address(dstp) + dst->offset;
+	cryptlen = req->cryptlen;
 
-	if (ivsize) {
+	if (ivsize && !(aead_request_flags(req) & CRYPTO_TFM_REQ_SG_HAS_IV)) {
 		sg_init_table(cipher, 2);
 		sg_set_buf(cipher, iv, ivsize);
 		authenc_chain(cipher, dst, vdst == iv + ivsize);
 		dst = cipher;
+		cryptlen += ivsize;
 	}
 
-	cryptlen = req->cryptlen + ivsize;
 	hash = crypto_authenc_hash(req, flags, dst, cryptlen);
 	if (IS_ERR(hash))
 		return PTR_ERR(hash);
@@ -369,6 +375,26 @@ static void crypto_authenc_exit_tfm(struct crypto_tfm *tfm)
 	crypto_free_ablkcipher(ctx->enc);
 }
 
+static int crypto_authenc_set_wrapper(struct crypto_instance *inst)
+{
+	int err = 0;
+
+	if (!wrapper)
+		goto out;
+
+	if (!strcmp(wrapper, "pcrypt")) {
+		inst->alg.cra_type = &crypto_nivaead_type;
+		inst->alg.cra_aead.geniv = "eseqiv";
+		inst->alg.cra_aead.wrapper = "pcrypt";
+
+		goto out;
+	}
+
+	err = -EINVAL;
+out:
+	return err;
+}
+
 static struct crypto_instance *crypto_authenc_alloc(struct rtattr **tb)
 {
 	struct crypto_attr_type *algt;
@@ -452,6 +478,9 @@ static struct crypto_instance *crypto_authenc_alloc(struct rtattr **tb)
 	inst->alg.cra_aead.decrypt = crypto_authenc_decrypt;
 	inst->alg.cra_aead.givencrypt = crypto_authenc_givencrypt;
 
+	if (crypto_authenc_set_wrapper(inst))
+		goto err_drop_enc;
+
 out:
 	crypto_mod_put(auth);
 	return inst;
-- 
1.5.4.2

[RFC] [PATCH 0/5] Parallel IPsec v4

[Steffen Klassert]

This patchset adds the 'pcrypt' parallel crypto template. With this template it
is possible to process the crypto requests of a transform in parallel without
getting request reorder. This is in particular interesting for IPsec.

The parallel crypto template is based on a generic parallelization/serialization
method. This method uses the remote softirq invocation infrastructure for
parallelization and serialization. With this method data objects can be
processed in parallel, starting at some given point.
After doing some expensive operations in parallel, it is possible to serialize
again. The parallelized data objects return after serialization in the order as
they were before the parallelization. In the case of IPsec, this makes it
possible to run the expensive parts in parallel without getting packet
reordering.

Changes from v3:

- The generic aead wrapper is dropped.

- tcrypt is extended to test algorithms by name. So it is possible to
  instantiate pcrypt by doing e.g.:
  modprobe tcrypt alg="pcrypt(authenc(hmac(sha1),cbc(aes)))" type=3

Changes from v2:

- The xfrm netlink configuration code is dropped,
  this will be an extra patchset.

- Add generic aead wrapper interface to be able to wrap an aead algorithm
  with an arbitrary crypto template. 

- Convert pcrypt to use the generic aead wrapper.

- Add support for aead algorithms to eseqiv.

- Add support for the pcrypt aead wrapper to authenc. It's now possible to
  choose for pcrypt as the default authenc wrapper with a module parameter.

- Patchset applies to linux-2.6 git current.

Changes from v1:

- cpu_chainiv is dropped, pcrypt uses eseqiv as it's IV generator now.

- Add a xfrm netlink message to be able to choose for pcrypt from userspace.

- Use pcrypt just if it is selected from userspace.

- Patchset applies to 2.6.30-rc3

Steffen

[RFC] [PATCH 1/5] padata: generic interface for parallel processing

[Steffen Klassert]

This patch introduces an interface to process data objects
in parallel. On request it is possible to serialize again.
The parallelized objects return after serialization in the
same order as they were before the parallelization.

Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
---
 include/linux/interrupt.h |    3 +-
 include/linux/padata.h    |  116 +++++++++++
 kernel/Makefile           |    2 +-
 kernel/padata.c           |  490 +++++++++++++++++++++++++++++++++++++++++++++
 kernel/softirq.c          |    2 +-
 5 files changed, 610 insertions(+), 3 deletions(-)
 create mode 100644 include/linux/padata.h
 create mode 100644 kernel/padata.c

diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
index 91bb76f..a17679c 100644
--- a/include/linux/interrupt.h
+++ b/include/linux/interrupt.h
@@ -338,7 +338,8 @@ enum
 	TASKLET_SOFTIRQ,
 	SCHED_SOFTIRQ,
 	HRTIMER_SOFTIRQ,
-	RCU_SOFTIRQ,	/* Preferable RCU should always be the last softirq */
+	PADATA_SOFTIRQ,
+	RCU_SOFTIRQ, 	/* Preferable RCU should always be the last softirq */
 
 	NR_SOFTIRQS
 };
diff --git a/include/linux/padata.h b/include/linux/padata.h
new file mode 100644
index 0000000..469359f
--- /dev/null
+++ b/include/linux/padata.h
@@ -0,0 +1,116 @@
+/*
+ * padata.h - header for the padata parallelization interface
+ *
+ * Copyright (C) 2008, 2009 secunet Security Networks AG
+ * Copyright (C) 2008, 2009 Steffen Klassert <steffen.klassert@secunet.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#ifndef PADATA_H
+#define PADATA_H
+
+#include <linux/interrupt.h>
+#include <linux/smp.h>
+#include <linux/list.h>
+
+enum
+{
+	NO_PADATA=0,
+	AEAD_ENC_PADATA,
+	AEAD_DEC_PADATA,
+	NR_PADATA
+};
+
+struct padata_priv {
+	struct list_head	list;
+	struct call_single_data	csd;
+	int			cb_cpu;
+	int			seq_nr;
+	unsigned int		nr;
+	int			info;
+	void                    (*parallel)(struct padata_priv *padata);
+	void                    (*serial)(struct padata_priv *padata);
+};
+
+struct padata_queue {
+	struct list_head        list;
+	atomic_t                num_obj;
+	int               	cpu_index;
+	spinlock_t              lock;
+};
+
+struct parallel_data {
+	struct work_struct	work;
+	struct padata_queue	*queue;
+	atomic_t		seq_nr;
+	atomic_t		queued_objects;
+	cpumask_t		cpu_map;
+	cpumask_t		new_cpu_map;
+	u8			flags;
+#define	PADATA_INIT			1
+#define	PADATA_FLUSH_HARD		2
+#define	PADATA_RESET_IN_PROGRESS	4
+	spinlock_t              lock;
+};
+
+#ifdef CONFIG_USE_GENERIC_SMP_HELPERS
+extern void __init padata_init(unsigned int nr, cpumask_t cpu_map);
+extern void padata_dont_wait(unsigned int nr, struct padata_priv *padata);
+extern int padata_do_parallel(unsigned int softirq_nr, unsigned int nr,
+			      struct padata_priv *padata, int cb_cpu);
+extern int padata_do_serial(unsigned int nr, struct padata_priv *padata);
+extern cpumask_t padata_get_cpumap(unsigned int nr);
+extern void padata_set_cpumap(unsigned int nr, cpumask_t cpu_map);
+extern void padata_add_cpu(unsigned int nr, int cpu);
+extern void padata_remove_cpu(unsigned int nr, int cpu);
+extern void padata_start(unsigned int nr);
+extern void padata_stop(unsigned int nr);
+#else
+static inline void padata_init(unsigned int nr,cpumask_t cpu_map)
+{
+}
+static inline void padata_dont_wait(unsigned int nr, struct padata_priv *padata)
+{
+}
+static inline int padata_do_parallel(unsigned int softirq_nr, unsigned int nr,
+				     struct padata_priv *padata, int cb_cpu)
+{
+	return 0;
+}
+static inline int padata_do_serial(unsigned int nr, struct padata_priv *padata)
+{
+	return 0;
+}
+static inline cpumask_t padata_get_cpumap(unsigned int nr)
+{
+	return cpu_online_map;
+}
+static inline void padata_set_cpumap(unsigned int nr, cpumask_t cpu_map)
+{
+}
+static inline padata_add_cpu(unsigned int nr, int cpu)
+{
+}
+static inline padata_remove_cpu(unsigned int nr, int cpu)
+{
+}
+static inline padata_start(unsigned int nr)
+{
+}
+static inline padata_stop(unsigned int nr)
+{
+}
+#endif
+#endif
diff --git a/kernel/Makefile b/kernel/Makefile
index 4242366..f2d7ea7 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -40,7 +40,7 @@ obj-$(CONFIG_RT_MUTEXES) += rtmutex.o
 obj-$(CONFIG_DEBUG_RT_MUTEXES) += rtmutex-debug.o
 obj-$(CONFIG_RT_MUTEX_TESTER) += rtmutex-tester.o
 obj-$(CONFIG_GENERIC_ISA_DMA) += dma.o
-obj-$(CONFIG_USE_GENERIC_SMP_HELPERS) += smp.o
+obj-$(CONFIG_USE_GENERIC_SMP_HELPERS) += smp.o padata.o
 ifneq ($(CONFIG_SMP),y)
 obj-y += up.o
 endif
diff --git a/kernel/padata.c b/kernel/padata.c
new file mode 100644
index 0000000..192c9a6
--- /dev/null
+++ b/kernel/padata.c
@@ -0,0 +1,490 @@
+/*
+ * padata.c - generic interface to process data streams in parallel
+ *
+ * Copyright (C) 2008, 2009 secunet Security Networks AG
+ * Copyright (C) 2008, 2009 Steffen Klassert <steffen.klassert@secunet.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include <linux/module.h>
+#include <linux/cpumask.h>
+#include <linux/err.h>
+#include <linux/padata.h>
+
+#define MAX_SEQ_NR  1000000000
+
+static struct parallel_data padata_vec[NR_PADATA];
+static struct padata_priv *padata_get_next(struct parallel_data *par_data);
+
+static void padata_flush_hard(struct parallel_data *par_data)
+{
+	int cpu;
+	struct padata_priv *padata;
+	struct padata_queue *queue;
+
+	for_each_cpu_mask(cpu, par_data->cpu_map) {
+		queue = per_cpu_ptr(par_data->queue, cpu);
+
+		while(!list_empty(&queue->list)) {
+			padata = list_entry(queue->list.next, struct padata_priv, list);
+
+			spin_lock(&queue->lock);
+			list_del_init(&padata->list);
+			spin_unlock(&queue->lock);
+
+			atomic_dec(&par_data->queued_objects);
+			padata->serial(padata);
+		}
+	}
+}
+
+static void padata_flush_order(struct parallel_data *par_data)
+{
+	struct padata_priv *padata;
+
+	while (1) {
+		padata = padata_get_next(par_data);
+
+		if (padata && !IS_ERR(padata))
+			padata->serial(padata);
+		else
+			break;
+	}
+
+	padata_flush_hard(par_data);
+}
+
+static void padata_reset_work(struct work_struct *work)
+{
+	int cpu, cpu_index;
+	struct padata_queue *queue;
+	struct parallel_data *par_data;
+
+	par_data = container_of(work, struct parallel_data, work);
+
+	if (par_data->flags & (PADATA_INIT|PADATA_RESET_IN_PROGRESS))
+		return;
+
+	spin_lock_bh(&par_data->lock);
+	par_data->flags |= PADATA_RESET_IN_PROGRESS;
+
+	if (!(par_data->flags & PADATA_FLUSH_HARD))
+		padata_flush_order(par_data);
+	else
+		padata_flush_hard(par_data);
+
+	cpu_index = 0;
+
+	par_data->cpu_map = par_data->new_cpu_map;
+
+	for_each_cpu_mask(cpu, par_data->cpu_map) {
+		queue = per_cpu_ptr(par_data->queue, cpu);
+
+		atomic_set(&queue->num_obj, 0);
+		queue->cpu_index = cpu_index;
+		cpu_index++;
+	}
+	spin_unlock_bh(&par_data->lock);
+
+	atomic_set(&par_data->seq_nr, -1);
+	par_data->flags &= ~PADATA_RESET_IN_PROGRESS;
+	par_data->flags |= PADATA_INIT;
+}
+
+static struct padata_priv *padata_get_next(struct parallel_data *par_data)
+{
+	int cpu, num_cpus, empty;
+	int seq_nr, calc_seq_nr, next_nr;
+	struct padata_queue *queue, *next_queue;
+	struct padata_priv *padata;
+
+	empty = 0;
+	next_nr = -1;
+	next_queue = NULL;
+
+	num_cpus = cpus_weight(par_data->cpu_map);
+
+	for_each_cpu_mask(cpu, par_data->cpu_map) {
+		queue = per_cpu_ptr(par_data->queue, cpu);
+
+		/*
+		 * Calculate the seq_nr of the object that should be
+		 * next in this queue.
+		 */
+		calc_seq_nr = (atomic_read(&queue->num_obj) * num_cpus)
+			+ queue->cpu_index;
+
+		if (!list_empty(&queue->list)) {
+			padata = list_entry(queue->list.next,
+					    struct padata_priv, list);
+
+			seq_nr  = padata->seq_nr;
+
+			if (unlikely(calc_seq_nr != seq_nr)) {
+				par_data->flags &= ~PADATA_INIT;
+				par_data->flags |= PADATA_FLUSH_HARD;
+				padata = NULL;
+				goto out;
+			}
+		} else {
+			seq_nr = calc_seq_nr;
+			empty++;
+		}
+
+		if (next_nr < 0 || seq_nr < next_nr) {
+			next_nr = seq_nr;
+			next_queue = queue;
+		}
+	}
+
+	padata = NULL;
+
+	if (empty == num_cpus)
+		goto out;
+
+	if (!list_empty(&next_queue->list)) {
+		padata = list_entry(next_queue->list.next,
+				    struct padata_priv, list);
+
+		spin_lock(&next_queue->lock);
+		list_del_init(&padata->list);
+		spin_unlock(&next_queue->lock);
+
+		atomic_dec(&par_data->queued_objects);
+		atomic_inc(&next_queue->num_obj);
+
+		goto out;
+	}
+
+	if (next_nr % num_cpus == next_queue->cpu_index) {
+		padata = ERR_PTR(-ENODATA);
+		goto out;
+	}
+
+	padata = ERR_PTR(-EINPROGRESS);
+out:
+	return padata;
+}
+
+static void padata_action(struct softirq_action *h)
+{
+	struct list_head *cpu_list, local_list;
+
+	cpu_list = &__get_cpu_var(softirq_work_list[PADATA_SOFTIRQ]);
+
+	local_irq_disable();
+	list_replace_init(cpu_list, &local_list);
+	local_irq_enable();
+
+	while (!list_empty(&local_list)) {
+		struct padata_priv *padata;
+
+		padata = list_entry(local_list.next,
+				    struct padata_priv, csd.list);
+
+		list_del_init(&padata->csd.list);
+
+		padata->serial(padata);
+	}
+}
+
+static int padata_cpu_hash(unsigned int nr, struct padata_priv *padata)
+{
+	int cpu, target_cpu, this_cpu, cpu_index;
+
+	this_cpu = smp_processor_id();
+
+	if (padata->nr != 0)
+		return this_cpu;
+
+	if (!(padata_vec[nr].flags & PADATA_INIT))
+		return this_cpu;
+
+	padata->seq_nr = atomic_inc_return(&padata_vec[nr].seq_nr);
+
+	if (padata->seq_nr > MAX_SEQ_NR) {
+		padata_vec[nr].flags &= ~PADATA_INIT;
+		padata->seq_nr = 0;
+		schedule_work(&padata_vec[nr].work);
+		return this_cpu;
+	}
+
+	padata->nr = nr;
+
+	/*
+	 * Hash the sequence numbers to the cpus by taking
+	 * seq_nr mod. number of cpus in use.
+	 */
+	cpu_index =  padata->seq_nr % cpus_weight(padata_vec[nr].cpu_map);
+
+	target_cpu = first_cpu(padata_vec[nr].cpu_map);
+	for (cpu = 0; cpu < cpu_index; cpu++)
+		target_cpu = next_cpu(target_cpu, padata_vec[nr].cpu_map);
+
+	return target_cpu;
+}
+
+/*
+ * padata_dont_wait - must be called if an object that runs in parallel will
+ * not be serialized with padata_do_serial
+ *
+ * @nr: number of the padata instance
+ * @padata: object that will not be seen by padata_do_serial
+ */
+void padata_dont_wait(unsigned int nr, struct padata_priv *padata)
+{
+	struct padata_queue *queue;
+
+	if (!(padata_vec[nr].flags & PADATA_INIT))
+		return;
+
+	if (padata->nr == 0 || padata->nr != nr)
+		return;
+
+	queue = per_cpu_ptr(padata_vec[nr].queue, smp_processor_id());
+	atomic_inc(&queue->num_obj);
+
+	padata->nr = 0;
+	padata->seq_nr = 0;
+}
+EXPORT_SYMBOL(padata_dont_wait);
+
+/*
+ * padata_do_parallel - padata parallelization function
+ *
+ * @softirq_nr: number of the softirq that will do the parallelization
+ * @nr: number of the padata instance
+ * @padata: object to be parallelized
+ * @cb_cpu: cpu number on which the serialization callback function will run
+ */
+int padata_do_parallel(unsigned int softirq_nr, unsigned int nr,
+		       struct padata_priv *padata, int cb_cpu)
+{
+	int target_cpu;
+
+	padata->cb_cpu = cb_cpu;
+
+	local_bh_disable();
+	target_cpu = padata_cpu_hash(nr, padata);
+	local_bh_enable();
+
+	send_remote_softirq(&padata->csd, target_cpu, softirq_nr);
+
+	return 1;
+}
+EXPORT_SYMBOL(padata_do_parallel);
+
+/*
+ * padata_do_serial - padata serialization function
+ *
+ * @nr: number of the padata instance
+ * @padata: object to be serialized
+ *
+ * returns 1 if the serialization callback function will be called
+ * from padata, 0 else
+ */
+int padata_do_serial(unsigned int nr, struct padata_priv *padata)
+{
+	int cpu;
+	struct padata_queue *reorder_queue;
+
+	if (!(padata_vec[nr].flags & PADATA_INIT))
+		return 0;
+
+	if (padata->nr != nr || padata->nr == 0) {
+		padata->serial(padata);
+		return 1;
+	}
+
+	cpu = smp_processor_id();
+
+	reorder_queue = per_cpu_ptr(padata_vec[nr].queue, cpu);
+
+	spin_lock(&reorder_queue->lock);
+	list_add_tail(&padata->list, &reorder_queue->list);
+	spin_unlock(&reorder_queue->lock);
+
+	atomic_inc(&padata_vec[nr].queued_objects);
+
+try_again:
+	if (!spin_trylock(&padata_vec[nr].lock))
+		goto out;
+
+	while(1) {
+		padata = padata_get_next(&padata_vec[nr]);
+
+		if (!padata || PTR_ERR(padata) == -EINPROGRESS)
+			break;
+		if (PTR_ERR(padata) == -ENODATA) {
+			spin_unlock(&padata_vec[nr].lock);
+			goto out;
+		}
+
+		send_remote_softirq(&padata->csd, padata->cb_cpu,
+				    PADATA_SOFTIRQ);
+	}
+
+	if (unlikely(!(padata_vec[nr].flags & PADATA_INIT))) {
+		spin_unlock(&padata_vec[nr].lock);
+		goto reset_out;
+	}
+
+	spin_unlock(&padata_vec[nr].lock);
+
+	if (atomic_read(&padata_vec[nr].queued_objects))
+		goto try_again;
+
+out:
+	return 1;
+reset_out:
+	schedule_work(&padata_vec[nr].work);
+	return 1;
+}
+EXPORT_SYMBOL(padata_do_serial);
+
+/*
+ * padata_get_cpumap - get the cpu map that is actually in use
+ *
+ * @nr: number of the padata instance
+ */
+cpumask_t padata_get_cpumap(unsigned int nr)
+{
+	return padata_vec[nr].cpu_map;
+}
+EXPORT_SYMBOL(padata_get_cpumap);
+
+/*
+ * padata_set_cpumap - set the cpu map that padata uses
+ *
+ * @nr: number of the padata instance
+ * @cpu_map: the cpu map to use
+ */
+void padata_set_cpumap(unsigned int nr, cpumask_t cpu_map)
+{
+	padata_vec[nr].new_cpu_map = cpu_map;
+	padata_vec[nr].flags &= ~PADATA_INIT;
+	padata_vec[nr].flags |= PADATA_FLUSH_HARD;
+
+	schedule_work(&padata_vec[nr].work);
+}
+EXPORT_SYMBOL(padata_set_cpumap);
+
+/*
+ * padata_add_cpu - add a cpu to the padata cpu map
+ *
+ * @nr: number of the padata instance
+ * @cpu: cpu to remove
+ */
+void padata_add_cpu(unsigned int nr, int cpu)
+{
+	cpumask_t cpu_map = padata_vec[nr].cpu_map;
+
+	cpu_set(cpu, cpu_map);
+	padata_set_cpumap(nr, cpu_map);
+}
+EXPORT_SYMBOL(padata_add_cpu);
+
+/*
+ * padata_remove_cpu - remove a cpu from the padata cpu map
+ *
+ * @nr: number of the padata instance
+ * @cpu: cpu to remove
+ */
+void padata_remove_cpu(unsigned int nr, int cpu)
+{
+	cpumask_t cpu_map = padata_vec[nr].cpu_map;
+
+	cpu_clear(cpu, cpu_map);
+	padata_set_cpumap(nr, cpu_map);
+}
+EXPORT_SYMBOL(padata_remove_cpu);
+
+/*
+ * padata_start - start the parallel processing
+ *
+ * @nr: number of the padata instance
+ */
+void padata_start(unsigned int nr)
+{
+	if (padata_vec[nr].flags & PADATA_INIT)
+		return;
+
+	schedule_work(&padata_vec[nr].work);
+}
+EXPORT_SYMBOL(padata_start);
+
+/*
+ * padata_stop - stop the parallel processing
+ *
+ * @nr: number of the padata instance
+ */
+void padata_stop(unsigned int nr)
+{
+	padata_vec[nr].flags &= ~PADATA_INIT;
+}
+EXPORT_SYMBOL(padata_stop);
+
+/*
+ * padata_init - initialize a padata instance
+ *
+ * @nr: number of the padata instance
+ * @cpu_map: map of the cpu set that padata uses for parallelization
+ */
+void __init padata_init(unsigned int nr, cpumask_t cpu_map)
+{
+	int cpu, cpu_index;
+	struct padata_queue *percpu_queue, *queue;
+
+	percpu_queue = alloc_percpu(struct padata_queue);
+
+	if (!percpu_queue) {
+		printk("padata_init: Failed to alloc the serialization"
+		       "queues for padata nr %d, exiting!\n", nr);
+		return;
+	}
+
+	cpu_index = 0;
+
+	for_each_possible_cpu(cpu) {
+		queue = per_cpu_ptr(percpu_queue, cpu);
+
+		if (cpu_isset(cpu, cpu_map)) {
+			queue->cpu_index = cpu_index;
+			cpu_index++;
+		}
+
+		INIT_LIST_HEAD(&queue->list);
+		spin_lock_init(&queue->lock);
+		atomic_set(&queue->num_obj, 0);
+	}
+
+	INIT_WORK(&padata_vec[nr].work, padata_reset_work);
+
+	atomic_set(&padata_vec[nr].seq_nr, -1);
+	atomic_set(&padata_vec[nr].queued_objects, 0);
+	padata_vec[nr].cpu_map = cpu_map;
+	padata_vec[nr].new_cpu_map = cpu_map;
+	padata_vec[nr].queue = percpu_queue;
+	padata_vec[nr].flags = 0;
+	spin_lock_init(&padata_vec[nr].lock);
+}
+
+static int __init padata_initcall(void)
+{
+	open_softirq(PADATA_SOFTIRQ, padata_action);
+
+	return 0;
+}
+subsys_initcall(padata_initcall);
diff --git a/kernel/softirq.c b/kernel/softirq.c
index b525dd3..12c9b64 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -56,7 +56,7 @@ static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
 
 char *softirq_to_name[NR_SOFTIRQS] = {
 	"HI", "TIMER", "NET_TX", "NET_RX", "BLOCK",
-	"TASKLET", "SCHED", "HRTIMER",	"RCU"
+	"TASKLET", "SCHED", "HRTIMER", "PADATA", "RCU"
 };
 
 /*
-- 
1.5.4.2