* [PATCH RFC 3/4] md/raid5: replace async_tx with synchronous xor/pq operations
2026-10-15 19:22 [PATCH RFC 0/4] async_tx/md: Remove async_tx framework Frank.Li
2026-10-15 19:22 ` [PATCH RFC 1/4] md/dm-integrity: replace async_tx with synchronous xor for journal Frank.Li
2026-10-15 19:22 ` [PATCH RFC 2/4] md/raid5-ppl: replace async_tx with synchronous xor Frank.Li
@ 2026-10-15 19:22 ` Frank.Li
2026-10-15 19:22 ` [PATCH RFC 4/4] crypto: remove async_tx framework Frank.Li
2026-10-15 19:51 ` [PATCH RFC 0/4] async_tx/md: Remove " Eric Biggers
4 siblings, 0 replies; 6+ messages in thread
From: Frank.Li @ 2026-10-15 19:22 UTC (permalink / raw)
To: Alasdair Kergon, Mike Snitzer, Mikulas Patocka,
Benjamin Marzinski, Song Liu, Yu Kuai, Li Nan, Xiao Ni,
Dan Williams, Herbert Xu, David S. Miller, Jonathan Corbet,
Shuah Khan, Randy Dunlap
Cc: dm-devel, linux-kernel, linux-raid, linux-crypto, linux-doc,
Anatolij Gustschin, Saeed Bishara, Rameshwar Prasad Sahu,
Thomas Petazzoni, Anup Patel, Frank Li
From: Frank Li <Frank.Li@nxp.com>
Remove all async_tx usage from drivers/md/raid5.c. Replace each
async operation with its synchronous equivalent:
- async_copy_data() -> sync_copy_data() using kmap_local_page/memcpy
- async_memcpy() -> memcpy() directly on page_address()
- async_xor_offs() ZERO_DST -> memset(dst,0) + xor_gen()
- async_xor_offs() DROP_DST -> xor_gen() (dest included in src ptrs)
- async_gen_syndrome() -> raid6_gen_syndrome()
- async_gen_syndrome() PQ_XOR_DST -> raid6_xor_syndrome()
- async_raid6_datap_recov() -> raid6_datap_recov()
- async_raid6_2data_recov() -> raid6_2data_recov()
- async_xor_val_offs() -> xor_gen() + memchr_inv()
- async_syndrome_val() -> xor_gen() + raid6_gen_syndrome() + memcmp()
- async_trigger_callback() -> direct callback invocation
- async_tx_quiesce() -> removed (synchronous, no-op needed)
- async_tx_issue_pending_all() -> removed (no-op)
Also remove the addr_conv_t region from the scribble buffer and
the to_addr_conv() helper since DMA address conversion is no
longer needed.
async_tx adds significant complexity and maintenance overhead while
providing little practical benefit on modern systems, where RAID parity
computation is typically performed more efficiently by CPU-based XOR/PQ
implementations than by DMA engine offload.
Replace #include <linux/async_tx.h> with #include <linux/raid/xor.h>.
---
drivers/md/raid5-log.h | 4 +-
drivers/md/raid5.c | 467 ++++++++++++++++++++++++++-----------------------
2 files changed, 251 insertions(+), 220 deletions(-)
diff --git a/drivers/md/raid5-log.h b/drivers/md/raid5-log.h
index c8332502669e5..52408067a40cb 100644
--- a/drivers/md/raid5-log.h
+++ b/drivers/md/raid5-log.h
@@ -31,9 +31,7 @@ void r5c_update_on_rdev_error(struct mddev *mddev, struct md_rdev *rdev);
bool r5c_big_stripe_cached(struct r5conf *conf, sector_t sect);
int r5l_start(struct r5l_log *log);
-struct dma_async_tx_descriptor *
-ops_run_partial_parity(struct stripe_head *sh, struct raid5_percpu *percpu,
- struct dma_async_tx_descriptor *tx);
+void ops_run_partial_parity(struct stripe_head *sh, struct raid5_percpu *percpu);
int ppl_init_log(struct r5conf *conf);
void ppl_exit_log(struct r5conf *conf);
int ppl_write_stripe(struct r5conf *conf, struct stripe_head *sh);
diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
index b91545ce090de..b1121e6f4bfbe 100644
--- a/drivers/md/raid5.c
+++ b/drivers/md/raid5.c
@@ -38,7 +38,7 @@
#include <linux/blkdev.h>
#include <linux/kthread.h>
#include <linux/raid/pq.h>
-#include <linux/async_tx.h>
+#include <linux/raid/xor.h>
#include <linux/module.h>
#include <linux/async.h>
#include <linux/seq_file.h>
@@ -1373,17 +1373,14 @@ static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s)
defer_issue_bios(conf, head_sh->sector, &pending_bios);
}
-static struct dma_async_tx_descriptor *
-async_copy_data(int frombio, struct bio *bio, struct page **page,
- unsigned int poff, sector_t sector, struct dma_async_tx_descriptor *tx,
- struct stripe_head *sh, int no_skipcopy)
+static void sync_copy_data(int frombio, struct bio *bio, struct page **page,
+ unsigned int poff, sector_t sector, struct stripe_head *sh,
+ int no_skipcopy)
{
struct bio_vec bvl;
struct bvec_iter iter;
struct page *bio_page;
int page_offset;
- struct async_submit_ctl submit;
- enum async_tx_flags flags = 0;
struct r5conf *conf = sh->raid_conf;
if (bio->bi_iter.bi_sector >= sector)
@@ -1391,10 +1388,6 @@ async_copy_data(int frombio, struct bio *bio, struct page **page,
else
page_offset = (signed)(sector - bio->bi_iter.bi_sector) * -512;
- if (frombio)
- flags |= ASYNC_TX_FENCE;
- init_async_submit(&submit, flags, tx, NULL, NULL, NULL);
-
bio_for_each_segment(bvl, bio, iter) {
int len = bvl.bv_len;
int clen;
@@ -1412,30 +1405,34 @@ async_copy_data(int frombio, struct bio *bio, struct page **page,
clen = len;
if (clen > 0) {
+ void *dst, *src;
+
b_offset += bvl.bv_offset;
bio_page = bvl.bv_page;
if (frombio) {
if (conf->skip_copy &&
b_offset == 0 && page_offset == 0 &&
clen == RAID5_STRIPE_SIZE(conf) &&
- !no_skipcopy)
+ !no_skipcopy) {
*page = bio_page;
- else
- tx = async_memcpy(*page, bio_page, page_offset + poff,
- b_offset, clen, &submit);
- } else
- tx = async_memcpy(bio_page, *page, b_offset,
- page_offset + poff, clen, &submit);
+ } else {
+ src = kmap_local_page(bio_page) + b_offset;
+ dst = page_address(*page) + page_offset + poff;
+ memcpy(dst, src, clen);
+ kunmap_local(src);
+ }
+ } else {
+ dst = kmap_local_page(bio_page) + b_offset;
+ src = page_address(*page) + page_offset + poff;
+ memcpy(dst, src, clen);
+ kunmap_local(dst);
+ }
}
- /* chain the operations */
- submit.depend_tx = tx;
if (clen < len) /* hit end of page */
break;
- page_offset += len;
+ page_offset += len;
}
-
- return tx;
}
static void ops_complete_biofill(void *stripe_head_ref)
@@ -1478,8 +1475,6 @@ static void ops_complete_biofill(void *stripe_head_ref)
static void ops_run_biofill(struct stripe_head *sh)
{
- struct dma_async_tx_descriptor *tx = NULL;
- struct async_submit_ctl submit;
int i;
struct r5conf *conf = sh->raid_conf;
@@ -1497,17 +1492,15 @@ static void ops_run_biofill(struct stripe_head *sh)
spin_unlock_irq(&sh->stripe_lock);
while (rbi && rbi->bi_iter.bi_sector <
dev->sector + RAID5_STRIPE_SECTORS(conf)) {
- tx = async_copy_data(0, rbi, &dev->page,
- dev->offset,
- dev->sector, tx, sh, 0);
+ sync_copy_data(0, rbi, &dev->page,
+ dev->offset, dev->sector, sh, 0);
rbi = r5_next_bio(conf, rbi, dev->sector);
}
}
}
atomic_inc(&sh->count);
- init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL);
- async_trigger_callback(&submit);
+ ops_complete_biofill(sh);
}
static void mark_target_uptodate(struct stripe_head *sh, int target)
@@ -1541,30 +1534,20 @@ static void ops_complete_compute(void *stripe_head_ref)
raid5_release_stripe(sh);
}
-/* return a pointer to the address conversion region of the scribble buffer */
+/* return a pointer to the page-pointer region of the scribble buffer */
static struct page **to_addr_page(struct raid5_percpu *percpu, int i)
{
return percpu->scribble + i * percpu->scribble_obj_size;
}
-/* return a pointer to the address conversion region of the scribble buffer */
-static addr_conv_t *to_addr_conv(struct stripe_head *sh,
- struct raid5_percpu *percpu, int i)
-{
- return (void *) (to_addr_page(percpu, i) + sh->disks + 2);
-}
-
-/*
- * Return a pointer to record offset address.
- */
+/* return a pointer to the offset array in the scribble buffer */
static unsigned int *
to_addr_offs(struct stripe_head *sh, struct raid5_percpu *percpu)
{
- return (unsigned int *) (to_addr_conv(sh, percpu, 0) + sh->disks + 2);
+ return (unsigned int *) (to_addr_page(percpu, 0) + sh->disks + 2);
}
-static struct dma_async_tx_descriptor *
-ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu)
+static void ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu)
{
int disks = sh->disks;
struct page **xor_srcs = to_addr_page(percpu, 0);
@@ -1573,9 +1556,10 @@ ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu)
struct r5dev *tgt = &sh->dev[target];
struct page *xor_dest = tgt->page;
unsigned int off_dest = tgt->offset;
+ size_t len = RAID5_STRIPE_SIZE(sh->raid_conf);
int count = 0;
- struct dma_async_tx_descriptor *tx;
- struct async_submit_ctl submit;
+ void **ptrs = (void **)xor_srcs;
+ void *dst;
int i;
BUG_ON(sh->batch_head);
@@ -1593,16 +1577,17 @@ ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu)
atomic_inc(&sh->count);
- init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, NULL,
- ops_complete_compute, sh, to_addr_conv(sh, percpu, 0));
- if (unlikely(count == 1))
- tx = async_memcpy(xor_dest, xor_srcs[0], off_dest, off_srcs[0],
- RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
- else
- tx = async_xor_offs(xor_dest, off_dest, xor_srcs, off_srcs, count,
- RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
+ dst = page_address(xor_dest) + off_dest;
+ if (unlikely(count == 1)) {
+ memcpy(dst, page_address(xor_srcs[0]) + off_srcs[0], len);
+ } else {
+ for (i = 0; i < count; i++)
+ ptrs[i] = page_address(xor_srcs[i]) + off_srcs[i];
+ memset(dst, 0, len);
+ xor_gen(dst, ptrs, count, len);
+ }
- return tx;
+ ops_complete_compute(sh);
}
/* set_syndrome_sources - populate source buffers for gen_syndrome
@@ -1660,19 +1645,18 @@ static int set_syndrome_sources(struct page **srcs,
return syndrome_disks;
}
-static struct dma_async_tx_descriptor *
-ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu)
+static void ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu)
{
int disks = sh->disks;
struct page **blocks = to_addr_page(percpu, 0);
unsigned int *offs = to_addr_offs(sh, percpu);
int target;
int qd_idx = sh->qd_idx;
- struct dma_async_tx_descriptor *tx;
- struct async_submit_ctl submit;
struct r5dev *tgt;
struct page *dest;
unsigned int dest_off;
+ size_t len = RAID5_STRIPE_SIZE(sh->raid_conf);
+ void **ptrs = (void **)blocks;
int i;
int count;
@@ -1699,13 +1683,17 @@ ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu)
count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_ALL);
blocks[count] = NULL; /* regenerating p is not necessary */
BUG_ON(blocks[count+1] != dest); /* q should already be set */
- init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
- ops_complete_compute, sh,
- to_addr_conv(sh, percpu, 0));
- tx = async_gen_syndrome(blocks, offs, count+2,
- RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
+ for (i = 0; i < count+2; i++) {
+ if (blocks[i])
+ ptrs[i] = page_address(blocks[i]) + offs[i];
+ else
+ ptrs[i] = NULL;
+ }
+ raid6_gen_syndrome(count+2, len, ptrs);
} else {
/* Compute any data- or p-drive using XOR */
+ void *dst = page_address(dest) + dest_off;
+
count = 0;
for (i = disks; i-- ; ) {
if (i == target || i == qd_idx)
@@ -1713,19 +1701,16 @@ ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu)
offs[count] = sh->dev[i].offset;
blocks[count++] = sh->dev[i].page;
}
-
- init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
- NULL, ops_complete_compute, sh,
- to_addr_conv(sh, percpu, 0));
- tx = async_xor_offs(dest, dest_off, blocks, offs, count,
- RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
+ for (i = 0; i < count; i++)
+ ptrs[i] = page_address(blocks[i]) + offs[i];
+ memset(dst, 0, len);
+ xor_gen(dst, ptrs, count, len);
}
- return tx;
+ ops_complete_compute(sh);
}
-static struct dma_async_tx_descriptor *
-ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu)
+static void ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu)
{
int i, count, disks = sh->disks;
int syndrome_disks = sh->ddf_layout ? disks : disks-2;
@@ -1735,10 +1720,10 @@ ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu)
int target2 = sh->ops.target2;
struct r5dev *tgt = &sh->dev[target];
struct r5dev *tgt2 = &sh->dev[target2];
- struct dma_async_tx_descriptor *tx;
struct page **blocks = to_addr_page(percpu, 0);
unsigned int *offs = to_addr_offs(sh, percpu);
- struct async_submit_ctl submit;
+ size_t len = RAID5_STRIPE_SIZE(sh->raid_conf);
+ void **ptrs = (void **)blocks;
BUG_ON(sh->batch_head);
pr_debug("%s: stripe %llu block1: %d block2: %d\n",
@@ -1781,17 +1766,19 @@ ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu)
/* Q disk is one of the missing disks */
if (faila == syndrome_disks) {
/* Missing P+Q, just recompute */
- init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
- ops_complete_compute, sh,
- to_addr_conv(sh, percpu, 0));
- return async_gen_syndrome(blocks, offs, syndrome_disks+2,
- RAID5_STRIPE_SIZE(sh->raid_conf),
- &submit);
+ for (i = 0; i < syndrome_disks+2; i++) {
+ if (blocks[i])
+ ptrs[i] = page_address(blocks[i]) + offs[i];
+ else
+ ptrs[i] = NULL;
+ }
+ raid6_gen_syndrome(syndrome_disks+2, len, ptrs);
} else {
struct page *dest;
unsigned int dest_off;
int data_target;
int qd_idx = sh->qd_idx;
+ void *d;
/* Missing D+Q: recompute D from P, then recompute Q */
if (target == qd_idx)
@@ -1808,40 +1795,38 @@ ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu)
}
dest = sh->dev[data_target].page;
dest_off = sh->dev[data_target].offset;
- init_async_submit(&submit,
- ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
- NULL, NULL, NULL,
- to_addr_conv(sh, percpu, 0));
- tx = async_xor_offs(dest, dest_off, blocks, offs, count,
- RAID5_STRIPE_SIZE(sh->raid_conf),
- &submit);
+ d = page_address(dest) + dest_off;
+ for (i = 0; i < count; i++)
+ ptrs[i] = page_address(blocks[i]) + offs[i];
+ memset(d, 0, len);
+ xor_gen(d, ptrs, count, len);
count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_ALL);
- init_async_submit(&submit, ASYNC_TX_FENCE, tx,
- ops_complete_compute, sh,
- to_addr_conv(sh, percpu, 0));
- return async_gen_syndrome(blocks, offs, count+2,
- RAID5_STRIPE_SIZE(sh->raid_conf),
- &submit);
+ for (i = 0; i < count+2; i++) {
+ if (blocks[i])
+ ptrs[i] = page_address(blocks[i]) + offs[i];
+ else
+ ptrs[i] = NULL;
+ }
+ raid6_gen_syndrome(count+2, len, ptrs);
}
} else {
- init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
- ops_complete_compute, sh,
- to_addr_conv(sh, percpu, 0));
+ for (i = 0; i < syndrome_disks+2; i++) {
+ if (blocks[i])
+ ptrs[i] = page_address(blocks[i]) + offs[i];
+ else
+ ptrs[i] = NULL;
+ }
if (failb == syndrome_disks) {
/* We're missing D+P. */
- return async_raid6_datap_recov(syndrome_disks+2,
- RAID5_STRIPE_SIZE(sh->raid_conf),
- faila,
- blocks, offs, &submit);
+ raid6_recov_datap(syndrome_disks+2, len, faila, ptrs);
} else {
/* We're missing D+D. */
- return async_raid6_2data_recov(syndrome_disks+2,
- RAID5_STRIPE_SIZE(sh->raid_conf),
- faila, failb,
- blocks, offs, &submit);
+ raid6_recov_2data(syndrome_disks+2, len, faila, failb, ptrs);
}
}
+
+ ops_complete_compute(sh);
}
static void ops_complete_prexor(void *stripe_head_ref)
@@ -1859,15 +1844,14 @@ static void ops_complete_prexor(void *stripe_head_ref)
r5c_release_extra_page(sh);
}
-static struct dma_async_tx_descriptor *
-ops_run_prexor5(struct stripe_head *sh, struct raid5_percpu *percpu,
- struct dma_async_tx_descriptor *tx)
+static void ops_run_prexor5(struct stripe_head *sh, struct raid5_percpu *percpu)
{
int disks = sh->disks;
struct page **xor_srcs = to_addr_page(percpu, 0);
unsigned int *off_srcs = to_addr_offs(sh, percpu);
int count = 0, pd_idx = sh->pd_idx, i;
- struct async_submit_ctl submit;
+ size_t len = RAID5_STRIPE_SIZE(sh->raid_conf);
+ void **ptrs = (void **)xor_srcs;
/* existing parity data subtracted */
unsigned int off_dest = off_srcs[count] = sh->dev[pd_idx].offset;
@@ -1893,38 +1877,55 @@ ops_run_prexor5(struct stripe_head *sh, struct raid5_percpu *percpu,
}
}
- init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
- ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));
- tx = async_xor_offs(xor_dest, off_dest, xor_srcs, off_srcs, count,
- RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
+ /* XOR_DROP_DST: dest (parity) is at ptrs[0], data pages follow.
+ * xor_gen(dst, ptrs, count, len) computes dst ^= ptrs[0..count-1],
+ * which cancels the old parity and folds in all written data. */
+ for (i = 0; i < count; i++)
+ ptrs[i] = page_address(xor_srcs[i]) + off_srcs[i];
+ xor_gen(page_address(xor_dest) + off_dest, ptrs, count, len);
- return tx;
+ ops_complete_prexor(sh);
}
-static struct dma_async_tx_descriptor *
-ops_run_prexor6(struct stripe_head *sh, struct raid5_percpu *percpu,
- struct dma_async_tx_descriptor *tx)
+static void ops_run_prexor6(struct stripe_head *sh, struct raid5_percpu *percpu)
{
struct page **blocks = to_addr_page(percpu, 0);
unsigned int *offs = to_addr_offs(sh, percpu);
- int count;
- struct async_submit_ctl submit;
+ size_t len = RAID5_STRIPE_SIZE(sh->raid_conf);
+ void **ptrs = (void **)blocks;
+ int count, i, start = -1, stop = -1;
pr_debug("%s: stripe %llu\n", __func__,
(unsigned long long)sh->sector);
count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_WANT_DRAIN);
- init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_PQ_XOR_DST, tx,
- ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));
- tx = async_gen_syndrome(blocks, offs, count+2,
- RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
+ /* PQ_XOR_DST: use xor_syndrome to fold in only the changed disks */
+ for (i = 0; i < count; i++) {
+ if (blocks[i]) {
+ if (start < 0)
+ start = i;
+ stop = i;
+ }
+ if (blocks[i])
+ ptrs[i] = page_address(blocks[i]) + offs[i];
+ else
+ ptrs[i] = NULL;
+ }
+ /* P slot */
+ ptrs[count] = blocks[count] ? page_address(blocks[count]) + offs[count] : NULL;
+ /* Q slot */
+ ptrs[count+1] = blocks[count+1] ? page_address(blocks[count+1]) + offs[count+1] : NULL;
+
+ if (start >= 0 && raid6_can_xor_syndrome())
+ raid6_xor_syndrome(count+2, start, stop, len, ptrs);
+ else
+ raid6_gen_syndrome(count+2, len, ptrs);
- return tx;
+ ops_complete_prexor(sh);
}
-static struct dma_async_tx_descriptor *
-ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)
+static void ops_run_biodrain(struct stripe_head *sh)
{
struct r5conf *conf = sh->raid_conf;
int disks = sh->disks;
@@ -1967,9 +1968,9 @@ ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)
if (bio_op(wbi) == REQ_OP_DISCARD)
set_bit(R5_Discard, &dev->flags);
else {
- tx = async_copy_data(1, wbi, &dev->page,
+ sync_copy_data(1, wbi, &dev->page,
dev->offset,
- dev->sector, tx, sh,
+ dev->sector, sh,
r5c_is_writeback(conf->log));
if (dev->page != dev->orig_page &&
!r5c_is_writeback(conf->log)) {
@@ -1991,8 +1992,6 @@ ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)
}
}
}
-
- return tx;
}
static void ops_complete_reconstruct(void *stripe_head_ref)
@@ -2043,18 +2042,16 @@ static void ops_complete_reconstruct(void *stripe_head_ref)
}
static void
-ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu,
- struct dma_async_tx_descriptor *tx)
+ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu)
{
int disks = sh->disks;
struct page **xor_srcs;
unsigned int *off_srcs;
- struct async_submit_ctl submit;
int count, pd_idx = sh->pd_idx, i;
struct page *xor_dest;
unsigned int off_dest;
+ size_t len = RAID5_STRIPE_SIZE(sh->raid_conf);
int prexor = 0;
- unsigned long flags;
int j = 0;
struct stripe_head *head_sh = sh;
int last_stripe;
@@ -2113,45 +2110,48 @@ ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu,
last_stripe = !head_sh->batch_head ||
list_first_entry(&sh->batch_list,
struct stripe_head, batch_list) == head_sh;
- if (last_stripe) {
- flags = ASYNC_TX_ACK |
- (prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST);
- atomic_inc(&head_sh->count);
- init_async_submit(&submit, flags, tx, ops_complete_reconstruct, head_sh,
- to_addr_conv(sh, percpu, j));
- } else {
- flags = prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST;
- init_async_submit(&submit, flags, tx, NULL, NULL,
- to_addr_conv(sh, percpu, j));
+ {
+ void **ptrs = (void **)xor_srcs;
+ void *dst = page_address(xor_dest) + off_dest;
+
+ if (unlikely(count == 1)) {
+ if (prexor)
+ memcpy(dst, page_address(xor_srcs[0]) + off_srcs[0], len);
+ else
+ memcpy(dst, page_address(xor_srcs[0]) + off_srcs[0], len);
+ } else {
+ int k;
+
+ for (k = 0; k < count; k++)
+ ptrs[k] = page_address(xor_srcs[k]) + off_srcs[k];
+ if (!prexor)
+ memset(dst, 0, len);
+ xor_gen(dst, ptrs, count, len);
+ }
}
- if (unlikely(count == 1))
- tx = async_memcpy(xor_dest, xor_srcs[0], off_dest, off_srcs[0],
- RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
- else
- tx = async_xor_offs(xor_dest, off_dest, xor_srcs, off_srcs, count,
- RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
if (!last_stripe) {
j++;
sh = list_first_entry(&sh->batch_list, struct stripe_head,
batch_list);
goto again;
}
+
+ atomic_inc(&head_sh->count);
+ ops_complete_reconstruct(head_sh);
}
static void
-ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu,
- struct dma_async_tx_descriptor *tx)
+ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu)
{
- struct async_submit_ctl submit;
struct page **blocks;
unsigned int *offs;
int count, i, j = 0;
struct stripe_head *head_sh = sh;
int last_stripe;
- int synflags;
- unsigned long txflags;
+ size_t len = RAID5_STRIPE_SIZE(sh->raid_conf);
+ int prexor_mode;
pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector);
@@ -2173,34 +2173,43 @@ ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu,
blocks = to_addr_page(percpu, j);
offs = to_addr_offs(sh, percpu);
- if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) {
- synflags = SYNDROME_SRC_WRITTEN;
- txflags = ASYNC_TX_ACK | ASYNC_TX_PQ_XOR_DST;
- } else {
- synflags = SYNDROME_SRC_ALL;
- txflags = ASYNC_TX_ACK;
- }
-
- count = set_syndrome_sources(blocks, offs, sh, synflags);
+ prexor_mode = (sh->reconstruct_state == reconstruct_state_prexor_drain_run);
+ count = set_syndrome_sources(blocks, offs, sh,
+ prexor_mode ? SYNDROME_SRC_WRITTEN : SYNDROME_SRC_ALL);
last_stripe = !head_sh->batch_head ||
list_first_entry(&sh->batch_list,
struct stripe_head, batch_list) == head_sh;
- if (last_stripe) {
- atomic_inc(&head_sh->count);
- init_async_submit(&submit, txflags, tx, ops_complete_reconstruct,
- head_sh, to_addr_conv(sh, percpu, j));
- } else
- init_async_submit(&submit, 0, tx, NULL, NULL,
- to_addr_conv(sh, percpu, j));
- tx = async_gen_syndrome(blocks, offs, count+2,
- RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
+ {
+ void **ptrs = (void **)blocks;
+ int start = -1, stop = -1;
+
+ for (i = 0; i < count; i++) {
+ if (blocks[i]) {
+ if (start < 0)
+ start = i;
+ stop = i;
+ }
+ ptrs[i] = blocks[i] ? page_address(blocks[i]) + offs[i] : NULL;
+ }
+ ptrs[count] = blocks[count] ? page_address(blocks[count]) + offs[count] : NULL;
+ ptrs[count+1] = blocks[count+1] ? page_address(blocks[count+1]) + offs[count+1] : NULL;
+
+ if (prexor_mode && start >= 0 && raid6_can_xor_syndrome())
+ raid6_xor_syndrome(count+2, start, stop, len, ptrs);
+ else
+ raid6_gen_syndrome(count+2, len, ptrs);
+ }
+
if (!last_stripe) {
j++;
sh = list_first_entry(&sh->batch_list, struct stripe_head,
batch_list);
goto again;
}
+
+ atomic_inc(&head_sh->count);
+ ops_complete_reconstruct(head_sh);
}
static void ops_complete_check(void *stripe_head_ref)
@@ -2224,8 +2233,8 @@ static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu)
unsigned int off_dest;
struct page **xor_srcs = to_addr_page(percpu, 0);
unsigned int *off_srcs = to_addr_offs(sh, percpu);
- struct dma_async_tx_descriptor *tx;
- struct async_submit_ctl submit;
+ void **ptrs = (void **)xor_srcs;
+ size_t len = RAID5_STRIPE_SIZE(sh->raid_conf);
int count;
int i;
@@ -2245,23 +2254,27 @@ static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu)
xor_srcs[count++] = sh->dev[i].page;
}
- init_async_submit(&submit, 0, NULL, NULL, NULL,
- to_addr_conv(sh, percpu, 0));
- tx = async_xor_val_offs(xor_dest, off_dest, xor_srcs, off_srcs, count,
- RAID5_STRIPE_SIZE(sh->raid_conf),
- &sh->ops.zero_sum_result, &submit);
+ /* XOR_DROP_DST: xor_dest is ptrs[0]; xor_gen computes
+ * xor_dest ^= xor_dest ^ data0 ^ data1 ^ ... = data XOR.
+ * Then check whether the result is zero. */
+ for (i = 0; i < count; i++)
+ ptrs[i] = page_address(xor_srcs[i]) + off_srcs[i];
+ xor_gen(page_address(xor_dest) + off_dest, ptrs, count, len);
+ sh->ops.zero_sum_result =
+ !!memchr_inv(page_address(xor_dest) + off_dest, 0, len) << SUM_CHECK_P;
atomic_inc(&sh->count);
- init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL);
- tx = async_trigger_callback(&submit);
+ ops_complete_check(sh);
}
static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp)
{
struct page **srcs = to_addr_page(percpu, 0);
unsigned int *offs = to_addr_offs(sh, percpu);
- struct async_submit_ctl submit;
- int count;
+ size_t len = RAID5_STRIPE_SIZE(sh->raid_conf);
+ struct page *spare = percpu->spare_page;
+ void **ptrs = (void **)srcs;
+ int count, i;
pr_debug("%s: stripe %llu checkp: %d\n", __func__,
(unsigned long long)sh->sector, checkp);
@@ -2271,18 +2284,52 @@ static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu
if (!checkp)
srcs[count] = NULL;
+ sh->ops.zero_sum_result = 0;
atomic_inc(&sh->count);
- init_async_submit(&submit, ASYNC_TX_ACK, NULL, ops_complete_check,
- sh, to_addr_conv(sh, percpu, 0));
- async_syndrome_val(srcs, offs, count+2,
- RAID5_STRIPE_SIZE(sh->raid_conf),
- &sh->ops.zero_sum_result, percpu->spare_page, 0, &submit);
+
+ if (srcs[count]) {
+ /* Check P: XOR all data into spare, compare with P. */
+ int xcnt = 0;
+ void *s = page_address(spare);
+ struct page *p_src = srcs[count];
+ unsigned int p_off = offs[count];
+
+ for (i = 0; i < count; i++) {
+ if (srcs[i])
+ ptrs[xcnt++] = page_address(srcs[i]) + offs[i];
+ }
+ memset(s, 0, len);
+ if (xcnt)
+ xor_gen(s, ptrs, xcnt, len);
+ sh->ops.zero_sum_result |=
+ !!memcmp(page_address(p_src) + p_off, s, len) << SUM_CHECK_P;
+ }
+
+ if (srcs[count+1]) {
+ /* Check Q: recompute syndrome, compare with Q. */
+ struct page *q_src = srcs[count+1];
+ unsigned int q_off = offs[count+1];
+ void *s = page_address(spare);
+
+ /* Re-populate srcs for Q-only syndrome computation. */
+ count = set_syndrome_sources(srcs, offs, sh, SYNDROME_SRC_ALL);
+ srcs[count] = NULL; /* skip P regen */
+ srcs[count+1] = spare;
+ offs[count+1] = 0;
+ for (i = 0; i <= count+1; i++) {
+ ptrs[i] = srcs[i] ? page_address(srcs[i]) + offs[i] : NULL;
+ }
+ raid6_gen_syndrome(count+2, len, ptrs);
+ sh->ops.zero_sum_result |=
+ !!memcmp(page_address(q_src) + q_off, s, len) << SUM_CHECK_Q;
+ }
+
+ ops_complete_check(sh);
}
static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request)
{
int overlap_clear = 0, i, disks = sh->disks;
- struct dma_async_tx_descriptor *tx = NULL;
struct r5conf *conf = sh->raid_conf;
int level = conf->level;
struct raid5_percpu *percpu;
@@ -2296,38 +2343,35 @@ static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request)
if (test_bit(STRIPE_OP_COMPUTE_BLK, &ops_request)) {
if (level < 6)
- tx = ops_run_compute5(sh, percpu);
+ ops_run_compute5(sh, percpu);
else {
if (sh->ops.target2 < 0 || sh->ops.target < 0)
- tx = ops_run_compute6_1(sh, percpu);
+ ops_run_compute6_1(sh, percpu);
else
- tx = ops_run_compute6_2(sh, percpu);
+ ops_run_compute6_2(sh, percpu);
}
- /* terminate the chain if reconstruct is not set to be run */
- if (tx && !test_bit(STRIPE_OP_RECONSTRUCT, &ops_request))
- async_tx_ack(tx);
}
if (test_bit(STRIPE_OP_PREXOR, &ops_request)) {
if (level < 6)
- tx = ops_run_prexor5(sh, percpu, tx);
+ ops_run_prexor5(sh, percpu);
else
- tx = ops_run_prexor6(sh, percpu, tx);
+ ops_run_prexor6(sh, percpu);
}
if (test_bit(STRIPE_OP_PARTIAL_PARITY, &ops_request))
- tx = ops_run_partial_parity(sh, percpu, tx);
+ ops_run_partial_parity(sh, percpu);
if (test_bit(STRIPE_OP_BIODRAIN, &ops_request)) {
- tx = ops_run_biodrain(sh, tx);
+ ops_run_biodrain(sh);
overlap_clear++;
}
if (test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) {
if (level < 6)
- ops_run_reconstruct5(sh, percpu, tx);
+ ops_run_reconstruct5(sh, percpu);
else
- ops_run_reconstruct6(sh, percpu, tx);
+ ops_run_reconstruct6(sh, percpu);
}
if (test_bit(STRIPE_OP_CHECK, &ops_request)) {
@@ -2455,8 +2499,7 @@ static int grow_stripes(struct r5conf *conf, int num)
*
* The scribble buffer size must be enough to contain:
* 1/ a struct page pointer for each device in the array +2
- * 2/ room to convert each entry in (1) to its corresponding dma
- * (dma_map_page()) or page (page_address()) address.
+ * 2/ an unsigned int offset for each entry in (1)
*
* Note: the +2 is for the destination buffers of the ddf/raid6 case where we
* calculate over all devices (not just the data blocks), using zeros in place
@@ -2467,7 +2510,6 @@ static int scribble_alloc(struct raid5_percpu *percpu,
{
size_t obj_size =
sizeof(struct page *) * (num + 2) +
- sizeof(addr_conv_t) * (num + 2) +
sizeof(unsigned int) * (num + 2);
void *scribble;
@@ -4629,14 +4671,12 @@ static void handle_stripe_expansion(struct r5conf *conf, struct stripe_head *sh)
/* We have read all the blocks in this stripe and now we need to
* copy some of them into a target stripe for expand.
*/
- struct dma_async_tx_descriptor *tx = NULL;
BUG_ON(sh->batch_head);
clear_bit(STRIPE_EXPAND_SOURCE, &sh->state);
for (i = 0; i < sh->disks; i++)
if (i != sh->pd_idx && i != sh->qd_idx) {
int dd_idx, j;
struct stripe_head *sh2;
- struct async_submit_ctl submit;
sector_t bn = raid5_compute_blocknr(sh, i, 1);
sector_t s = raid5_compute_sector(conf, bn, 0,
@@ -4656,12 +4696,9 @@ static void handle_stripe_expansion(struct r5conf *conf, struct stripe_head *sh)
continue;
}
- /* place all the copies on one channel */
- init_async_submit(&submit, 0, tx, NULL, NULL, NULL);
- tx = async_memcpy(sh2->dev[dd_idx].page,
- sh->dev[i].page, sh2->dev[dd_idx].offset,
- sh->dev[i].offset, RAID5_STRIPE_SIZE(conf),
- &submit);
+ memcpy(page_address(sh2->dev[dd_idx].page) + sh2->dev[dd_idx].offset,
+ page_address(sh->dev[i].page) + sh->dev[i].offset,
+ RAID5_STRIPE_SIZE(conf));
set_bit(R5_Expanded, &sh2->dev[dd_idx].flags);
set_bit(R5_UPTODATE, &sh2->dev[dd_idx].flags);
@@ -4677,8 +4714,6 @@ static void handle_stripe_expansion(struct r5conf *conf, struct stripe_head *sh)
raid5_release_stripe(sh2);
}
- /* done submitting copies, wait for them to complete */
- async_tx_quiesce(&tx);
}
static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s)
@@ -6917,7 +6952,6 @@ static void raid5_do_work(struct work_struct *work)
r5l_flush_stripe_to_raid(conf->log);
- async_tx_issue_pending_all();
blk_finish_plug(&plug);
pr_debug("--- raid5worker inactive\n");
@@ -7011,7 +7045,6 @@ static void raid5d(struct md_thread *thread)
r5l_flush_stripe_to_raid(conf->log);
- async_tx_issue_pending_all();
blk_finish_plug(&plug);
pr_debug("--- raid5d inactive\n");
--
2.43.0
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH RFC 4/4] crypto: remove async_tx framework
2026-10-15 19:22 [PATCH RFC 0/4] async_tx/md: Remove async_tx framework Frank.Li
` (2 preceding siblings ...)
2026-10-15 19:22 ` [PATCH RFC 3/4] md/raid5: replace async_tx with synchronous xor/pq operations Frank.Li
@ 2026-10-15 19:22 ` Frank.Li
2026-10-15 19:51 ` [PATCH RFC 0/4] async_tx/md: Remove " Eric Biggers
4 siblings, 0 replies; 6+ messages in thread
From: Frank.Li @ 2026-10-15 19:22 UTC (permalink / raw)
To: Alasdair Kergon, Mike Snitzer, Mikulas Patocka,
Benjamin Marzinski, Song Liu, Yu Kuai, Li Nan, Xiao Ni,
Dan Williams, Herbert Xu, David S. Miller, Jonathan Corbet,
Shuah Khan, Randy Dunlap
Cc: dm-devel, linux-kernel, linux-raid, linux-crypto, linux-doc,
Anatolij Gustschin, Saeed Bishara, Rameshwar Prasad Sahu,
Thomas Petazzoni, Anup Patel, Frank Li
From: Frank Li <Frank.Li@nxp.com>
Remove the async_tx API and all associated files now that the last
consumer (md/raid5) has been converted to synchronous operations:
Signed-off-by: Frank Li <Frank.Li@nxp.com>
---
Documentation/crypto/async-tx-api.rst | 270 ---------------
Documentation/crypto/index.rst | 1 -
crypto/Kconfig | 5 -
crypto/Makefile | 4 -
crypto/async_tx/Kconfig | 28 --
crypto/async_tx/Makefile | 7 -
crypto/async_tx/async_memcpy.c | 97 ------
crypto/async_tx/async_pq.c | 464 --------------------------
crypto/async_tx/async_raid6_recov.c | 595 ----------------------------------
crypto/async_tx/async_tx.c | 299 -----------------
crypto/async_tx/async_xor.c | 380 ----------------------
crypto/async_tx/raid6test.c | 250 --------------
include/linux/async_tx.h | 187 -----------
13 files changed, 2587 deletions(-)
diff --git a/Documentation/crypto/async-tx-api.rst b/Documentation/crypto/async-tx-api.rst
deleted file mode 100644
index 49fcfc66314ac..0000000000000
--- a/Documentation/crypto/async-tx-api.rst
+++ /dev/null
@@ -1,270 +0,0 @@
-.. SPDX-License-Identifier: GPL-2.0
-
-=====================================
-Asynchronous Transfers/Transforms API
-=====================================
-
-.. Contents
-
- 1. INTRODUCTION
-
- 2 GENEALOGY
-
- 3 USAGE
- 3.1 General format of the API
- 3.2 Supported operations
- 3.3 Descriptor management
- 3.4 When does the operation execute?
- 3.5 When does the operation complete?
- 3.6 Constraints
- 3.7 Example
-
- 4 DMAENGINE DRIVER DEVELOPER NOTES
- 4.1 Conformance points
- 4.2 "My application needs exclusive control of hardware channels"
-
- 5 SOURCE
-
-1. Introduction
-===============
-
-The async_tx API provides methods for describing a chain of asynchronous
-bulk memory transfers/transforms with support for inter-transactional
-dependencies. It is implemented as a dmaengine client that smooths over
-the details of different hardware offload engine implementations. Code
-that is written to the API can optimize for asynchronous operation and
-the API will fit the chain of operations to the available offload
-resources.
-
-2.Genealogy
-===========
-
-The API was initially designed to offload the memory copy and
-xor-parity-calculations of the md-raid5 driver using the offload engines
-present in the Intel(R) Xscale series of I/O processors. It also built
-on the 'dmaengine' layer developed for offloading memory copies in the
-network stack using Intel(R) I/OAT engines. The following design
-features surfaced as a result:
-
-1. implicit synchronous path: users of the API do not need to know if
- the platform they are running on has offload capabilities. The
- operation will be offloaded when an engine is available and carried out
- in software otherwise.
-2. cross channel dependency chains: the API allows a chain of dependent
- operations to be submitted, like xor->copy->xor in the raid5 case. The
- API automatically handles cases where the transition from one operation
- to another implies a hardware channel switch.
-3. dmaengine extensions to support multiple clients and operation types
- beyond 'memcpy'
-
-3. Usage
-========
-
-3.1 General format of the API
------------------------------
-
-::
-
- struct dma_async_tx_descriptor *
- async_<operation>(<op specific parameters>, struct async_submit_ctl *submit)
-
-3.2 Supported operations
-------------------------
-
-======== ====================================================================
-memcpy memory copy between a source and a destination buffer
-memset fill a destination buffer with a byte value
-xor xor a series of source buffers and write the result to a
- destination buffer
-xor_val xor a series of source buffers and set a flag if the
- result is zero. The implementation attempts to prevent
- writes to memory
-pq generate the p+q (raid6 syndrome) from a series of source buffers
-pq_val validate that a p and or q buffer are in sync with a given series of
- sources
-datap (raid6_recov_datap) recover a raid6 data block and the p block
- from the given sources
-2data (raid6_recov_2data) recover 2 raid6 data blocks from the given
- sources
-======== ====================================================================
-
-3.3 Descriptor management
--------------------------
-
-The return value is non-NULL and points to a 'descriptor' when the operation
-has been queued to execute asynchronously. Descriptors are recycled
-resources, under control of the offload engine driver, to be reused as
-operations complete. When an application needs to submit a chain of
-operations it must guarantee that the descriptor is not automatically recycled
-before the dependency is submitted. This requires that all descriptors be
-acknowledged by the application before the offload engine driver is allowed to
-recycle (or free) the descriptor. A descriptor can be acked by one of the
-following methods:
-
-1. setting the ASYNC_TX_ACK flag if no child operations are to be submitted
-2. submitting an unacknowledged descriptor as a dependency to another
- async_tx call will implicitly set the acknowledged state.
-3. calling async_tx_ack() on the descriptor.
-
-3.4 When does the operation execute?
-------------------------------------
-
-Operations do not immediately issue after return from the
-async_<operation> call. Offload engine drivers batch operations to
-improve performance by reducing the number of mmio cycles needed to
-manage the channel. Once a driver-specific threshold is met the driver
-automatically issues pending operations. An application can force this
-event by calling async_tx_issue_pending_all(). This operates on all
-channels since the application has no knowledge of channel to operation
-mapping.
-
-3.5 When does the operation complete?
--------------------------------------
-
-There are two methods for an application to learn about the completion
-of an operation.
-
-1. Call dma_wait_for_async_tx(). This call causes the CPU to spin while
- it polls for the completion of the operation. It handles dependency
- chains and issuing pending operations.
-2. Specify a completion callback. The callback routine runs in tasklet
- context if the offload engine driver supports interrupts, or it is
- called in application context if the operation is carried out
- synchronously in software. The callback can be set in the call to
- async_<operation>, or when the application needs to submit a chain of
- unknown length it can use the async_trigger_callback() routine to set a
- completion interrupt/callback at the end of the chain.
-
-3.6 Constraints
----------------
-
-1. Calls to async_<operation> are not permitted in IRQ context. Other
- contexts are permitted provided constraint #2 is not violated.
-2. Completion callback routines cannot submit new operations. This
- results in recursion in the synchronous case and spin_locks being
- acquired twice in the asynchronous case.
-
-3.7 Example
------------
-
-Perform a xor->copy->xor operation where each operation depends on the
-result from the previous operation::
-
- #include <linux/async_tx.h>
-
- static void callback(void *param)
- {
- complete(param);
- }
-
- #define NDISKS 2
-
- static void run_xor_copy_xor(struct page **xor_srcs,
- struct page *xor_dest,
- size_t xor_len,
- struct page *copy_src,
- struct page *copy_dest,
- size_t copy_len)
- {
- struct dma_async_tx_descriptor *tx;
- struct async_submit_ctl submit;
- addr_conv_t addr_conv[NDISKS];
- struct completion cmp;
-
- init_async_submit(&submit, ASYNC_TX_XOR_DROP_DST, NULL, NULL, NULL,
- addr_conv);
- tx = async_xor(xor_dest, xor_srcs, 0, NDISKS, xor_len, &submit);
-
- submit.depend_tx = tx;
- tx = async_memcpy(copy_dest, copy_src, 0, 0, copy_len, &submit);
-
- init_completion(&cmp);
- init_async_submit(&submit, ASYNC_TX_XOR_DROP_DST | ASYNC_TX_ACK, tx,
- callback, &cmp, addr_conv);
- tx = async_xor(xor_dest, xor_srcs, 0, NDISKS, xor_len, &submit);
-
- async_tx_issue_pending_all();
-
- wait_for_completion(&cmp);
- }
-
-See include/linux/async_tx.h for more information on the flags. See the
-ops_run_* and ops_complete_* routines in drivers/md/raid5.c for more
-implementation examples.
-
-4. Driver Development Notes
-===========================
-
-4.1 Conformance points
-----------------------
-
-There are a few conformance points required in dmaengine drivers to
-accommodate assumptions made by applications using the async_tx API:
-
-1. Completion callbacks are expected to happen in tasklet context
-2. dma_async_tx_descriptor fields are never manipulated in IRQ context
-3. Use async_tx_run_dependencies() in the descriptor clean up path to
- handle submission of dependent operations
-
-4.2 "My application needs exclusive control of hardware channels"
------------------------------------------------------------------
-
-Primarily this requirement arises from cases where a DMA engine driver
-is being used to support device-to-memory operations. A channel that is
-performing these operations cannot, for many platform specific reasons,
-be shared. For these cases the dma_request_channel() interface is
-provided.
-
-The interface is::
-
- struct dma_chan *dma_request_channel(dma_cap_mask_t mask,
- dma_filter_fn filter_fn,
- void *filter_param);
-
-Where dma_filter_fn is defined as::
-
- typedef bool (*dma_filter_fn)(struct dma_chan *chan, void *filter_param);
-
-When the optional 'filter_fn' parameter is set to NULL
-dma_request_channel simply returns the first channel that satisfies the
-capability mask. Otherwise, when the mask parameter is insufficient for
-specifying the necessary channel, the filter_fn routine can be used to
-disposition the available channels in the system. The filter_fn routine
-is called once for each free channel in the system. Upon seeing a
-suitable channel filter_fn returns DMA_ACK which flags that channel to
-be the return value from dma_request_channel. A channel allocated via
-this interface is exclusive to the caller, until dma_release_channel()
-is called.
-
-The DMA_PRIVATE capability flag is used to tag dma devices that should
-not be used by the general-purpose allocator. It can be set at
-initialization time if it is known that a channel will always be
-private. Alternatively, it is set when dma_request_channel() finds an
-unused "public" channel.
-
-A couple caveats to note when implementing a driver and consumer:
-
-1. Once a channel has been privately allocated it will no longer be
- considered by the general-purpose allocator even after a call to
- dma_release_channel().
-2. Since capabilities are specified at the device level a dma_device
- with multiple channels will either have all channels public, or all
- channels private.
-
-5. Source
----------
-
-include/linux/dmaengine.h:
- core header file for DMA drivers and api users
-drivers/dma/dmaengine.c:
- offload engine channel management routines
-drivers/dma/:
- location for offload engine drivers
-include/linux/async_tx.h:
- core header file for the async_tx api
-crypto/async_tx/async_tx.c:
- async_tx interface to dmaengine and common code
-crypto/async_tx/async_memcpy.c:
- copy offload
-crypto/async_tx/async_xor.c:
- xor and xor zero sum offload
diff --git a/Documentation/crypto/index.rst b/Documentation/crypto/index.rst
index 705f186d662ba..7cd3b0281963d 100644
--- a/Documentation/crypto/index.rst
+++ b/Documentation/crypto/index.rst
@@ -18,7 +18,6 @@ for cryptographic use cases, as well as programming examples.
api-intro
architecture
- async-tx-api
asymmetric-keys
devel-algos
userspace-if
diff --git a/crypto/Kconfig b/crypto/Kconfig
index 981d5dc422d4c..e1570f8cbc1be 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -3,11 +3,6 @@
# Generic algorithms support
#
-#
-# async_tx api: hardware offloaded memory transfer/transform support
-#
-source "crypto/async_tx/Kconfig"
-
#
# Cryptographic API Configuration
#
diff --git a/crypto/Makefile b/crypto/Makefile
index 33bb5ad595e8f..4a6fe43c3d22f 100644
--- a/crypto/Makefile
+++ b/crypto/Makefile
@@ -188,10 +188,6 @@ ecrdsa_generic-y += ecrdsa_params.asn1.o
ecrdsa_generic-y += ecrdsa_pub_key.asn1.o
obj-$(CONFIG_CRYPTO_ECRDSA) += ecrdsa_generic.o
-#
-# generic algorithms and the async_tx api
-#
-obj-$(CONFIG_ASYNC_CORE) += async_tx/
obj-$(CONFIG_ASYMMETRIC_KEY_TYPE) += asymmetric_keys/
crypto_simd-y := simd.o
obj-$(CONFIG_CRYPTO_SIMD) += crypto_simd.o
diff --git a/crypto/async_tx/Kconfig b/crypto/async_tx/Kconfig
deleted file mode 100644
index 89bafa2efc300..0000000000000
--- a/crypto/async_tx/Kconfig
+++ /dev/null
@@ -1,28 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-config ASYNC_CORE
- tristate
-
-config ASYNC_MEMCPY
- tristate
- select ASYNC_CORE
-
-config ASYNC_XOR
- tristate
- select ASYNC_CORE
- select XOR_BLOCKS
-
-config ASYNC_PQ
- tristate
- select ASYNC_CORE
-
-config ASYNC_RAID6_RECOV
- tristate
- select ASYNC_CORE
- select ASYNC_PQ
- select ASYNC_XOR
-
-config ASYNC_TX_DISABLE_PQ_VAL_DMA
- bool
-
-config ASYNC_TX_DISABLE_XOR_VAL_DMA
- bool
diff --git a/crypto/async_tx/Makefile b/crypto/async_tx/Makefile
deleted file mode 100644
index 056e482453730..0000000000000
--- a/crypto/async_tx/Makefile
+++ /dev/null
@@ -1,7 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-obj-$(CONFIG_ASYNC_CORE) += async_tx.o
-obj-$(CONFIG_ASYNC_MEMCPY) += async_memcpy.o
-obj-$(CONFIG_ASYNC_XOR) += async_xor.o
-obj-$(CONFIG_ASYNC_PQ) += async_pq.o
-obj-$(CONFIG_ASYNC_RAID6_RECOV) += async_raid6_recov.o
-obj-$(CONFIG_ASYNC_RAID6_TEST) += raid6test.o
diff --git a/crypto/async_tx/async_memcpy.c b/crypto/async_tx/async_memcpy.c
deleted file mode 100644
index c538e30e99094..0000000000000
--- a/crypto/async_tx/async_memcpy.c
+++ /dev/null
@@ -1,97 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * copy offload engine support
- *
- * Copyright © 2006, Intel Corporation.
- *
- * Dan Williams <dan.j.williams@intel.com>
- *
- * with architecture considerations by:
- * Neil Brown <neilb@suse.de>
- * Jeff Garzik <jeff@garzik.org>
- */
-#include <linux/kernel.h>
-#include <linux/highmem.h>
-#include <linux/module.h>
-#include <linux/mm.h>
-#include <linux/dma-mapping.h>
-#include <linux/async_tx.h>
-
-/**
- * async_memcpy - attempt to copy memory with a dma engine.
- * @dest: destination page
- * @src: src page
- * @dest_offset: offset into 'dest' to start transaction
- * @src_offset: offset into 'src' to start transaction
- * @len: length in bytes
- * @submit: submission / completion modifiers
- *
- * honored flags: ASYNC_TX_ACK
- */
-struct dma_async_tx_descriptor *
-async_memcpy(struct page *dest, struct page *src, unsigned int dest_offset,
- unsigned int src_offset, size_t len,
- struct async_submit_ctl *submit)
-{
- struct dma_chan *chan = async_tx_find_channel(submit, DMA_MEMCPY,
- &dest, 1, &src, 1, len);
- struct dma_device *device = chan ? chan->device : NULL;
- struct dma_async_tx_descriptor *tx = NULL;
- struct dmaengine_unmap_data *unmap = NULL;
-
- if (device)
- unmap = dmaengine_get_unmap_data(device->dev, 2, GFP_NOWAIT);
-
- if (unmap && is_dma_copy_aligned(device, src_offset, dest_offset, len)) {
- unsigned long dma_prep_flags = 0;
-
- if (submit->cb_fn)
- dma_prep_flags |= DMA_PREP_INTERRUPT;
- if (submit->flags & ASYNC_TX_FENCE)
- dma_prep_flags |= DMA_PREP_FENCE;
-
- unmap->to_cnt = 1;
- unmap->addr[0] = dma_map_page(device->dev, src, src_offset, len,
- DMA_TO_DEVICE);
- unmap->from_cnt = 1;
- unmap->addr[1] = dma_map_page(device->dev, dest, dest_offset, len,
- DMA_FROM_DEVICE);
- unmap->len = len;
-
- tx = device->device_prep_dma_memcpy(chan, unmap->addr[1],
- unmap->addr[0], len,
- dma_prep_flags);
- }
-
- if (tx) {
- pr_debug("%s: (async) len: %zu\n", __func__, len);
-
- dma_set_unmap(tx, unmap);
- async_tx_submit(chan, tx, submit);
- } else {
- void *dest_buf, *src_buf;
- pr_debug("%s: (sync) len: %zu\n", __func__, len);
-
- /* wait for any prerequisite operations */
- async_tx_quiesce(&submit->depend_tx);
-
- dest_buf = kmap_atomic(dest) + dest_offset;
- src_buf = kmap_atomic(src) + src_offset;
-
- memcpy(dest_buf, src_buf, len);
-
- kunmap_atomic(src_buf);
- kunmap_atomic(dest_buf);
-
- async_tx_sync_epilog(submit);
- }
-
- dmaengine_unmap_put(unmap);
-
- return tx;
-}
-EXPORT_SYMBOL_GPL(async_memcpy);
-
-MODULE_AUTHOR("Intel Corporation");
-MODULE_DESCRIPTION("asynchronous memcpy api");
-MODULE_LICENSE("GPL");
diff --git a/crypto/async_tx/async_pq.c b/crypto/async_tx/async_pq.c
deleted file mode 100644
index 27f99349e310b..0000000000000
--- a/crypto/async_tx/async_pq.c
+++ /dev/null
@@ -1,464 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * Copyright(c) 2007 Yuri Tikhonov <yur@emcraft.com>
- * Copyright(c) 2009 Intel Corporation
- */
-#include <linux/kernel.h>
-#include <linux/interrupt.h>
-#include <linux/module.h>
-#include <linux/dma-mapping.h>
-#include <linux/raid/pq.h>
-#include <linux/raid/pq_tables.h>
-#include <linux/async_tx.h>
-#include <linux/gfp.h>
-
-/*
- * struct pq_scribble_page - space to hold throwaway P or Q buffer for
- * synchronous gen_syndrome
- */
-static struct page *pq_scribble_page;
-
-/* the struct page *blocks[] parameter passed to async_gen_syndrome()
- * and async_syndrome_val() contains the 'P' destination address at
- * blocks[disks-2] and the 'Q' destination address at blocks[disks-1]
- *
- * note: these are macros as they are used as lvalues
- */
-#define P(b, d) (b[d-2])
-#define Q(b, d) (b[d-1])
-
-#define MAX_DISKS 255
-
-/*
- * do_async_gen_syndrome - asynchronously calculate P and/or Q
- */
-static __async_inline struct dma_async_tx_descriptor *
-do_async_gen_syndrome(struct dma_chan *chan,
- const unsigned char *scfs, int disks,
- struct dmaengine_unmap_data *unmap,
- enum dma_ctrl_flags dma_flags,
- struct async_submit_ctl *submit)
-{
- struct dma_async_tx_descriptor *tx = NULL;
- struct dma_device *dma = chan->device;
- enum async_tx_flags flags_orig = submit->flags;
- dma_async_tx_callback cb_fn_orig = submit->cb_fn;
- dma_async_tx_callback cb_param_orig = submit->cb_param;
- int src_cnt = disks - 2;
- unsigned short pq_src_cnt;
- dma_addr_t dma_dest[2];
- int src_off = 0;
-
- while (src_cnt > 0) {
- submit->flags = flags_orig;
- pq_src_cnt = min(src_cnt, dma_maxpq(dma, dma_flags));
- /* if we are submitting additional pqs, leave the chain open,
- * clear the callback parameters, and leave the destination
- * buffers mapped
- */
- if (src_cnt > pq_src_cnt) {
- submit->flags &= ~ASYNC_TX_ACK;
- submit->flags |= ASYNC_TX_FENCE;
- submit->cb_fn = NULL;
- submit->cb_param = NULL;
- } else {
- submit->cb_fn = cb_fn_orig;
- submit->cb_param = cb_param_orig;
- if (cb_fn_orig)
- dma_flags |= DMA_PREP_INTERRUPT;
- }
- if (submit->flags & ASYNC_TX_FENCE)
- dma_flags |= DMA_PREP_FENCE;
-
- /* Drivers force forward progress in case they can not provide
- * a descriptor
- */
- for (;;) {
- dma_dest[0] = unmap->addr[disks - 2];
- dma_dest[1] = unmap->addr[disks - 1];
- tx = dma->device_prep_dma_pq(chan, dma_dest,
- &unmap->addr[src_off],
- pq_src_cnt,
- &scfs[src_off], unmap->len,
- dma_flags);
- if (likely(tx))
- break;
- async_tx_quiesce(&submit->depend_tx);
- dma_async_issue_pending(chan);
- }
-
- dma_set_unmap(tx, unmap);
- async_tx_submit(chan, tx, submit);
- submit->depend_tx = tx;
-
- /* drop completed sources */
- src_cnt -= pq_src_cnt;
- src_off += pq_src_cnt;
-
- dma_flags |= DMA_PREP_CONTINUE;
- }
-
- return tx;
-}
-
-/*
- * do_sync_gen_syndrome - synchronously calculate a raid6 syndrome
- */
-static void
-do_sync_gen_syndrome(struct page **blocks, unsigned int *offsets, int disks,
- size_t len, struct async_submit_ctl *submit)
-{
- void **srcs;
- int i;
- int start = -1, stop = disks - 3;
-
- if (submit->scribble)
- srcs = submit->scribble;
- else
- srcs = (void **) blocks;
-
- for (i = 0; i < disks; i++) {
- if (blocks[i] == NULL) {
- BUG_ON(i > disks - 3); /* P or Q can't be zero */
- srcs[i] = page_address(ZERO_PAGE(0));
- } else {
- srcs[i] = page_address(blocks[i]) + offsets[i];
-
- if (i < disks - 2) {
- stop = i;
- if (start == -1)
- start = i;
- }
- }
- }
- if (submit->flags & ASYNC_TX_PQ_XOR_DST) {
- BUG_ON(!raid6_can_xor_syndrome());
- if (start >= 0)
- raid6_xor_syndrome(disks, start, stop, len, srcs);
- } else
- raid6_gen_syndrome(disks, len, srcs);
- async_tx_sync_epilog(submit);
-}
-
-static inline bool
-is_dma_pq_aligned_offs(struct dma_device *dev, unsigned int *offs,
- int src_cnt, size_t len)
-{
- int i;
-
- for (i = 0; i < src_cnt; i++) {
- if (!is_dma_pq_aligned(dev, offs[i], 0, len))
- return false;
- }
- return true;
-}
-
-/**
- * async_gen_syndrome - asynchronously calculate a raid6 syndrome
- * @blocks: source blocks from idx 0..disks-3, P @ disks-2 and Q @ disks-1
- * @offsets: offset array into each block (src and dest) to start transaction
- * @disks: number of blocks (including missing P or Q, see below)
- * @len: length of operation in bytes
- * @submit: submission/completion modifiers
- *
- * General note: This routine assumes a field of GF(2^8) with a
- * primitive polynomial of 0x11d and a generator of {02}.
- *
- * 'disks' note: callers can optionally omit either P or Q (but not
- * both) from the calculation by setting blocks[disks-2] or
- * blocks[disks-1] to NULL. When P or Q is omitted 'len' must be <=
- * PAGE_SIZE as a temporary buffer of this size is used in the
- * synchronous path. 'disks' always accounts for both destination
- * buffers. If any source buffers (blocks[i] where i < disks - 2) are
- * set to NULL those buffers will be replaced with the raid6_zero_page
- * in the synchronous path and omitted in the hardware-asynchronous
- * path.
- */
-struct dma_async_tx_descriptor *
-async_gen_syndrome(struct page **blocks, unsigned int *offsets, int disks,
- size_t len, struct async_submit_ctl *submit)
-{
- int src_cnt = disks - 2;
- struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ,
- &P(blocks, disks), 2,
- blocks, src_cnt, len);
- struct dma_device *device = chan ? chan->device : NULL;
- struct dmaengine_unmap_data *unmap = NULL;
-
- BUG_ON(disks > MAX_DISKS || !(P(blocks, disks) || Q(blocks, disks)));
-
- if (device)
- unmap = dmaengine_get_unmap_data(device->dev, disks, GFP_NOWAIT);
-
- /* XORing P/Q is only implemented in software */
- if (unmap && !(submit->flags & ASYNC_TX_PQ_XOR_DST) &&
- (src_cnt <= dma_maxpq(device, 0) ||
- dma_maxpq(device, DMA_PREP_CONTINUE) > 0) &&
- is_dma_pq_aligned_offs(device, offsets, disks, len)) {
- struct dma_async_tx_descriptor *tx;
- enum dma_ctrl_flags dma_flags = 0;
- unsigned char coefs[MAX_DISKS];
- int i, j;
-
- /* run the p+q asynchronously */
- pr_debug("%s: (async) disks: %d len: %zu\n",
- __func__, disks, len);
-
- /* convert source addresses being careful to collapse 'empty'
- * sources and update the coefficients accordingly
- */
- unmap->len = len;
- for (i = 0, j = 0; i < src_cnt; i++) {
- if (blocks[i] == NULL)
- continue;
- unmap->addr[j] = dma_map_page(device->dev, blocks[i],
- offsets[i], len, DMA_TO_DEVICE);
- coefs[j] = raid6_gfexp[i];
- unmap->to_cnt++;
- j++;
- }
-
- /*
- * DMAs use destinations as sources,
- * so use BIDIRECTIONAL mapping
- */
- unmap->bidi_cnt++;
- if (P(blocks, disks))
- unmap->addr[j++] = dma_map_page(device->dev, P(blocks, disks),
- P(offsets, disks),
- len, DMA_BIDIRECTIONAL);
- else {
- unmap->addr[j++] = 0;
- dma_flags |= DMA_PREP_PQ_DISABLE_P;
- }
-
- unmap->bidi_cnt++;
- if (Q(blocks, disks))
- unmap->addr[j++] = dma_map_page(device->dev, Q(blocks, disks),
- Q(offsets, disks),
- len, DMA_BIDIRECTIONAL);
- else {
- unmap->addr[j++] = 0;
- dma_flags |= DMA_PREP_PQ_DISABLE_Q;
- }
-
- tx = do_async_gen_syndrome(chan, coefs, j, unmap, dma_flags, submit);
- dmaengine_unmap_put(unmap);
- return tx;
- }
-
- dmaengine_unmap_put(unmap);
-
- /* run the pq synchronously */
- pr_debug("%s: (sync) disks: %d len: %zu\n", __func__, disks, len);
-
- /* wait for any prerequisite operations */
- async_tx_quiesce(&submit->depend_tx);
-
- if (!P(blocks, disks)) {
- P(blocks, disks) = pq_scribble_page;
- P(offsets, disks) = 0;
- }
- if (!Q(blocks, disks)) {
- Q(blocks, disks) = pq_scribble_page;
- Q(offsets, disks) = 0;
- }
- do_sync_gen_syndrome(blocks, offsets, disks, len, submit);
-
- return NULL;
-}
-EXPORT_SYMBOL_GPL(async_gen_syndrome);
-
-static inline struct dma_chan *
-pq_val_chan(struct async_submit_ctl *submit, struct page **blocks, int disks, size_t len)
-{
- #ifdef CONFIG_ASYNC_TX_DISABLE_PQ_VAL_DMA
- return NULL;
- #endif
- return async_tx_find_channel(submit, DMA_PQ_VAL, NULL, 0, blocks,
- disks, len);
-}
-
-/**
- * async_syndrome_val - asynchronously validate a raid6 syndrome
- * @blocks: source blocks from idx 0..disks-3, P @ disks-2 and Q @ disks-1
- * @offsets: common offset into each block (src and dest) to start transaction
- * @disks: number of blocks (including missing P or Q, see below)
- * @len: length of operation in bytes
- * @pqres: on val failure SUM_CHECK_P_RESULT and/or SUM_CHECK_Q_RESULT are set
- * @spare: temporary result buffer for the synchronous case
- * @s_off: spare buffer page offset
- * @submit: submission / completion modifiers
- *
- * The same notes from async_gen_syndrome apply to the 'blocks',
- * and 'disks' parameters of this routine. The synchronous path
- * requires a temporary result buffer and submit->scribble to be
- * specified.
- */
-struct dma_async_tx_descriptor *
-async_syndrome_val(struct page **blocks, unsigned int *offsets, int disks,
- size_t len, enum sum_check_flags *pqres, struct page *spare,
- unsigned int s_off, struct async_submit_ctl *submit)
-{
- struct dma_chan *chan = pq_val_chan(submit, blocks, disks, len);
- struct dma_device *device = chan ? chan->device : NULL;
- struct dma_async_tx_descriptor *tx;
- unsigned char coefs[MAX_DISKS];
- enum dma_ctrl_flags dma_flags = submit->cb_fn ? DMA_PREP_INTERRUPT : 0;
- struct dmaengine_unmap_data *unmap = NULL;
-
- BUG_ON(disks < 4 || disks > MAX_DISKS);
-
- if (device)
- unmap = dmaengine_get_unmap_data(device->dev, disks, GFP_NOWAIT);
-
- if (unmap && disks <= dma_maxpq(device, 0) &&
- is_dma_pq_aligned_offs(device, offsets, disks, len)) {
- struct device *dev = device->dev;
- dma_addr_t pq[2];
- int i, j = 0, src_cnt = 0;
-
- pr_debug("%s: (async) disks: %d len: %zu\n",
- __func__, disks, len);
-
- unmap->len = len;
- for (i = 0; i < disks-2; i++)
- if (likely(blocks[i])) {
- unmap->addr[j] = dma_map_page(dev, blocks[i],
- offsets[i], len,
- DMA_TO_DEVICE);
- coefs[j] = raid6_gfexp[i];
- unmap->to_cnt++;
- src_cnt++;
- j++;
- }
-
- if (!P(blocks, disks)) {
- pq[0] = 0;
- dma_flags |= DMA_PREP_PQ_DISABLE_P;
- } else {
- pq[0] = dma_map_page(dev, P(blocks, disks),
- P(offsets, disks), len,
- DMA_TO_DEVICE);
- unmap->addr[j++] = pq[0];
- unmap->to_cnt++;
- }
- if (!Q(blocks, disks)) {
- pq[1] = 0;
- dma_flags |= DMA_PREP_PQ_DISABLE_Q;
- } else {
- pq[1] = dma_map_page(dev, Q(blocks, disks),
- Q(offsets, disks), len,
- DMA_TO_DEVICE);
- unmap->addr[j++] = pq[1];
- unmap->to_cnt++;
- }
-
- if (submit->flags & ASYNC_TX_FENCE)
- dma_flags |= DMA_PREP_FENCE;
- for (;;) {
- tx = device->device_prep_dma_pq_val(chan, pq,
- unmap->addr,
- src_cnt,
- coefs,
- len, pqres,
- dma_flags);
- if (likely(tx))
- break;
- async_tx_quiesce(&submit->depend_tx);
- dma_async_issue_pending(chan);
- }
-
- dma_set_unmap(tx, unmap);
- async_tx_submit(chan, tx, submit);
- } else {
- struct page *p_src = P(blocks, disks);
- unsigned int p_off = P(offsets, disks);
- struct page *q_src = Q(blocks, disks);
- unsigned int q_off = Q(offsets, disks);
- enum async_tx_flags flags_orig = submit->flags;
- dma_async_tx_callback cb_fn_orig = submit->cb_fn;
- void *scribble = submit->scribble;
- void *cb_param_orig = submit->cb_param;
- void *p, *q, *s;
-
- pr_debug("%s: (sync) disks: %d len: %zu\n",
- __func__, disks, len);
-
- /* caller must provide a temporary result buffer and
- * allow the input parameters to be preserved
- */
- BUG_ON(!spare || !scribble);
-
- /* wait for any prerequisite operations */
- async_tx_quiesce(&submit->depend_tx);
-
- /* recompute p and/or q into the temporary buffer and then
- * check to see the result matches the current value
- */
- tx = NULL;
- *pqres = 0;
- if (p_src) {
- init_async_submit(submit, ASYNC_TX_XOR_ZERO_DST, NULL,
- NULL, NULL, scribble);
- tx = async_xor_offs(spare, s_off,
- blocks, offsets, disks-2, len, submit);
- async_tx_quiesce(&tx);
- p = page_address(p_src) + p_off;
- s = page_address(spare) + s_off;
- *pqres |= !!memcmp(p, s, len) << SUM_CHECK_P;
- }
-
- if (q_src) {
- P(blocks, disks) = NULL;
- Q(blocks, disks) = spare;
- Q(offsets, disks) = s_off;
- init_async_submit(submit, 0, NULL, NULL, NULL, scribble);
- tx = async_gen_syndrome(blocks, offsets, disks,
- len, submit);
- async_tx_quiesce(&tx);
- q = page_address(q_src) + q_off;
- s = page_address(spare) + s_off;
- *pqres |= !!memcmp(q, s, len) << SUM_CHECK_Q;
- }
-
- /* restore P, Q and submit */
- P(blocks, disks) = p_src;
- P(offsets, disks) = p_off;
- Q(blocks, disks) = q_src;
- Q(offsets, disks) = q_off;
-
- submit->cb_fn = cb_fn_orig;
- submit->cb_param = cb_param_orig;
- submit->flags = flags_orig;
- async_tx_sync_epilog(submit);
- tx = NULL;
- }
- dmaengine_unmap_put(unmap);
-
- return tx;
-}
-EXPORT_SYMBOL_GPL(async_syndrome_val);
-
-static int __init async_pq_init(void)
-{
- pq_scribble_page = alloc_page(GFP_KERNEL);
-
- if (pq_scribble_page)
- return 0;
-
- pr_err("%s: failed to allocate required spare page\n", __func__);
-
- return -ENOMEM;
-}
-
-static void __exit async_pq_exit(void)
-{
- __free_page(pq_scribble_page);
-}
-
-module_init(async_pq_init);
-module_exit(async_pq_exit);
-
-MODULE_DESCRIPTION("asynchronous raid6 syndrome generation/validation");
-MODULE_LICENSE("GPL");
diff --git a/crypto/async_tx/async_raid6_recov.c b/crypto/async_tx/async_raid6_recov.c
deleted file mode 100644
index e53870d84bc55..0000000000000
--- a/crypto/async_tx/async_raid6_recov.c
+++ /dev/null
@@ -1,595 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * Asynchronous RAID-6 recovery calculations ASYNC_TX API.
- * Copyright(c) 2009 Intel Corporation
- *
- * based on raid6recov.c:
- * Copyright 2002 H. Peter Anvin
- */
-#include <linux/kernel.h>
-#include <linux/interrupt.h>
-#include <linux/module.h>
-#include <linux/dma-mapping.h>
-#include <linux/raid/pq.h>
-#include <linux/raid/pq_tables.h>
-#include <linux/async_tx.h>
-#include <linux/dmaengine.h>
-
-static struct dma_async_tx_descriptor *
-async_sum_product(struct page *dest, unsigned int d_off,
- struct page **srcs, unsigned int *src_offs, unsigned char *coef,
- size_t len, struct async_submit_ctl *submit)
-{
- struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ,
- &dest, 1, srcs, 2, len);
- struct dma_device *dma = chan ? chan->device : NULL;
- struct dmaengine_unmap_data *unmap = NULL;
- const u8 *amul, *bmul;
- u8 ax, bx;
- u8 *a, *b, *c;
-
- if (dma)
- unmap = dmaengine_get_unmap_data(dma->dev, 3, GFP_NOWAIT);
-
- if (unmap) {
- struct device *dev = dma->dev;
- dma_addr_t pq[2];
- struct dma_async_tx_descriptor *tx;
- enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P;
-
- if (submit->flags & ASYNC_TX_FENCE)
- dma_flags |= DMA_PREP_FENCE;
- unmap->addr[0] = dma_map_page(dev, srcs[0], src_offs[0],
- len, DMA_TO_DEVICE);
- unmap->addr[1] = dma_map_page(dev, srcs[1], src_offs[1],
- len, DMA_TO_DEVICE);
- unmap->to_cnt = 2;
-
- unmap->addr[2] = dma_map_page(dev, dest, d_off,
- len, DMA_BIDIRECTIONAL);
- unmap->bidi_cnt = 1;
- /* engine only looks at Q, but expects it to follow P */
- pq[1] = unmap->addr[2];
-
- unmap->len = len;
- tx = dma->device_prep_dma_pq(chan, pq, unmap->addr, 2, coef,
- len, dma_flags);
- if (tx) {
- dma_set_unmap(tx, unmap);
- async_tx_submit(chan, tx, submit);
- dmaengine_unmap_put(unmap);
- return tx;
- }
-
- /* could not get a descriptor, unmap and fall through to
- * the synchronous path
- */
- dmaengine_unmap_put(unmap);
- }
-
- /* run the operation synchronously */
- async_tx_quiesce(&submit->depend_tx);
- amul = raid6_gfmul[coef[0]];
- bmul = raid6_gfmul[coef[1]];
- a = page_address(srcs[0]) + src_offs[0];
- b = page_address(srcs[1]) + src_offs[1];
- c = page_address(dest) + d_off;
-
- while (len--) {
- ax = amul[*a++];
- bx = bmul[*b++];
- *c++ = ax ^ bx;
- }
-
- return NULL;
-}
-
-static struct dma_async_tx_descriptor *
-async_mult(struct page *dest, unsigned int d_off, struct page *src,
- unsigned int s_off, u8 coef, size_t len,
- struct async_submit_ctl *submit)
-{
- struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ,
- &dest, 1, &src, 1, len);
- struct dma_device *dma = chan ? chan->device : NULL;
- struct dmaengine_unmap_data *unmap = NULL;
- const u8 *qmul; /* Q multiplier table */
- u8 *d, *s;
-
- if (dma)
- unmap = dmaengine_get_unmap_data(dma->dev, 3, GFP_NOWAIT);
-
- if (unmap) {
- dma_addr_t dma_dest[2];
- struct device *dev = dma->dev;
- struct dma_async_tx_descriptor *tx;
- enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P;
-
- if (submit->flags & ASYNC_TX_FENCE)
- dma_flags |= DMA_PREP_FENCE;
- unmap->addr[0] = dma_map_page(dev, src, s_off,
- len, DMA_TO_DEVICE);
- unmap->to_cnt++;
- unmap->addr[1] = dma_map_page(dev, dest, d_off,
- len, DMA_BIDIRECTIONAL);
- dma_dest[1] = unmap->addr[1];
- unmap->bidi_cnt++;
- unmap->len = len;
-
- /* this looks funny, but the engine looks for Q at
- * dma_dest[1] and ignores dma_dest[0] as a dest
- * due to DMA_PREP_PQ_DISABLE_P
- */
- tx = dma->device_prep_dma_pq(chan, dma_dest, unmap->addr,
- 1, &coef, len, dma_flags);
-
- if (tx) {
- dma_set_unmap(tx, unmap);
- dmaengine_unmap_put(unmap);
- async_tx_submit(chan, tx, submit);
- return tx;
- }
-
- /* could not get a descriptor, unmap and fall through to
- * the synchronous path
- */
- dmaengine_unmap_put(unmap);
- }
-
- /* no channel available, or failed to allocate a descriptor, so
- * perform the operation synchronously
- */
- async_tx_quiesce(&submit->depend_tx);
- qmul = raid6_gfmul[coef];
- d = page_address(dest) + d_off;
- s = page_address(src) + s_off;
-
- while (len--)
- *d++ = qmul[*s++];
-
- return NULL;
-}
-
-static struct dma_async_tx_descriptor *
-__2data_recov_4(int disks, size_t bytes, int faila, int failb,
- struct page **blocks, unsigned int *offs,
- struct async_submit_ctl *submit)
-{
- struct dma_async_tx_descriptor *tx = NULL;
- struct page *p, *q, *a, *b;
- unsigned int p_off, q_off, a_off, b_off;
- struct page *srcs[2];
- unsigned int src_offs[2];
- unsigned char coef[2];
- enum async_tx_flags flags = submit->flags;
- dma_async_tx_callback cb_fn = submit->cb_fn;
- void *cb_param = submit->cb_param;
- void *scribble = submit->scribble;
-
- p = blocks[disks-2];
- p_off = offs[disks-2];
- q = blocks[disks-1];
- q_off = offs[disks-1];
-
- a = blocks[faila];
- a_off = offs[faila];
- b = blocks[failb];
- b_off = offs[failb];
-
- /* in the 4 disk case P + Pxy == P and Q + Qxy == Q */
- /* Dx = A*(P+Pxy) + B*(Q+Qxy) */
- srcs[0] = p;
- src_offs[0] = p_off;
- srcs[1] = q;
- src_offs[1] = q_off;
- coef[0] = raid6_gfexi[failb-faila];
- coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
- init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
- tx = async_sum_product(b, b_off, srcs, src_offs, coef, bytes, submit);
-
- /* Dy = P+Pxy+Dx */
- srcs[0] = p;
- src_offs[0] = p_off;
- srcs[1] = b;
- src_offs[1] = b_off;
- init_async_submit(submit, flags | ASYNC_TX_XOR_ZERO_DST, tx, cb_fn,
- cb_param, scribble);
- tx = async_xor_offs(a, a_off, srcs, src_offs, 2, bytes, submit);
-
- return tx;
-
-}
-
-static struct dma_async_tx_descriptor *
-__2data_recov_5(int disks, size_t bytes, int faila, int failb,
- struct page **blocks, unsigned int *offs,
- struct async_submit_ctl *submit)
-{
- struct dma_async_tx_descriptor *tx = NULL;
- struct page *p, *q, *g, *dp, *dq;
- unsigned int p_off, q_off, g_off, dp_off, dq_off;
- struct page *srcs[2];
- unsigned int src_offs[2];
- unsigned char coef[2];
- enum async_tx_flags flags = submit->flags;
- dma_async_tx_callback cb_fn = submit->cb_fn;
- void *cb_param = submit->cb_param;
- void *scribble = submit->scribble;
- int good_srcs, good, i;
-
- good_srcs = 0;
- good = -1;
- for (i = 0; i < disks-2; i++) {
- if (blocks[i] == NULL)
- continue;
- if (i == faila || i == failb)
- continue;
- good = i;
- good_srcs++;
- }
- BUG_ON(good_srcs > 1);
-
- p = blocks[disks-2];
- p_off = offs[disks-2];
- q = blocks[disks-1];
- q_off = offs[disks-1];
- g = blocks[good];
- g_off = offs[good];
-
- /* Compute syndrome with zero for the missing data pages
- * Use the dead data pages as temporary storage for delta p and
- * delta q
- */
- dp = blocks[faila];
- dp_off = offs[faila];
- dq = blocks[failb];
- dq_off = offs[failb];
-
- init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
- tx = async_memcpy(dp, g, dp_off, g_off, bytes, submit);
- init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
- tx = async_mult(dq, dq_off, g, g_off,
- raid6_gfexp[good], bytes, submit);
-
- /* compute P + Pxy */
- srcs[0] = dp;
- src_offs[0] = dp_off;
- srcs[1] = p;
- src_offs[1] = p_off;
- init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
- NULL, NULL, scribble);
- tx = async_xor_offs(dp, dp_off, srcs, src_offs, 2, bytes, submit);
-
- /* compute Q + Qxy */
- srcs[0] = dq;
- src_offs[0] = dq_off;
- srcs[1] = q;
- src_offs[1] = q_off;
- init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
- NULL, NULL, scribble);
- tx = async_xor_offs(dq, dq_off, srcs, src_offs, 2, bytes, submit);
-
- /* Dx = A*(P+Pxy) + B*(Q+Qxy) */
- srcs[0] = dp;
- src_offs[0] = dp_off;
- srcs[1] = dq;
- src_offs[1] = dq_off;
- coef[0] = raid6_gfexi[failb-faila];
- coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
- init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
- tx = async_sum_product(dq, dq_off, srcs, src_offs, coef, bytes, submit);
-
- /* Dy = P+Pxy+Dx */
- srcs[0] = dp;
- src_offs[0] = dp_off;
- srcs[1] = dq;
- src_offs[1] = dq_off;
- init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
- cb_param, scribble);
- tx = async_xor_offs(dp, dp_off, srcs, src_offs, 2, bytes, submit);
-
- return tx;
-}
-
-static struct dma_async_tx_descriptor *
-__2data_recov_n(int disks, size_t bytes, int faila, int failb,
- struct page **blocks, unsigned int *offs,
- struct async_submit_ctl *submit)
-{
- struct dma_async_tx_descriptor *tx = NULL;
- struct page *p, *q, *dp, *dq;
- unsigned int p_off, q_off, dp_off, dq_off;
- struct page *srcs[2];
- unsigned int src_offs[2];
- unsigned char coef[2];
- enum async_tx_flags flags = submit->flags;
- dma_async_tx_callback cb_fn = submit->cb_fn;
- void *cb_param = submit->cb_param;
- void *scribble = submit->scribble;
-
- p = blocks[disks-2];
- p_off = offs[disks-2];
- q = blocks[disks-1];
- q_off = offs[disks-1];
-
- /* Compute syndrome with zero for the missing data pages
- * Use the dead data pages as temporary storage for
- * delta p and delta q
- */
- dp = blocks[faila];
- dp_off = offs[faila];
- blocks[faila] = NULL;
- blocks[disks-2] = dp;
- offs[disks-2] = dp_off;
- dq = blocks[failb];
- dq_off = offs[failb];
- blocks[failb] = NULL;
- blocks[disks-1] = dq;
- offs[disks-1] = dq_off;
-
- init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
- tx = async_gen_syndrome(blocks, offs, disks, bytes, submit);
-
- /* Restore pointer table */
- blocks[faila] = dp;
- offs[faila] = dp_off;
- blocks[failb] = dq;
- offs[failb] = dq_off;
- blocks[disks-2] = p;
- offs[disks-2] = p_off;
- blocks[disks-1] = q;
- offs[disks-1] = q_off;
-
- /* compute P + Pxy */
- srcs[0] = dp;
- src_offs[0] = dp_off;
- srcs[1] = p;
- src_offs[1] = p_off;
- init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
- NULL, NULL, scribble);
- tx = async_xor_offs(dp, dp_off, srcs, src_offs, 2, bytes, submit);
-
- /* compute Q + Qxy */
- srcs[0] = dq;
- src_offs[0] = dq_off;
- srcs[1] = q;
- src_offs[1] = q_off;
- init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
- NULL, NULL, scribble);
- tx = async_xor_offs(dq, dq_off, srcs, src_offs, 2, bytes, submit);
-
- /* Dx = A*(P+Pxy) + B*(Q+Qxy) */
- srcs[0] = dp;
- src_offs[0] = dp_off;
- srcs[1] = dq;
- src_offs[1] = dq_off;
- coef[0] = raid6_gfexi[failb-faila];
- coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
- init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
- tx = async_sum_product(dq, dq_off, srcs, src_offs, coef, bytes, submit);
-
- /* Dy = P+Pxy+Dx */
- srcs[0] = dp;
- src_offs[0] = dp_off;
- srcs[1] = dq;
- src_offs[1] = dq_off;
- init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
- cb_param, scribble);
- tx = async_xor_offs(dp, dp_off, srcs, src_offs, 2, bytes, submit);
-
- return tx;
-}
-
-/**
- * async_raid6_2data_recov - asynchronously calculate two missing data blocks
- * @disks: number of disks in the RAID-6 array
- * @bytes: block size
- * @faila: first failed drive index
- * @failb: second failed drive index
- * @blocks: array of source pointers where the last two entries are p and q
- * @offs: array of offset for pages in blocks
- * @submit: submission/completion modifiers
- */
-struct dma_async_tx_descriptor *
-async_raid6_2data_recov(int disks, size_t bytes, int faila, int failb,
- struct page **blocks, unsigned int *offs,
- struct async_submit_ctl *submit)
-{
- void *scribble = submit->scribble;
- int non_zero_srcs, i;
-
- BUG_ON(faila == failb);
- if (failb < faila)
- swap(faila, failb);
-
- pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes);
-
- /* if a dma resource is not available or a scribble buffer is not
- * available punt to the synchronous path. In the 'dma not
- * available' case be sure to use the scribble buffer to
- * preserve the content of 'blocks' as the caller intended.
- */
- if (!async_dma_find_channel(DMA_PQ) || !scribble) {
- void **ptrs = scribble ? scribble : (void **) blocks;
-
- async_tx_quiesce(&submit->depend_tx);
- for (i = 0; i < disks; i++)
- if (blocks[i] == NULL)
- ptrs[i] = page_address(ZERO_PAGE(0));
- else
- ptrs[i] = page_address(blocks[i]) + offs[i];
-
- raid6_recov_2data(disks, bytes, faila, failb, ptrs);
-
- async_tx_sync_epilog(submit);
-
- return NULL;
- }
-
- non_zero_srcs = 0;
- for (i = 0; i < disks-2 && non_zero_srcs < 4; i++)
- if (blocks[i])
- non_zero_srcs++;
- switch (non_zero_srcs) {
- case 0:
- case 1:
- /* There must be at least 2 sources - the failed devices. */
- BUG();
-
- case 2:
- /* dma devices do not uniformly understand a zero source pq
- * operation (in contrast to the synchronous case), so
- * explicitly handle the special case of a 4 disk array with
- * both data disks missing.
- */
- return __2data_recov_4(disks, bytes, faila, failb,
- blocks, offs, submit);
- case 3:
- /* dma devices do not uniformly understand a single
- * source pq operation (in contrast to the synchronous
- * case), so explicitly handle the special case of a 5 disk
- * array with 2 of 3 data disks missing.
- */
- return __2data_recov_5(disks, bytes, faila, failb,
- blocks, offs, submit);
- default:
- return __2data_recov_n(disks, bytes, faila, failb,
- blocks, offs, submit);
- }
-}
-EXPORT_SYMBOL_GPL(async_raid6_2data_recov);
-
-/**
- * async_raid6_datap_recov - asynchronously calculate a data and the 'p' block
- * @disks: number of disks in the RAID-6 array
- * @bytes: block size
- * @faila: failed drive index
- * @blocks: array of source pointers where the last two entries are p and q
- * @offs: array of offset for pages in blocks
- * @submit: submission/completion modifiers
- */
-struct dma_async_tx_descriptor *
-async_raid6_datap_recov(int disks, size_t bytes, int faila,
- struct page **blocks, unsigned int *offs,
- struct async_submit_ctl *submit)
-{
- struct dma_async_tx_descriptor *tx = NULL;
- struct page *p, *q, *dq;
- unsigned int p_off, q_off, dq_off;
- u8 coef;
- enum async_tx_flags flags = submit->flags;
- dma_async_tx_callback cb_fn = submit->cb_fn;
- void *cb_param = submit->cb_param;
- void *scribble = submit->scribble;
- int good_srcs, good, i;
- struct page *srcs[2];
- unsigned int src_offs[2];
-
- pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes);
-
- /* if a dma resource is not available or a scribble buffer is not
- * available punt to the synchronous path. In the 'dma not
- * available' case be sure to use the scribble buffer to
- * preserve the content of 'blocks' as the caller intended.
- */
- if (!async_dma_find_channel(DMA_PQ) || !scribble) {
- void **ptrs = scribble ? scribble : (void **) blocks;
-
- async_tx_quiesce(&submit->depend_tx);
- for (i = 0; i < disks; i++)
- if (blocks[i] == NULL)
- ptrs[i] = page_address(ZERO_PAGE(0));
- else
- ptrs[i] = page_address(blocks[i]) + offs[i];
-
- raid6_recov_datap(disks, bytes, faila, ptrs);
-
- async_tx_sync_epilog(submit);
-
- return NULL;
- }
-
- good_srcs = 0;
- good = -1;
- for (i = 0; i < disks-2; i++) {
- if (i == faila)
- continue;
- if (blocks[i]) {
- good = i;
- good_srcs++;
- if (good_srcs > 1)
- break;
- }
- }
- BUG_ON(good_srcs == 0);
-
- p = blocks[disks-2];
- p_off = offs[disks-2];
- q = blocks[disks-1];
- q_off = offs[disks-1];
-
- /* Compute syndrome with zero for the missing data page
- * Use the dead data page as temporary storage for delta q
- */
- dq = blocks[faila];
- dq_off = offs[faila];
- blocks[faila] = NULL;
- blocks[disks-1] = dq;
- offs[disks-1] = dq_off;
-
- /* in the 4-disk case we only need to perform a single source
- * multiplication with the one good data block.
- */
- if (good_srcs == 1) {
- struct page *g = blocks[good];
- unsigned int g_off = offs[good];
-
- init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
- scribble);
- tx = async_memcpy(p, g, p_off, g_off, bytes, submit);
-
- init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
- scribble);
- tx = async_mult(dq, dq_off, g, g_off,
- raid6_gfexp[good], bytes, submit);
- } else {
- init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
- scribble);
- tx = async_gen_syndrome(blocks, offs, disks, bytes, submit);
- }
-
- /* Restore pointer table */
- blocks[faila] = dq;
- offs[faila] = dq_off;
- blocks[disks-1] = q;
- offs[disks-1] = q_off;
-
- /* calculate g^{-faila} */
- coef = raid6_gfinv[raid6_gfexp[faila]];
-
- srcs[0] = dq;
- src_offs[0] = dq_off;
- srcs[1] = q;
- src_offs[1] = q_off;
- init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
- NULL, NULL, scribble);
- tx = async_xor_offs(dq, dq_off, srcs, src_offs, 2, bytes, submit);
-
- init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
- tx = async_mult(dq, dq_off, dq, dq_off, coef, bytes, submit);
-
- srcs[0] = p;
- src_offs[0] = p_off;
- srcs[1] = dq;
- src_offs[1] = dq_off;
- init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
- cb_param, scribble);
- tx = async_xor_offs(p, p_off, srcs, src_offs, 2, bytes, submit);
-
- return tx;
-}
-EXPORT_SYMBOL_GPL(async_raid6_datap_recov);
-
-MODULE_AUTHOR("Dan Williams <dan.j.williams@intel.com>");
-MODULE_DESCRIPTION("asynchronous RAID-6 recovery api");
-MODULE_LICENSE("GPL");
diff --git a/crypto/async_tx/async_tx.c b/crypto/async_tx/async_tx.c
deleted file mode 100644
index 7d3009c5c3605..0000000000000
--- a/crypto/async_tx/async_tx.c
+++ /dev/null
@@ -1,299 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * core routines for the asynchronous memory transfer/transform api
- *
- * Copyright © 2006, Intel Corporation.
- *
- * Dan Williams <dan.j.williams@intel.com>
- *
- * with architecture considerations by:
- * Neil Brown <neilb@suse.de>
- * Jeff Garzik <jeff@garzik.org>
- */
-#include <linux/rculist.h>
-#include <linux/module.h>
-#include <linux/kernel.h>
-#include <linux/async_tx.h>
-#include "../../drivers/dma/dmaengine.h"
-
-#ifdef CONFIG_DMA_ENGINE
-static int __init async_tx_init(void)
-{
- async_dmaengine_get();
-
- printk(KERN_INFO "async_tx: api initialized (async)\n");
-
- return 0;
-}
-
-static void __exit async_tx_exit(void)
-{
- async_dmaengine_put();
-}
-
-module_init(async_tx_init);
-module_exit(async_tx_exit);
-
-/**
- * __async_tx_find_channel - find a channel to carry out the operation or let
- * the transaction execute synchronously
- * @submit: transaction dependency and submission modifiers
- * @tx_type: transaction type
- */
-struct dma_chan *
-__async_tx_find_channel(struct async_submit_ctl *submit,
- enum dma_transaction_type tx_type)
-{
- struct dma_async_tx_descriptor *depend_tx = submit->depend_tx;
-
- /* see if we can keep the chain on one channel */
- if (depend_tx &&
- dma_has_cap(tx_type, depend_tx->chan->device->cap_mask))
- return depend_tx->chan;
- return async_dma_find_channel(tx_type);
-}
-EXPORT_SYMBOL_GPL(__async_tx_find_channel);
-#endif
-
-
-/**
- * async_tx_channel_switch - queue an interrupt descriptor with a dependency
- * pre-attached.
- * @depend_tx: the operation that must finish before the new operation runs
- * @tx: the new operation
- */
-static void
-async_tx_channel_switch(struct dma_async_tx_descriptor *depend_tx,
- struct dma_async_tx_descriptor *tx)
-{
- struct dma_chan *chan = depend_tx->chan;
- struct dma_device *device = chan->device;
- struct dma_async_tx_descriptor *intr_tx = (void *) ~0;
-
- /* first check to see if we can still append to depend_tx */
- txd_lock(depend_tx);
- if (txd_parent(depend_tx) && depend_tx->chan == tx->chan) {
- txd_chain(depend_tx, tx);
- intr_tx = NULL;
- }
- txd_unlock(depend_tx);
-
- /* attached dependency, flush the parent channel */
- if (!intr_tx) {
- device->device_issue_pending(chan);
- return;
- }
-
- /* see if we can schedule an interrupt
- * otherwise poll for completion
- */
- if (dma_has_cap(DMA_INTERRUPT, device->cap_mask))
- intr_tx = device->device_prep_dma_interrupt(chan, 0);
- else
- intr_tx = NULL;
-
- if (intr_tx) {
- dmaengine_desc_set_callback(intr_tx, NULL, NULL, NULL);
- /* safe to chain outside the lock since we know we are
- * not submitted yet
- */
- txd_chain(intr_tx, tx);
-
- /* check if we need to append */
- txd_lock(depend_tx);
- if (txd_parent(depend_tx)) {
- txd_chain(depend_tx, intr_tx);
- async_tx_ack(intr_tx);
- intr_tx = NULL;
- }
- txd_unlock(depend_tx);
-
- if (intr_tx) {
- txd_clear_parent(intr_tx);
- intr_tx->tx_submit(intr_tx);
- async_tx_ack(intr_tx);
- }
- device->device_issue_pending(chan);
- } else {
- if (dma_wait_for_async_tx(depend_tx) != DMA_COMPLETE)
- panic("%s: DMA error waiting for depend_tx\n",
- __func__);
- tx->tx_submit(tx);
- }
-}
-
-
-/**
- * enum submit_disposition - flags for routing an incoming operation
- * @ASYNC_TX_SUBMITTED: we were able to append the new operation under the lock
- * @ASYNC_TX_CHANNEL_SWITCH: when the lock is dropped schedule a channel switch
- * @ASYNC_TX_DIRECT_SUBMIT: when the lock is dropped submit directly
- *
- * while holding depend_tx->lock we must avoid submitting new operations
- * to prevent a circular locking dependency with drivers that already
- * hold a channel lock when calling async_tx_run_dependencies.
- */
-enum submit_disposition {
- ASYNC_TX_SUBMITTED,
- ASYNC_TX_CHANNEL_SWITCH,
- ASYNC_TX_DIRECT_SUBMIT,
-};
-
-void
-async_tx_submit(struct dma_chan *chan, struct dma_async_tx_descriptor *tx,
- struct async_submit_ctl *submit)
-{
- struct dma_async_tx_descriptor *depend_tx = submit->depend_tx;
-
- dmaengine_desc_set_callback(tx, submit->cb_fn, NULL, submit->cb_param);
-
- if (depend_tx) {
- enum submit_disposition s;
-
- /* sanity check the dependency chain:
- * 1/ if ack is already set then we cannot be sure
- * we are referring to the correct operation
- * 2/ dependencies are 1:1 i.e. two transactions can
- * not depend on the same parent
- */
- BUG_ON(async_tx_test_ack(depend_tx) || txd_next(depend_tx) ||
- txd_parent(tx));
-
- /* the lock prevents async_tx_run_dependencies from missing
- * the setting of ->next when ->parent != NULL
- */
- txd_lock(depend_tx);
- if (txd_parent(depend_tx)) {
- /* we have a parent so we can not submit directly
- * if we are staying on the same channel: append
- * else: channel switch
- */
- if (depend_tx->chan == chan) {
- txd_chain(depend_tx, tx);
- s = ASYNC_TX_SUBMITTED;
- } else
- s = ASYNC_TX_CHANNEL_SWITCH;
- } else {
- /* we do not have a parent so we may be able to submit
- * directly if we are staying on the same channel
- */
- if (depend_tx->chan == chan)
- s = ASYNC_TX_DIRECT_SUBMIT;
- else
- s = ASYNC_TX_CHANNEL_SWITCH;
- }
- txd_unlock(depend_tx);
-
- switch (s) {
- case ASYNC_TX_SUBMITTED:
- break;
- case ASYNC_TX_CHANNEL_SWITCH:
- async_tx_channel_switch(depend_tx, tx);
- break;
- case ASYNC_TX_DIRECT_SUBMIT:
- txd_clear_parent(tx);
- tx->tx_submit(tx);
- break;
- }
- } else {
- txd_clear_parent(tx);
- tx->tx_submit(tx);
- }
-
- if (submit->flags & ASYNC_TX_ACK)
- async_tx_ack(tx);
-
- if (depend_tx)
- async_tx_ack(depend_tx);
-}
-EXPORT_SYMBOL_GPL(async_tx_submit);
-
-/**
- * async_tx_issue_pending - send pending descriptor to the hardware channel
- * @tx: descriptor handle to retrieve hardware context
- *
- * Note: any dependent operations will have already been issued by
- * async_tx_channel_switch, or (in the case of no channel switch) will
- * be already pending on this channel.
- */
-void async_tx_issue_pending(struct dma_async_tx_descriptor *tx)
-{
- if (likely(tx)) {
- struct dma_chan *chan = tx->chan;
- struct dma_device *dma = chan->device;
-
- dma->device_issue_pending(chan);
- }
-}
-EXPORT_SYMBOL_GPL(async_tx_issue_pending);
-
-/**
- * async_trigger_callback - schedules the callback function to be run
- * @submit: submission and completion parameters
- *
- * honored flags: ASYNC_TX_ACK
- *
- * The callback is run after any dependent operations have completed.
- */
-struct dma_async_tx_descriptor *
-async_trigger_callback(struct async_submit_ctl *submit)
-{
- struct dma_chan *chan;
- struct dma_device *device;
- struct dma_async_tx_descriptor *tx;
- struct dma_async_tx_descriptor *depend_tx = submit->depend_tx;
-
- if (depend_tx) {
- chan = depend_tx->chan;
- device = chan->device;
-
- /* see if we can schedule an interrupt
- * otherwise poll for completion
- */
- if (device && !dma_has_cap(DMA_INTERRUPT, device->cap_mask))
- device = NULL;
-
- tx = device ? device->device_prep_dma_interrupt(chan, 0) : NULL;
- } else
- tx = NULL;
-
- if (tx) {
- pr_debug("%s: (async)\n", __func__);
-
- async_tx_submit(chan, tx, submit);
- } else {
- pr_debug("%s: (sync)\n", __func__);
-
- /* wait for any prerequisite operations */
- async_tx_quiesce(&submit->depend_tx);
-
- async_tx_sync_epilog(submit);
- }
-
- return tx;
-}
-EXPORT_SYMBOL_GPL(async_trigger_callback);
-
-/**
- * async_tx_quiesce - ensure tx is complete and freeable upon return
- * @tx: transaction to quiesce
- */
-void async_tx_quiesce(struct dma_async_tx_descriptor **tx)
-{
- if (*tx) {
- /* if ack is already set then we cannot be sure
- * we are referring to the correct operation
- */
- BUG_ON(async_tx_test_ack(*tx));
- if (dma_wait_for_async_tx(*tx) != DMA_COMPLETE)
- panic("%s: DMA error waiting for transaction\n",
- __func__);
- async_tx_ack(*tx);
- *tx = NULL;
- }
-}
-EXPORT_SYMBOL_GPL(async_tx_quiesce);
-
-MODULE_AUTHOR("Intel Corporation");
-MODULE_DESCRIPTION("Asynchronous Bulk Memory Transactions API");
-MODULE_LICENSE("GPL");
diff --git a/crypto/async_tx/async_xor.c b/crypto/async_tx/async_xor.c
deleted file mode 100644
index 84458375b202b..0000000000000
--- a/crypto/async_tx/async_xor.c
+++ /dev/null
@@ -1,380 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * xor offload engine api
- *
- * Copyright © 2006, Intel Corporation.
- *
- * Dan Williams <dan.j.williams@intel.com>
- *
- * with architecture considerations by:
- * Neil Brown <neilb@suse.de>
- * Jeff Garzik <jeff@garzik.org>
- */
-#include <linux/kernel.h>
-#include <linux/interrupt.h>
-#include <linux/module.h>
-#include <linux/mm.h>
-#include <linux/dma-mapping.h>
-#include <linux/raid/xor.h>
-#include <linux/async_tx.h>
-
-/* do_async_xor - dma map the pages and perform the xor with an engine */
-static __async_inline struct dma_async_tx_descriptor *
-do_async_xor(struct dma_chan *chan, struct dmaengine_unmap_data *unmap,
- struct async_submit_ctl *submit)
-{
- struct dma_device *dma = chan->device;
- struct dma_async_tx_descriptor *tx = NULL;
- dma_async_tx_callback cb_fn_orig = submit->cb_fn;
- void *cb_param_orig = submit->cb_param;
- enum async_tx_flags flags_orig = submit->flags;
- enum dma_ctrl_flags dma_flags = 0;
- int src_cnt = unmap->to_cnt;
- int xor_src_cnt;
- dma_addr_t dma_dest = unmap->addr[unmap->to_cnt];
- dma_addr_t *src_list = unmap->addr;
-
- while (src_cnt) {
- dma_addr_t tmp;
-
- submit->flags = flags_orig;
- xor_src_cnt = min(src_cnt, (int)dma->max_xor);
- /* if we are submitting additional xors, leave the chain open
- * and clear the callback parameters
- */
- if (src_cnt > xor_src_cnt) {
- submit->flags &= ~ASYNC_TX_ACK;
- submit->flags |= ASYNC_TX_FENCE;
- submit->cb_fn = NULL;
- submit->cb_param = NULL;
- } else {
- submit->cb_fn = cb_fn_orig;
- submit->cb_param = cb_param_orig;
- }
- if (submit->cb_fn)
- dma_flags |= DMA_PREP_INTERRUPT;
- if (submit->flags & ASYNC_TX_FENCE)
- dma_flags |= DMA_PREP_FENCE;
-
- /* Drivers force forward progress in case they can not provide a
- * descriptor
- */
- tmp = src_list[0];
- if (src_list > unmap->addr)
- src_list[0] = dma_dest;
- tx = dma->device_prep_dma_xor(chan, dma_dest, src_list,
- xor_src_cnt, unmap->len,
- dma_flags);
-
- if (unlikely(!tx))
- async_tx_quiesce(&submit->depend_tx);
-
- /* spin wait for the preceding transactions to complete */
- while (unlikely(!tx)) {
- dma_async_issue_pending(chan);
- tx = dma->device_prep_dma_xor(chan, dma_dest,
- src_list,
- xor_src_cnt, unmap->len,
- dma_flags);
- }
- src_list[0] = tmp;
-
- dma_set_unmap(tx, unmap);
- async_tx_submit(chan, tx, submit);
- submit->depend_tx = tx;
-
- if (src_cnt > xor_src_cnt) {
- /* drop completed sources */
- src_cnt -= xor_src_cnt;
- /* use the intermediate result a source */
- src_cnt++;
- src_list += xor_src_cnt - 1;
- } else
- break;
- }
-
- return tx;
-}
-
-static void
-do_sync_xor_offs(struct page *dest, unsigned int offset,
- struct page **src_list, unsigned int *src_offs,
- int src_cnt, size_t len, struct async_submit_ctl *submit)
-{
- int i;
- int xor_src_cnt = 0;
- void *dest_buf;
- void **srcs;
-
- if (submit->scribble)
- srcs = submit->scribble;
- else
- srcs = (void **) src_list;
-
- /* convert to buffer pointers */
- for (i = 0; i < src_cnt; i++)
- if (src_list[i])
- srcs[xor_src_cnt++] = page_address(src_list[i]) +
- (src_offs ? src_offs[i] : offset);
-
- /* set destination address */
- dest_buf = page_address(dest) + offset;
- if (submit->flags & ASYNC_TX_XOR_ZERO_DST)
- memset(dest_buf, 0, len);
- xor_gen(dest_buf, srcs, xor_src_cnt, len);
- async_tx_sync_epilog(submit);
-}
-
-static inline bool
-dma_xor_aligned_offsets(struct dma_device *device, unsigned int offset,
- unsigned int *src_offs, int src_cnt, int len)
-{
- int i;
-
- if (!is_dma_xor_aligned(device, offset, 0, len))
- return false;
-
- if (!src_offs)
- return true;
-
- for (i = 0; i < src_cnt; i++) {
- if (!is_dma_xor_aligned(device, src_offs[i], 0, len))
- return false;
- }
- return true;
-}
-
-/**
- * async_xor_offs - attempt to xor a set of blocks with a dma engine.
- * @dest: destination page
- * @offset: dst offset to start transaction
- * @src_list: array of source pages
- * @src_offs: array of source pages offset, NULL means common src/dst offset
- * @src_cnt: number of source pages
- * @len: length in bytes
- * @submit: submission / completion modifiers
- *
- * honored flags: ASYNC_TX_ACK, ASYNC_TX_XOR_ZERO_DST, ASYNC_TX_XOR_DROP_DST
- *
- * xor_gen always uses the dest as a source so the ASYNC_TX_XOR_ZERO_DST flag
- * must be set to not include dest data in the calculation. The assumption with
- * dma engines is that they only use the destination buffer as a source when it
- * is explicitly specified in the source list.
- *
- * src_list note: if the dest is also a source it must be at index zero.
- * The contents of this array will be overwritten if a scribble region
- * is not specified.
- */
-struct dma_async_tx_descriptor *
-async_xor_offs(struct page *dest, unsigned int offset,
- struct page **src_list, unsigned int *src_offs,
- int src_cnt, size_t len, struct async_submit_ctl *submit)
-{
- struct dma_chan *chan = async_tx_find_channel(submit, DMA_XOR,
- &dest, 1, src_list,
- src_cnt, len);
- struct dma_device *device = chan ? chan->device : NULL;
- struct dmaengine_unmap_data *unmap = NULL;
-
- BUG_ON(src_cnt <= 1);
-
- if (device)
- unmap = dmaengine_get_unmap_data(device->dev, src_cnt+1, GFP_NOWAIT);
-
- if (unmap && dma_xor_aligned_offsets(device, offset,
- src_offs, src_cnt, len)) {
- struct dma_async_tx_descriptor *tx;
- int i, j;
-
- /* run the xor asynchronously */
- pr_debug("%s (async): len: %zu\n", __func__, len);
-
- unmap->len = len;
- for (i = 0, j = 0; i < src_cnt; i++) {
- if (!src_list[i])
- continue;
- unmap->to_cnt++;
- unmap->addr[j++] = dma_map_page(device->dev, src_list[i],
- src_offs ? src_offs[i] : offset,
- len, DMA_TO_DEVICE);
- }
-
- /* map it bidirectional as it may be re-used as a source */
- unmap->addr[j] = dma_map_page(device->dev, dest, offset, len,
- DMA_BIDIRECTIONAL);
- unmap->bidi_cnt = 1;
-
- tx = do_async_xor(chan, unmap, submit);
- dmaengine_unmap_put(unmap);
- return tx;
- } else {
- dmaengine_unmap_put(unmap);
- /* run the xor synchronously */
- pr_debug("%s (sync): len: %zu\n", __func__, len);
- WARN_ONCE(chan, "%s: no space for dma address conversion\n",
- __func__);
-
- /* in the sync case the dest is an implied source
- * (assumes the dest is the first source)
- */
- if (submit->flags & ASYNC_TX_XOR_DROP_DST) {
- src_cnt--;
- src_list++;
- if (src_offs)
- src_offs++;
- }
-
- /* wait for any prerequisite operations */
- async_tx_quiesce(&submit->depend_tx);
-
- do_sync_xor_offs(dest, offset, src_list, src_offs,
- src_cnt, len, submit);
-
- return NULL;
- }
-}
-EXPORT_SYMBOL_GPL(async_xor_offs);
-
-/**
- * async_xor - attempt to xor a set of blocks with a dma engine.
- * @dest: destination page
- * @src_list: array of source pages
- * @offset: common src/dst offset to start transaction
- * @src_cnt: number of source pages
- * @len: length in bytes
- * @submit: submission / completion modifiers
- *
- * honored flags: ASYNC_TX_ACK, ASYNC_TX_XOR_ZERO_DST, ASYNC_TX_XOR_DROP_DST
- *
- * xor_gen always uses the dest as a source so the ASYNC_TX_XOR_ZERO_DST flag
- * must be set to not include dest data in the calculation. The assumption with
- * dma engines is that they only use the destination buffer as a source when it
- * is explicitly specified in the source list.
- *
- * src_list note: if the dest is also a source it must be at index zero.
- * The contents of this array will be overwritten if a scribble region
- * is not specified.
- */
-struct dma_async_tx_descriptor *
-async_xor(struct page *dest, struct page **src_list, unsigned int offset,
- int src_cnt, size_t len, struct async_submit_ctl *submit)
-{
- return async_xor_offs(dest, offset, src_list, NULL,
- src_cnt, len, submit);
-}
-EXPORT_SYMBOL_GPL(async_xor);
-
-static int page_is_zero(struct page *p, unsigned int offset, size_t len)
-{
- return !memchr_inv(page_address(p) + offset, 0, len);
-}
-
-static inline struct dma_chan *
-xor_val_chan(struct async_submit_ctl *submit, struct page *dest,
- struct page **src_list, int src_cnt, size_t len)
-{
- #ifdef CONFIG_ASYNC_TX_DISABLE_XOR_VAL_DMA
- return NULL;
- #endif
- return async_tx_find_channel(submit, DMA_XOR_VAL, &dest, 1, src_list,
- src_cnt, len);
-}
-
-/**
- * async_xor_val_offs - attempt a xor parity check with a dma engine.
- * @dest: destination page used if the xor is performed synchronously
- * @offset: des offset in pages to start transaction
- * @src_list: array of source pages
- * @src_offs: array of source pages offset, NULL means common src/det offset
- * @src_cnt: number of source pages
- * @len: length in bytes
- * @result: 0 if sum == 0 else non-zero
- * @submit: submission / completion modifiers
- *
- * honored flags: ASYNC_TX_ACK
- *
- * src_list note: if the dest is also a source it must be at index zero.
- * The contents of this array will be overwritten if a scribble region
- * is not specified.
- */
-struct dma_async_tx_descriptor *
-async_xor_val_offs(struct page *dest, unsigned int offset,
- struct page **src_list, unsigned int *src_offs,
- int src_cnt, size_t len, enum sum_check_flags *result,
- struct async_submit_ctl *submit)
-{
- struct dma_chan *chan = xor_val_chan(submit, dest, src_list, src_cnt, len);
- struct dma_device *device = chan ? chan->device : NULL;
- struct dma_async_tx_descriptor *tx = NULL;
- struct dmaengine_unmap_data *unmap = NULL;
-
- BUG_ON(src_cnt <= 1);
-
- if (device)
- unmap = dmaengine_get_unmap_data(device->dev, src_cnt, GFP_NOWAIT);
-
- if (unmap && src_cnt <= device->max_xor &&
- dma_xor_aligned_offsets(device, offset, src_offs, src_cnt, len)) {
- unsigned long dma_prep_flags = 0;
- int i;
-
- pr_debug("%s: (async) len: %zu\n", __func__, len);
-
- if (submit->cb_fn)
- dma_prep_flags |= DMA_PREP_INTERRUPT;
- if (submit->flags & ASYNC_TX_FENCE)
- dma_prep_flags |= DMA_PREP_FENCE;
-
- for (i = 0; i < src_cnt; i++) {
- unmap->addr[i] = dma_map_page(device->dev, src_list[i],
- src_offs ? src_offs[i] : offset,
- len, DMA_TO_DEVICE);
- unmap->to_cnt++;
- }
- unmap->len = len;
-
- tx = device->device_prep_dma_xor_val(chan, unmap->addr, src_cnt,
- len, result,
- dma_prep_flags);
- if (unlikely(!tx)) {
- async_tx_quiesce(&submit->depend_tx);
-
- while (!tx) {
- dma_async_issue_pending(chan);
- tx = device->device_prep_dma_xor_val(chan,
- unmap->addr, src_cnt, len, result,
- dma_prep_flags);
- }
- }
- dma_set_unmap(tx, unmap);
- async_tx_submit(chan, tx, submit);
- } else {
- enum async_tx_flags flags_orig = submit->flags;
-
- pr_debug("%s: (sync) len: %zu\n", __func__, len);
- WARN_ONCE(device && src_cnt <= device->max_xor,
- "%s: no space for dma address conversion\n",
- __func__);
-
- submit->flags |= ASYNC_TX_XOR_DROP_DST;
- submit->flags &= ~ASYNC_TX_ACK;
-
- tx = async_xor_offs(dest, offset, src_list, src_offs,
- src_cnt, len, submit);
-
- async_tx_quiesce(&tx);
-
- *result = !page_is_zero(dest, offset, len) << SUM_CHECK_P;
-
- async_tx_sync_epilog(submit);
- submit->flags = flags_orig;
- }
- dmaengine_unmap_put(unmap);
-
- return tx;
-}
-EXPORT_SYMBOL_GPL(async_xor_val_offs);
-
-MODULE_AUTHOR("Intel Corporation");
-MODULE_DESCRIPTION("asynchronous xor/xor-zero-sum api");
-MODULE_LICENSE("GPL");
diff --git a/crypto/async_tx/raid6test.c b/crypto/async_tx/raid6test.c
deleted file mode 100644
index d3fbee1e03e55..0000000000000
--- a/crypto/async_tx/raid6test.c
+++ /dev/null
@@ -1,250 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * asynchronous raid6 recovery self test
- * Copyright (c) 2009, Intel Corporation.
- *
- * based on drivers/md/raid6test/test.c:
- * Copyright 2002-2007 H. Peter Anvin
- */
-#include <linux/async_tx.h>
-#include <linux/gfp.h>
-#include <linux/mm.h>
-#include <linux/random.h>
-#include <linux/module.h>
-
-#undef pr
-#define pr(fmt, args...) pr_info("raid6test: " fmt, ##args)
-
-#define NDISKS 64 /* Including P and Q */
-
-static struct page *dataptrs[NDISKS];
-unsigned int dataoffs[NDISKS];
-static addr_conv_t addr_conv[NDISKS];
-static struct page *data[NDISKS+3];
-static struct page *spare;
-static struct page *recovi;
-static struct page *recovj;
-
-static void callback(void *param)
-{
- struct completion *cmp = param;
-
- complete(cmp);
-}
-
-static void makedata(int disks)
-{
- int i;
-
- for (i = 0; i < disks; i++) {
- get_random_bytes(page_address(data[i]), PAGE_SIZE);
- dataptrs[i] = data[i];
- dataoffs[i] = 0;
- }
-}
-
-static char disk_type(int d, int disks)
-{
- if (d == disks - 2)
- return 'P';
- else if (d == disks - 1)
- return 'Q';
- else
- return 'D';
-}
-
-/* Recover two failed blocks. */
-static void raid6_dual_recov(int disks, size_t bytes, int faila, int failb,
- struct page **ptrs, unsigned int *offs)
-{
- struct async_submit_ctl submit;
- struct completion cmp;
- struct dma_async_tx_descriptor *tx = NULL;
- enum sum_check_flags result = ~0;
-
- if (faila > failb)
- swap(faila, failb);
-
- if (failb == disks-1) {
- if (faila == disks-2) {
- /* P+Q failure. Just rebuild the syndrome. */
- init_async_submit(&submit, 0, NULL, NULL, NULL, addr_conv);
- tx = async_gen_syndrome(ptrs, offs,
- disks, bytes, &submit);
- } else {
- struct page *blocks[NDISKS];
- struct page *dest;
- int count = 0;
- int i;
-
- BUG_ON(disks > NDISKS);
-
- /* data+Q failure. Reconstruct data from P,
- * then rebuild syndrome
- */
- for (i = disks; i-- ; ) {
- if (i == faila || i == failb)
- continue;
- blocks[count++] = ptrs[i];
- }
- dest = ptrs[faila];
- init_async_submit(&submit, ASYNC_TX_XOR_ZERO_DST, NULL,
- NULL, NULL, addr_conv);
- tx = async_xor(dest, blocks, 0, count, bytes, &submit);
-
- init_async_submit(&submit, 0, tx, NULL, NULL, addr_conv);
- tx = async_gen_syndrome(ptrs, offs,
- disks, bytes, &submit);
- }
- } else {
- if (failb == disks-2) {
- /* data+P failure. */
- init_async_submit(&submit, 0, NULL, NULL, NULL, addr_conv);
- tx = async_raid6_datap_recov(disks, bytes,
- faila, ptrs, offs, &submit);
- } else {
- /* data+data failure. */
- init_async_submit(&submit, 0, NULL, NULL, NULL, addr_conv);
- tx = async_raid6_2data_recov(disks, bytes,
- faila, failb, ptrs, offs, &submit);
- }
- }
- init_completion(&cmp);
- init_async_submit(&submit, ASYNC_TX_ACK, tx, callback, &cmp, addr_conv);
- tx = async_syndrome_val(ptrs, offs,
- disks, bytes, &result, spare, 0, &submit);
- async_tx_issue_pending(tx);
-
- if (wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000)) == 0)
- pr("%s: timeout! (faila: %d failb: %d disks: %d)\n",
- __func__, faila, failb, disks);
-
- if (result != 0)
- pr("%s: validation failure! faila: %d failb: %d sum_check_flags: %x\n",
- __func__, faila, failb, result);
-}
-
-static int test_disks(int i, int j, int disks)
-{
- int erra, errb;
-
- memset(page_address(recovi), 0xf0, PAGE_SIZE);
- memset(page_address(recovj), 0xba, PAGE_SIZE);
-
- dataptrs[i] = recovi;
- dataptrs[j] = recovj;
-
- raid6_dual_recov(disks, PAGE_SIZE, i, j, dataptrs, dataoffs);
-
- erra = memcmp(page_address(data[i]), page_address(recovi), PAGE_SIZE);
- errb = memcmp(page_address(data[j]), page_address(recovj), PAGE_SIZE);
-
- pr("%s(%d, %d): faila=%3d(%c) failb=%3d(%c) %s\n",
- __func__, i, j, i, disk_type(i, disks), j, disk_type(j, disks),
- (!erra && !errb) ? "OK" : !erra ? "ERRB" : !errb ? "ERRA" : "ERRAB");
-
- dataptrs[i] = data[i];
- dataptrs[j] = data[j];
-
- return erra || errb;
-}
-
-static int test(int disks, int *tests)
-{
- struct dma_async_tx_descriptor *tx;
- struct async_submit_ctl submit;
- struct completion cmp;
- int err = 0;
- int i, j;
-
- recovi = data[disks];
- recovj = data[disks+1];
- spare = data[disks+2];
-
- makedata(disks);
-
- /* Nuke syndromes */
- memset(page_address(data[disks-2]), 0xee, PAGE_SIZE);
- memset(page_address(data[disks-1]), 0xee, PAGE_SIZE);
-
- /* Generate assumed good syndrome */
- init_completion(&cmp);
- init_async_submit(&submit, ASYNC_TX_ACK, NULL, callback, &cmp, addr_conv);
- tx = async_gen_syndrome(dataptrs, dataoffs, disks, PAGE_SIZE, &submit);
- async_tx_issue_pending(tx);
-
- if (wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000)) == 0) {
- pr("error: initial gen_syndrome(%d) timed out\n", disks);
- return 1;
- }
-
- pr("testing the %d-disk case...\n", disks);
- for (i = 0; i < disks-1; i++)
- for (j = i+1; j < disks; j++) {
- (*tests)++;
- err += test_disks(i, j, disks);
- }
-
- return err;
-}
-
-
-static int __init raid6_test(void)
-{
- int err = 0;
- int tests = 0;
- int i;
-
- for (i = 0; i < NDISKS+3; i++) {
- data[i] = alloc_page(GFP_KERNEL);
- if (!data[i]) {
- while (i--)
- put_page(data[i]);
- return -ENOMEM;
- }
- }
-
- /* the 4-disk and 5-disk cases are special for the recovery code */
- if (NDISKS > 4)
- err += test(4, &tests);
- if (NDISKS > 5)
- err += test(5, &tests);
- /* the 11 and 12 disk cases are special for ioatdma (p-disabled
- * q-continuation without extended descriptor)
- */
- if (NDISKS > 12) {
- err += test(11, &tests);
- err += test(12, &tests);
- }
-
- /* the 24 disk case is special for ioatdma as it is the boundary point
- * at which it needs to switch from 8-source ops to 16-source
- * ops for continuation (assumes DMA_HAS_PQ_CONTINUE is not set)
- */
- if (NDISKS > 24)
- err += test(24, &tests);
-
- err += test(NDISKS, &tests);
-
- pr("\n");
- pr("complete (%d tests, %d failure%s)\n",
- tests, err, err == 1 ? "" : "s");
-
- for (i = 0; i < NDISKS+3; i++)
- put_page(data[i]);
-
- return 0;
-}
-
-static void __exit raid6_test_exit(void)
-{
-}
-
-/* when compiled-in wait for drivers to load first (assumes dma drivers
- * are also compiled-in)
- */
-late_initcall(raid6_test);
-module_exit(raid6_test_exit);
-MODULE_AUTHOR("Dan Williams <dan.j.williams@intel.com>");
-MODULE_DESCRIPTION("asynchronous RAID-6 recovery self tests");
-MODULE_LICENSE("GPL");
diff --git a/include/linux/async_tx.h b/include/linux/async_tx.h
deleted file mode 100644
index d49540570ba09..0000000000000
--- a/include/linux/async_tx.h
+++ /dev/null
@@ -1,187 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright © 2006, Intel Corporation.
- */
-#ifndef _ASYNC_TX_H_
-#define _ASYNC_TX_H_
-#include <linux/dmaengine.h>
-#include <linux/spinlock.h>
-#include <linux/interrupt.h>
-
-/* on architectures without dma-mapping capabilities we need to ensure
- * that the asynchronous path compiles away
- */
-#ifdef CONFIG_HAS_DMA
-#define __async_inline
-#else
-#define __async_inline __always_inline
-#endif
-
-/**
- * dma_chan_ref - object used to manage dma channels received from the
- * dmaengine core.
- * @chan - the channel being tracked
- * @node - node for the channel to be placed on async_tx_master_list
- * @rcu - for list_del_rcu
- * @count - number of times this channel is listed in the pool
- * (for channels with multiple capabiities)
- */
-struct dma_chan_ref {
- struct dma_chan *chan;
- struct list_head node;
- struct rcu_head rcu;
- atomic_t count;
-};
-
-/**
- * async_tx_flags - modifiers for the async_* calls
- * @ASYNC_TX_XOR_ZERO_DST: this flag must be used for xor operations where the
- * destination address is not a source. The asynchronous case handles this
- * implicitly, the synchronous case needs to zero the destination block.
- * @ASYNC_TX_XOR_DROP_DST: this flag must be used if the destination address is
- * also one of the source addresses. In the synchronous case the destination
- * address is an implied source, whereas the asynchronous case it must be listed
- * as a source. The destination address must be the first address in the source
- * array.
- * @ASYNC_TX_ACK: immediately ack the descriptor, precludes setting up a
- * dependency chain
- * @ASYNC_TX_FENCE: specify that the next operation in the dependency
- * chain uses this operation's result as an input
- * @ASYNC_TX_PQ_XOR_DST: do not overwrite the syndrome but XOR it with the
- * input data. Required for rmw case.
- */
-enum async_tx_flags {
- ASYNC_TX_XOR_ZERO_DST = (1 << 0),
- ASYNC_TX_XOR_DROP_DST = (1 << 1),
- ASYNC_TX_ACK = (1 << 2),
- ASYNC_TX_FENCE = (1 << 3),
- ASYNC_TX_PQ_XOR_DST = (1 << 4),
-};
-
-/**
- * struct async_submit_ctl - async_tx submission/completion modifiers
- * @flags: submission modifiers
- * @depend_tx: parent dependency of the current operation being submitted
- * @cb_fn: callback routine to run at operation completion
- * @cb_param: parameter for the callback routine
- * @scribble: caller provided space for dma/page address conversions
- */
-struct async_submit_ctl {
- enum async_tx_flags flags;
- struct dma_async_tx_descriptor *depend_tx;
- dma_async_tx_callback cb_fn;
- void *cb_param;
- void *scribble;
-};
-
-#if defined(CONFIG_DMA_ENGINE) && !defined(CONFIG_ASYNC_TX_CHANNEL_SWITCH)
-#define async_tx_issue_pending_all dma_issue_pending_all
-
-void async_tx_issue_pending(struct dma_async_tx_descriptor *tx);
-
-#ifdef CONFIG_ARCH_HAS_ASYNC_TX_FIND_CHANNEL
-#include <asm/async_tx.h>
-#else
-#define async_tx_find_channel(dep, type, dst, dst_count, src, src_count, len) \
- __async_tx_find_channel(dep, type)
-struct dma_chan *
-__async_tx_find_channel(struct async_submit_ctl *submit,
- enum dma_transaction_type tx_type);
-#endif /* CONFIG_ARCH_HAS_ASYNC_TX_FIND_CHANNEL */
-#else
-static inline void async_tx_issue_pending_all(void)
-{
- do { } while (0);
-}
-
-static inline void async_tx_issue_pending(struct dma_async_tx_descriptor *tx)
-{
-}
-
-static inline struct dma_chan *
-async_tx_find_channel(struct async_submit_ctl *submit,
- enum dma_transaction_type tx_type, struct page **dst,
- int dst_count, struct page **src, int src_count,
- size_t len)
-{
- return NULL;
-}
-#endif
-
-/**
- * async_tx_sync_epilog - actions to take if an operation is run synchronously
- * @cb_fn: function to call when the transaction completes
- * @cb_fn_param: parameter to pass to the callback routine
- */
-static inline void
-async_tx_sync_epilog(struct async_submit_ctl *submit)
-{
- if (submit->cb_fn)
- submit->cb_fn(submit->cb_param);
-}
-
-typedef union {
- unsigned long addr;
- struct page *page;
- dma_addr_t dma;
-} addr_conv_t;
-
-static inline void
-init_async_submit(struct async_submit_ctl *args, enum async_tx_flags flags,
- struct dma_async_tx_descriptor *tx,
- dma_async_tx_callback cb_fn, void *cb_param,
- addr_conv_t *scribble)
-{
- args->flags = flags;
- args->depend_tx = tx;
- args->cb_fn = cb_fn;
- args->cb_param = cb_param;
- args->scribble = scribble;
-}
-
-void async_tx_submit(struct dma_chan *chan, struct dma_async_tx_descriptor *tx,
- struct async_submit_ctl *submit);
-
-struct dma_async_tx_descriptor *
-async_xor(struct page *dest, struct page **src_list, unsigned int offset,
- int src_cnt, size_t len, struct async_submit_ctl *submit);
-
-struct dma_async_tx_descriptor *
-async_xor_offs(struct page *dest, unsigned int offset,
- struct page **src_list, unsigned int *src_offset,
- int src_cnt, size_t len, struct async_submit_ctl *submit);
-
-struct dma_async_tx_descriptor *
-async_xor_val_offs(struct page *dest, unsigned int offset,
- struct page **src_list, unsigned int *src_offset,
- int src_cnt, size_t len, enum sum_check_flags *result,
- struct async_submit_ctl *submit);
-
-struct dma_async_tx_descriptor *
-async_memcpy(struct page *dest, struct page *src, unsigned int dest_offset,
- unsigned int src_offset, size_t len,
- struct async_submit_ctl *submit);
-
-struct dma_async_tx_descriptor *async_trigger_callback(struct async_submit_ctl *submit);
-
-struct dma_async_tx_descriptor *
-async_gen_syndrome(struct page **blocks, unsigned int *offsets, int src_cnt,
- size_t len, struct async_submit_ctl *submit);
-
-struct dma_async_tx_descriptor *
-async_syndrome_val(struct page **blocks, unsigned int *offsets, int src_cnt,
- size_t len, enum sum_check_flags *pqres, struct page *spare,
- unsigned int s_off, struct async_submit_ctl *submit);
-
-struct dma_async_tx_descriptor *
-async_raid6_2data_recov(int src_num, size_t bytes, int faila, int failb,
- struct page **ptrs, unsigned int *offs,
- struct async_submit_ctl *submit);
-
-struct dma_async_tx_descriptor *
-async_raid6_datap_recov(int src_num, size_t bytes, int faila,
- struct page **ptrs, unsigned int *offs,
- struct async_submit_ctl *submit);
-
-void async_tx_quiesce(struct dma_async_tx_descriptor **tx);
-#endif /* _ASYNC_TX_H_ */
--
2.43.0
^ permalink raw reply related [flat|nested] 6+ messages in thread