* [RFC 4/9] io_uring/zcrx: split frag handling loop
From: Pavel Begunkov @ 2026-07-11 9:22 UTC (permalink / raw)
To: David S . Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Simon Horman, netdev
Cc: io-uring, asml.silence
In-Reply-To: <cover.1783619193.git.asml.silence@gmail.com>
A preparation patch splitting the frag array handling loop into two,
where first we skip frags below the requested offset. It makes further
changes more readable.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
io_uring/zcrx.c | 32 +++++++++++++++++++-------------
1 file changed, 19 insertions(+), 13 deletions(-)
diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c
index 816a169b848e..162e67287916 100644
--- a/io_uring/zcrx.c
+++ b/io_uring/zcrx.c
@@ -1877,23 +1877,29 @@ static int __zcrx_recv_skb(read_descriptor_t *desc, struct sk_buff *skb,
const skb_frag_t *frag = &shi->frags[i];
unsigned frag_end = start + skb_frag_size(frag);
+ if (offset < frag_end)
+ break;
+ start = frag_end;
+ }
+
+ for (; i < shi->nr_frags; i++) {
+ const skb_frag_t *frag = &shi->frags[i];
+ unsigned frag_end = start + skb_frag_size(frag);
+ unsigned copy = min(frag_end - offset, len);
+ unsigned frag_off = offset - start;
+
if (WARN_ON(start > offset + len))
return -EFAULT;
+ start = frag_end;
- if (offset < frag_end) {
- unsigned copy = min(frag_end - offset, len);
- unsigned frag_off = offset - start;
-
- ret = io_zcrx_recv_frag(req, ifq, frag, frag_off, copy);
- if (ret < 0)
- goto out;
+ ret = io_zcrx_recv_frag(req, ifq, frag, frag_off, copy);
+ if (ret < 0)
+ goto out;
- offset += ret;
- len -= ret;
- if (len == 0 || ret != copy)
- goto out;
- }
- start = frag_end;
+ offset += ret;
+ len -= ret;
+ if (len == 0 || ret != copy)
+ goto out;
}
skb_walk_frags(skb, frag_iter) {
--
2.54.0
^ permalink raw reply related
* [RFC 3/9] io_uring/zcrx: don't save/restore count for frag skbs
From: Pavel Begunkov @ 2026-07-11 9:22 UTC (permalink / raw)
To: David S . Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Simon Horman, netdev
Cc: io-uring, asml.silence
In-Reply-To: <cover.1783619193.git.asml.silence@gmail.com>
We save and restore desc->count before recursing for frag skb
processing. Extract the handling into a separate function instead and
pass a flag.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
io_uring/zcrx.c | 22 ++++++++++++++--------
1 file changed, 14 insertions(+), 8 deletions(-)
diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c
index 0aa6455971d6..816a169b848e 100644
--- a/io_uring/zcrx.c
+++ b/io_uring/zcrx.c
@@ -1827,9 +1827,8 @@ static int io_zcrx_recv_frag(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
return len;
}
-static int
-io_zcrx_recv_skb(read_descriptor_t *desc, struct sk_buff *skb,
- unsigned int offset, size_t len)
+static int __zcrx_recv_skb(read_descriptor_t *desc, struct sk_buff *skb,
+ unsigned int offset, size_t len)
{
struct io_zcrx_args *args = desc->arg.data;
struct io_zcrx_ifq *ifq = args->ifq;
@@ -1907,11 +1906,8 @@ io_zcrx_recv_skb(read_descriptor_t *desc, struct sk_buff *skb,
if (offset < frag_end) {
unsigned copy = min(frag_end - offset, len);
unsigned frag_off = offset - start;
- size_t count;
- count = desc->count;
- ret = io_zcrx_recv_skb(desc, frag_iter, frag_off, copy);
- desc->count = count;
+ ret = __zcrx_recv_skb(desc, frag_iter, frag_off, copy);
if (ret < 0)
goto out;
@@ -1926,10 +1922,20 @@ io_zcrx_recv_skb(read_descriptor_t *desc, struct sk_buff *skb,
out:
if (offset == start_off)
return ret;
- desc->count -= (offset - start_off);
return offset - start_off;
}
+static
+int io_zcrx_recv_skb(read_descriptor_t *desc, struct sk_buff *skb,
+ unsigned int offset, size_t len)
+{
+ int ret;
+
+ ret = __zcrx_recv_skb(desc, skb, offset, len);
+ desc->count -= max(0, ret);
+ return ret;
+}
+
static int io_zcrx_tcp_recvmsg(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
struct sock *sk, int flags,
unsigned issue_flags, unsigned int *outlen)
--
2.54.0
^ permalink raw reply related
* [RFC 2/9] net: add provider specific net_iov field
From: Pavel Begunkov @ 2026-07-11 9:22 UTC (permalink / raw)
To: David S . Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Simon Horman, netdev
Cc: io-uring, asml.silence
In-Reply-To: <cover.1783619193.git.asml.silence@gmail.com>
Use a hole in struct net_iov to give some extra space to memory
providers like zcrx. Keeping some extra info in net_iov itself helps
with cache utilisation.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
include/net/netmem.h | 1 +
1 file changed, 1 insertion(+)
diff --git a/include/net/netmem.h b/include/net/netmem.h
index bccacd21b6c3..a564c510b484 100644
--- a/include/net/netmem.h
+++ b/include/net/netmem.h
@@ -95,6 +95,7 @@ enum net_iov_type {
struct net_iov {
struct netmem_desc desc;
enum net_iov_type type;
+ unsigned int mp_private;
struct net_iov_area *owner;
};
--
2.54.0
^ permalink raw reply related
* [RFC 1/9] net: allow __tcp_read_sock actors to steal skbs
From: Pavel Begunkov @ 2026-07-11 9:22 UTC (permalink / raw)
To: David S . Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Simon Horman, netdev
Cc: io-uring, asml.silence
In-Reply-To: <cover.1783619193.git.asml.silence@gmail.com>
Currently __tcp_read_sock() owns skbs and expects them to be present
when the actor function returns (modulo collapsing). For zcrx
optimisations I want to be able to take ownership of the skb in the
callback, add a helper doing that. It's only implemented for tcp, hence
keep "tcp" in the helper name. It could be later extended to other
protocols but would need some whitelisting mechanism.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
include/linux/net.h | 1 +
include/net/tcp.h | 13 +++++++++++++
net/ipv4/tcp.c | 11 +++++++++++
3 files changed, 25 insertions(+)
diff --git a/include/linux/net.h b/include/linux/net.h
index f268f395ce47..ed882aeac4a5 100644
--- a/include/linux/net.h
+++ b/include/linux/net.h
@@ -165,6 +165,7 @@ typedef struct {
void *data;
} arg;
int error;
+ bool stolen;
} read_descriptor_t;
struct vm_area_struct;
diff --git a/include/net/tcp.h b/include/net/tcp.h
index ecbadcb3a744..3d25707b73c3 100644
--- a/include/net/tcp.h
+++ b/include/net/tcp.h
@@ -3089,6 +3089,19 @@ static inline int tcp_recv_should_stop(struct sock *sk)
signal_pending(current);
}
+static inline bool tcp_read_sock_steal_skb(read_descriptor_t *desc,
+ struct sk_buff *skb,
+ struct sock *sk)
+{
+ if (skb_shared(skb))
+ return false;
+
+ desc->stolen = true;
+ __skb_unlink(skb, &sk->sk_receive_queue);
+ skb_orphan(skb);
+ return true;
+}
+
INDIRECT_CALLABLE_DECLARE(union tcp_seq_and_ts_off
tcp_v4_init_seq_and_ts_off(const struct net *net,
const struct sk_buff *skb));
diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index 432fa28e47d4..309a0e6b0173 100644
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -1677,6 +1677,7 @@ static int __tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
return -ENOTCONN;
while ((skb = tcp_recv_skb(sk, seq, &offset)) != NULL) {
if (offset < skb->len) {
+ u8 tcp_flags = TCP_SKB_CB(skb)->tcp_flags;
int used;
size_t len;
@@ -1689,6 +1690,7 @@ static int __tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
if (!len)
break;
}
+ desc->stolen = false;
used = recv_actor(desc, skb, offset, len);
if (used <= 0) {
if (!copied)
@@ -1701,6 +1703,14 @@ static int __tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
copied += used;
offset += used;
+ if (desc->stolen) {
+ if (tcp_flags & TCPHDR_FIN) {
+ ++seq;
+ break;
+ }
+ goto next;
+ }
+
/* If recv_actor drops the lock (e.g. TCP splice
* receive) the skb pointer might be invalid when
* getting here: tcp_collapse might have deleted it
@@ -1721,6 +1731,7 @@ static int __tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
break;
}
tcp_eat_recv_skb(sk, skb);
+next:
if (!desc->count)
break;
WRITE_ONCE(*copied_seq, seq);
--
2.54.0
^ permalink raw reply related
* [RFC 0/9] optimise zcrx refs cache bouncing
From: Pavel Begunkov @ 2026-07-11 9:22 UTC (permalink / raw)
To: David S . Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Simon Horman, netdev
Cc: io-uring, asml.silence
zcrx works well when user space and NAPI run on the same CPU but loses a
chunk of performance otherwise. It's caused by cache bounces from
1) zcrx "user" refs, which track whether buffers are given to the user
2) niov refs, as zcrx bumps them on recv(), and hence
skb_attempt_defer_free() doesn't help.
In this patchset, zcrx steals received skbs, delays their destruction
similar to skb_attempt_defer_free(), and processes them in
io_pp_zc_alloc_netmems(). This moves all aforementioned refs
modifications for the hot path to the NAPI context.
For the networking side the most interesting bits are patches 1 and 8,
and patch 6 around the call to tcp_read_sock_steal_skb(). I'm looking
to get opinions on whether tcp_read_sock_steal_skb() is fine or what
kind of helpers / API would work better.
Tested with liburing/examples/{zcrx + send-zerocopy},
200Gbit/s NICs, rx_page=4KB
before: MB/s=18948
CPU %usr %sys %iowait %irq %soft %idle
0 4.92 63.68 0.00 1.64 2.84 26.91
7 0.00 0.00 0.00 0.30 89.50 10.20
after: MB/s=21034
CPU %usr %sys %iowait %irq %soft %idle
0 5.59 50.18 0.00 2.26 2.73 39.24
7 0.00 0.00 0.00 0.20 87.49 12.31
Helps in a similar way to 32KB rx page size, and also improves numbers
when NAPI and user space run on the same CPU.
kernel:
url: https://github.com/isilence/linux/tree/zcrx/skb-stealing
git: https://github.com/isilence/linux.git zcrx/skb-stealing
liburing (can be used any other version):
url: https://github.com/isilence/liburing/tree/zcrx/test-skb-steal
git: https://github.com/isilence/liburing.git zcrx/test-skb-steal
Pavel Begunkov (9):
net: allow __tcp_read_sock actors to steal skbs
net: add provider specific net_iov field
io_uring/zcrx: don't save/restore count for frag skbs
io_uring/zcrx: split frag handling loop
io_uring/zcrx: split io_zcrx_recv_frag()
io_uring/zcrx: implement skb stealing
io_uring/zcrx: don't lock for single producer ptr ring
io_uring/zcrx: steal niov refs
io_uring/zcrx: add rq_lock cache of "user" niov refs
include/linux/net.h | 1 +
include/net/netmem.h | 1 +
include/net/tcp.h | 13 +++
io_uring/zcrx.c | 210 ++++++++++++++++++++++++++++++++++---------
io_uring/zcrx.h | 4 +
net/ipv4/tcp.c | 11 +++
6 files changed, 200 insertions(+), 40 deletions(-)
--
2.54.0
^ permalink raw reply
* [PATCH review-only 16/17] io_urint/zcrx: narrow var scope in io_zcrx_recv_skb()
From: Pavel Begunkov @ 2026-07-11 9:11 UTC (permalink / raw)
To: io-uring; +Cc: asml.silence, netdev
In-Reply-To: <cover.1783616211.git.asml.silence@gmail.com>
A preparation patch that limits scopes of a couple variables in
io_zcrx_recv_skb() and rename them, it makes it easier to reason about
the code.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
io_uring/zcrx.c | 35 ++++++++++++++++-------------------
1 file changed, 16 insertions(+), 19 deletions(-)
diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c
index f7592a3c058d..74046a09911a 100644
--- a/io_uring/zcrx.c
+++ b/io_uring/zcrx.c
@@ -1836,8 +1836,7 @@ io_zcrx_recv_skb(read_descriptor_t *desc, struct sk_buff *skb,
struct io_kiocb *req = args->req;
struct sk_buff *frag_iter;
unsigned start, start_off = offset;
- int i, copy, end, off;
- int ret = 0;
+ int i, ret = 0;
len = min_t(size_t, len, desc->count);
/*
@@ -1875,20 +1874,19 @@ io_zcrx_recv_skb(read_descriptor_t *desc, struct sk_buff *skb,
for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
const skb_frag_t *frag;
+ unsigned frag_end;
if (WARN_ON(start > offset + len))
return -EFAULT;
frag = &skb_shinfo(skb)->frags[i];
- end = start + skb_frag_size(frag);
+ frag_end = start + skb_frag_size(frag);
- if (offset < end) {
- copy = end - offset;
- if (copy > len)
- copy = len;
+ if (offset < frag_end) {
+ unsigned copy = min(frag_end - offset, len);
+ unsigned frag_off = offset - start;
- off = offset - start;
- ret = io_zcrx_recv_frag(req, ifq, frag, off, copy);
+ ret = io_zcrx_recv_frag(req, ifq, frag, frag_off, copy);
if (ret < 0)
goto out;
@@ -1897,24 +1895,23 @@ io_zcrx_recv_skb(read_descriptor_t *desc, struct sk_buff *skb,
if (len == 0 || ret != copy)
goto out;
}
- start = end;
+ start = frag_end;
}
skb_walk_frags(skb, frag_iter) {
+ unsigned frag_end;
+
if (WARN_ON(start > offset + len))
return -EFAULT;
- end = start + frag_iter->len;
- if (offset < end) {
+ frag_end = start + frag_iter->len;
+ if (offset < frag_end) {
+ unsigned copy = min(frag_end - offset, len);
+ unsigned frag_off = offset - start;
size_t count;
- copy = end - offset;
- if (copy > len)
- copy = len;
-
- off = offset - start;
count = desc->count;
- ret = io_zcrx_recv_skb(desc, frag_iter, off, copy);
+ ret = io_zcrx_recv_skb(desc, frag_iter, frag_off, copy);
desc->count = count;
if (ret < 0)
goto out;
@@ -1924,7 +1921,7 @@ io_zcrx_recv_skb(read_descriptor_t *desc, struct sk_buff *skb,
if (len == 0 || ret != copy)
goto out;
}
- start = end;
+ start = frag_end;
}
out:
--
2.54.0
^ permalink raw reply related
* [PATCH review-only 04/17] io_uring/zcrx: cache RQ tail
From: Pavel Begunkov @ 2026-07-11 9:11 UTC (permalink / raw)
To: io-uring; +Cc: asml.silence, netdev
In-Reply-To: <cover.1783616211.git.asml.silence@gmail.com>
The RQ tail is updated by the user space. Cache it to reduce cache line
bouncing. Refilling now tries to exhaust the previous batch of rqes, but
since it could be too low, the iterator is allowed to recalculate the
rqes to process once after synching the tail value.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
io_uring/zcrx.c | 27 +++++++++++++++++++++------
io_uring/zcrx.h | 1 +
2 files changed, 22 insertions(+), 6 deletions(-)
diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c
index 45b178afbbc3..1b8d748b35e7 100644
--- a/io_uring/zcrx.c
+++ b/io_uring/zcrx.c
@@ -1090,16 +1090,22 @@ void io_unregister_zcrx(struct io_ring_ctx *ctx)
struct zcrx_rq_iter {
int rqes_left;
+ bool flushed;
};
-static inline u32 zcrx_rq_entries(struct zcrx_rq *rq)
+static inline u32 __zcrx_rq_entries(struct zcrx_rq *rq)
{
- u32 entries;
+ u32 entries = rq->cached_tail - rq->cached_head;
- entries = smp_load_acquire(&rq->ring->tail) - rq->cached_head;
return min(entries, rq->nr_entries);
}
+static inline u32 zcrx_rq_entries(struct zcrx_rq *rq)
+{
+ rq->cached_tail = smp_load_acquire(&rq->ring->tail);
+ return __zcrx_rq_entries(rq);
+}
+
static struct io_uring_zcrx_rqe *zcrx_next_rqe(struct zcrx_rq *rq, unsigned mask)
{
unsigned int idx = rq->cached_head++ & mask;
@@ -1110,7 +1116,8 @@ static struct io_uring_zcrx_rqe *zcrx_next_rqe(struct zcrx_rq *rq, unsigned mask
static inline void zcrx_rq_iter_init(struct zcrx_rq_iter *it,
struct zcrx_rq *rq)
{
- it->rqes_left = min_t(unsigned, zcrx_rq_entries(rq), ZCRX_REFILL_CAP);
+ it->rqes_left = min_t(unsigned, __zcrx_rq_entries(rq), ZCRX_REFILL_CAP);
+ it->flushed = false;
}
static inline bool zcrx_rq_iter_next(struct zcrx_rq_iter *it,
@@ -1118,8 +1125,16 @@ static inline bool zcrx_rq_iter_next(struct zcrx_rq_iter *it,
struct io_uring_zcrx_rqe **rqe)
{
it->rqes_left--;
- if (unlikely(it->rqes_left < 0))
- return false;
+ if (unlikely(it->rqes_left < 0)) {
+ if (it->flushed)
+ return false;
+ rq->cached_tail = smp_load_acquire(&rq->ring->tail);
+ it->rqes_left = min_t(unsigned, __zcrx_rq_entries(rq),
+ ZCRX_REFILL_CAP);
+ it->flushed = true;
+ if (--it->rqes_left < 0)
+ return false;
+ }
*rqe = zcrx_next_rqe(rq, rq->nr_entries - 1);
return true;
diff --git a/io_uring/zcrx.h b/io_uring/zcrx.h
index 3cdfa4415d62..0eb7ea35a9ff 100644
--- a/io_uring/zcrx.h
+++ b/io_uring/zcrx.h
@@ -53,6 +53,7 @@ struct zcrx_rq {
struct zcrx_rq_hdr *ring;
struct io_uring_zcrx_rqe *rqes;
u32 cached_head;
+ u32 cached_tail;
u32 nr_entries;
};
--
2.54.0
^ permalink raw reply related
* [PATCH review-only 15/17] io_uring/zcrx: add dynamic area creation
From: Pavel Begunkov @ 2026-07-11 9:11 UTC (permalink / raw)
To: io-uring; +Cc: asml.silence, netdev
In-Reply-To: <cover.1783616211.git.asml.silence@gmail.com>
It's not always possible for the user to predict during registration how
much memory zcrx will need to sustain the traffic. Allow to dynamically
add more areas with a new ctrl code ZCRX_CTRL_ADD_AREA.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
include/uapi/linux/io_uring/zcrx.h | 7 +++
io_uring/zcrx.c | 84 +++++++++++++++++++++++++-----
2 files changed, 79 insertions(+), 12 deletions(-)
diff --git a/include/uapi/linux/io_uring/zcrx.h b/include/uapi/linux/io_uring/zcrx.h
index 15c05c45ce36..08cdb173b04b 100644
--- a/include/uapi/linux/io_uring/zcrx.h
+++ b/include/uapi/linux/io_uring/zcrx.h
@@ -116,6 +116,7 @@ enum zcrx_ctrl_op {
ZCRX_CTRL_FLUSH_RQ,
ZCRX_CTRL_EXPORT,
ZCRX_CTRL_ARM_NOTIFICATION,
+ ZCRX_CTRL_ADD_AREA,
__ZCRX_CTRL_LAST,
};
@@ -134,6 +135,11 @@ struct zcrx_ctrl_arm_notif {
__u32 __resv[11];
};
+struct zcrx_ctrl_add_area {
+ __u64 area_ptr; /* pointer to struct io_uring_zcrx_area_reg */
+ __u64 __resv[5];
+};
+
struct zcrx_ctrl {
__u32 zcrx_id;
__u32 op; /* see enum zcrx_ctrl_op */
@@ -143,6 +149,7 @@ struct zcrx_ctrl {
struct zcrx_ctrl_export zc_export;
struct zcrx_ctrl_flush_rq zc_flush;
struct zcrx_ctrl_arm_notif zc_arm_notif;
+ struct zcrx_ctrl_add_area zc_area;
};
};
diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c
index 3f61f942c393..f7592a3c058d 100644
--- a/io_uring/zcrx.c
+++ b/io_uring/zcrx.c
@@ -36,6 +36,7 @@
#define ZCRX_REFILL_CAP MIN(64 * ZCRX_MAX_FRAGS_PER_PAGE, 1024)
#define IO_ZCRX_AREA_SUPPORTED_FLAGS (IORING_ZCRX_AREA_DMABUF)
+#define ZCRX_MAX_AREAS 1024
#define IO_DMA_ATTR (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING)
@@ -46,7 +47,7 @@ static inline u64 zcrx_area_id_to_token(u32 area_id)
static inline u32 zcrx_next_area_id(struct io_zcrx_ifq *zcrx)
{
- return zcrx->nr_areas;
+ return READ_ONCE(zcrx->nr_areas);
}
static inline struct io_zcrx_ifq *io_pp_to_ifq(struct page_pool *pp)
@@ -295,8 +296,6 @@ static int io_import_area(struct io_zcrx_ifq *ifq,
if (area_reg->flags & ~IO_ZCRX_AREA_SUPPORTED_FLAGS)
return -EINVAL;
- if (area_reg->rq_area_token)
- return -EINVAL;
if (area_reg->__resv2[0] || area_reg->__resv2[1])
return -EINVAL;
@@ -311,15 +310,11 @@ static int io_import_area(struct io_zcrx_ifq *ifq,
return io_import_umem(ifq, mem, area_reg);
}
-static void io_zcrx_unmap_area(struct io_zcrx_ifq *ifq,
- struct io_zcrx_area *area)
+static void __io_zcrx_unmap_area(struct io_zcrx_ifq *ifq,
+ struct io_zcrx_area *area)
{
int i;
- if (!area)
- return;
-
- guard(mutex)(&ifq->pp_lock);
if (!area->is_mapped)
return;
area->is_mapped = false;
@@ -337,6 +332,15 @@ static void io_zcrx_unmap_area(struct io_zcrx_ifq *ifq,
}
}
+static void io_zcrx_unmap_area(struct io_zcrx_ifq *ifq,
+ struct io_zcrx_area *area)
+{
+ if (!area)
+ return;
+ guard(mutex)(&ifq->pp_lock);
+ __io_zcrx_unmap_area(ifq, area);
+}
+
static void io_zcrx_unmap_areas(struct io_zcrx_ifq *ifq)
{
unsigned area_idx;
@@ -475,7 +479,9 @@ static int io_zcrx_append_area(struct io_zcrx_ifq *ifq,
struct io_zcrx_area **areas, **old_areas;
unsigned old_nr;
- if (WARN_ON_ONCE(ifq->kern_readable != kern_readable))
+ if (ifq->kern_readable != kern_readable)
+ return -EINVAL;
+ if (ifq->nr_areas + 1 > ZCRX_MAX_AREAS)
return -EINVAL;
if (WARN_ON_ONCE(area->area_id != zcrx_next_area_id(ifq)))
return -EINVAL;
@@ -516,7 +522,7 @@ static int __zcrx_create_area(struct io_zcrx_ifq *ifq,
return -EINVAL;
buf_size_shift = ilog2(rx_buf_len);
}
- if (WARN_ON_ONCE(ifq->niov_shift))
+ if (ifq->niov_shift && ifq->niov_shift != buf_size_shift)
return -EINVAL;
if (!ifq->dev && buf_size_shift != PAGE_SHIFT)
return -EOPNOTSUPP;
@@ -578,7 +584,7 @@ static int __zcrx_create_area(struct io_zcrx_ifq *ifq,
return 0;
err:
if (area) {
- io_zcrx_unmap_area(ifq, area);
+ __io_zcrx_unmap_area(ifq, area);
io_zcrx_free_area(ifq, area);
}
return ret;
@@ -1012,6 +1018,8 @@ int io_register_zcrx(struct io_ring_ctx *ctx,
if (copy_from_user(&area, u64_to_user_ptr(reg.area_ptr), sizeof(area)))
return -EFAULT;
+ if (area.rq_area_token)
+ return -EINVAL;
memset(¬if, 0, sizeof(notif));
if (reg.notif_desc && copy_from_user(¬if, u64_to_user_ptr(reg.notif_desc),
@@ -1074,6 +1082,8 @@ int io_register_zcrx(struct io_ring_ctx *ctx,
goto err;
}
+ WARN_ON_ONCE(!ifq->niov_shift);
+
reg.zcrx_id = id;
scoped_guard(mutex, &ctx->mmap_lock) {
@@ -1559,6 +1569,54 @@ static int zcrx_arm_notif(struct io_ring_ctx *ctx, struct io_zcrx_ifq *zcrx,
return 0;
}
+static int zcrx_ctrl_add_area(struct io_ring_ctx *ctx, struct io_zcrx_ifq *ifq,
+ struct zcrx_ctrl *ctrl)
+{
+ struct zcrx_ctrl_add_area *ctrl_add = &ctrl->zc_area;
+ struct io_uring_zcrx_area_reg __user *area_uptr;
+ struct io_uring_zcrx_area_reg area_reg;
+ struct io_zcrx_area *area = NULL;
+ int ret;
+
+ area_uptr = u64_to_user_ptr(ctrl_add->area_ptr);
+ if (copy_from_user(&area_reg, area_uptr, sizeof(area_reg)))
+ return -EFAULT;
+ if (!mem_is_zero(&ctrl_add->__resv, sizeof(ctrl_add->__resv)))
+ return -EINVAL;
+ if (area_reg.rq_area_token)
+ return -EINVAL;
+
+ while (true) {
+ u32 area_id = zcrx_next_area_id(ifq);
+
+ /*
+ * It's hard to roll back append and page faults under
+ * ->pp_lock is a bad idea. Grab and post an unstable area id
+ * first, and then check-retry under the lock.
+ */
+ area_reg.rq_area_token = zcrx_area_id_to_token(area_id);
+ if (copy_to_user(area_uptr, &area_reg, sizeof(area_reg)))
+ return -EFAULT;
+
+ guard(mutex)(&ifq->pp_lock);
+ if (area_id != zcrx_next_area_id(ifq))
+ continue;
+
+ ret = __zcrx_create_area(ifq, &area_reg, &area,
+ 1U << ifq->niov_shift, area_id);
+ if (ret)
+ break;
+
+ ret = io_zcrx_append_area(ifq, area);
+ if (ret)
+ __io_zcrx_unmap_area(ifq, area);
+ break;
+ }
+ if (ret && area)
+ io_zcrx_free_area(ifq, area);
+ return ret;
+}
+
int io_zcrx_ctrl(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args)
{
struct zcrx_ctrl ctrl;
@@ -1585,6 +1643,8 @@ int io_zcrx_ctrl(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args)
return zcrx_export(ctx, zcrx, &ctrl, arg);
case ZCRX_CTRL_ARM_NOTIFICATION:
return zcrx_arm_notif(ctx, zcrx, &ctrl);
+ case ZCRX_CTRL_ADD_AREA:
+ return zcrx_ctrl_add_area(ctx, zcrx, &ctrl);
}
return -EOPNOTSUPP;
--
2.54.0
^ permalink raw reply related
* [PATCH review-only 14/17] io_uring/zcrx: pass area_id to __zcrx_create_area()
From: Pavel Begunkov @ 2026-07-11 9:11 UTC (permalink / raw)
To: io-uring; +Cc: asml.silence, netdev
In-Reply-To: <cover.1783616211.git.asml.silence@gmail.com>
Instead of generating an area id inside of __zcrx_create_area(), let the
caller to pass it. It needs the id to derive the user token, and we
might need to know it before creating and publishing the area.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
io_uring/zcrx.c | 21 +++++++++++++++------
1 file changed, 15 insertions(+), 6 deletions(-)
diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c
index 474ffc217b0b..3f61f942c393 100644
--- a/io_uring/zcrx.c
+++ b/io_uring/zcrx.c
@@ -44,6 +44,11 @@ static inline u64 zcrx_area_id_to_token(u32 area_id)
return (u64)area_id << IORING_ZCRX_AREA_SHIFT;
}
+static inline u32 zcrx_next_area_id(struct io_zcrx_ifq *zcrx)
+{
+ return zcrx->nr_areas;
+}
+
static inline struct io_zcrx_ifq *io_pp_to_ifq(struct page_pool *pp)
{
return pp->mp_priv;
@@ -472,6 +477,8 @@ static int io_zcrx_append_area(struct io_zcrx_ifq *ifq,
if (WARN_ON_ONCE(ifq->kern_readable != kern_readable))
return -EINVAL;
+ if (WARN_ON_ONCE(area->area_id != zcrx_next_area_id(ifq)))
+ return -EINVAL;
old_areas = ifq->areas;
old_nr = ifq->nr_areas;
@@ -494,9 +501,10 @@ static int io_zcrx_append_area(struct io_zcrx_ifq *ifq,
}
static int __zcrx_create_area(struct io_zcrx_ifq *ifq,
- struct io_uring_zcrx_area_reg *area_reg,
+ const struct io_uring_zcrx_area_reg *area_reg,
struct io_zcrx_area **res_area,
- u32 rx_buf_len)
+ u32 rx_buf_len,
+ u32 area_id)
{
int buf_size_shift = PAGE_SHIFT;
struct io_zcrx_area *area;
@@ -565,9 +573,7 @@ static int __zcrx_create_area(struct io_zcrx_ifq *ifq,
}
area->free_count = nr_iovs;
- /* we're only supporting one area per ifq for now */
- area->area_id = 0;
- area_reg->rq_area_token = zcrx_area_id_to_token(area->area_id);
+ area->area_id = area_id;
*res_area = area;
return 0;
err:
@@ -583,9 +589,12 @@ static int io_zcrx_create_area(struct io_zcrx_ifq *ifq,
struct io_uring_zcrx_ifq_reg *reg)
{
struct io_zcrx_area *area;
+ u32 id = zcrx_next_area_id(ifq);
int ret;
- ret = __zcrx_create_area(ifq, area_reg, &area, reg->rx_buf_len);
+ area_reg->rq_area_token = zcrx_area_id_to_token(id);
+
+ ret = __zcrx_create_area(ifq, area_reg, &area, reg->rx_buf_len, id);
if (ret)
return ret;
--
2.54.0
^ permalink raw reply related
* [PATCH review-only 13/17] io_uring/zcrx: array of areas
From: Pavel Begunkov @ 2026-07-11 9:11 UTC (permalink / raw)
To: io-uring; +Cc: asml.silence, netdev
In-Reply-To: <cover.1783616211.git.asml.silence@gmail.com>
Currently, we have only a one area per zcrx instance, and struct
io_zcrx_ifq stores a single pointer. To prepare for adding more areas,
replace it with an array of areas.
We'll be creating them at runtime, and the array is protected by 3
locks: ->pp_lock, ->alloc_lock and ->rq.lock. It takes all of them when
switching arrays, and readers should hold either of them.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
io_uring/zcrx.c | 95 ++++++++++++++++++++++++++++++++++++-------------
io_uring/zcrx.h | 5 ++-
2 files changed, 75 insertions(+), 25 deletions(-)
diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c
index 81520bda230d..474ffc217b0b 100644
--- a/io_uring/zcrx.c
+++ b/io_uring/zcrx.c
@@ -332,6 +332,14 @@ static void io_zcrx_unmap_area(struct io_zcrx_ifq *ifq,
}
}
+static void io_zcrx_unmap_areas(struct io_zcrx_ifq *ifq)
+{
+ unsigned area_idx;
+
+ for (area_idx = 0; area_idx < ifq->nr_areas; area_idx++)
+ io_zcrx_unmap_area(ifq, ifq->areas[area_idx]);
+}
+
static void zcrx_sync_for_device(struct page_pool *pp, struct io_zcrx_ifq *zcrx,
netmem_ref *netmems, unsigned nr)
{
@@ -459,13 +467,29 @@ static int io_zcrx_append_area(struct io_zcrx_ifq *ifq,
struct io_zcrx_area *area)
{
bool kern_readable = !area->mem.is_dmabuf;
+ struct io_zcrx_area **areas, **old_areas;
+ unsigned old_nr;
- if (WARN_ON_ONCE(ifq->area))
- return -EINVAL;
if (WARN_ON_ONCE(ifq->kern_readable != kern_readable))
return -EINVAL;
- ifq->area = area;
+ old_areas = ifq->areas;
+ old_nr = ifq->nr_areas;
+
+ areas = kmalloc_array(old_nr + 1, sizeof(areas[0]),
+ GFP_KERNEL_ACCOUNT | __GFP_ZERO);
+ if (!areas)
+ return -ENOMEM;
+ if (old_areas)
+ memcpy(areas, old_areas, old_nr * sizeof(areas[0]));
+ areas[old_nr] = area;
+
+ scoped_guard(spinlock_bh, &ifq->rq.lock) {
+ guard(spinlock_bh)(&ifq->alloc_lock);
+ ifq->areas = areas;
+ ifq->nr_areas = old_nr + 1;
+ }
+ kfree(old_areas);
return 0;
}
@@ -621,7 +645,7 @@ static void io_close_queue(struct io_zcrx_ifq *ifq)
if (ifq->if_rxq != -1)
netif_mp_close_rxq(netdev, ifq->if_rxq, &p);
- io_zcrx_unmap_area(ifq, ifq->area);
+ io_zcrx_unmap_areas(ifq);
netdev_unlock(netdev);
netdev_put(netdev, &netdev_tracker);
}
@@ -630,6 +654,8 @@ static void io_close_queue(struct io_zcrx_ifq *ifq)
static void io_zcrx_ifq_free(struct io_zcrx_ifq *ifq)
{
+ int i;
+
if (WARN_ON_ONCE(ifq->if_rxq != -1))
return;
if (WARN_ON_ONCE(ifq->netdev != NULL))
@@ -637,8 +663,8 @@ static void io_zcrx_ifq_free(struct io_zcrx_ifq *ifq)
if (WARN_ON_ONCE(ifq->master_ctx))
return;
- if (ifq->area)
- io_zcrx_free_area(ifq, ifq->area);
+ for (i = 0; i < ifq->nr_areas; i++)
+ io_zcrx_free_area(ifq, ifq->areas[i]);
if (ifq->mm_account)
mmdrop(ifq->mm_account);
if (ifq->dev)
@@ -647,6 +673,7 @@ static void io_zcrx_ifq_free(struct io_zcrx_ifq *ifq)
io_free_rbuf_ring(ifq);
free_uid(ifq->user);
mutex_destroy(&ifq->pp_lock);
+ kfree(ifq->areas);
kfree(ifq);
}
@@ -692,14 +719,10 @@ static void io_zcrx_return_niov(struct net_iov *niov)
page_pool_put_unrefed_netmem(niov->desc.pp, netmem, -1, false);
}
-static void io_zcrx_scrub(struct io_zcrx_ifq *ifq)
+static void io_zcrx_scrub_area(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area)
{
- struct io_zcrx_area *area = ifq->area;
int i;
- if (!area)
- return;
-
/* Reclaim back all buffers given to the user space. */
for (i = 0; i < area->nia.num_niovs; i++) {
struct net_iov *niov = &area->nia.niovs[i];
@@ -713,6 +736,15 @@ static void io_zcrx_scrub(struct io_zcrx_ifq *ifq)
}
}
+static void io_zcrx_scrub(struct io_zcrx_ifq *ifq)
+{
+ int i;
+
+ guard(mutex)(&ifq->pp_lock);
+ for (i = 0; i < ifq->nr_areas; i++)
+ io_zcrx_scrub_area(ifq, ifq->areas[i]);
+}
+
static void zcrx_unregister_user(struct io_zcrx_ifq *ifq, struct io_ring_ctx *ctx)
{
scoped_guard(spinlock_bh, &ifq->ctx_lock) {
@@ -1185,12 +1217,15 @@ static inline bool io_parse_rqe(struct io_uring_zcrx_rqe *rqe,
unsigned niov_idx, area_idx;
struct io_zcrx_area *area;
+ lockdep_assert_held(&ifq->rq.lock);
+
area_idx = off >> IORING_ZCRX_AREA_SHIFT;
niov_idx = (off & ~IORING_ZCRX_AREA_MASK) >> ifq->niov_shift;
- if (unlikely(rqe->__pad || area_idx))
+ if (unlikely(rqe->__pad || area_idx >= ifq->nr_areas))
return false;
- area = ifq->area;
+ area_idx = array_index_nospec(area_idx, ifq->nr_areas);
+ area = ifq->areas[area_idx];
if (unlikely(niov_idx >= area->nia.num_niovs))
return false;
@@ -1260,18 +1295,24 @@ static unsigned io_zcrx_ring_refill(struct page_pool *pp,
static unsigned io_zcrx_refill_slow(struct page_pool *pp, struct io_zcrx_ifq *ifq,
netmem_ref *netmems, unsigned to_alloc)
{
- struct io_zcrx_area *area = ifq->area;
+ unsigned area_idx = 0;
unsigned allocated = 0;
guard(spinlock_bh)(&ifq->alloc_lock);
- for (allocated = 0; allocated < to_alloc; allocated++) {
- struct net_iov *niov = zcrx_get_free_niov(area);
+ while (allocated < to_alloc) {
+ struct net_iov *niov = zcrx_get_free_niov(ifq->areas[area_idx]);
+
+ if (!niov) {
+ area_idx++;
+ if (area_idx >= ifq->nr_areas)
+ break;
+ continue;
+ }
- if (!niov)
- break;
net_mp_niov_set_page_pool(pp, niov);
netmems[allocated] = net_iov_to_netmem(niov);
+ allocated++;
}
return allocated;
}
@@ -1407,8 +1448,8 @@ static void io_pp_uninstall(void *mp_priv, struct netdev_rx_queue *rxq)
struct pp_memory_provider_params *p = &rxq->mp_params;
struct io_zcrx_ifq *ifq = mp_priv;
+ io_zcrx_unmap_areas(ifq);
io_zcrx_drop_netdev(ifq);
- io_zcrx_unmap_area(ifq, ifq->area);
p->mp_ops = NULL;
p->mp_priv = NULL;
@@ -1569,16 +1610,22 @@ static bool io_zcrx_queue_cqe(struct io_kiocb *req, struct net_iov *niov,
static struct net_iov *io_alloc_fallback_niov(struct io_zcrx_ifq *ifq)
{
struct net_iov *niov = NULL;
+ unsigned area_idx;
if (!ifq->kern_readable)
return NULL;
- scoped_guard(spinlock_bh, &ifq->alloc_lock)
- niov = zcrx_get_free_niov(ifq->area);
+ guard(spinlock_bh)(&ifq->alloc_lock);
+
+ for (area_idx = 0; area_idx < ifq->nr_areas; area_idx++) {
+ niov = zcrx_get_free_niov(ifq->areas[area_idx]);
+ if (niov) {
+ page_pool_fragment_netmem(net_iov_to_netmem(niov), 1);
+ return niov;
+ }
+ }
- if (niov)
- page_pool_fragment_netmem(net_iov_to_netmem(niov), 1);
- return niov;
+ return NULL;
}
struct io_copy_cache {
diff --git a/io_uring/zcrx.h b/io_uring/zcrx.h
index 302659669ba4..05598f08eda0 100644
--- a/io_uring/zcrx.h
+++ b/io_uring/zcrx.h
@@ -57,7 +57,10 @@ struct zcrx_rq {
};
struct io_zcrx_ifq {
- struct io_zcrx_area *area;
+ /* read-protected by any of: ->pp_lock, ->alloc_lock, ->rq.lock */
+ struct io_zcrx_area **areas;
+ unsigned nr_areas;
+
unsigned niov_shift;
struct user_struct *user;
struct mm_struct *mm_account;
--
2.54.0
^ permalink raw reply related
* [PATCH review-only 11/17] io_uring/zcrx: split append out of area creation
From: Pavel Begunkov @ 2026-07-11 9:11 UTC (permalink / raw)
To: io-uring; +Cc: asml.silence, netdev
In-Reply-To: <cover.1783616211.git.asml.silence@gmail.com>
A preparation patch, move appending an area from __zcrx_create_area()
to the caller.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
io_uring/zcrx.c | 21 ++++++++++++++++-----
1 file changed, 16 insertions(+), 5 deletions(-)
diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c
index 4936d92f6339..40cabf4384d1 100644
--- a/io_uring/zcrx.c
+++ b/io_uring/zcrx.c
@@ -471,6 +471,7 @@ static int io_zcrx_append_area(struct io_zcrx_ifq *ifq,
static int __zcrx_create_area(struct io_zcrx_ifq *ifq,
struct io_uring_zcrx_area_reg *area_reg,
+ struct io_zcrx_area **res_area,
u32 rx_buf_len)
{
int buf_size_shift = PAGE_SHIFT;
@@ -544,10 +545,8 @@ static int __zcrx_create_area(struct io_zcrx_ifq *ifq,
area->area_id = 0;
area_reg->rq_area_token = zcrx_area_id_to_token(area->area_id);
spin_lock_init(&area->freelist_lock);
-
- ret = io_zcrx_append_area(ifq, area);
- if (!ret)
- return 0;
+ *res_area = area;
+ return 0;
err:
if (area) {
io_zcrx_unmap_area(ifq, area);
@@ -560,7 +559,19 @@ static int io_zcrx_create_area(struct io_zcrx_ifq *ifq,
struct io_uring_zcrx_area_reg *area_reg,
struct io_uring_zcrx_ifq_reg *reg)
{
- return __zcrx_create_area(ifq, area_reg, reg->rx_buf_len);
+ struct io_zcrx_area *area;
+ int ret;
+
+ ret = __zcrx_create_area(ifq, area_reg, &area, reg->rx_buf_len);
+ if (ret)
+ return ret;
+
+ ret = io_zcrx_append_area(ifq, area);
+ if (ret) {
+ io_zcrx_free_area(ifq, area);
+ return ret;
+ }
+ return 0;
}
static struct io_zcrx_ifq *io_zcrx_ifq_alloc(struct io_ring_ctx *ctx)
--
2.54.0
^ permalink raw reply related
* [PATCH review-only 12/17] io_uring/zcrx: move freelist lock to struct zcrx
From: Pavel Begunkov @ 2026-07-11 9:11 UTC (permalink / raw)
To: io-uring; +Cc: asml.silence, netdev
In-Reply-To: <cover.1783616211.git.asml.silence@gmail.com>
freelist_lock, which protects slow path allocations, is currently stored
in struct io_zcrx_area. Once we add support for multiple queues, we'll
need a lock in the zcrx ctx, move it there.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
io_uring/zcrx.c | 14 +++++++-------
io_uring/zcrx.h | 2 +-
2 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c
index 40cabf4384d1..81520bda230d 100644
--- a/io_uring/zcrx.c
+++ b/io_uring/zcrx.c
@@ -544,7 +544,6 @@ static int __zcrx_create_area(struct io_zcrx_ifq *ifq,
/* we're only supporting one area per ifq for now */
area->area_id = 0;
area_reg->rq_area_token = zcrx_area_id_to_token(area->area_id);
- spin_lock_init(&area->freelist_lock);
*res_area = area;
return 0;
err:
@@ -585,6 +584,7 @@ static struct io_zcrx_ifq *io_zcrx_ifq_alloc(struct io_ring_ctx *ctx)
ifq->if_rxq = -1;
spin_lock_init(&ifq->ctx_lock);
spin_lock_init(&ifq->rq.lock);
+ spin_lock_init(&ifq->alloc_lock);
mutex_init(&ifq->pp_lock);
refcount_set(&ifq->refs, 1);
refcount_set(&ifq->user_refs, 1);
@@ -659,8 +659,9 @@ static void io_put_zcrx_ifq(struct io_zcrx_ifq *ifq)
static void io_zcrx_return_niov_freelist(struct net_iov *niov)
{
struct io_zcrx_area *area = io_zcrx_iov_to_area(niov);
+ struct io_zcrx_ifq *ifq = area->ifq;
- guard(spinlock_bh)(&area->freelist_lock);
+ guard(spinlock_bh)(&ifq->alloc_lock);
if (WARN_ON_ONCE(area->free_count >= area->nia.num_niovs))
return;
area->freelist[area->free_count++] = net_iov_idx(niov);
@@ -670,7 +671,7 @@ static struct net_iov *zcrx_get_free_niov(struct io_zcrx_area *area)
{
unsigned niov_idx;
- lockdep_assert_held(&area->freelist_lock);
+ lockdep_assert_held(&area->ifq->alloc_lock);
if (unlikely(!area->free_count))
return NULL;
@@ -1262,7 +1263,7 @@ static unsigned io_zcrx_refill_slow(struct page_pool *pp, struct io_zcrx_ifq *if
struct io_zcrx_area *area = ifq->area;
unsigned allocated = 0;
- guard(spinlock_bh)(&area->freelist_lock);
+ guard(spinlock_bh)(&ifq->alloc_lock);
for (allocated = 0; allocated < to_alloc; allocated++) {
struct net_iov *niov = zcrx_get_free_niov(area);
@@ -1567,14 +1568,13 @@ static bool io_zcrx_queue_cqe(struct io_kiocb *req, struct net_iov *niov,
static struct net_iov *io_alloc_fallback_niov(struct io_zcrx_ifq *ifq)
{
- struct io_zcrx_area *area = ifq->area;
struct net_iov *niov = NULL;
if (!ifq->kern_readable)
return NULL;
- scoped_guard(spinlock_bh, &area->freelist_lock)
- niov = zcrx_get_free_niov(area);
+ scoped_guard(spinlock_bh, &ifq->alloc_lock)
+ niov = zcrx_get_free_niov(ifq->area);
if (niov)
page_pool_fragment_netmem(net_iov_to_netmem(niov), 1);
diff --git a/io_uring/zcrx.h b/io_uring/zcrx.h
index 0eb7ea35a9ff..302659669ba4 100644
--- a/io_uring/zcrx.h
+++ b/io_uring/zcrx.h
@@ -36,7 +36,6 @@ struct io_zcrx_area {
u16 area_id;
/* freelist */
- spinlock_t freelist_lock ____cacheline_aligned_in_smp;
u32 free_count;
u32 *freelist;
@@ -65,6 +64,7 @@ struct io_zcrx_ifq {
bool kern_readable;
struct zcrx_rq rq ____cacheline_aligned_in_smp;
+ spinlock_t alloc_lock ____cacheline_aligned_in_smp;
u32 if_rxq;
struct device *dev;
--
2.54.0
^ permalink raw reply related
* [PATCH review-only 10/17] io_uring/zcrx: unmap under netdev lock
From: Pavel Begunkov @ 2026-07-11 9:11 UTC (permalink / raw)
To: io-uring; +Cc: asml.silence, netdev
In-Reply-To: <cover.1783616211.git.asml.silence@gmail.com>
Make sure we unmap areas while closing a queue.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
io_uring/zcrx.c | 22 ++++++++++++++--------
1 file changed, 14 insertions(+), 8 deletions(-)
diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c
index 86e8046e98c4..4936d92f6339 100644
--- a/io_uring/zcrx.c
+++ b/io_uring/zcrx.c
@@ -311,6 +311,9 @@ static void io_zcrx_unmap_area(struct io_zcrx_ifq *ifq,
{
int i;
+ if (!area)
+ return;
+
guard(mutex)(&ifq->pp_lock);
if (!area->is_mapped)
return;
@@ -438,7 +441,8 @@ static void io_free_rbuf_ring(struct io_zcrx_ifq *ifq)
static void io_zcrx_free_area(struct io_zcrx_ifq *ifq,
struct io_zcrx_area *area)
{
- io_zcrx_unmap_area(ifq, area);
+ if (WARN_ON_ONCE(area->is_mapped))
+ return;
io_release_area_mem(&area->mem);
if (area->mem.account_pages)
@@ -545,8 +549,10 @@ static int __zcrx_create_area(struct io_zcrx_ifq *ifq,
if (!ret)
return 0;
err:
- if (area)
+ if (area) {
+ io_zcrx_unmap_area(ifq, area);
io_zcrx_free_area(ifq, area);
+ }
return ret;
}
@@ -600,11 +606,12 @@ static void io_close_queue(struct io_zcrx_ifq *ifq)
}
if (netdev) {
- if (ifq->if_rxq != -1) {
- netdev_lock(netdev);
+ netdev_lock(netdev);
+ if (ifq->if_rxq != -1)
netif_mp_close_rxq(netdev, ifq->if_rxq, &p);
- netdev_unlock(netdev);
- }
+
+ io_zcrx_unmap_area(ifq, ifq->area);
+ netdev_unlock(netdev);
netdev_put(netdev, &netdev_tracker);
}
ifq->if_rxq = -1;
@@ -1389,8 +1396,7 @@ static void io_pp_uninstall(void *mp_priv, struct netdev_rx_queue *rxq)
struct io_zcrx_ifq *ifq = mp_priv;
io_zcrx_drop_netdev(ifq);
- if (ifq->area)
- io_zcrx_unmap_area(ifq, ifq->area);
+ io_zcrx_unmap_area(ifq, ifq->area);
p->mp_ops = NULL;
p->mp_priv = NULL;
--
2.54.0
^ permalink raw reply related
* [PATCH review-only 09/17] io_uring/zcrx: split dmabuf unmap and release
From: Pavel Begunkov @ 2026-07-11 9:11 UTC (permalink / raw)
To: io-uring; +Cc: asml.silence, netdev
In-Reply-To: <cover.1783616211.git.asml.silence@gmail.com>
Until now unmapping and destroying dmabuf were the same thing. To keep
it consistent with non-dmabuf, split it into two separate helpers. Unmap
destroys mappings and attachements as it should, and release only
putting down the dmabuf fd reference.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
io_uring/zcrx.c | 17 +++++++++++------
1 file changed, 11 insertions(+), 6 deletions(-)
diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c
index 79099a78f8cd..86e8046e98c4 100644
--- a/io_uring/zcrx.c
+++ b/io_uring/zcrx.c
@@ -121,21 +121,25 @@ static int io_populate_area_dma(struct io_zcrx_ifq *ifq,
return 0;
}
-static void io_release_dmabuf(struct io_zcrx_mem *mem)
+static void io_unmap_dmabuf(struct io_zcrx_mem *mem)
{
if (!IS_ENABLED(CONFIG_DMA_SHARED_BUFFER))
return;
-
if (mem->sgt)
dma_buf_unmap_attachment_unlocked(mem->attach, mem->sgt,
DMA_FROM_DEVICE);
if (mem->attach)
dma_buf_detach(mem->dmabuf, mem->attach);
- if (mem->dmabuf)
- dma_buf_put(mem->dmabuf);
-
mem->sgt = NULL;
mem->attach = NULL;
+}
+
+static void io_release_dmabuf(struct io_zcrx_mem *mem)
+{
+ if (!IS_ENABLED(CONFIG_DMA_SHARED_BUFFER))
+ return;
+ if (mem->dmabuf)
+ dma_buf_put(mem->dmabuf);
mem->dmabuf = NULL;
}
@@ -190,6 +194,7 @@ static int io_import_dmabuf(struct io_zcrx_ifq *ifq,
mem->size = len;
return 0;
err:
+ io_unmap_dmabuf(mem);
io_release_dmabuf(mem);
return ret;
}
@@ -317,7 +322,7 @@ static void io_zcrx_unmap_area(struct io_zcrx_ifq *ifq,
}
if (area->mem.is_dmabuf) {
- io_release_dmabuf(&area->mem);
+ io_unmap_dmabuf(&area->mem);
} else {
dma_unmap_sgtable(ifq->dev, &area->mem.page_sg_table,
DMA_FROM_DEVICE, IO_DMA_ATTR);
--
2.54.0
^ permalink raw reply related
* [PATCH review-only 08/17] io_uring/zcrx: don't pass ifq_reg to area creation
From: Pavel Begunkov @ 2026-07-11 9:11 UTC (permalink / raw)
To: io-uring; +Cc: asml.silence, netdev
In-Reply-To: <cover.1783616211.git.asml.silence@gmail.com>
We might want to create an area without having an instance of struct
io_uring_zcrx_ifq_reg. Extract a helper that doesn't have the ifq
registration structure as an argument but takes the buf length
explicitly.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
io_uring/zcrx.c | 20 ++++++++++++++------
1 file changed, 14 insertions(+), 6 deletions(-)
diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c
index cfbfbd262f90..79099a78f8cd 100644
--- a/io_uring/zcrx.c
+++ b/io_uring/zcrx.c
@@ -460,21 +460,22 @@ static int io_zcrx_append_area(struct io_zcrx_ifq *ifq,
return 0;
}
-static int io_zcrx_create_area(struct io_zcrx_ifq *ifq,
+static int __zcrx_create_area(struct io_zcrx_ifq *ifq,
struct io_uring_zcrx_area_reg *area_reg,
- struct io_uring_zcrx_ifq_reg *reg)
+ u32 rx_buf_len)
{
int buf_size_shift = PAGE_SHIFT;
struct io_zcrx_area *area;
unsigned nr_iovs;
int i, ret;
- if (reg->rx_buf_len) {
- if (!is_power_of_2(reg->rx_buf_len) ||
- reg->rx_buf_len < PAGE_SIZE)
+ if (rx_buf_len) {
+ if (!is_power_of_2(rx_buf_len) || rx_buf_len < PAGE_SIZE)
return -EINVAL;
- buf_size_shift = ilog2(reg->rx_buf_len);
+ buf_size_shift = ilog2(rx_buf_len);
}
+ if (WARN_ON_ONCE(ifq->niov_shift))
+ return -EINVAL;
if (!ifq->dev && buf_size_shift != PAGE_SHIFT)
return -EOPNOTSUPP;
@@ -544,6 +545,13 @@ static int io_zcrx_create_area(struct io_zcrx_ifq *ifq,
return ret;
}
+static int io_zcrx_create_area(struct io_zcrx_ifq *ifq,
+ struct io_uring_zcrx_area_reg *area_reg,
+ struct io_uring_zcrx_ifq_reg *reg)
+{
+ return __zcrx_create_area(ifq, area_reg, reg->rx_buf_len);
+}
+
static struct io_zcrx_ifq *io_zcrx_ifq_alloc(struct io_ring_ctx *ctx)
{
struct io_zcrx_ifq *ifq;
--
2.54.0
^ permalink raw reply related
* [PATCH review-only 06/17] io_uring/zcrx: constify area_reg on import
From: Pavel Begunkov @ 2026-07-11 9:11 UTC (permalink / raw)
To: io-uring; +Cc: asml.silence, netdev
In-Reply-To: <cover.1783616211.git.asml.silence@gmail.com>
io_import_area() doesn't modify its struct io_uring_zcrx_area_reg
argument, add const to enforce that, it'll make later modifications
easier.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
io_uring/zcrx.c | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c
index cb73dca3c1ee..9f21ae61b862 100644
--- a/io_uring/zcrx.c
+++ b/io_uring/zcrx.c
@@ -136,7 +136,7 @@ static void io_release_dmabuf(struct io_zcrx_mem *mem)
static int io_import_dmabuf(struct io_zcrx_ifq *ifq,
struct io_zcrx_mem *mem,
- struct io_uring_zcrx_area_reg *area_reg)
+ const struct io_uring_zcrx_area_reg *area_reg)
{
unsigned long off = (unsigned long)area_reg->addr;
unsigned long len = (unsigned long)area_reg->len;
@@ -208,7 +208,7 @@ static unsigned long io_count_account_pages(struct page **pages, unsigned nr_pag
static int io_import_umem(struct io_zcrx_ifq *ifq,
struct io_zcrx_mem *mem,
- struct io_uring_zcrx_area_reg *area_reg)
+ const struct io_uring_zcrx_area_reg *area_reg)
{
struct page **pages;
int nr_pages, ret;
@@ -274,7 +274,7 @@ static void io_release_area_mem(struct io_zcrx_mem *mem)
static int io_import_area(struct io_zcrx_ifq *ifq,
struct io_zcrx_mem *mem,
- struct io_uring_zcrx_area_reg *area_reg)
+ const struct io_uring_zcrx_area_reg *area_reg)
{
int ret;
--
2.54.0
^ permalink raw reply related
* [PATCH review-only 07/17] io_uring/zcrx: add helper for deriving area token
From: Pavel Begunkov @ 2026-07-11 9:11 UTC (permalink / raw)
To: io-uring; +Cc: asml.silence, netdev
In-Reply-To: <cover.1783616211.git.asml.silence@gmail.com>
Add zcrx_area_id_to_token() to deduplicate the way the area token is
calculated out of the area index.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
io_uring/zcrx.c | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c
index 9f21ae61b862..cfbfbd262f90 100644
--- a/io_uring/zcrx.c
+++ b/io_uring/zcrx.c
@@ -39,6 +39,11 @@
#define IO_DMA_ATTR (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING)
+static inline u64 zcrx_area_id_to_token(u32 area_id)
+{
+ return (u64)area_id << IORING_ZCRX_AREA_SHIFT;
+}
+
static inline struct io_zcrx_ifq *io_pp_to_ifq(struct page_pool *pp)
{
return pp->mp_priv;
@@ -527,7 +532,7 @@ static int io_zcrx_create_area(struct io_zcrx_ifq *ifq,
area->free_count = nr_iovs;
/* we're only supporting one area per ifq for now */
area->area_id = 0;
- area_reg->rq_area_token = (u64)area->area_id << IORING_ZCRX_AREA_SHIFT;
+ area_reg->rq_area_token = zcrx_area_id_to_token(area->area_id);
spin_lock_init(&area->freelist_lock);
ret = io_zcrx_append_area(ifq, area);
@@ -1525,7 +1530,7 @@ static bool io_zcrx_queue_cqe(struct io_kiocb *req, struct net_iov *niov,
area = io_zcrx_iov_to_area(niov);
offset = off + (net_iov_idx(niov) << ifq->niov_shift);
rcqe = (struct io_uring_zcrx_cqe *)(cqe + 1);
- rcqe->off = offset + ((u64)area->area_id << IORING_ZCRX_AREA_SHIFT);
+ rcqe->off = offset + zcrx_area_id_to_token(area->area_id);
rcqe->__pad = 0;
return true;
}
--
2.54.0
^ permalink raw reply related
* [PATCH review-only 05/17] io_uring/zcrx: coalesce same-niov RQEs on refill
From: Pavel Begunkov @ 2026-07-11 9:11 UTC (permalink / raw)
To: io-uring; +Cc: asml.silence, netdev
In-Reply-To: <cover.1783616211.git.asml.silence@gmail.com>
With large rx piages I often see >10 sequential RQEs referring to the
same niov. Instead of putting them one by one, count such RQEs during
parsing and batch refcounting for the niov.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
io_uring/zcrx.c | 56 +++++++++++++++++++++++++++++++------------------
1 file changed, 36 insertions(+), 20 deletions(-)
diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c
index 1b8d748b35e7..cb73dca3c1ee 100644
--- a/io_uring/zcrx.c
+++ b/io_uring/zcrx.c
@@ -359,16 +359,16 @@ static inline atomic_t *io_get_user_counter(struct net_iov *niov)
return &area->user_refs[net_iov_idx(niov)];
}
-static bool io_zcrx_put_niov_uref(struct net_iov *niov)
+static bool io_zcrx_put_niov_uref(struct net_iov *niov, unsigned refs)
{
atomic_t *uref = io_get_user_counter(niov);
int old;
old = atomic_read(uref);
do {
- if (unlikely(old == 0))
+ if (unlikely(old < refs))
return false;
- } while (!atomic_try_cmpxchg(uref, &old, old - 1));
+ } while (!atomic_try_cmpxchg(uref, &old, old - refs));
return true;
}
@@ -1163,6 +1163,22 @@ static inline bool io_parse_rqe(struct io_uring_zcrx_rqe *rqe,
return true;
}
+static bool zcrx_put_refill_niov(struct net_iov *niov, struct page_pool *pp,
+ unsigned refs)
+{
+ netmem_ref netmem = net_iov_to_netmem(niov);
+
+ if (!io_zcrx_put_niov_uref(niov, refs))
+ return false;
+ if (page_pool_unref_netmem(netmem, refs) != 0)
+ return false;
+ if (unlikely(niov->desc.pp != pp)) {
+ io_zcrx_return_niov(niov);
+ return false;
+ }
+ return true;
+}
+
static unsigned io_zcrx_ring_refill(struct page_pool *pp,
struct io_zcrx_ifq *ifq,
netmem_ref *netmems, unsigned to_alloc)
@@ -1170,34 +1186,34 @@ static unsigned io_zcrx_ring_refill(struct page_pool *pp,
struct zcrx_rq *rq = &ifq->rq;
struct io_uring_zcrx_rqe *rqe;
struct zcrx_rq_iter it;
+ struct net_iov *niov = NULL;
+ unsigned niov_refs = 0;
unsigned allocated = 0;
guard(spinlock_bh)(&rq->lock);
zcrx_rq_iter_init(&it, rq);
- while (zcrx_rq_iter_next(&it, rq, &rqe)) {
- struct net_iov *niov;
- netmem_ref netmem;
+ while (allocated < to_alloc - 1 && zcrx_rq_iter_next(&it, rq, &rqe)) {
+ struct net_iov *next_niov;
- if (!io_parse_rqe(rqe, ifq, &niov))
- continue;
- if (!io_zcrx_put_niov_uref(niov))
+ if (!io_parse_rqe(rqe, ifq, &next_niov))
continue;
-
- netmem = net_iov_to_netmem(niov);
- if (!page_pool_unref_and_test(netmem))
- continue;
-
- if (unlikely(niov->desc.pp != pp)) {
- io_zcrx_return_niov(niov);
+ if (niov == next_niov) {
+ niov_refs++;
continue;
}
+ if (niov && zcrx_put_refill_niov(niov, pp, niov_refs)) {
+ netmems[allocated] = net_iov_to_netmem(niov);
+ allocated++;
+ }
+ niov = next_niov;
+ niov_refs = 1;
+ }
- netmems[allocated] = netmem;
+ if (niov && zcrx_put_refill_niov(niov, pp, niov_refs)) {
+ netmems[allocated] = net_iov_to_netmem(niov);
allocated++;
- if (allocated >= to_alloc)
- break;
}
smp_store_release(&rq->ring->head, rq->cached_head);
@@ -1401,7 +1417,7 @@ static void zcrx_return_buffers(netmem_ref *netmems, unsigned nr)
netmem_ref netmem = netmems[i];
struct net_iov *niov = netmem_to_net_iov(netmem);
- if (!io_zcrx_put_niov_uref(niov))
+ if (!io_zcrx_put_niov_uref(niov, 1))
continue;
if (!page_pool_unref_and_test(netmem))
continue;
--
2.54.0
^ permalink raw reply related
* [PATCH review-only 03/17] io_uring/zcrx: add RQ iterator
From: Pavel Begunkov @ 2026-07-11 9:11 UTC (permalink / raw)
To: io-uring; +Cc: asml.silence, netdev
In-Reply-To: <cover.1783616211.git.asml.silence@gmail.com>
Add a iterator structure and helper functions for the refill queue
processing to avoid polluting io_zcrx_ring_refill() with extra state
and logic once it's extended in following patches.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
io_uring/zcrx.c | 32 ++++++++++++++++++++++++++------
1 file changed, 26 insertions(+), 6 deletions(-)
diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c
index c4a9a663eba4..45b178afbbc3 100644
--- a/io_uring/zcrx.c
+++ b/io_uring/zcrx.c
@@ -1088,6 +1088,10 @@ void io_unregister_zcrx(struct io_ring_ctx *ctx)
xa_destroy(&ctx->zcrx_ctxs);
}
+struct zcrx_rq_iter {
+ int rqes_left;
+};
+
static inline u32 zcrx_rq_entries(struct zcrx_rq *rq)
{
u32 entries;
@@ -1103,6 +1107,24 @@ static struct io_uring_zcrx_rqe *zcrx_next_rqe(struct zcrx_rq *rq, unsigned mask
return &rq->rqes[idx];
}
+static inline void zcrx_rq_iter_init(struct zcrx_rq_iter *it,
+ struct zcrx_rq *rq)
+{
+ it->rqes_left = min_t(unsigned, zcrx_rq_entries(rq), ZCRX_REFILL_CAP);
+}
+
+static inline bool zcrx_rq_iter_next(struct zcrx_rq_iter *it,
+ struct zcrx_rq *rq,
+ struct io_uring_zcrx_rqe **rqe)
+{
+ it->rqes_left--;
+ if (unlikely(it->rqes_left < 0))
+ return false;
+
+ *rqe = zcrx_next_rqe(rq, rq->nr_entries - 1);
+ return true;
+}
+
static inline bool io_parse_rqe(struct io_uring_zcrx_rqe *rqe,
struct io_zcrx_ifq *ifq,
struct net_iov **ret_niov)
@@ -1131,17 +1153,15 @@ static unsigned io_zcrx_ring_refill(struct page_pool *pp,
netmem_ref *netmems, unsigned to_alloc)
{
struct zcrx_rq *rq = &ifq->rq;
- unsigned int mask = rq->nr_entries - 1;
- unsigned int rqes_left;
+ struct io_uring_zcrx_rqe *rqe;
+ struct zcrx_rq_iter it;
unsigned allocated = 0;
guard(spinlock_bh)(&rq->lock);
- rqes_left = zcrx_rq_entries(rq);
- rqes_left = min_t(unsigned, rqes_left, ZCRX_REFILL_CAP);
+ zcrx_rq_iter_init(&it, rq);
- for (; rqes_left; rqes_left--) {
- struct io_uring_zcrx_rqe *rqe = zcrx_next_rqe(rq, mask);
+ while (zcrx_rq_iter_next(&it, rq, &rqe)) {
struct net_iov *niov;
netmem_ref netmem;
--
2.54.0
^ permalink raw reply related
* [PATCH review-only 02/17] io_uring/zcrx: move RQ head/tail to separate cache lines
From: Pavel Begunkov @ 2026-07-11 9:11 UTC (permalink / raw)
To: io-uring; +Cc: asml.silence, netdev
In-Reply-To: <cover.1783616211.git.asml.silence@gmail.com>
RQ head and tail are currently put into the same cache line, which can
cause false sharing problems when refill is run on another CPU. Put them
into separate cache lines.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
io_uring/query.c | 2 +-
io_uring/zcrx.c | 8 ++++----
io_uring/zcrx.h | 7 ++++++-
3 files changed, 11 insertions(+), 6 deletions(-)
diff --git a/io_uring/query.c b/io_uring/query.c
index d529d94aa8f4..2e7b893cc8f0 100644
--- a/io_uring/query.c
+++ b/io_uring/query.c
@@ -38,7 +38,7 @@ static ssize_t io_query_zcrx(union io_query_data *data)
e->register_flags = ZCRX_SUPPORTED_REG_FLAGS;
e->area_flags = IORING_ZCRX_AREA_DMABUF;
e->nr_ctrl_opcodes = __ZCRX_CTRL_LAST;
- e->rq_hdr_size = sizeof(struct io_uring);
+ e->rq_hdr_size = sizeof(struct zcrx_rq_hdr);
e->rq_hdr_alignment = L1_CACHE_BYTES;
e->features = ZCRX_FEATURES;
e->__resv2 = 0;
diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c
index 8348413d6d24..c4a9a663eba4 100644
--- a/io_uring/zcrx.c
+++ b/io_uring/zcrx.c
@@ -380,9 +380,9 @@ static void io_zcrx_get_niov_uref(struct net_iov *niov)
static void io_fill_zcrx_offsets(struct io_uring_zcrx_offsets *offsets)
{
- offsets->head = offsetof(struct io_uring, head);
- offsets->tail = offsetof(struct io_uring, tail);
- offsets->rqes = ALIGN(sizeof(struct io_uring), L1_CACHE_BYTES);
+ offsets->head = offsetof(struct zcrx_rq_hdr, head);
+ offsets->tail = offsetof(struct zcrx_rq_hdr, tail);
+ offsets->rqes = ALIGN(sizeof(struct zcrx_rq_hdr), L1_CACHE_BYTES);
}
static int io_allocate_rbuf_ring(struct io_ring_ctx *ctx,
@@ -410,7 +410,7 @@ static int io_allocate_rbuf_ring(struct io_ring_ctx *ctx,
return ret;
ptr = io_region_get_ptr(&ifq->rq_region);
- ifq->rq.ring = (struct io_uring *)ptr;
+ ifq->rq.ring = (struct zcrx_rq_hdr *)ptr;
ifq->rq.rqes = (struct io_uring_zcrx_rqe *)(ptr + off);
memset(ifq->rq.ring, 0, sizeof(*ifq->rq.ring));
diff --git a/io_uring/zcrx.h b/io_uring/zcrx.h
index fa00900e479e..3cdfa4415d62 100644
--- a/io_uring/zcrx.h
+++ b/io_uring/zcrx.h
@@ -43,9 +43,14 @@ struct io_zcrx_area {
struct io_zcrx_mem mem;
};
+struct zcrx_rq_hdr {
+ u32 head ____cacheline_aligned_in_smp;
+ u32 tail ____cacheline_aligned_in_smp;
+};
+
struct zcrx_rq {
spinlock_t lock;
- struct io_uring *ring;
+ struct zcrx_rq_hdr *ring;
struct io_uring_zcrx_rqe *rqes;
u32 cached_head;
u32 nr_entries;
--
2.54.0
^ permalink raw reply related
* [PATCH review-only 01/17] io_uring/zcrx: scale refilling with large pages
From: Pavel Begunkov @ 2026-07-11 9:11 UTC (permalink / raw)
To: io-uring; +Cc: asml.silence, netdev
In-Reply-To: <cover.1783616211.git.asml.silence@gmail.com>
io_zcrx_ring_refill() caps the loop by mixing the max number of
allocated netmems and the number of available RQEs together, which
caps the number of entries to process the pp cache size. As a result,
when niovs are heavily fragmented, the refilling logic allocates only a
small number of niovs per call on average and sometimes even none.
Keep a separate counter for the number of processed RQ entries, which is
capped by a roughly calculated from the page size value to keep the
cache full. And separately break if it allocates enough niovs.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
io_uring/zcrx.c | 21 ++++++++++++++-------
1 file changed, 14 insertions(+), 7 deletions(-)
diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c
index 6bd71435e475..8348413d6d24 100644
--- a/io_uring/zcrx.c
+++ b/io_uring/zcrx.c
@@ -28,6 +28,13 @@
#include "zcrx.h"
#include "rsrc.h"
+#define ZCRX_MAX_FRAGS_PER_PAGE MAX(PAGE_SIZE / 1024, 1)
+/*
+ * We need a reasonable limit to be able to fill in 64 entries on average
+ * for 1500 byte MTU. Over-estimate it to keep it pow2.
+ */
+#define ZCRX_REFILL_CAP MIN(64 * ZCRX_MAX_FRAGS_PER_PAGE, 1024)
+
#define IO_ZCRX_AREA_SUPPORTED_FLAGS (IORING_ZCRX_AREA_DMABUF)
#define IO_DMA_ATTR (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING)
@@ -1125,17 +1132,15 @@ static unsigned io_zcrx_ring_refill(struct page_pool *pp,
{
struct zcrx_rq *rq = &ifq->rq;
unsigned int mask = rq->nr_entries - 1;
- unsigned int entries;
+ unsigned int rqes_left;
unsigned allocated = 0;
guard(spinlock_bh)(&rq->lock);
- entries = zcrx_rq_entries(rq);
- entries = min_t(unsigned, entries, to_alloc);
- if (unlikely(!entries))
- return 0;
+ rqes_left = zcrx_rq_entries(rq);
+ rqes_left = min_t(unsigned, rqes_left, ZCRX_REFILL_CAP);
- do {
+ for (; rqes_left; rqes_left--) {
struct io_uring_zcrx_rqe *rqe = zcrx_next_rqe(rq, mask);
struct net_iov *niov;
netmem_ref netmem;
@@ -1156,7 +1161,9 @@ static unsigned io_zcrx_ring_refill(struct page_pool *pp,
netmems[allocated] = netmem;
allocated++;
- } while (--entries);
+ if (allocated >= to_alloc)
+ break;
+ }
smp_store_release(&rq->ring->head, rq->cached_head);
return allocated;
--
2.54.0
^ permalink raw reply related
* [PATCH review-only 00/17] zcrx RQ improvements and dynamic memory provisioning
From: Pavel Begunkov @ 2026-07-11 9:11 UTC (permalink / raw)
To: io-uring; +Cc: asml.silence, netdev
Sending it out mainly to trigger review bots. The first half improves
the refill queue implementation and improves refilling limits, which
shows up when niovs are heavily fragmented like with large rx pages.
The 2nd half adds dynamic backing memory provisioning.
Pavel Begunkov (17):
io_uring/zcrx: scale refilling with large pages
io_uring/zcrx: move RQ head/tail to separate cache lines
io_uring/zcrx: add RQ iterator
io_uring/zcrx: cache RQ tail
io_uring/zcrx: coalesce same-niov RQEs on refill
io_uring/zcrx: constify area_reg on import
io_uring/zcrx: add helper for deriving area token
io_uring/zcrx: don't pass ifq_reg to area creation
io_uring/zcrx: split dmabuf unmap and release
io_uring/zcrx: unmap under netdev lock
io_uring/zcrx: split append out of area creation
io_uring/zcrx: move freelist lock to struct zcrx
io_uring/zcrx: array of areas
io_uring/zcrx: pass area_id to __zcrx_create_area()
io_uring/zcrx: add dynamic area creation
io_urint/zcrx: narrow var scope in io_zcrx_recv_skb()
io_uring/zcrx: don't reload skb_shinfo
include/uapi/linux/io_uring/zcrx.h | 7 +
io_uring/query.c | 2 +-
io_uring/zcrx.c | 445 +++++++++++++++++++++--------
io_uring/zcrx.h | 15 +-
4 files changed, 345 insertions(+), 124 deletions(-)
--
2.54.0
^ permalink raw reply
* Re: Ethtool is missing C2C link modes
From: Maxime Chevallier @ 2026-07-11 8:42 UTC (permalink / raw)
To: David Laight, Andrew Lunn
Cc: D H, Siddaraju, Michal Kubecek, netdev@vger.kernel.org,
Chintalapalle, Balaji, Das, Shubham, Srinivasan, Vijay,
Samudrala, Sridhar, Keller, Jacob E, Nguyen, Anthony L,
singhai.anjali55@gmail.com, Brandeburg, Jesse
In-Reply-To: <20260711073128.7f94e0ac@pumpkin>
Hi,
On 7/11/26 08:31, David Laight wrote:
> On Sat, 11 Jul 2026 00:54:00 +0200
> Andrew Lunn <andrew@lunn.ch> wrote:
>
>> On Fri, Jul 10, 2026 at 09:45:43PM +0000, D H, Siddaraju wrote:
>>> Hello Linux Ethernet team, Maxime, Andrew & Michal,
>>>
>>> The IEEE AUI chip-to-chip (C2C) is the accepted standard for connecting
>>> chips that handle subfunctions within the OSI physical layer. Just to
>>> pick, the C2C is widely used when connecting Ethernet SoCs with retimers
>>> and PCS SerDes terminated external-phys to offload PHY sublayer functions.
>>
>> It cannot be that widely used if Linux does not support it yet :-)
>
> It also seems like something that is fixed for a physical board.
> So while a common MAC driver would need to be told how to configure
> its output, the user wouldn't be changing the value so it would
> be more of a DT parameter than an ethtool one.
>
> ...
>> Also, an architecture question...
>>
>> It sounds like you use this between the MAC and the PCS. The PCS can
>> then be connected to a PHY, and the PHY then has a line side. (I'm
>> being a bit loose with the terms here, i should probably be saying
>> PMA, PMD etc.)
>>
>> Should ethtool be saying:
>>
>> Settings for eth0:
>> Supported ports: [ TP MII ]
>> Supported link modes: 25000baseC2C
>>
>> or should it be reporting:
>>
>> Settings for eth0:
>> Supported ports: [ TP MII ]
>> Supported link modes: 25000baseSR
>>
>> I _think_ ethtool reports the media, not some intermediary format.
>
> You'd want to use ethtool to set the final link parameters of the
> external phy?
> So I think you's still want to be able to select (say) 100MHDX
> for a TP link.
>
> Remember the history.
> The parameter was originally used to select between the the AUI, COAX and TP
> connectors on a 10M ethernet card.
> Then the internal TP gained extra speeds.
> We then get MII for external 10M and 100M PHY, later RGMII for external Ge PHY.
> But you rarely get boards (not MAC chips) that have a choice of interfaces
> any more.
Note that there's ongoing work[1] to better support nics with multiple connectors,
which also includes reporting what the 'media-side' of a MAC / PHY can do :
[1]: https://lore.kernel.org/netdev/20260701110427.143945-1-maxime.chevallier@bootlin.com/#t
Taking the example of a combo-port (SFP + RJ45), it would look like that :
ethtool --show-ports eth1
Port for eth1:
Port id: 1
Supported link modes: 10baseT/Half 10baseT/Full
100baseT/Half 100baseT/Full
1000baseT/Full
10000baseT/Full
2500baseT/Full
5000baseT/Full
Port type: mdi
Port for eth1:
Port id: 2
Supported MII interfaces : 10gbase-r
Port type: sfp
or even
# ethtool --show-ports eth3
Port for eth3:
Port id: 1
Supported MII interfaces : sgmii, 1000base-x, 2500base-x
Port type: sfp
Now, with this infrastucture also comes the ability to list the media-side
interfaces that are not MDI but rather MII.
The main goal is combo-port support, but also media-converters, e.g. things
like :
MAC ------ PHY ------- <something>
rgmii sgmii
(for now, the ongoing series focus on supporting this through SFP, but extending
that to other link types is something I'd like to achieve)
As Andrew says, when you use "ethtool ethX", the list you get is the media-side
modes that you can use at the connector, so it's an aggregated list of the MDI
that are usable based on the MAC, PCS, PHY you're using (MAC and PCS would limit
speed/encoding, PHY defines the actual MDI modes)
For the C2C modes, it's not clear to me if these are MDI modes, i.e:
ETHTOOL_LINK_MODE_25000baseC2C
Or a phy_interface_t, i.e:
PHY_INTERFACE_MODE_25GAUI
We already have XAUI and RXAUI as phy_interface_t as of today, so it looks like
we don't want an ethool linkmode for that but rather a phy_interface_t
Maxime
^ permalink raw reply
* [PATCH v2 8/8] clk: sunxi-ng: a733: Add reset lines
From: Junhui Liu @ 2026-07-11 8:10 UTC (permalink / raw)
To: Michael Turquette, Stephen Boyd, Brian Masney, Rob Herring,
Krzysztof Kozlowski, Conor Dooley, Chen-Yu Tsai, Jernej Skrabec,
Samuel Holland, Philipp Zabel, Junhui Liu, Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Richard Cochran
Cc: linux-clk, devicetree, linux-arm-kernel, linux-sunxi,
linux-kernel, linux-riscv, netdev, Jerome Brunet
In-Reply-To: <20260711-a733-clk-v2-0-974d188cbe0c@pigmoral.tech>
Add the reset lines for the Allwinner A733 SoC. These reset control bits
are integrated into the Bus Gate Reset (BGR) registers, typically
sharing the same register address with their corresponding bus clock
gates. Integrate them into the main CCU driver using the existing
sunxi-ng ccu_reset framework, allowing the CCU to also function as a
reset controller for the SoC.
Tested-by: Jerome Brunet <jbrunet@baylibre.com>
Signed-off-by: Junhui Liu <junhui.liu@pigmoral.tech>
---
drivers/clk/sunxi-ng/ccu-sun60i-a733.c | 126 +++++++++++++++++++++++++++++++++
1 file changed, 126 insertions(+)
diff --git a/drivers/clk/sunxi-ng/ccu-sun60i-a733.c b/drivers/clk/sunxi-ng/ccu-sun60i-a733.c
index 48d7c0395ae8..ff64c35206d2 100644
--- a/drivers/clk/sunxi-ng/ccu-sun60i-a733.c
+++ b/drivers/clk/sunxi-ng/ccu-sun60i-a733.c
@@ -17,6 +17,7 @@
#include "../clk.h"
#include "ccu_common.h"
+#include "ccu_reset.h"
#include "ccu_div.h"
#include "ccu_gate.h"
@@ -2156,11 +2157,136 @@ static struct clk_hw_onecell_data sun60i_a733_hw_clks = {
.num = CLK_FANOUT3 + 1,
};
+static struct ccu_reset_map sun60i_a733_ccu_resets[] = {
+ [RST_BUS_ITS_PCIE0] = { 0x574, BIT(16) },
+ [RST_BUS_IOMMU0_SYS] = { 0x58c, BIT(16) },
+ [RST_BUS_MSI_LITE0_AHB] = { 0x594, BIT(16) },
+ [RST_BUS_MSI_LITE0_MBUS] = { 0x594, BIT(17) },
+ [RST_BUS_MSI_LITE1_AHB] = { 0x59c, BIT(16) },
+ [RST_BUS_MSI_LITE1_MBUS] = { 0x59c, BIT(17) },
+ [RST_BUS_MSI_LITE2_AHB] = { 0x5a4, BIT(16) },
+ [RST_BUS_MSI_LITE2_MBUS] = { 0x5a4, BIT(17) },
+ [RST_BUS_IOMMU1_SYS] = { 0x5b4, BIT(16) },
+ [RST_BUS_DMA0] = { 0x704, BIT(16) },
+ [RST_BUS_DMA1] = { 0x70c, BIT(16) },
+ [RST_BUS_SPINLOCK] = { 0x724, BIT(16) },
+ [RST_BUS_MSGBOX] = { 0x744, BIT(16) },
+ [RST_BUS_PWM0] = { 0x784, BIT(16) },
+ [RST_BUS_PWM1] = { 0x78c, BIT(16) },
+ [RST_BUS_DBG] = { 0x7a4, BIT(16) },
+ [RST_BUS_SYSDAP] = { 0x7ac, BIT(16) },
+ [RST_BUS_TIMER0] = { 0x850, BIT(16) },
+ [RST_BUS_DE0] = { 0xa04, BIT(16) },
+ [RST_BUS_DI] = { 0xa24, BIT(16) },
+ [RST_BUS_G2D] = { 0xa44, BIT(16) },
+ [RST_BUS_EINK] = { 0xa6c, BIT(16) },
+ [RST_BUS_DE_SYS] = { 0xa74, BIT(16) },
+ [RST_BUS_VE_ENC0] = { 0xa8c, BIT(16) },
+ [RST_BUS_VE_DEC0] = { 0xa8c, BIT(18) },
+ [RST_BUS_CE] = { 0xac4, BIT(16) },
+ [RST_BUS_CE_SYS] = { 0xac4, BIT(17) },
+ [RST_BUS_NPU_CORE] = { 0xb04, BIT(16) },
+ [RST_BUS_NPU_AXI] = { 0xb04, BIT(17) },
+ [RST_BUS_NPU_AHB] = { 0xb04, BIT(18) },
+ [RST_BUS_NPU_SRAM] = { 0xb04, BIT(19) },
+ [RST_BUS_GPU0] = { 0xb24, BIT(16) },
+ [RST_BUS_DRAM0] = { 0xc0c, BIT(16) },
+ [RST_BUS_NAND0] = { 0xc8c, BIT(16) },
+ [RST_BUS_MMC0] = { 0xd0c, BIT(16) },
+ [RST_BUS_MMC1] = { 0xd1c, BIT(16) },
+ [RST_BUS_MMC2] = { 0xd2c, BIT(16) },
+ [RST_BUS_MMC3] = { 0xd3c, BIT(16) },
+ [RST_BUS_UFS_AHB] = { 0xd8c, BIT(16) },
+ [RST_BUS_UFS_AXI] = { 0xd8c, BIT(17) },
+ [RST_BUS_UFS_PHY] = { 0xd8c, BIT(18) },
+ [RST_BUS_UFS_CORE] = { 0xd8c, BIT(19) },
+ [RST_BUS_UART0] = { 0xe00, BIT(16) },
+ [RST_BUS_UART1] = { 0xe04, BIT(16) },
+ [RST_BUS_UART2] = { 0xe08, BIT(16) },
+ [RST_BUS_UART3] = { 0xe0c, BIT(16) },
+ [RST_BUS_UART4] = { 0xe10, BIT(16) },
+ [RST_BUS_UART5] = { 0xe14, BIT(16) },
+ [RST_BUS_UART6] = { 0xe18, BIT(16) },
+ [RST_BUS_I2C0] = { 0xe80, BIT(16) },
+ [RST_BUS_I2C1] = { 0xe84, BIT(16) },
+ [RST_BUS_I2C2] = { 0xe88, BIT(16) },
+ [RST_BUS_I2C3] = { 0xe8c, BIT(16) },
+ [RST_BUS_I2C4] = { 0xe90, BIT(16) },
+ [RST_BUS_I2C5] = { 0xe94, BIT(16) },
+ [RST_BUS_I2C6] = { 0xe98, BIT(16) },
+ [RST_BUS_I2C7] = { 0xe9c, BIT(16) },
+ [RST_BUS_I2C8] = { 0xea0, BIT(16) },
+ [RST_BUS_I2C9] = { 0xea4, BIT(16) },
+ [RST_BUS_I2C10] = { 0xea8, BIT(16) },
+ [RST_BUS_I2C11] = { 0xeac, BIT(16) },
+ [RST_BUS_I2C12] = { 0xeb0, BIT(16) },
+ [RST_BUS_SPI0] = { 0xf04, BIT(16) },
+ [RST_BUS_SPI1] = { 0xf0c, BIT(16) },
+ [RST_BUS_SPI2] = { 0xf14, BIT(16) },
+ [RST_BUS_SPIF] = { 0xf1c, BIT(16) },
+ [RST_BUS_SPI3] = { 0xf24, BIT(16) },
+ [RST_BUS_SPI4] = { 0xf2c, BIT(16) }, /* From the vendor kernel. */
+ [RST_BUS_GPADC0] = { 0xfc4, BIT(16) },
+ [RST_BUS_THS0] = { 0xfe4, BIT(16) },
+ [RST_BUS_IRRX] = { 0x1004, BIT(16) },
+ [RST_BUS_IRTX] = { 0x100c, BIT(16) },
+ [RST_BUS_LRADC] = { 0x1024, BIT(16) },
+ [RST_BUS_SGPIO] = { 0x1064, BIT(16) }, /* From the vendor kernel. */
+ [RST_BUS_LPC] = { 0x1084, BIT(16) }, /* From the vendor kernel. */
+ [RST_BUS_I2SPCM0] = { 0x120c, BIT(16) },
+ [RST_BUS_I2SPCM1] = { 0x121c, BIT(16) },
+ [RST_BUS_I2SPCM2] = { 0x122c, BIT(16) },
+ [RST_BUS_I2SPCM3] = { 0x123c, BIT(16) },
+ [RST_BUS_I2SPCM4] = { 0x124c, BIT(16) },
+ [RST_BUS_SPDIF] = { 0x128c, BIT(16) },
+ [RST_BUS_DMIC] = { 0x12cc, BIT(16) },
+ [RST_USB_PHY0] = { 0x1300, BIT(30) },
+ [RST_BUS_OHCI0] = { 0x1304, BIT(16) },
+ [RST_BUS_EHCI0] = { 0x1304, BIT(20) },
+ [RST_BUS_OTG] = { 0x1304, BIT(24) },
+ [RST_USB_PHY1] = { 0x1308, BIT(30) },
+ [RST_BUS_OHCI1] = { 0x130c, BIT(16) },
+ [RST_BUS_EHCI1] = { 0x130c, BIT(20) },
+ [RST_BUS_USB2] = { 0x135c, BIT(16) },
+ [RST_BUS_PCIE_PWRUP] = { 0x138c, BIT(16) },
+ [RST_BUS_PCIE] = { 0x138c, BIT(17) },
+ [RST_BUS_SERDES] = { 0x13c4, BIT(16) },
+ [RST_BUS_GMAC0] = { 0x141c, BIT(16) },
+ [RST_BUS_GMAC0_AXI] = { 0x141c, BIT(17) },
+ [RST_BUS_GMAC1] = { 0x142c, BIT(16) }, /* From the vendor kernel. */
+ [RST_BUS_GMAC1_AXI] = { 0x142c, BIT(17) }, /* From the vendor kernel. */
+ [RST_BUS_TCON_LCD0] = { 0x1504, BIT(16) },
+ [RST_BUS_TCON_LCD1] = { 0x150c, BIT(16) },
+ [RST_BUS_TCON_LCD2] = { 0x1514, BIT(16) }, /* From the vendor kernel. */
+ [RST_BUS_LVDS0] = { 0x1544, BIT(16) },
+ [RST_BUS_LVDS1] = { 0x154c, BIT(16) },
+ [RST_BUS_DSI0] = { 0x1584, BIT(16) },
+ [RST_BUS_DSI1] = { 0x158c, BIT(16) },
+ [RST_BUS_TCON_TV0] = { 0x1604, BIT(16) },
+ [RST_BUS_TCON_TV1] = { 0x160c, BIT(16) },
+ [RST_BUS_EDP] = { 0x164c, BIT(16) },
+ [RST_BUS_HDMI_MAIN] = { 0x168c, BIT(16) },
+ [RST_BUS_HDMI_SUB] = { 0x168c, BIT(17) },
+ [RST_BUS_HDMI_HDCP] = { 0x168c, BIT(18) },
+ [RST_BUS_DPSS_TOP0] = { 0x16c4, BIT(16) },
+ [RST_BUS_DPSS_TOP1] = { 0x16cc, BIT(16) },
+ [RST_BUS_VIDEO_OUT0] = { 0x16e4, BIT(16) },
+ [RST_BUS_VIDEO_OUT1] = { 0x16ec, BIT(16) },
+ [RST_BUS_LEDC] = { 0x1704, BIT(16) },
+ [RST_BUS_DSC] = { 0x1744, BIT(16) },
+ [RST_BUS_CSI] = { 0x1844, BIT(16) },
+ [RST_BUS_VIDEO_IN] = { 0x1884, BIT(16) },
+ [RST_BUS_APB2JTAG] = { 0x1c04, BIT(16) },
+};
+
static const struct sunxi_ccu_desc sun60i_a733_ccu_desc = {
.ccu_clks = sun60i_a733_ccu_clks,
.num_ccu_clks = ARRAY_SIZE(sun60i_a733_ccu_clks),
.hw_clks = &sun60i_a733_hw_clks,
+
+ .resets = sun60i_a733_ccu_resets,
+ .num_resets = ARRAY_SIZE(sun60i_a733_ccu_resets),
};
static const u32 pll_regs[] = {
--
2.54.0
^ permalink raw reply related
* [PATCH v2 7/8] clk: sunxi-ng: a733: Add bus clock gates
From: Junhui Liu @ 2026-07-11 8:10 UTC (permalink / raw)
To: Michael Turquette, Stephen Boyd, Brian Masney, Rob Herring,
Krzysztof Kozlowski, Conor Dooley, Chen-Yu Tsai, Jernej Skrabec,
Samuel Holland, Philipp Zabel, Junhui Liu, Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Richard Cochran
Cc: linux-clk, devicetree, linux-arm-kernel, linux-sunxi,
linux-kernel, linux-riscv, netdev, Jerome Brunet
In-Reply-To: <20260711-a733-clk-v2-0-974d188cbe0c@pigmoral.tech>
Add the bus clock gates that control access to the devices' register
interface on the Allwinner A733 SoC. These clocks are typically
single-bit controls in the BGR registers, covering UARTs, SPI, I2C, and
various multimedia engines. It also includes bus gates for system
components like the IOMMU and MSI-lite interfaces.
Also mark ahb-store and mbus-store clocks as critical, since disabling
either gate breaks access to boot/storage devices such as MMC and SPI
NOR when unused clocks are disabled.
Tested-by: Jerome Brunet <jbrunet@baylibre.com>
Signed-off-by: Junhui Liu <junhui.liu@pigmoral.tech>
---
drivers/clk/sunxi-ng/ccu-sun60i-a733.c | 488 ++++++++++++++++++++++++++++++++-
1 file changed, 487 insertions(+), 1 deletion(-)
diff --git a/drivers/clk/sunxi-ng/ccu-sun60i-a733.c b/drivers/clk/sunxi-ng/ccu-sun60i-a733.c
index bf26e310f08a..48d7c0395ae8 100644
--- a/drivers/clk/sunxi-ng/ccu-sun60i-a733.c
+++ b/drivers/clk/sunxi-ng/ccu-sun60i-a733.c
@@ -412,16 +412,19 @@ static SUNXI_CCU_M_DATA_WITH_MUX(ahb_clk, "ahb", ahb_apb_parents, 0x500,
0, 5, /* M */
24, 2, /* mux */
0);
+static const struct clk_hw *ahb_hws[] = { &ahb_clk.common.hw };
static SUNXI_CCU_M_DATA_WITH_MUX(apb0_clk, "apb0", ahb_apb_parents, 0x510,
0, 5, /* M */
24, 2, /* mux */
0);
+static const struct clk_hw *apb0_hws[] = { &apb0_clk.common.hw };
static SUNXI_CCU_M_DATA_WITH_MUX(apb1_clk, "apb1", ahb_apb_parents, 0x518,
0, 5, /* M */
24, 2, /* mux */
0);
+static const struct clk_hw *apb1_hws[] = { &apb1_clk.common.hw };
static const struct clk_parent_data apb_uart_parents[] = {
{ .hw = &sys_24M_clk.hw },
@@ -434,6 +437,9 @@ static SUNXI_CCU_M_DATA_WITH_MUX(apb_uart_clk, "apb-uart", apb_uart_parents, 0x5
0, 5, /* M */
24, 3, /* mux */
0);
+static const struct clk_hw *apb_uart_hws[] = {
+ &apb_uart_clk.common.hw
+};
static const struct clk_parent_data trace_parents[] = {
{ .hw = &sys_24M_clk.hw },
@@ -448,6 +454,8 @@ static SUNXI_CCU_M_DATA_WITH_MUX_GATE(trace_clk, "trace", trace_parents, 0x540,
BIT(31), /* gate */
0);
+static SUNXI_CCU_GATE_DATA(bus_its_pcie0_aclk_clk, "bus-its-pcie0-aclk", hosc, 0x574, BIT(1), 0);
+
static const struct clk_parent_data mbus_parents[] = {
{ .hw = &sys_24M_clk.hw },
{ .hw = &pll_periph1_600M_clk.hw },
@@ -463,9 +471,118 @@ static SUNXI_CCU_MP_DATA_WITH_MUX_GATE_FEAT(mbus_clk, "mbus", mbus_parents, 0x58
BIT(31), /* gate */
CLK_IS_CRITICAL,
CCU_FEATURE_UPDATE_BIT);
+static const struct clk_hw *mbus_hws[] = { &mbus_clk.common.hw };
+
+static SUNXI_CCU_GATE_HWS(mbus_iommu0_sys_clk, "mbus-iommu0-sys", mbus_hws, 0x58c, BIT(0), 0);
+static SUNXI_CCU_GATE_HWS(apb_iommu0_sys_clk, "apb-iommu0-sys", apb0_hws, 0x58c, BIT(1), 0);
+static SUNXI_CCU_GATE_HWS(ahb_iommu0_sys_clk, "ahb-iommu0-sys", ahb_hws, 0x58c, BIT(2), 0);
+
+static SUNXI_CCU_GATE_DATA(bus_msi_lite0_clk, "bus-msi-lite0", hosc, 0x594, BIT(0), 0);
+static SUNXI_CCU_GATE_DATA(bus_msi_lite1_clk, "bus-msi-lite1", hosc, 0x59c, BIT(0), 0);
+static SUNXI_CCU_GATE_DATA(bus_msi_lite2_clk, "bus-msi-lite2", hosc, 0x5a4, BIT(0), 0);
+
+static SUNXI_CCU_GATE_HWS(mbus_iommu1_sys_clk, "mbus-iommu1-sys", mbus_hws, 0x5b4, BIT(0), 0);
+static SUNXI_CCU_GATE_HWS(apb_iommu1_sys_clk, "apb_iommu1-sys", apb0_hws, 0x5b4, BIT(1), 0);
+static SUNXI_CCU_GATE_HWS(ahb_iommu1_sys_clk, "ahb_iommu1-sys", ahb_hws, 0x5b4, BIT(2), 0);
+
+static SUNXI_CCU_GATE_HWS(ahb_ve_dec_clk, "ahb-ve-dec", ahb_hws,
+ 0x5c0, BIT(0), 0);
+static SUNXI_CCU_GATE_HWS(ahb_ve_enc_clk, "ahb-ve-enc", ahb_hws,
+ 0x5c0, BIT(1), 0);
+static SUNXI_CCU_GATE_HWS(ahb_vid_in_clk, "ahb-vid-in", ahb_hws,
+ 0x5c0, BIT(2), 0);
+static SUNXI_CCU_GATE_HWS(ahb_vid_cout0_clk, "ahb-vid-cout0", ahb_hws,
+ 0x5c0, BIT(3), 0);
+static SUNXI_CCU_GATE_HWS(ahb_vid_cout1_clk, "ahb-vid-cout1", ahb_hws,
+ 0x5c0, BIT(4), 0);
+static SUNXI_CCU_GATE_HWS(ahb_de_clk, "ahb-de", ahb_hws,
+ 0x5c0, BIT(5), 0);
+static SUNXI_CCU_GATE_HWS(ahb_npu_clk, "ahb-npu", ahb_hws,
+ 0x5c0, BIT(6), 0);
+static SUNXI_CCU_GATE_HWS(ahb_gpu0_clk, "ahb-gpu0", ahb_hws,
+ 0x5c0, BIT(7), 0);
+static SUNXI_CCU_GATE_HWS(ahb_serdes_clk, "ahb-serdes", ahb_hws,
+ 0x5c0, BIT(8), 0);
+static SUNXI_CCU_GATE_HWS(ahb_usb_sys_clk, "ahb-usb-sys", ahb_hws,
+ 0x5c0, BIT(9), 0);
+static SUNXI_CCU_GATE_HWS(ahb_msi_lite0_clk, "ahb-msi-lite0", ahb_hws,
+ 0x5c0, BIT(16), 0);
+static SUNXI_CCU_GATE_HWS(ahb_store_clk, "ahb-store", ahb_hws,
+ 0x5c0, BIT(24), CLK_IS_CRITICAL);
+static SUNXI_CCU_GATE_HWS(ahb_cpus_clk, "ahb-cpus", ahb_hws,
+ 0x5c0, BIT(28), 0);
+
+static SUNXI_CCU_GATE_HWS(mbus_iommu0_clk, "mbus-iommu0", mbus_hws,
+ 0x5e0, BIT(0), 0);
+static SUNXI_CCU_GATE_HWS(mbus_iommu1_clk, "mbus-iommu1", mbus_hws,
+ 0x5e0, BIT(1), 0);
+static SUNXI_CCU_GATE_HWS(mbus_desys_clk, "mbus-desys", mbus_hws,
+ 0x5e0, BIT(11), 0);
+static SUNXI_CCU_GATE_HWS(mbus_ve_enc0_gate_clk, "mbus-ve-enc0-gate", mbus_hws,
+ 0x5e0, BIT(12), 0);
+static SUNXI_CCU_GATE_HWS(mbus_ve_dec0_gate_clk, "mbus-ve-dec0-gate", mbus_hws,
+ 0x5e0, BIT(14), 0);
+static SUNXI_CCU_GATE_HWS(mbus_gpu0_clk, "mbus-gpu0", mbus_hws,
+ 0x5e0, BIT(16), 0);
+static SUNXI_CCU_GATE_HWS(mbus_npu_clk, "mbus-npu", mbus_hws,
+ 0x5e0, BIT(18), 0);
+static SUNXI_CCU_GATE_HWS(mbus_vid_in_clk, "mbus-vid-in", mbus_hws,
+ 0x5e0, BIT(24), 0);
+static SUNXI_CCU_GATE_HWS(mbus_serdes_clk, "mbus-serdes", mbus_hws,
+ 0x5e0, BIT(28), 0);
+static SUNXI_CCU_GATE_HWS(mbus_msi_lite0_clk, "mbus-msi-lite0", mbus_hws,
+ 0x5e0, BIT(29), 0);
+static SUNXI_CCU_GATE_HWS(mbus_store_clk, "mbus-store", mbus_hws,
+ 0x5e0, BIT(30), CLK_IS_CRITICAL);
+static SUNXI_CCU_GATE_HWS(mbus_msi_lite2_clk, "mbus-msi-lite2", mbus_hws,
+ 0x5e0, BIT(31), 0);
+
+static SUNXI_CCU_GATE_HWS(mbus_dma0_clk, "mbus-dma0", mbus_hws,
+ 0x5e4, BIT(0), 0);
+static SUNXI_CCU_GATE_HWS(mbus_ve_enc0_clk, "mbus-ve-enc0", mbus_hws,
+ 0x5e4, BIT(1), 0);
+static SUNXI_CCU_GATE_HWS(mbus_ce_clk, "mbus-ce", mbus_hws,
+ 0x5e4, BIT(2), 0);
+static SUNXI_CCU_GATE_HWS(mbus_dma1_clk, "mbus-dma1", mbus_hws,
+ 0x5e4, BIT(3), 0);
+static SUNXI_CCU_GATE_HWS(mbus_nand_clk, "mbus-nand", mbus_hws,
+ 0x5e4, BIT(5), 0);
+static SUNXI_CCU_GATE_HWS(mbus_csi_clk, "mbus-csi", mbus_hws,
+ 0x5e4, BIT(8), 0);
+static SUNXI_CCU_GATE_HWS(mbus_isp_clk, "mbus-isp", mbus_hws,
+ 0x5e4, BIT(9), 0);
+static SUNXI_CCU_GATE_HWS(mbus_gmac0_clk, "mbus-gmac0", mbus_hws,
+ 0x5e4, BIT(11), 0);
+/* Undocumented, taken from the vendor kernel. */
+static SUNXI_CCU_GATE_HWS(mbus_gmac1_clk, "mbus-gmac1", mbus_hws,
+ 0x5e4, BIT(12), 0);
+static SUNXI_CCU_GATE_HWS(mbus_ve_dec0_clk, "mbus-ve-dec0", mbus_hws,
+ 0x5e4, BIT(18), 0);
+
+static SUNXI_CCU_GATE_HWS(bus_dma0_clk, "bus-dma0", ahb_hws,
+ 0x704, BIT(0), 0);
+static SUNXI_CCU_GATE_HWS(bus_dma1_clk, "bus-dma1", ahb_hws,
+ 0x70c, BIT(0), 0);
+
+static SUNXI_CCU_GATE_HWS(bus_spinlock_clk, "bus-spinlock", ahb_hws,
+ 0x724, BIT(0), 0);
+
+static SUNXI_CCU_GATE_HWS(bus_msgbox0_clk, "bus-msgbox0", ahb_hws,
+ 0x744, BIT(0), 0);
+
+static SUNXI_CCU_GATE_HWS(bus_pwm0_clk, "bus-pwm0", apb0_hws,
+ 0x784, BIT(0), 0);
+static SUNXI_CCU_GATE_HWS(bus_pwm1_clk, "bus-pwm1", apb0_hws,
+ 0x78c, BIT(0), 0);
+
+static SUNXI_CCU_GATE_HWS(bus_dbg_clk, "bus-dbg", sys_24M_hws,
+ 0x7a4, BIT(0), 0);
+
+static SUNXI_CCU_GATE_HWS(bus_sysdap_clk, "bus-sysdap", apb1_hws,
+ 0x7ac, BIT(0), 0);
/**************************************************************************
- * mod clocks *
+ * mod clocks with gates *
**************************************************************************/
static const struct clk_parent_data timer_parents[] = {
@@ -535,6 +652,7 @@ static SUNXI_CCU_MP_DATA_WITH_MUX_GATE(timer9_clk, "timer9", timer_parents, 0x82
24, 3, /* mux */
BIT(31), /* gate */
0);
+static SUNXI_CCU_GATE_HWS(bus_timer_clk, "bus-timer", ahb_hws, 0x850, BIT(0), 0);
/* Undocumented, taken from the vendor kernel. */
static const struct clk_parent_data avs_parents[] = {
@@ -560,6 +678,7 @@ static SUNXI_CCU_M_HW_WITH_MUX_GATE(de0_clk, "de0", de_parents, 0xa00,
24, 3, /* mux */
BIT(31), /* gate */
CLK_SET_RATE_PARENT);
+static SUNXI_CCU_GATE_HWS(bus_de0_clk, "bus-de0", ahb_hws, 0xa04, BIT(0), 0);
static const struct clk_hw *di_parents[] = {
&pll_periph0_600M_clk.hw,
@@ -573,6 +692,7 @@ static SUNXI_CCU_M_HW_WITH_MUX_GATE(di_clk, "di", di_parents, 0xa20,
24, 3, /* mux */
BIT(31), /* gate */
CLK_SET_RATE_PARENT);
+static SUNXI_CCU_GATE_HWS(bus_di_clk, "bus-di", ahb_hws, 0xa24, BIT(0), 0);
static const struct clk_hw *g2d_parents[] = {
&pll_periph0_400M_clk.hw,
@@ -585,6 +705,7 @@ static SUNXI_CCU_M_HW_WITH_MUX_GATE(g2d_clk, "g2d", g2d_parents, 0xa40,
24, 3, /* mux */
BIT(31), /* gate */
CLK_SET_RATE_PARENT);
+static SUNXI_CCU_GATE_HWS(bus_g2d_clk, "bus-g2d", ahb_hws, 0xa44, BIT(0), 0);
static const struct clk_hw *eink_parents[] = {
&pll_periph0_480M_clk.common.hw,
@@ -608,6 +729,7 @@ static SUNXI_CCU_M_HW_WITH_MUX_GATE(eink_panel_clk, "eink-panel", eink_panel_par
24, 3, /* mux */
BIT(31), /* gate */
CLK_SET_RATE_PARENT);
+static SUNXI_CCU_GATE_HWS(bus_eink_clk, "bus-eink", ahb_hws, 0xa6c, BIT(0), 0);
static const struct clk_hw *ve_enc_parents[] = {
&pll_ve0_clk.common.hw,
@@ -639,6 +761,9 @@ static SUNXI_CCU_M_HW_WITH_MUX_GATE(ve_dec0_clk, "ve-dec0", ve_dec_parents, 0xa8
BIT(31), /* gate */
CLK_SET_RATE_PARENT);
+static SUNXI_CCU_GATE_HWS(bus_ve_enc0_clk, "bus-ve-enc0", ahb_hws, 0xa8c, BIT(0), 0);
+static SUNXI_CCU_GATE_HWS(bus_ve_dec0_clk, "bus-ve-dec0", ahb_hws, 0xa8c, BIT(2), 0);
+
static const struct clk_hw *ce_parents[] = {
&sys_24M_clk.hw,
&pll_periph0_400M_clk.hw,
@@ -649,6 +774,8 @@ static SUNXI_CCU_M_HW_WITH_MUX_GATE(ce_clk, "ce", ce_parents, 0xac0,
24, 3, /* mux */
BIT(31), /* gate */
0);
+static SUNXI_CCU_GATE_HWS(bus_ce_clk, "bus-ce", ahb_hws, 0xac4, BIT(0), 0);
+static SUNXI_CCU_GATE_HWS(bus_ce_sys_clk, "bus-ce-sys", ahb_hws, 0xac4, BIT(1), 0);
static const struct clk_hw *npu_parents[] = {
&pll_npu_clk.common.hw,
@@ -664,6 +791,7 @@ static SUNXI_CCU_M_HW_WITH_MUX_GATE(npu_clk, "npu", npu_parents, 0xb00,
24, 3, /* mux */
BIT(31), /* gate */
0);
+static SUNXI_CCU_GATE_DATA(bus_npu_clk, "bus-npu", hosc, 0xb04, BIT(0), 0);
/*
* GPU_CLK = ClockSource * ((16 - M) / 16)
@@ -696,6 +824,7 @@ static struct ccu_div gpu0_clk = {
&ccu_div_ops, 0),
}
};
+static SUNXI_CCU_GATE_HWS(bus_gpu0_clk, "bus-gpu0", ahb_hws, 0xb24, BIT(0), 0);
static const struct clk_parent_data dram_parents[] = {
{ .hw = &pll_ddr_clk.common.hw, },
@@ -711,6 +840,8 @@ static SUNXI_CCU_MP_DATA_WITH_MUX_GATE_FEAT(dram0_clk, "dram0", dram_parents, 0x
BIT(31), /* gate */
CLK_IS_CRITICAL,
CCU_FEATURE_UPDATE_BIT);
+static SUNXI_CCU_GATE_HWS(bus_dram0_clk, "bus-dram0", ahb_hws, 0xc0c,
+ BIT(0), CLK_IS_CRITICAL);
static const struct clk_parent_data nand_mmc_parents[] = {
{ .hw = &sys_24M_clk.hw, },
@@ -729,6 +860,7 @@ static SUNXI_CCU_M_DATA_WITH_MUX_GATE(nand0_clk1_clk, "nand0-clk1", nand_mmc_par
24, 3, /* mux */
BIT(31), /* gate */
0);
+static SUNXI_CCU_GATE_HWS(bus_nand0_clk, "bus-nand0", ahb_hws, 0xc8c, BIT(0), 0);
static SUNXI_CCU_MP_MUX_GATE_POSTDIV_DUALDIV(mmc0_clk, "mmc0", nand_mmc_parents, 0xd00,
0, 5, /* M */
@@ -737,6 +869,8 @@ static SUNXI_CCU_MP_MUX_GATE_POSTDIV_DUALDIV(mmc0_clk, "mmc0", nand_mmc_parents,
BIT(31), /* gate */
2, /* post div */
0);
+static SUNXI_CCU_GATE_HWS(bus_mmc0_clk, "bus-mmc0", ahb_hws, 0xd0c, BIT(0), 0);
+
static SUNXI_CCU_MP_MUX_GATE_POSTDIV_DUALDIV(mmc1_clk, "mmc1", nand_mmc_parents, 0xd10,
0, 5, /* M */
8, 5, /* P */
@@ -744,6 +878,7 @@ static SUNXI_CCU_MP_MUX_GATE_POSTDIV_DUALDIV(mmc1_clk, "mmc1", nand_mmc_parents,
BIT(31), /* gate */
2, /* post div */
0);
+static SUNXI_CCU_GATE_HWS(bus_mmc1_clk, "bus-mmc1", ahb_hws, 0xd1c, BIT(0), 0);
static const struct clk_parent_data mmc2_mmc3_parents[] = {
{ .hw = &sys_24M_clk.hw, },
@@ -759,6 +894,8 @@ static SUNXI_CCU_MP_MUX_GATE_POSTDIV_DUALDIV(mmc2_clk, "mmc2", mmc2_mmc3_parents
BIT(31), /* gate */
2, /* post div */
0);
+static SUNXI_CCU_GATE_HWS(bus_mmc2_clk, "bus-mmc2", ahb_hws, 0xd2c, BIT(0), 0);
+
static SUNXI_CCU_MP_MUX_GATE_POSTDIV_DUALDIV(mmc3_clk, "mmc3", mmc2_mmc3_parents, 0xd30,
0, 5, /* M */
8, 5, /* P */
@@ -766,6 +903,7 @@ static SUNXI_CCU_MP_MUX_GATE_POSTDIV_DUALDIV(mmc3_clk, "mmc3", mmc2_mmc3_parents
BIT(31), /* gate */
2, /* post div */
0);
+static SUNXI_CCU_GATE_HWS(bus_mmc3_clk, "bus-mmc3", ahb_hws, 0xd3c, BIT(0), 0);
static const struct clk_hw *ufs_axi_parents[] = {
&pll_periph0_300M_clk.hw,
@@ -786,6 +924,29 @@ static SUNXI_CCU_M_DATA_WITH_MUX_GATE(ufs_cfg_clk, "ufs-cfg", ufs_cfg_parents, 0
24, 3, /* mux */
BIT(31), /* gate */
0);
+static SUNXI_CCU_GATE_DATA(bus_ufs_clk, "bus-ufs", hosc, 0xd8c, BIT(0), 0);
+
+static SUNXI_CCU_GATE_HWS(bus_uart0_clk, "bus-uart0", apb_uart_hws, 0xe00, BIT(0), 0);
+static SUNXI_CCU_GATE_HWS(bus_uart1_clk, "bus-uart1", apb_uart_hws, 0xe04, BIT(0), 0);
+static SUNXI_CCU_GATE_HWS(bus_uart2_clk, "bus-uart2", apb_uart_hws, 0xe08, BIT(0), 0);
+static SUNXI_CCU_GATE_HWS(bus_uart3_clk, "bus-uart3", apb_uart_hws, 0xe0c, BIT(0), 0);
+static SUNXI_CCU_GATE_HWS(bus_uart4_clk, "bus-uart4", apb_uart_hws, 0xe10, BIT(0), 0);
+static SUNXI_CCU_GATE_HWS(bus_uart5_clk, "bus-uart5", apb_uart_hws, 0xe14, BIT(0), 0);
+static SUNXI_CCU_GATE_HWS(bus_uart6_clk, "bus-uart6", apb_uart_hws, 0xe18, BIT(0), 0);
+
+static SUNXI_CCU_GATE_HWS(bus_i2c0_clk, "bus-i2c0", apb1_hws, 0xe80, BIT(0), 0);
+static SUNXI_CCU_GATE_HWS(bus_i2c1_clk, "bus-i2c1", apb1_hws, 0xe84, BIT(0), 0);
+static SUNXI_CCU_GATE_HWS(bus_i2c2_clk, "bus-i2c2", apb1_hws, 0xe88, BIT(0), 0);
+static SUNXI_CCU_GATE_HWS(bus_i2c3_clk, "bus-i2c3", apb1_hws, 0xe8c, BIT(0), 0);
+static SUNXI_CCU_GATE_HWS(bus_i2c4_clk, "bus-i2c4", apb1_hws, 0xe90, BIT(0), 0);
+static SUNXI_CCU_GATE_HWS(bus_i2c5_clk, "bus-i2c5", apb1_hws, 0xe94, BIT(0), 0);
+static SUNXI_CCU_GATE_HWS(bus_i2c6_clk, "bus-i2c6", apb1_hws, 0xe98, BIT(0), 0);
+static SUNXI_CCU_GATE_HWS(bus_i2c7_clk, "bus-i2c7", apb1_hws, 0xe9c, BIT(0), 0);
+static SUNXI_CCU_GATE_HWS(bus_i2c8_clk, "bus-i2c8", apb1_hws, 0xea0, BIT(0), 0);
+static SUNXI_CCU_GATE_HWS(bus_i2c9_clk, "bus-i2c9", apb1_hws, 0xea4, BIT(0), 0);
+static SUNXI_CCU_GATE_HWS(bus_i2c10_clk, "bus-i2c10", apb1_hws, 0xea8, BIT(0), 0);
+static SUNXI_CCU_GATE_HWS(bus_i2c11_clk, "bus-i2c11", apb1_hws, 0xeac, BIT(0), 0);
+static SUNXI_CCU_GATE_HWS(bus_i2c12_clk, "bus-i2c12", apb1_hws, 0xeb0, BIT(0), 0);
static const struct clk_parent_data spi_parents[] = {
{ .hw = &sys_24M_clk.hw },
@@ -803,18 +964,23 @@ static SUNXI_CCU_DUALDIV_MUX_GATE(spi0_clk, "spi0", spi_parents, 0xf00,
24, 3, /* mux */
BIT(31), /* gate */
0);
+static SUNXI_CCU_GATE_HWS(bus_spi0_clk, "bus-spi0", ahb_hws, 0xf04, BIT(0), 0);
+
static SUNXI_CCU_DUALDIV_MUX_GATE(spi1_clk, "spi1", spi_parents, 0xf08,
0, 5, /* M */
8, 5, /* N */
24, 3, /* mux */
BIT(31), /* gate */
0);
+static SUNXI_CCU_GATE_HWS(bus_spi1_clk, "bus-spi1", ahb_hws, 0xf0c, BIT(0), 0);
+
static SUNXI_CCU_DUALDIV_MUX_GATE(spi2_clk, "spi2", spi_parents, 0xf10,
0, 5, /* M */
8, 5, /* N */
24, 3, /* mux */
BIT(31), /* gate */
0);
+static SUNXI_CCU_GATE_HWS(bus_spi2_clk, "bus-spi2", ahb_hws, 0xf14, BIT(0), 0);
static const struct clk_parent_data spif_parents[] = {
{ .hw = &sys_24M_clk.hw },
@@ -832,12 +998,16 @@ static SUNXI_CCU_DUALDIV_MUX_GATE(spif_clk, "spif", spif_parents, 0xf18,
24, 3, /* mux */
BIT(31), /* gate */
0);
+static SUNXI_CCU_GATE_HWS(bus_spif_clk, "bus-spif", ahb_hws, 0xf1c, BIT(0), 0);
+
static SUNXI_CCU_DUALDIV_MUX_GATE(spi3_clk, "spi3", spi_parents, 0xf20,
0, 5, /* M */
8, 5, /* N */
24, 3, /* mux */
BIT(31), /* gate */
0);
+static SUNXI_CCU_GATE_HWS(bus_spi3_clk, "bus-spi3", ahb_hws, 0xf24, BIT(0), 0);
+
/* Undocumented, taken from the vendor kernel. */
static SUNXI_CCU_DUALDIV_MUX_GATE(spi4_clk, "spi4", spi_parents, 0xf28,
0, 5, /* M */
@@ -845,6 +1015,7 @@ static SUNXI_CCU_DUALDIV_MUX_GATE(spi4_clk, "spi4", spi_parents, 0xf28,
24, 3, /* mux */
BIT(31), /* gate */
0);
+static SUNXI_CCU_GATE_HWS(bus_spi4_clk, "bus-spi4", ahb_hws, 0xf2c, BIT(0), 0);
static const struct clk_parent_data gpadc0_24m_parents[] = {
{ .hw = &sys_24M_clk.hw },
@@ -855,6 +1026,9 @@ static SUNXI_CCU_M_DATA_WITH_MUX_GATE(gpadc0_24m_clk, "gpadc0-24m", gpadc0_24m_p
24, 3, /* mux */
BIT(31), /* gate */
0);
+static SUNXI_CCU_GATE_HWS(bus_gpadc0_clk, "bus-gpadc0", ahb_hws, 0xfc4, BIT(0), 0);
+
+static SUNXI_CCU_GATE_HWS(bus_ths0_clk, "bus-ths0", apb0_hws, 0xfe4, BIT(0), 0);
static const struct clk_parent_data irrx_parents[] = {
{ .fw_name = "losc"},
@@ -866,6 +1040,7 @@ static SUNXI_CCU_M_DATA_WITH_MUX_GATE(irrx_clk, "irrx", irrx_parents, 0x1000,
24, 3, /* mux */
BIT(31), /* gate */
0);
+static SUNXI_CCU_GATE_HWS(bus_irrx_clk, "bus-irrx", apb0_hws, 0x1004, BIT(0), 0);
static const struct clk_parent_data irtx_parents[] = {
{ .fw_name = "losc"},
@@ -877,6 +1052,9 @@ static SUNXI_CCU_M_DATA_WITH_MUX_GATE(irtx_clk, "irtx", irtx_parents, 0x1008,
24, 3, /* mux */
BIT(31), /* gate */
0);
+static SUNXI_CCU_GATE_HWS(bus_irtx_clk, "bus-irtx", apb0_hws, 0x100c, BIT(0), 0);
+
+static SUNXI_CCU_GATE_HWS(bus_lradc_clk, "bus-lradc", apb0_hws, 0x1024, BIT(0), 0);
/* Undocumented, taken from the vendor kernel. */
static const struct clk_parent_data sgpio_parents[] = {
@@ -888,6 +1066,7 @@ static SUNXI_CCU_M_DATA_WITH_MUX_GATE(sgpio_clk, "sgpio", sgpio_parents, 0x1060,
24, 3, /* mux */
BIT(31), /* gate */
0);
+static SUNXI_CCU_GATE_DATA(bus_sgpio_clk, "bus-sgpio", hosc, 0x1064, BIT(0), 0);
/* Undocumented, taken from the vendor kernel. */
static const struct clk_hw *lpc_parents[] = {
@@ -901,6 +1080,7 @@ static SUNXI_CCU_M_HW_WITH_MUX_GATE(lpc_clk, "lpc", lpc_parents, 0x1080,
24, 3, /* mux */
BIT(31), /* gate */
0);
+static SUNXI_CCU_GATE_DATA(bus_lpc_clk, "bus-lpc", hosc, 0x1084, BIT(0), 0);
static const struct clk_hw *i2spcm_parents[] = {
&pll_audio0_4x_clk.common.hw,
@@ -913,11 +1093,15 @@ static SUNXI_CCU_M_HW_WITH_MUX_GATE(i2spcm0_clk, "i2spcm0", i2spcm_parents, 0x12
24, 3, /* mux */
BIT(31), /* gate */
0);
+static SUNXI_CCU_GATE_DATA(bus_i2spcm0_clk, "bus-i2spcm0", hosc, 0x120c, BIT(0), 0);
+
static SUNXI_CCU_M_HW_WITH_MUX_GATE(i2spcm1_clk, "i2spcm1", i2spcm_parents, 0x1210,
0, 5, /* M */
24, 3, /* mux */
BIT(31), /* gate */
0);
+static SUNXI_CCU_GATE_DATA(bus_i2spcm1_clk, "bus-i2spcm1", hosc, 0x121c, BIT(0), 0);
+
static SUNXI_CCU_M_HW_WITH_MUX_GATE(i2spcm2_clk, "i2spcm2", i2spcm_parents, 0x1220,
0, 5, /* M */
24, 3, /* mux */
@@ -936,16 +1120,22 @@ static SUNXI_CCU_M_HW_WITH_MUX_GATE(i2spcm2_asrc_clk, "i2spcm2_asrc", i2spcm2_as
24, 3, /* mux */
BIT(31), /* gate */
0);
+
+static SUNXI_CCU_GATE_DATA(bus_i2spcm2_clk, "bus-i2spcm2", hosc, 0x122c, BIT(0), 0);
+
static SUNXI_CCU_M_HW_WITH_MUX_GATE(i2spcm3_clk, "i2spcm3", i2spcm_parents, 0x1230,
0, 5, /* M */
24, 3, /* mux */
BIT(31), /* gate */
0);
+static SUNXI_CCU_GATE_DATA(bus_i2spcm3_clk, "bus-i2spcm3", hosc, 0x123c, BIT(0), 0);
+
static SUNXI_CCU_M_HW_WITH_MUX_GATE(i2spcm4_clk, "i2spcm4", i2spcm_parents, 0x1240,
0, 5, /* M */
24, 3, /* mux */
BIT(31), /* gate */
0);
+static SUNXI_CCU_GATE_DATA(bus_i2spcm4_clk, "bus-i2spcm4", hosc, 0x124c, BIT(0), 0);
static const struct clk_hw *spdif_tx_parents[] = {
&pll_audio0_4x_clk.common.hw,
@@ -969,6 +1159,8 @@ static SUNXI_CCU_M_HW_WITH_MUX_GATE(spdif_rx_clk, "spdif-rx", spdif_rx_parents,
BIT(31), /* gate */
0);
+static SUNXI_CCU_GATE_HWS(bus_spdif_clk, "bus-spdif", apb1_hws, 0x128c, BIT(0), 0);
+
static const struct clk_hw *dmic_parents[] = {
&pll_audio0_4x_clk.common.hw,
&pll_audio1_div2_clk.common.hw,
@@ -980,6 +1172,8 @@ static SUNXI_CCU_M_HW_WITH_MUX_GATE(dmic_clk, "dmic", dmic_parents, 0x12c0,
BIT(31), /* gate */
0);
+static SUNXI_CCU_GATE_HWS(bus_dmic_clk, "bus-dmic", apb1_hws, 0x12cc, BIT(0), 0);
+
/*
* The first parent is a 48 MHz input clock divided by 4. That 48 MHz clock is
* a 2x multiplier from pll-ref synchronized by pll-periph0, and is also used by
@@ -1011,6 +1205,9 @@ static struct ccu_mux usb_ohci0_clk = {
&ccu_mux_ops, 0),
},
};
+static SUNXI_CCU_GATE_HWS(bus_ohci0_clk, "bus-ohci0", ahb_hws, 0x1304, BIT(0), 0);
+static SUNXI_CCU_GATE_HWS(bus_ehci0_clk, "bus-ehci0", ahb_hws, 0x1304, BIT(4), 0);
+static SUNXI_CCU_GATE_HWS(bus_otg_clk, "bus-otg", ahb_hws, 0x1304, BIT(8), 0);
static struct ccu_mux usb_ohci1_clk = {
.enable = BIT(31),
@@ -1027,6 +1224,8 @@ static struct ccu_mux usb_ohci1_clk = {
&ccu_mux_ops, 0),
},
};
+static SUNXI_CCU_GATE_HWS(bus_ohci1_clk, "bus-ohci1", ahb_hws, 0x130c, BIT(0), 0);
+static SUNXI_CCU_GATE_HWS(bus_ehci1_clk, "bus-ehci1", ahb_hws, 0x130c, BIT(4), 0);
static const struct clk_parent_data usb01_ref_parents[] = {
{ .hw = &sys_24M_clk.hw },
@@ -1129,11 +1328,14 @@ static SUNXI_CCU_M_HWS_WITH_GATE(gmac0_phy_clk, "gmac0-phy", pll_periph0_150M_hw
0, 5, /* M */
BIT(31), /* gate */
0);
+static SUNXI_CCU_GATE_HWS(bus_gmac0_clk, "bus-gmac0", ahb_hws, 0x141c, BIT(0), 0);
+
/* Undocumented, taken from the vendor kernel. */
static SUNXI_CCU_M_HWS_WITH_GATE(gmac1_phy_clk, "gmac1-phy", pll_periph0_150M_hws, 0x1420,
0, 5, /* M */
BIT(31), /* gate */
0);
+static SUNXI_CCU_GATE_HWS(bus_gmac1_clk, "bus-gmac1", ahb_hws, 0x142c, BIT(0), 0);
static const struct clk_hw *tcon_lcd_parents[] = {
&pll_video0_4x_clk.common.hw,
@@ -1146,17 +1348,22 @@ static SUNXI_CCU_M_HW_WITH_MUX_GATE(tcon_lcd0_clk, "tcon-lcd0", tcon_lcd_parents
24, 3, /* mux */
BIT(31), /* gate */
0);
+static SUNXI_CCU_GATE_HWS(bus_tcon_lcd0_clk, "bus-tcon-lcd0", ahb_hws, 0x1504, BIT(0), 0);
+
static SUNXI_CCU_M_HW_WITH_MUX_GATE(tcon_lcd1_clk, "tcon-lcd1", tcon_lcd_parents, 0x1508,
0, 5, /* M */
24, 3, /* mux */
BIT(31), /* gate */
0);
+static SUNXI_CCU_GATE_HWS(bus_tcon_lcd1_clk, "bus-tcon-lcd1", ahb_hws, 0x150c, BIT(0), 0);
+
/* Undocumented, taken from the vendor kernel. */
static SUNXI_CCU_M_HW_WITH_MUX_GATE(tcon_lcd2_clk, "tcon-lcd2", tcon_lcd_parents, 0x1510,
0, 5, /* M */
24, 3, /* mux */
BIT(31), /* gate */
0);
+static SUNXI_CCU_GATE_HWS(bus_tcon_lcd2_clk, "bus-tcon-lcd2", ahb_hws, 0x1514, BIT(0), 0);
static const struct clk_hw *dsi_parents[] = {
&sys_24M_clk.hw,
@@ -1168,11 +1375,14 @@ static SUNXI_CCU_M_HW_WITH_MUX_GATE(dsi0_clk, "dsi0", dsi_parents, 0x1580,
24, 3, /* mux */
BIT(31), /* gate */
0);
+static SUNXI_CCU_GATE_HWS(bus_dsi0_clk, "bus-dsi0", ahb_hws, 0x1584, BIT(0), 0);
+
static SUNXI_CCU_M_HW_WITH_MUX_GATE(dsi1_clk, "dsi1", dsi_parents, 0x1588,
0, 5, /* M */
24, 3, /* mux */
BIT(31), /* gate */
0);
+static SUNXI_CCU_GATE_HWS(bus_dsi1_clk, "bus-dsi1", ahb_hws, 0x158c, BIT(0), 0);
static const struct clk_hw *combphy_parents[] = {
&pll_video0_4x_clk.common.hw,
@@ -1192,6 +1402,9 @@ static SUNXI_CCU_M_HW_WITH_MUX_GATE(combphy1_clk, "combphy1", combphy_parents, 0
BIT(31), /* gate */
0);
+static SUNXI_CCU_GATE_HWS(bus_tcon_tv0_clk, "bus-tcon-tv0", ahb_hws, 0x1604, BIT(0), 0);
+static SUNXI_CCU_GATE_HWS(bus_tcon_tv1_clk, "bus-tcon-tv1", ahb_hws, 0x160c, BIT(0), 0);
+
static const struct clk_hw *edp_tv_parents[] = {
&pll_video0_4x_clk.common.hw,
&pll_video1_4x_clk.common.hw,
@@ -1203,6 +1416,7 @@ static SUNXI_CCU_M_HW_WITH_MUX_GATE(edp_tv_clk, "edp-tv", edp_tv_parents, 0x1640
24, 3, /* mux */
BIT(31), /* gate */
0);
+static SUNXI_CCU_GATE_HWS(bus_edp_tv_clk, "bus-edp-tv", ahb_hws, 0x164c, BIT(0), 0);
static SUNXI_CCU_GATE_HWS_WITH_PREDIV(hdmi_cec_32k_clk, "hdmi-cec-32k", pll_periph0_2x_hws, 0x1680,
BIT(30), /* gate */
@@ -1230,6 +1444,7 @@ static SUNXI_CCU_DUALDIV_MUX_GATE(hdmi_tv_clk, "hdmi-tv", hdmi_tv_parents, 0x168
24, 3, /* mux */
BIT(31), /* gate */
0);
+static SUNXI_CCU_GATE_HWS(bus_hdmi_tv_clk, "bus-hdmi-tv", ahb_hws, 0x168c, BIT(0), 0);
static const struct clk_parent_data hdmi_sfr_parents[] = {
{ .hw = &sys_24M_clk.hw },
@@ -1242,6 +1457,9 @@ static SUNXI_CCU_MUX_DATA_WITH_GATE(hdmi_sfr_clk, "hdmi-sfr", hdmi_sfr_parents,
static SUNXI_CCU_GATE_HWS(hdcp_esm_clk, "hdcp-esm", pll_periph0_300M_hws, 0x1694, BIT(31), 0);
+static SUNXI_CCU_GATE_HWS(bus_dpss_top0_clk, "bus-dpss-top0", ahb_hws, 0x16c4, BIT(0), 0);
+static SUNXI_CCU_GATE_HWS(bus_dpss_top1_clk, "bus-dpss-top1", ahb_hws, 0x16cc, BIT(0), 0);
+
static const struct clk_parent_data ledc_parents[] = {
{ .hw = &sys_24M_clk.hw },
{ .hw = &pll_periph0_600M_clk.hw },
@@ -1252,6 +1470,9 @@ static SUNXI_CCU_M_DATA_WITH_MUX_GATE(ledc_clk, "ledc", ledc_parents, 0x1700,
24, 3, /* mux */
BIT(31), /* gate */
0);
+static SUNXI_CCU_GATE_HWS(bus_ledc_clk, "bus-ledc", apb0_hws, 0x1704, BIT(0), 0);
+
+static SUNXI_CCU_GATE_HWS(bus_dsc_clk, "bus-dsc", ahb_hws, 0x1744, BIT(0), 0);
static const struct clk_parent_data csi_master_parents[] = {
{ .hw = &sys_24M_clk.hw },
@@ -1293,6 +1514,7 @@ static SUNXI_CCU_M_HW_WITH_MUX_GATE(csi_clk, "csi", csi_parents, 0x1840,
24, 3, /* mux */
BIT(31), /* gate */
0);
+static SUNXI_CCU_GATE_HWS(bus_csi_clk, "bus-csi", ahb_hws, 0x1844, BIT(0), 0);
static const struct clk_hw *isp_parents[] = {
&pll_video2_4x_clk.common.hw,
@@ -1422,7 +1644,60 @@ static struct ccu_common *sun60i_a733_ccu_clks[] = {
&apb1_clk.common,
&apb_uart_clk.common,
&trace_clk.common,
+ &bus_its_pcie0_aclk_clk.common,
&mbus_clk.common,
+ &mbus_iommu0_sys_clk.common,
+ &apb_iommu0_sys_clk.common,
+ &ahb_iommu0_sys_clk.common,
+ &bus_msi_lite0_clk.common,
+ &bus_msi_lite1_clk.common,
+ &bus_msi_lite2_clk.common,
+ &mbus_iommu1_sys_clk.common,
+ &apb_iommu1_sys_clk.common,
+ &ahb_iommu1_sys_clk.common,
+ &ahb_ve_dec_clk.common,
+ &ahb_ve_enc_clk.common,
+ &ahb_vid_in_clk.common,
+ &ahb_vid_cout0_clk.common,
+ &ahb_vid_cout1_clk.common,
+ &ahb_de_clk.common,
+ &ahb_npu_clk.common,
+ &ahb_gpu0_clk.common,
+ &ahb_serdes_clk.common,
+ &ahb_usb_sys_clk.common,
+ &ahb_msi_lite0_clk.common,
+ &ahb_store_clk.common,
+ &ahb_cpus_clk.common,
+ &mbus_iommu0_clk.common,
+ &mbus_iommu1_clk.common,
+ &mbus_desys_clk.common,
+ &mbus_ve_enc0_gate_clk.common,
+ &mbus_ve_dec0_gate_clk.common,
+ &mbus_gpu0_clk.common,
+ &mbus_npu_clk.common,
+ &mbus_vid_in_clk.common,
+ &mbus_serdes_clk.common,
+ &mbus_msi_lite0_clk.common,
+ &mbus_store_clk.common,
+ &mbus_msi_lite2_clk.common,
+ &mbus_dma0_clk.common,
+ &mbus_ve_enc0_clk.common,
+ &mbus_ce_clk.common,
+ &mbus_dma1_clk.common,
+ &mbus_nand_clk.common,
+ &mbus_csi_clk.common,
+ &mbus_isp_clk.common,
+ &mbus_gmac0_clk.common,
+ &mbus_gmac1_clk.common,
+ &mbus_ve_dec0_clk.common,
+ &bus_dma0_clk.common,
+ &bus_dma1_clk.common,
+ &bus_spinlock_clk.common,
+ &bus_msgbox0_clk.common,
+ &bus_pwm0_clk.common,
+ &bus_pwm1_clk.common,
+ &bus_dbg_clk.common,
+ &bus_sysdap_clk.common,
&timer0_clk.common,
&timer1_clk.common,
&timer2_clk.common,
@@ -1433,48 +1708,111 @@ static struct ccu_common *sun60i_a733_ccu_clks[] = {
&timer7_clk.common,
&timer8_clk.common,
&timer9_clk.common,
+ &bus_timer_clk.common,
&avs_clk.common,
&de0_clk.common,
+ &bus_de0_clk.common,
&di_clk.common,
+ &bus_di_clk.common,
&g2d_clk.common,
+ &bus_g2d_clk.common,
&eink_clk.common,
&eink_panel_clk.common,
+ &bus_eink_clk.common,
&ve_enc0_clk.common,
&ve_dec0_clk.common,
+ &bus_ve_enc0_clk.common,
+ &bus_ve_dec0_clk.common,
&ce_clk.common,
+ &bus_ce_clk.common,
+ &bus_ce_sys_clk.common,
&npu_clk.common,
+ &bus_npu_clk.common,
&gpu0_clk.common,
+ &bus_gpu0_clk.common,
&dram0_clk.common,
+ &bus_dram0_clk.common,
&nand0_clk0_clk.common,
&nand0_clk1_clk.common,
+ &bus_nand0_clk.common,
&mmc0_clk.common,
+ &bus_mmc0_clk.common,
&mmc1_clk.common,
+ &bus_mmc1_clk.common,
&mmc2_clk.common,
+ &bus_mmc2_clk.common,
&mmc3_clk.common,
+ &bus_mmc3_clk.common,
&ufs_axi_clk.common,
&ufs_cfg_clk.common,
+ &bus_ufs_clk.common,
+ &bus_uart0_clk.common,
+ &bus_uart1_clk.common,
+ &bus_uart2_clk.common,
+ &bus_uart3_clk.common,
+ &bus_uart4_clk.common,
+ &bus_uart5_clk.common,
+ &bus_uart6_clk.common,
+ &bus_i2c0_clk.common,
+ &bus_i2c1_clk.common,
+ &bus_i2c2_clk.common,
+ &bus_i2c3_clk.common,
+ &bus_i2c4_clk.common,
+ &bus_i2c5_clk.common,
+ &bus_i2c6_clk.common,
+ &bus_i2c7_clk.common,
+ &bus_i2c8_clk.common,
+ &bus_i2c9_clk.common,
+ &bus_i2c10_clk.common,
+ &bus_i2c11_clk.common,
+ &bus_i2c12_clk.common,
&spi0_clk.common,
+ &bus_spi0_clk.common,
&spi1_clk.common,
+ &bus_spi1_clk.common,
&spi2_clk.common,
+ &bus_spi2_clk.common,
&spif_clk.common,
+ &bus_spif_clk.common,
&spi3_clk.common,
+ &bus_spi3_clk.common,
&spi4_clk.common,
+ &bus_spi4_clk.common,
&gpadc0_24m_clk.common,
+ &bus_gpadc0_clk.common,
+ &bus_ths0_clk.common,
&irrx_clk.common,
+ &bus_irrx_clk.common,
&irtx_clk.common,
+ &bus_irtx_clk.common,
+ &bus_lradc_clk.common,
&sgpio_clk.common,
+ &bus_sgpio_clk.common,
&lpc_clk.common,
+ &bus_lpc_clk.common,
&i2spcm0_clk.common,
+ &bus_i2spcm0_clk.common,
&i2spcm1_clk.common,
+ &bus_i2spcm1_clk.common,
&i2spcm2_clk.common,
&i2spcm2_asrc_clk.common,
+ &bus_i2spcm2_clk.common,
&i2spcm3_clk.common,
+ &bus_i2spcm3_clk.common,
&i2spcm4_clk.common,
+ &bus_i2spcm4_clk.common,
&spdif_tx_clk.common,
&spdif_rx_clk.common,
+ &bus_spdif_clk.common,
&dmic_clk.common,
+ &bus_dmic_clk.common,
&usb_ohci0_clk.common,
+ &bus_ohci0_clk.common,
+ &bus_ehci0_clk.common,
+ &bus_otg_clk.common,
&usb_ohci1_clk.common,
+ &bus_ohci1_clk.common,
+ &bus_ehci1_clk.common,
&usb01_ref_clk.common,
&usb2_u2_ref_clk.common,
&usb2_suspend_clk.common,
@@ -1486,25 +1824,41 @@ static struct ccu_common *sun60i_a733_ccu_clks[] = {
&serdes_phy_clk.common,
&gmac_ptp_clk.common,
&gmac0_phy_clk.common,
+ &bus_gmac0_clk.common,
&gmac1_phy_clk.common,
+ &bus_gmac1_clk.common,
&tcon_lcd0_clk.common,
+ &bus_tcon_lcd0_clk.common,
&tcon_lcd1_clk.common,
+ &bus_tcon_lcd1_clk.common,
&tcon_lcd2_clk.common,
+ &bus_tcon_lcd2_clk.common,
&dsi0_clk.common,
+ &bus_dsi0_clk.common,
&dsi1_clk.common,
+ &bus_dsi1_clk.common,
&combphy0_clk.common,
&combphy1_clk.common,
+ &bus_tcon_tv0_clk.common,
+ &bus_tcon_tv1_clk.common,
&edp_tv_clk.common,
+ &bus_edp_tv_clk.common,
&hdmi_cec_32k_clk.common,
&hdmi_cec_clk.common,
&hdmi_tv_clk.common,
+ &bus_hdmi_tv_clk.common,
&hdmi_sfr_clk.common,
&hdcp_esm_clk.common,
+ &bus_dpss_top0_clk.common,
+ &bus_dpss_top1_clk.common,
&ledc_clk.common,
+ &bus_ledc_clk.common,
+ &bus_dsc_clk.common,
&csi_master0_clk.common,
&csi_master1_clk.common,
&csi_master2_clk.common,
&csi_clk.common,
+ &bus_csi_clk.common,
&isp_clk.common,
&res_dcap_24m_clk.common,
&apb2jtag_clk.common,
@@ -1570,7 +1924,60 @@ static struct clk_hw_onecell_data sun60i_a733_hw_clks = {
[CLK_APB1] = &apb1_clk.common.hw,
[CLK_APB_UART] = &apb_uart_clk.common.hw,
[CLK_TRACE] = &trace_clk.common.hw,
+ [CLK_BUS_ITS_PCIE0_ACLK] = &bus_its_pcie0_aclk_clk.common.hw,
[CLK_MBUS] = &mbus_clk.common.hw,
+ [CLK_MBUS_IOMMU0_SYS] = &mbus_iommu0_sys_clk.common.hw,
+ [CLK_APB_IOMMU0_SYS] = &apb_iommu0_sys_clk.common.hw,
+ [CLK_AHB_IOMMU0_SYS] = &ahb_iommu0_sys_clk.common.hw,
+ [CLK_BUS_MSI_LITE0] = &bus_msi_lite0_clk.common.hw,
+ [CLK_BUS_MSI_LITE1] = &bus_msi_lite1_clk.common.hw,
+ [CLK_BUS_MSI_LITE2] = &bus_msi_lite2_clk.common.hw,
+ [CLK_MBUS_IOMMU1_SYS] = &mbus_iommu1_sys_clk.common.hw,
+ [CLK_APB_IOMMU1_SYS] = &apb_iommu1_sys_clk.common.hw,
+ [CLK_AHB_IOMMU1_SYS] = &ahb_iommu1_sys_clk.common.hw,
+ [CLK_AHB_VE_DEC] = &ahb_ve_dec_clk.common.hw,
+ [CLK_AHB_VE_ENC] = &ahb_ve_enc_clk.common.hw,
+ [CLK_AHB_VID_IN] = &ahb_vid_in_clk.common.hw,
+ [CLK_AHB_VID_COUT0] = &ahb_vid_cout0_clk.common.hw,
+ [CLK_AHB_VID_COUT1] = &ahb_vid_cout1_clk.common.hw,
+ [CLK_AHB_DE] = &ahb_de_clk.common.hw,
+ [CLK_AHB_NPU] = &ahb_npu_clk.common.hw,
+ [CLK_AHB_GPU0] = &ahb_gpu0_clk.common.hw,
+ [CLK_AHB_SERDES] = &ahb_serdes_clk.common.hw,
+ [CLK_AHB_USB_SYS] = &ahb_usb_sys_clk.common.hw,
+ [CLK_AHB_MSI_LITE0] = &ahb_msi_lite0_clk.common.hw,
+ [CLK_AHB_STORE] = &ahb_store_clk.common.hw,
+ [CLK_AHB_CPUS] = &ahb_cpus_clk.common.hw,
+ [CLK_MBUS_IOMMU0] = &mbus_iommu0_clk.common.hw,
+ [CLK_MBUS_IOMMU1] = &mbus_iommu1_clk.common.hw,
+ [CLK_MBUS_DESYS] = &mbus_desys_clk.common.hw,
+ [CLK_MBUS_VE_ENC0_GATE] = &mbus_ve_enc0_gate_clk.common.hw,
+ [CLK_MBUS_VE_DEC0_GATE] = &mbus_ve_dec0_gate_clk.common.hw,
+ [CLK_MBUS_GPU0] = &mbus_gpu0_clk.common.hw,
+ [CLK_MBUS_NPU] = &mbus_npu_clk.common.hw,
+ [CLK_MBUS_VID_IN] = &mbus_vid_in_clk.common.hw,
+ [CLK_MBUS_SERDES] = &mbus_serdes_clk.common.hw,
+ [CLK_MBUS_MSI_LITE0] = &mbus_msi_lite0_clk.common.hw,
+ [CLK_MBUS_STORE] = &mbus_store_clk.common.hw,
+ [CLK_MBUS_MSI_LITE2] = &mbus_msi_lite2_clk.common.hw,
+ [CLK_MBUS_DMA0] = &mbus_dma0_clk.common.hw,
+ [CLK_MBUS_VE_ENC0] = &mbus_ve_enc0_clk.common.hw,
+ [CLK_MBUS_CE] = &mbus_ce_clk.common.hw,
+ [CLK_MBUS_DMA1] = &mbus_dma1_clk.common.hw,
+ [CLK_MBUS_NAND] = &mbus_nand_clk.common.hw,
+ [CLK_MBUS_CSI] = &mbus_csi_clk.common.hw,
+ [CLK_MBUS_ISP] = &mbus_isp_clk.common.hw,
+ [CLK_MBUS_GMAC0] = &mbus_gmac0_clk.common.hw,
+ [CLK_MBUS_GMAC1] = &mbus_gmac1_clk.common.hw,
+ [CLK_MBUS_VE_DEC0] = &mbus_ve_dec0_clk.common.hw,
+ [CLK_BUS_DMA0] = &bus_dma0_clk.common.hw,
+ [CLK_BUS_DMA1] = &bus_dma1_clk.common.hw,
+ [CLK_BUS_SPINLOCK] = &bus_spinlock_clk.common.hw,
+ [CLK_BUS_MSGBOX0] = &bus_msgbox0_clk.common.hw,
+ [CLK_BUS_PWM0] = &bus_pwm0_clk.common.hw,
+ [CLK_BUS_PWM1] = &bus_pwm1_clk.common.hw,
+ [CLK_BUS_DBG] = &bus_dbg_clk.common.hw,
+ [CLK_BUS_SYSDAP] = &bus_sysdap_clk.common.hw,
[CLK_TIMER0] = &timer0_clk.common.hw,
[CLK_TIMER1] = &timer1_clk.common.hw,
[CLK_TIMER2] = &timer2_clk.common.hw,
@@ -1581,48 +1988,111 @@ static struct clk_hw_onecell_data sun60i_a733_hw_clks = {
[CLK_TIMER7] = &timer7_clk.common.hw,
[CLK_TIMER8] = &timer8_clk.common.hw,
[CLK_TIMER9] = &timer9_clk.common.hw,
+ [CLK_BUS_TIMER] = &bus_timer_clk.common.hw,
[CLK_AVS] = &avs_clk.common.hw,
[CLK_DE0] = &de0_clk.common.hw,
+ [CLK_BUS_DE0] = &bus_de0_clk.common.hw,
[CLK_DI] = &di_clk.common.hw,
+ [CLK_BUS_DI] = &bus_di_clk.common.hw,
[CLK_G2D] = &g2d_clk.common.hw,
+ [CLK_BUS_G2D] = &bus_g2d_clk.common.hw,
[CLK_EINK] = &eink_clk.common.hw,
[CLK_EINK_PANEL] = &eink_panel_clk.common.hw,
+ [CLK_BUS_EINK] = &bus_eink_clk.common.hw,
[CLK_VE_ENC0] = &ve_enc0_clk.common.hw,
[CLK_VE_DEC0] = &ve_dec0_clk.common.hw,
+ [CLK_BUS_VE_ENC0] = &bus_ve_enc0_clk.common.hw,
+ [CLK_BUS_VE_DEC0] = &bus_ve_dec0_clk.common.hw,
[CLK_CE] = &ce_clk.common.hw,
+ [CLK_BUS_CE] = &bus_ce_clk.common.hw,
+ [CLK_BUS_CE_SYS] = &bus_ce_sys_clk.common.hw,
[CLK_NPU] = &npu_clk.common.hw,
+ [CLK_BUS_NPU] = &bus_npu_clk.common.hw,
[CLK_GPU0] = &gpu0_clk.common.hw,
+ [CLK_BUS_GPU0] = &bus_gpu0_clk.common.hw,
[CLK_DRAM0] = &dram0_clk.common.hw,
+ [CLK_BUS_DRAM0] = &bus_dram0_clk.common.hw,
[CLK_NAND0_CLK0] = &nand0_clk0_clk.common.hw,
[CLK_NAND0_CLK1] = &nand0_clk1_clk.common.hw,
+ [CLK_BUS_NAND0] = &bus_nand0_clk.common.hw,
[CLK_MMC0] = &mmc0_clk.common.hw,
+ [CLK_BUS_MMC0] = &bus_mmc0_clk.common.hw,
[CLK_MMC1] = &mmc1_clk.common.hw,
+ [CLK_BUS_MMC1] = &bus_mmc1_clk.common.hw,
[CLK_MMC2] = &mmc2_clk.common.hw,
+ [CLK_BUS_MMC2] = &bus_mmc2_clk.common.hw,
[CLK_MMC3] = &mmc3_clk.common.hw,
+ [CLK_BUS_MMC3] = &bus_mmc3_clk.common.hw,
[CLK_UFS_AXI] = &ufs_axi_clk.common.hw,
[CLK_UFS_CFG] = &ufs_cfg_clk.common.hw,
+ [CLK_BUS_UFS] = &bus_ufs_clk.common.hw,
+ [CLK_BUS_UART0] = &bus_uart0_clk.common.hw,
+ [CLK_BUS_UART1] = &bus_uart1_clk.common.hw,
+ [CLK_BUS_UART2] = &bus_uart2_clk.common.hw,
+ [CLK_BUS_UART3] = &bus_uart3_clk.common.hw,
+ [CLK_BUS_UART4] = &bus_uart4_clk.common.hw,
+ [CLK_BUS_UART5] = &bus_uart5_clk.common.hw,
+ [CLK_BUS_UART6] = &bus_uart6_clk.common.hw,
+ [CLK_BUS_I2C0] = &bus_i2c0_clk.common.hw,
+ [CLK_BUS_I2C1] = &bus_i2c1_clk.common.hw,
+ [CLK_BUS_I2C2] = &bus_i2c2_clk.common.hw,
+ [CLK_BUS_I2C3] = &bus_i2c3_clk.common.hw,
+ [CLK_BUS_I2C4] = &bus_i2c4_clk.common.hw,
+ [CLK_BUS_I2C5] = &bus_i2c5_clk.common.hw,
+ [CLK_BUS_I2C6] = &bus_i2c6_clk.common.hw,
+ [CLK_BUS_I2C7] = &bus_i2c7_clk.common.hw,
+ [CLK_BUS_I2C8] = &bus_i2c8_clk.common.hw,
+ [CLK_BUS_I2C9] = &bus_i2c9_clk.common.hw,
+ [CLK_BUS_I2C10] = &bus_i2c10_clk.common.hw,
+ [CLK_BUS_I2C11] = &bus_i2c11_clk.common.hw,
+ [CLK_BUS_I2C12] = &bus_i2c12_clk.common.hw,
[CLK_SPI0] = &spi0_clk.common.hw,
+ [CLK_BUS_SPI0] = &bus_spi0_clk.common.hw,
[CLK_SPI1] = &spi1_clk.common.hw,
+ [CLK_BUS_SPI1] = &bus_spi1_clk.common.hw,
[CLK_SPI2] = &spi2_clk.common.hw,
+ [CLK_BUS_SPI2] = &bus_spi2_clk.common.hw,
[CLK_SPIF] = &spif_clk.common.hw,
+ [CLK_BUS_SPIF] = &bus_spif_clk.common.hw,
[CLK_SPI3] = &spi3_clk.common.hw,
+ [CLK_BUS_SPI3] = &bus_spi3_clk.common.hw,
[CLK_SPI4] = &spi4_clk.common.hw,
+ [CLK_BUS_SPI4] = &bus_spi4_clk.common.hw,
[CLK_GPADC0_24M] = &gpadc0_24m_clk.common.hw,
+ [CLK_BUS_GPADC0] = &bus_gpadc0_clk.common.hw,
+ [CLK_BUS_THS0] = &bus_ths0_clk.common.hw,
[CLK_IRRX] = &irrx_clk.common.hw,
+ [CLK_BUS_IRRX] = &bus_irrx_clk.common.hw,
[CLK_IRTX] = &irtx_clk.common.hw,
+ [CLK_BUS_IRTX] = &bus_irtx_clk.common.hw,
+ [CLK_BUS_LRADC] = &bus_lradc_clk.common.hw,
[CLK_SGPIO] = &sgpio_clk.common.hw,
+ [CLK_BUS_SGPIO] = &bus_sgpio_clk.common.hw,
[CLK_LPC] = &lpc_clk.common.hw,
+ [CLK_BUS_LPC] = &bus_lpc_clk.common.hw,
[CLK_I2SPCM0] = &i2spcm0_clk.common.hw,
+ [CLK_BUS_I2SPCM0] = &bus_i2spcm0_clk.common.hw,
[CLK_I2SPCM1] = &i2spcm1_clk.common.hw,
+ [CLK_BUS_I2SPCM1] = &bus_i2spcm1_clk.common.hw,
[CLK_I2SPCM2] = &i2spcm2_clk.common.hw,
[CLK_I2SPCM2_ASRC] = &i2spcm2_asrc_clk.common.hw,
+ [CLK_BUS_I2SPCM2] = &bus_i2spcm2_clk.common.hw,
[CLK_I2SPCM3] = &i2spcm3_clk.common.hw,
+ [CLK_BUS_I2SPCM3] = &bus_i2spcm3_clk.common.hw,
[CLK_I2SPCM4] = &i2spcm4_clk.common.hw,
+ [CLK_BUS_I2SPCM4] = &bus_i2spcm4_clk.common.hw,
[CLK_SPDIF_TX] = &spdif_tx_clk.common.hw,
[CLK_SPDIF_RX] = &spdif_rx_clk.common.hw,
+ [CLK_BUS_SPDIF] = &bus_spdif_clk.common.hw,
[CLK_DMIC] = &dmic_clk.common.hw,
+ [CLK_BUS_DMIC] = &bus_dmic_clk.common.hw,
[CLK_USB_OHCI0] = &usb_ohci0_clk.common.hw,
+ [CLK_BUS_OHCI0] = &bus_ohci0_clk.common.hw,
+ [CLK_BUS_EHCI0] = &bus_ehci0_clk.common.hw,
+ [CLK_BUS_OTG] = &bus_otg_clk.common.hw,
[CLK_USB_OHCI1] = &usb_ohci1_clk.common.hw,
+ [CLK_BUS_OHCI1] = &bus_ohci1_clk.common.hw,
+ [CLK_BUS_EHCI1] = &bus_ehci1_clk.common.hw,
[CLK_USB01_REF] = &usb01_ref_clk.common.hw,
[CLK_USB2_U2_REF] = &usb2_u2_ref_clk.common.hw,
[CLK_USB2_SUSPEND] = &usb2_suspend_clk.common.hw,
@@ -1634,25 +2104,41 @@ static struct clk_hw_onecell_data sun60i_a733_hw_clks = {
[CLK_SERDES_PHY] = &serdes_phy_clk.common.hw,
[CLK_GMAC_PTP] = &gmac_ptp_clk.common.hw,
[CLK_GMAC0_PHY] = &gmac0_phy_clk.common.hw,
+ [CLK_BUS_GMAC0] = &bus_gmac0_clk.common.hw,
[CLK_GMAC1_PHY] = &gmac1_phy_clk.common.hw,
+ [CLK_BUS_GMAC1] = &bus_gmac1_clk.common.hw,
[CLK_TCON_LCD0] = &tcon_lcd0_clk.common.hw,
+ [CLK_BUS_TCON_LCD0] = &bus_tcon_lcd0_clk.common.hw,
[CLK_TCON_LCD1] = &tcon_lcd1_clk.common.hw,
+ [CLK_BUS_TCON_LCD1] = &bus_tcon_lcd1_clk.common.hw,
[CLK_TCON_LCD2] = &tcon_lcd2_clk.common.hw,
+ [CLK_BUS_TCON_LCD2] = &bus_tcon_lcd2_clk.common.hw,
[CLK_DSI0] = &dsi0_clk.common.hw,
+ [CLK_BUS_DSI0] = &bus_dsi0_clk.common.hw,
[CLK_DSI1] = &dsi1_clk.common.hw,
+ [CLK_BUS_DSI1] = &bus_dsi1_clk.common.hw,
[CLK_COMBPHY0] = &combphy0_clk.common.hw,
[CLK_COMBPHY1] = &combphy1_clk.common.hw,
+ [CLK_BUS_TCON_TV0] = &bus_tcon_tv0_clk.common.hw,
+ [CLK_BUS_TCON_TV1] = &bus_tcon_tv1_clk.common.hw,
[CLK_EDP_TV] = &edp_tv_clk.common.hw,
+ [CLK_BUS_EDP_TV] = &bus_edp_tv_clk.common.hw,
[CLK_HDMI_CEC_32K] = &hdmi_cec_32k_clk.common.hw,
[CLK_HDMI_CEC] = &hdmi_cec_clk.common.hw,
[CLK_HDMI_TV] = &hdmi_tv_clk.common.hw,
+ [CLK_BUS_HDMI_TV] = &bus_hdmi_tv_clk.common.hw,
[CLK_HDMI_SFR] = &hdmi_sfr_clk.common.hw,
[CLK_HDCP_ESM] = &hdcp_esm_clk.common.hw,
+ [CLK_BUS_DPSS_TOP0] = &bus_dpss_top0_clk.common.hw,
+ [CLK_BUS_DPSS_TOP1] = &bus_dpss_top1_clk.common.hw,
[CLK_LEDC] = &ledc_clk.common.hw,
+ [CLK_BUS_LEDC] = &bus_ledc_clk.common.hw,
+ [CLK_BUS_DSC] = &bus_dsc_clk.common.hw,
[CLK_CSI_MASTER0] = &csi_master0_clk.common.hw,
[CLK_CSI_MASTER1] = &csi_master1_clk.common.hw,
[CLK_CSI_MASTER2] = &csi_master2_clk.common.hw,
[CLK_CSI] = &csi_clk.common.hw,
+ [CLK_BUS_CSI] = &bus_csi_clk.common.hw,
[CLK_ISP] = &isp_clk.common.hw,
[CLK_RES_DCAP_24M] = &res_dcap_24m_clk.common.hw,
[CLK_APB2JTAG] = &apb2jtag_clk.common.hw,
--
2.54.0
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox