* Re: [RFC PATCH v3 09/12] net: add support for skbs with unreadable frags
From: Stanislav Fomichev @ 2023-11-07 1:06 UTC (permalink / raw)
To: Mina Almasry
Cc: Willem de Bruijn, David Ahern, netdev, linux-kernel, linux-arch,
linux-kselftest, linux-media, dri-devel, linaro-mm-sig,
David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Jesper Dangaard Brouer, Ilias Apalodimas, Arnd Bergmann,
Shuah Khan, Sumit Semwal, Christian König, Shakeel Butt,
Jeroen de Borst, Praveen Kaligineedi, Willem de Bruijn,
Kaiyuan Zhang
In-Reply-To: <CAHS8izNxKHhW5uCqmfau6n3c18=hE3RXzA+ng5LEGiKj12nGcg@mail.gmail.com>
On 11/06, Mina Almasry wrote:
> On Mon, Nov 6, 2023 at 4:08 PM Willem de Bruijn
> <willemdebruijn.kernel@gmail.com> wrote:
> >
> > On Mon, Nov 6, 2023 at 3:55 PM Stanislav Fomichev <sdf@google.com> wrote:
> > >
> > > On Mon, Nov 6, 2023 at 3:27 PM Mina Almasry <almasrymina@google.com> wrote:
> > > >
> > > > On Mon, Nov 6, 2023 at 2:59 PM Stanislav Fomichev <sdf@google.com> wrote:
> > > > >
> > > > > On 11/06, Mina Almasry wrote:
> > > > > > On Mon, Nov 6, 2023 at 1:59 PM Stanislav Fomichev <sdf@google.com> wrote:
> > > > > > >
> > > > > > > On 11/06, Mina Almasry wrote:
> > > > > > > > On Mon, Nov 6, 2023 at 11:34 AM David Ahern <dsahern@kernel.org> wrote:
> > > > > > > > >
> > > > > > > > > On 11/6/23 11:47 AM, Stanislav Fomichev wrote:
> > > > > > > > > > On 11/05, Mina Almasry wrote:
> > > > > > > > > >> For device memory TCP, we expect the skb headers to be available in host
> > > > > > > > > >> memory for access, and we expect the skb frags to be in device memory
> > > > > > > > > >> and unaccessible to the host. We expect there to be no mixing and
> > > > > > > > > >> matching of device memory frags (unaccessible) with host memory frags
> > > > > > > > > >> (accessible) in the same skb.
> > > > > > > > > >>
> > > > > > > > > >> Add a skb->devmem flag which indicates whether the frags in this skb
> > > > > > > > > >> are device memory frags or not.
> > > > > > > > > >>
> > > > > > > > > >> __skb_fill_page_desc() now checks frags added to skbs for page_pool_iovs,
> > > > > > > > > >> and marks the skb as skb->devmem accordingly.
> > > > > > > > > >>
> > > > > > > > > >> Add checks through the network stack to avoid accessing the frags of
> > > > > > > > > >> devmem skbs and avoid coalescing devmem skbs with non devmem skbs.
> > > > > > > > > >>
> > > > > > > > > >> Signed-off-by: Willem de Bruijn <willemb@google.com>
> > > > > > > > > >> Signed-off-by: Kaiyuan Zhang <kaiyuanz@google.com>
> > > > > > > > > >> Signed-off-by: Mina Almasry <almasrymina@google.com>
> > > > > > > > > >>
> > > > > > > > > >> ---
> > > > > > > > > >> include/linux/skbuff.h | 14 +++++++-
> > > > > > > > > >> include/net/tcp.h | 5 +--
> > > > > > > > > >> net/core/datagram.c | 6 ++++
> > > > > > > > > >> net/core/gro.c | 5 ++-
> > > > > > > > > >> net/core/skbuff.c | 77 ++++++++++++++++++++++++++++++++++++------
> > > > > > > > > >> net/ipv4/tcp.c | 6 ++++
> > > > > > > > > >> net/ipv4/tcp_input.c | 13 +++++--
> > > > > > > > > >> net/ipv4/tcp_output.c | 5 ++-
> > > > > > > > > >> net/packet/af_packet.c | 4 +--
> > > > > > > > > >> 9 files changed, 115 insertions(+), 20 deletions(-)
> > > > > > > > > >>
> > > > > > > > > >> diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
> > > > > > > > > >> index 1fae276c1353..8fb468ff8115 100644
> > > > > > > > > >> --- a/include/linux/skbuff.h
> > > > > > > > > >> +++ b/include/linux/skbuff.h
> > > > > > > > > >> @@ -805,6 +805,8 @@ typedef unsigned char *sk_buff_data_t;
> > > > > > > > > >> * @csum_level: indicates the number of consecutive checksums found in
> > > > > > > > > >> * the packet minus one that have been verified as
> > > > > > > > > >> * CHECKSUM_UNNECESSARY (max 3)
> > > > > > > > > >> + * @devmem: indicates that all the fragments in this skb are backed by
> > > > > > > > > >> + * device memory.
> > > > > > > > > >> * @dst_pending_confirm: need to confirm neighbour
> > > > > > > > > >> * @decrypted: Decrypted SKB
> > > > > > > > > >> * @slow_gro: state present at GRO time, slower prepare step required
> > > > > > > > > >> @@ -991,7 +993,7 @@ struct sk_buff {
> > > > > > > > > >> #if IS_ENABLED(CONFIG_IP_SCTP)
> > > > > > > > > >> __u8 csum_not_inet:1;
> > > > > > > > > >> #endif
> > > > > > > > > >> -
> > > > > > > > > >> + __u8 devmem:1;
> > > > > > > > > >> #if defined(CONFIG_NET_SCHED) || defined(CONFIG_NET_XGRESS)
> > > > > > > > > >> __u16 tc_index; /* traffic control index */
> > > > > > > > > >> #endif
> > > > > > > > > >> @@ -1766,6 +1768,12 @@ static inline void skb_zcopy_downgrade_managed(struct sk_buff *skb)
> > > > > > > > > >> __skb_zcopy_downgrade_managed(skb);
> > > > > > > > > >> }
> > > > > > > > > >>
> > > > > > > > > >> +/* Return true if frags in this skb are not readable by the host. */
> > > > > > > > > >> +static inline bool skb_frags_not_readable(const struct sk_buff *skb)
> > > > > > > > > >> +{
> > > > > > > > > >> + return skb->devmem;
> > > > > > > > > >
> > > > > > > > > > bikeshedding: should we also rename 'devmem' sk_buff flag to 'not_readable'?
> > > > > > > > > > It better communicates the fact that the stack shouldn't dereference the
> > > > > > > > > > frags (because it has 'devmem' fragments or for some other potential
> > > > > > > > > > future reason).
> > > > > > > > >
> > > > > > > > > +1.
> > > > > > > > >
> > > > > > > > > Also, the flag on the skb is an optimization - a high level signal that
> > > > > > > > > one or more frags is in unreadable memory. There is no requirement that
> > > > > > > > > all of the frags are in the same memory type.
> > > > > > >
> > > > > > > David: maybe there should be such a requirement (that they all are
> > > > > > > unreadable)? Might be easier to support initially; we can relax later
> > > > > > > on.
> > > > > > >
> > > > > >
> > > > > > Currently devmem == not_readable, and the restriction is that all the
> > > > > > frags in the same skb must be either all readable or all unreadable
> > > > > > (all devmem or all non-devmem).
> > > > > >
> > > > > > > > The flag indicates that the skb contains all devmem dma-buf memory
> > > > > > > > specifically, not generic 'not_readable' frags as the comment says:
> > > > > > > >
> > > > > > > > + * @devmem: indicates that all the fragments in this skb are backed by
> > > > > > > > + * device memory.
> > > > > > > >
> > > > > > > > The reason it's not a generic 'not_readable' flag is because handing
> > > > > > > > off a generic not_readable skb to the userspace is semantically not
> > > > > > > > what we're doing. recvmsg() is augmented in this patch series to
> > > > > > > > return a devmem skb to the user via a cmsg_devmem struct which refers
> > > > > > > > specifically to the memory in the dma-buf. recvmsg() in this patch
> > > > > > > > series is not augmented to give any 'not_readable' skb to the
> > > > > > > > userspace.
> > > > > > > >
> > > > > > > > IMHO skb->devmem + an skb_frags_not_readable() as implemented is
> > > > > > > > correct. If a new type of unreadable skbs are introduced to the stack,
> > > > > > > > I imagine the stack would implement:
> > > > > > > >
> > > > > > > > 1. new header flag: skb->newmem
> > > > > > > > 2.
> > > > > > > >
> > > > > > > > static inline bool skb_frags_not_readable(const struct skb_buff *skb)
> > > > > > > > {
> > > > > > > > return skb->devmem || skb->newmem;
> > > > > > > > }
> > > > > > > >
> > > > > > > > 3. tcp_recvmsg_devmem() would handle skb->devmem skbs is in this patch
> > > > > > > > series, but tcp_recvmsg_newmem() would handle skb->newmem skbs.
> > > > > > >
> > > > > > > You copy it to the userspace in a special way because your frags
> > > > > > > are page_is_page_pool_iov(). I agree with David, the skb bit is
> > > > > > > just and optimization.
> > > > > > >
> > > > > > > For most of the core stack, it doesn't matter why your skb is not
> > > > > > > readable. For a few places where it matters (recvmsg?), you can
> > > > > > > double-check your frags (all or some) with page_is_page_pool_iov.
> > > > > > >
> > > > > >
> > > > > > I see, we can do that then. I.e. make the header flag 'not_readable'
> > > > > > and check the frags to decide to delegate to tcp_recvmsg_devmem() or
> > > > > > something else. We can even assume not_readable == devmem because
> > > > > > currently devmem is the only type of unreadable frag currently.
> > > > > >
> > > > > > > Unrelated: we probably need socket to dmabuf association as well (via
> > > > > > > netlink or something).
> > > > > >
> > > > > > Not sure this is possible. The dma-buf is bound to the rx-queue, and
> > > > > > any packets that land on that rx-queue are bound to that dma-buf,
> > > > > > regardless of which socket that packet belongs to. So the association
> > > > > > IMO must be rx-queue to dma-buf, not socket to dma-buf.
> > > > >
> > > > > But there is still always 1 dmabuf to 1 socket association (on rx), right?
> > > > > Because otherwise, there is no way currently to tell, at recvmsg, which
> > > > > dmabuf the received token belongs to.
> > > > >
> > > >
> > > > Yes, but this 1 dma-buf to 1 socket association happens because the
> > > > user binds the dma-buf to an rx-queue and configures flow steering of
> > > > the socket to that rx-queue.
> > >
> > > It's still fixed and won't change during the socket lifetime, right?
>
> Technically, no.
>
> The user is free to modify or delete flow steering rules outside of
> the lifetime of the socket. Technically it's possible for the user to
> reconfigure flow steering while the socket is simultaneously
> receiving, and the result will be packets switching
> from devmem to non-devmem. For a reasonably correctly configured
> application the application would probably want to steer 1 flow to 1
> dma-buf and never change it, but this is not something we enforce, but
> rather the user orchestrates. In theory someone can find a use case
> for configuring and unconfigure flow steering during a connection.
If we do want to support this flexible configuration then we also
should export some dmabuf id along with the token?
> > > And the socket has to know this association; otherwise those tokens
> > > are useless since they don't carry anything to identify the dmabuf.
> > >
> > > I think my other issue with MSG_SOCK_DEVMEM being on recvmsg is that
> > > it somehow implies that I have an option of passing or not passing it
> > > for an individual system call.
>
> You do have the option of passing it or not passing it per system
> call. The MSG_SOCK_DEVMEM says the application is willing to receive
> devmem cmsgs - that's all. The application doesn't get to decide
> whether it's actually going to receive a devmem cmsg or not, because
> that's dictated by the type of skb that is present in the receive
> queue, and not up to the application. I should explain this in the
> commit message...
What would be the case of passing it or not passing it? Some fallback to
the host memory after flow steering update? Yeah, would be useful to
document those constrains. I'd lean on starting stricter and relaxing
those conditions if we find the use-cases.
> > > If we know that we're going to use dmabuf with the socket, maybe we
> > > should move this flag to the socket() syscall?
> > >
> > > fd = socket(AF_INET6, SOCK_STREAM, SOCK_DEVMEM);
> > >
> > > ?
> >
> > I think it should then be a setsockopt called before any data is
> > exchanged, with no change of modifying mode later. We generally use
> > setsockopts for the mode of a socket. This use of the protocol field
> > in socket() for setting a mode would be novel. Also, it might miss
> > passively opened connections, or be overly restrictive: one approach
> > for all accepted child sockets.
>
> We can definitely move SOCK_DEVMEM to a setsockopt(). Seems more than
> reasonable.
SG, added another suggestion for SO_DEVMEM_DONTNEED on another thread
with Willem. LMK what you think.
^ permalink raw reply
* Re: [RFC PATCH v3 09/12] net: add support for skbs with unreadable frags
From: David Ahern @ 2023-11-07 1:09 UTC (permalink / raw)
To: Mina Almasry, Willem de Bruijn
Cc: Stanislav Fomichev, netdev, linux-kernel, linux-arch,
linux-kselftest, linux-media, dri-devel, linaro-mm-sig,
David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Jesper Dangaard Brouer, Ilias Apalodimas, Arnd Bergmann,
Shuah Khan, Sumit Semwal, Christian König, Shakeel Butt,
Jeroen de Borst, Praveen Kaligineedi, Willem de Bruijn,
Kaiyuan Zhang
In-Reply-To: <CAHS8izNxKHhW5uCqmfau6n3c18=hE3RXzA+ng5LEGiKj12nGcg@mail.gmail.com>
On 11/6/23 5:20 PM, Mina Almasry wrote:
> The user is free to modify or delete flow steering rules outside of the
> lifetime of the socket. Technically it's possible for the user to
> reconfigure flow steering while the socket is simultaneously receiving,
> and the result will be packets switching from devmem to non-devmem.
generically, from one page pool to another (ie., devmem piece of that
statement is not relevant).
^ permalink raw reply
* [PATCH iproute2] libnetlink: validate nlmsg header length first
From: Max Kunzelmann @ 2023-11-07 1:20 UTC (permalink / raw)
To: netdev; +Cc: stephen, dsahern, Max Kunzelmann, Benny Baumann,
Robert Geislinger
Validate the nlmsg header length before accessing the nlmsg payload
length.
Fixes: 892a25e286fb ("libnetlink: break up dump function")
Signed-off-by: Max Kunzelmann <maxdev@posteo.de>
Reviewed-by: Benny Baumann <BenBE@geshi.org>
Reviewed-by: Robert Geislinger <github@crpykng.de>
---
lib/libnetlink.c | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/lib/libnetlink.c b/lib/libnetlink.c
index 7edcd285..01648229 100644
--- a/lib/libnetlink.c
+++ b/lib/libnetlink.c
@@ -727,13 +727,15 @@ int rtnl_dump_request_n(struct rtnl_handle *rth, struct nlmsghdr *n)
static int rtnl_dump_done(struct nlmsghdr *h,
const struct rtnl_dump_filter_arg *a)
{
- int len = *(int *)NLMSG_DATA(h);
+ int len;
if (h->nlmsg_len < NLMSG_LENGTH(sizeof(int))) {
fprintf(stderr, "DONE truncated\n");
return -1;
}
+ len = *(int *)NLMSG_DATA(h);
+
if (len < 0) {
errno = -len;
--
2.42.0
^ permalink raw reply related
* Re: [PATCH net] idpf: fix potential use-after-free in idpf_tso()
From: patchwork-bot+netdevbpf @ 2023-11-07 1:30 UTC (permalink / raw)
To: Eric Dumazet
Cc: davem, kuba, pabeni, netdev, eric.dumazet, joshua.a.hay,
alan.brady, madhu.chittim, phani.r.burra, sridhar.samudrala,
willemb, pavan.kumar.linga, anthony.l.nguyen, bcf
In-Reply-To: <20231103200451.514047-1-edumazet@google.com>
Hello:
This patch was applied to netdev/net.git (main)
by Jakub Kicinski <kuba@kernel.org>:
On Fri, 3 Nov 2023 20:04:51 +0000 you wrote:
> skb_cow_head() can change skb->head (and thus skb_shinfo(skb))
>
> We must not cache skb_shinfo(skb) before skb_cow_head().
>
> Fixes: 6818c4d5b3c2 ("idpf: add splitq start_xmit")
> Signed-off-by: Eric Dumazet <edumazet@google.com>
> Cc: Joshua Hay <joshua.a.hay@intel.com>
> Cc: Alan Brady <alan.brady@intel.com>
> Cc: Madhu Chittim <madhu.chittim@intel.com>
> Cc: Phani Burra <phani.r.burra@intel.com>
> Cc: Sridhar Samudrala <sridhar.samudrala@intel.com>
> Cc: Willem de Bruijn <willemb@google.com>
> Cc: Pavan Kumar Linga <pavan.kumar.linga@intel.com>
> Cc: Tony Nguyen <anthony.l.nguyen@intel.com>
> Cc: Bailey Forrest <bcf@google.com>
>
> [...]
Here is the summary with links:
- [net] idpf: fix potential use-after-free in idpf_tso()
https://git.kernel.org/netdev/net/c/115c0f4d5857
You are awesome, thank you!
--
Deet-doot-dot, I am a bot.
https://korg.docs.kernel.org/patchwork/pwbot.html
^ permalink raw reply
* Re: [PATCH v2] tg3: power down device only on SYSTEM_POWER_OFF
From: patchwork-bot+netdevbpf @ 2023-11-07 1:30 UTC (permalink / raw)
To: George Shuklin; +Cc: netdev, kai.heng.feng
In-Reply-To: <20231103115029.83273-1-george.shuklin@gmail.com>
Hello:
This patch was applied to netdev/net.git (main)
by Jakub Kicinski <kuba@kernel.org>:
On Fri, 3 Nov 2023 13:50:29 +0200 you wrote:
> Dell R650xs servers hangs on reboot if tg3 driver calls
> tg3_power_down.
>
> This happens only if network adapters (BCM5720 for R650xs) were
> initialized using SNP (e.g. by booting ipxe.efi).
>
> The actual problem is on Dell side, but this fix allows servers
> to come back alive after reboot.
>
> [...]
Here is the summary with links:
- [v2] tg3: power down device only on SYSTEM_POWER_OFF
https://git.kernel.org/netdev/net/c/9fc3bc764334
You are awesome, thank you!
--
Deet-doot-dot, I am a bot.
https://korg.docs.kernel.org/patchwork/pwbot.html
^ permalink raw reply
* [linus:master] [kselftest] 9c2a19f715: kernel-selftests.net.rtnetlink.sh.gretap.fail
From: kernel test robot @ 2023-11-07 2:00 UTC (permalink / raw)
To: Daniel Mendes
Cc: oe-lkp, lkp, linux-kernel, David S. Miller, netdev, oliver.sang
hi, Daniel Mendes,
we reported
"[linux-next:master] [kselftest] 9c2a19f715: kernel-selftests.net.rtnetlink.sh.gretap.fail"
in
https://lore.kernel.org/all/202310112125.c5889283-oliver.sang@intel.com/
when this commit is in linux-next/master
now we noticed the commit was merged into mainline. and we still saw the same
issue in our tests. just FYI.
Hello,
kernel test robot noticed "kernel-selftests.net.rtnetlink.sh.gretap.fail" on:
commit: 9c2a19f71515553a874e2bf31655ac2264a66e37 ("kselftest: rtnetlink.sh: add verbose flag")
https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git master
[test failed on linus/master 2c40c1c6adab90ee4660caf03722b3a3ec67767b]
[test failed on linux-next/master e27090b1413ff236ca1aec26d6b022149115de2c]
in testcase: kernel-selftests
version: kernel-selftests-x86_64-60acb023-1_20230329
with following parameters:
group: net
compiler: gcc-12
test machine: 36 threads 1 sockets Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz (Cascade Lake) with 32G memory
(please refer to attached dmesg/kmsg for entire log/backtrace)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <oliver.sang@intel.com>
| Closes: https://lore.kernel.org/oe-lkp/202311061651.88994df5-oliver.sang@intel.com
....
# timeout set to 1500
# selftests: net: rtnetlink.sh
# PASS: policy routing
# PASS: route get
# PASS: preferred_lft addresses have expired
# PASS: promote_secondaries complete
# PASS: tc htb hierarchy
# PASS: gre tunnel endpoint
# FAIL: gretap
# PASS: ip6gretap
# PASS: erspan
# PASS: ip6erspan
# PASS: bridge setup
# PASS: ipv6 addrlabel
# PASS: set ifalias f007c594-8cc4-4bfa-bf82-fef2dfb81ad2 for test-dummy0
# PASS: vrf
# PASS: macsec
# PASS: macsec_offload
# PASS: ipsec
# PASS: ipsec_offload
# PASS: bridge fdb get
# PASS: neigh get
# PASS: bridge_parent_id
# Error: either "local" is duplicate, or "proto" is a garbage.
# Error: either "local" is duplicate, or "proto" is a garbage.
# Error: either "local" is duplicate, or "proto" is a garbage.
# Error: either "dev" is duplicate, or "proto" is a garbage.
# Error: either "dev" is duplicate, or "proto" is a garbage.
# Error: either "dev" is duplicate, or "proto" is a garbage.
# Error: ipv4: Address not found.
# Error: ipv4: Address not found.
# FAIL: address proto IPv4
# Error: either "local" is duplicate, or "proto" is a garbage.
# Error: either "local" is duplicate, or "proto" is a garbage.
# Error: either "local" is duplicate, or "proto" is a garbage.
# Error: either "dev" is duplicate, or "proto" is a garbage.
# Error: either "dev" is duplicate, or "proto" is a garbage.
# Error: either "dev" is duplicate, or "proto" is a garbage.
# Error: ipv6: address not found.
# Error: ipv6: address not found.
# FAIL: address proto IPv6
not ok 15 selftests: net: rtnetlink.sh # exit=1
....
The kernel config and materials to reproduce are available at:
https://download.01.org/0day-ci/archive/20231106/202311061651.88994df5-oliver.sang@intel.com
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
^ permalink raw reply
* RE: [PATCH] netlink: introduce netlink poll to resolve fast return issue
From: Jong eon Park @ 2023-11-07 2:05 UTC (permalink / raw)
To: 'Jakub Kicinski', 'Paolo Abeni'
Cc: 'David S. Miller', 'Eric Dumazet', netdev,
linux-kernel, 'Dong ha Kang'
In-Reply-To: <20231106154812.14c470c2@kernel.org>
> -----Original Message-----
> From: Jakub Kicinski <kuba@kernel.org>
> Sent: Tuesday, November 7, 2023 8:48 AM
> To: Jong eon Park <jongeon.park@samsung.com>; Paolo Abeni
> <pabeni@redhat.com>
> Cc: David S. Miller <davem@davemloft.net>; Eric Dumazet
> <edumazet@google.com>; netdev@vger.kernel.org; linux-
> kernel@vger.kernel.org; Dong ha Kang <dongha7.kang@samsung.com>
> Subject: Re: [PATCH] netlink: introduce netlink poll to resolve fast
> return issue
>
> On Fri, 3 Nov 2023 16:22:09 +0900 Jong eon Park wrote:
> > In very rare cases, there was an issue where a user's poll function
> > waiting for a uevent would continuously return very quickly, causing
> > excessive CPU usage due to the following scenario.
> >
> > Once sk_rcvbuf becomes full netlink_broadcast_deliver returns an error
> > and netlink_overrun is called. However, if netlink_overrun was called
> > in a context just before a another context returns from the poll and
> > recv is invoked, emptying the rcvbuf, sk->sk_err = ENOBUF is written
> > to the netlink socket belatedly and it enters the NETLINK_S_CONGESTED
> state.
> > If the user does not check for POLLERR, they cannot consume and clean
> > sk_err and repeatedly enter the situation where they call poll again
> > but return immediately.
> >
> > To address this issue, I would like to introduce the following netlink
> > poll.
> >
> > After calling the datagram_poll, netlink poll checks the
> > NETLINK_S_CONGESTED status and rcv queue, and this make the user to be
> > readable once more even if the user has already emptied rcv queue.
> > This allows the user to be able to consume sk->sk_err value through
> > netlink_recvmsg, thus the situation described above can be avoided
>
> The explanation makes sense, but I'm not able to make the jump in
> understanding how this is a netlink problem. datagram_poll() returns
> EPOLLERR because sk_err is set, what makes netlink special?
> The fact that we can have an sk_err with nothing in the recv queue?
>
> Paolo understands this better, maybe he can weigh in tomorrow...
Perhaps my explanation was not comprehensive enough.
The issue at hand is that once it occurs, users cannot escape from this
"busy running" situation, and the inadequate handling of EPOLLERR by users
imposes a heavy burden on the entire system, which seems quite harsh.
The reason for a separate netlink poll is related to the netlink state.
When it enters the NETLINK_S_CONGESTED state, sk can no longer receive or
deliver skb, and the receive_queue must be completely emptied to clear the
state. However, it was found that the NETLINK_S_CONGESTED state was still
maintained even when the receive_queue was empty, which was incorrect, and
that's why I implemented the handling in poll.
I don't consider this approach to be the best way, so if you have any
recommendations for a better solution, I would appreciate it.
Regards.
JE Park.
^ permalink raw reply
* Re: [PATCH] staging: Revert "staging: qlge: Retire the driver"
From: Benjamin Poirier @ 2023-11-07 2:15 UTC (permalink / raw)
To: Greg Kroah-Hartman
Cc: Kira, David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Jonathan Corbet, Manish Chopra, GR-Linux-NIC-Dev, Coiby Xu,
James E.J. Bottomley, Helge Deller, Sven Joachim, Ian Kent,
netdev, linux-doc, linux-kernel, linux-parisc, linux-staging
In-Reply-To: <2023110655-swarm-parka-177d@gregkh>
On 2023-11-06 07:54 +0100, Greg Kroah-Hartman wrote:
> On Tue, Oct 31, 2023 at 02:04:00AM +1100, Benjamin Poirier wrote:
> > This reverts commit 875be090928d19ff4ae7cbaadb54707abb3befdf.
> >
> > On All Hallows' Eve, fear and cower for it is the return of the undead
> > driver.
> >
> > There was a report [1] from a user of a QLE8142 device. They would like for
> > the driver to remain in the kernel. Therefore, revert the removal of the
> > qlge driver.
> >
> > [1] https://lore.kernel.org/netdev/566c0155-4f80-43ec-be2c-2d1ad631bf25@gmail.com/
>
> <snip>
>
> > --- /dev/null
> > +++ b/drivers/staging/qlge/TODO
> > @@ -0,0 +1,28 @@
> > +* commit 7c734359d350 ("qlge: Size RX buffers based on MTU.", v2.6.33-rc1)
> > + introduced dead code in the receive routines, which should be rewritten
> > + anyways by the admission of the author himself, see the comment above
> > + qlge_build_rx_skb(). That function is now used exclusively to handle packets
> > + that underwent header splitting but it still contains code to handle non
> > + split cases.
> > +* truesize accounting is incorrect (ex: a 9000B frame has skb->truesize 10280
> > + while containing two frags of order-1 allocations, ie. >16K)
> > +* while in that area, using two 8k buffers to store one 9k frame is a poor
> > + choice of buffer size.
> > +* in the "chain of large buffers" case, the driver uses an skb allocated with
> > + head room but only puts data in the frags.
> > +* rename "rx" queues to "completion" queues. Calling tx completion queues "rx
> > + queues" is confusing.
> > +* struct rx_ring is used for rx and tx completions, with some members relevant
> > + to one case only
> > +* the flow control implementation in firmware is buggy (sends a flood of pause
> > + frames, resets the link, device and driver buffer queues become
> > + desynchronized), disable it by default
> > +* the driver has a habit of using runtime checks where compile time checks are
> > + possible (ex. qlge_free_rx_buffers())
> > +* reorder struct members to avoid holes if it doesn't impact performance
> > +* use better-suited apis (ex. use pci_iomap() instead of ioremap())
> > +* remove duplicate and useless comments
> > +* fix weird line wrapping (all over, ex. the qlge_set_routing_reg() calls in
> > + qlge_set_multicast_list()).
> > +* remove useless casts (ex. memset((void *)mac_iocb_ptr, ...))
> > +* fix checkpatch issues
>
> In looking at this again, are you sure you all want this in the tree?
> I'm glad to take the revert but ONLY if you are willing to then take a
> "move this to drivers/net/" patch for the code as well, WITH an actual
> maintainer and developer who is willing to do the work for this code.
>
> In all the years that this has been in the staging tree, the listed
> maintainers have not been active at all from what I can remember, and
> obviously the above list of "things to fix" have not really been worked
> on at all.
>
> So why should it be added back? I understand there is at least one
> reported user, but for drivers in the staging tree, that's not a good
> reason to keep them around if there is not an actual maintainer that is
> willing to do the work.
>
> Which reminds me, we should probably sweep the drivers/staging/ tree
> again to see what we can remove given a lack of real development.
> Normally we do that every other year or so, and this driver would fall
> into the "no one is doing anything with it" category and should be
> dropped.
Thank you for revisiting this topic. I agree with you that it's better
not to add orphaned code back into the kernel. I didn't want users to be
left out in the cold by the removal of the driver, so I just created a
dkms package as a fallback:
https://github.com/gobenji/qlge-dkms
People who want to use qlge with the latest kernel can use that package.
Since the driver code is not mainline quality and there isn't much
willingness to invest in its improvement, I think it's fitting that the
code lives out of tree. Of course, if somebody takes ownership of the
code and substantially improves it, they can submit it back to netdev.
^ permalink raw reply
* Re: [RFC PATCH v3 09/12] net: add support for skbs with unreadable frags
From: Willem de Bruijn @ 2023-11-07 2:23 UTC (permalink / raw)
To: Stanislav Fomichev
Cc: Mina Almasry, David Ahern, netdev, linux-kernel, linux-arch,
linux-kselftest, linux-media, dri-devel, linaro-mm-sig,
David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Jesper Dangaard Brouer, Ilias Apalodimas, Arnd Bergmann,
Shuah Khan, Sumit Semwal, Christian König, Shakeel Butt,
Jeroen de Borst, Praveen Kaligineedi, Willem de Bruijn,
Kaiyuan Zhang
In-Reply-To: <ZUmMBZpLPQkRS9bg@google.com>
> > > > I think my other issue with MSG_SOCK_DEVMEM being on recvmsg is that
> > > > it somehow implies that I have an option of passing or not passing it
> > > > for an individual system call.
> > > > If we know that we're going to use dmabuf with the socket, maybe we
> > > > should move this flag to the socket() syscall?
> > > >
> > > > fd = socket(AF_INET6, SOCK_STREAM, SOCK_DEVMEM);
> > > >
> > > > ?
> > >
> > > I think it should then be a setsockopt called before any data is
> > > exchanged, with no change of modifying mode later. We generally use
> > > setsockopts for the mode of a socket. This use of the protocol field
> > > in socket() for setting a mode would be novel. Also, it might miss
> > > passively opened connections, or be overly restrictive: one approach
> > > for all accepted child sockets.
> >
> > I was thinking this is similar to SOCK_CLOEXEC or SOCK_NONBLOCK? There
> > are plenty of bits we can grab. But setsockopt works as well!
>
> To follow up: if we have this flag on a socket, not on a per-message
> basis, can we also use recvmsg for the recycling part maybe?
>
> while (true) {
> memset(msg, 0, ...);
>
> /* receive the tokens */
> ret = recvmsg(fd, &msg, 0);
>
> /* recycle the tokens from the above recvmsg() */
> ret = recvmsg(fd, &msg, MSG_RECYCLE);
> }
>
> recvmsg + MSG_RECYCLE can parse the same format that regular recvmsg
> exports (SO_DEVMEM_OFFSET) and we can also add extra cmsg option
> to recycle a range.
>
> Will this be more straightforward than a setsockopt(SO_DEVMEM_DONTNEED)?
> Or is it more confusing?
It would have to be sendmsg, as recvmsg is a copy_to_user operation.
I am not aware of any precedent in multiplexing the data stream and a
control operation stream in this manner. It would also require adding
a branch in the sendmsg hot path.
The memory is associated with the socket, freed when the socket is
closed as well as on SO_DEVMEM_DONTNEED. Fundamentally it is a socket
state operation, for which setsockopt is the socket interface.
Is your request purely a dislike, or is there some technical concern
with BPF and setsockopt?
^ permalink raw reply
* Re: [PATCH] can: netlink: Fix TDCO calculation using the old data bittiming
From: Vincent MAILHOL @ 2023-11-07 2:26 UTC (permalink / raw)
To: Maxime Jayat
Cc: Wolfgang Grandegger, Marc Kleine-Budde, linux-can, netdev,
linux-kernel
In-Reply-To: <40579c18-63c0-43a4-8d4c-f3a6c1c0b417@munic.io>
On Tue. 7 Nov. 2023 at 03:02, Maxime Jayat
<maxime.jayat@mobile-devices.fr> wrote:
> The TDCO calculation was done using the currently applied data bittiming,
> instead of the newly computed data bittiming, which means that the TDCO
> had an invalid value unless setting the same data bittiming twice.
Nice catch!
Moving the can_calc_tdco() before the memcpy(&priv->data_bittiming,
&dbt, sizeof(dbt)) was one of the last changes I made. And the last
batch of tests did not catch that. Thanks for the patch!
> Fixes: d99755f71a80 ("can: netlink: add interface for CAN-FD Transmitter Delay Compensation (TDC)")
> Signed-off-by: Maxime Jayat <maxime.jayat@mobile-devices.fr>
Reviewed-by: Vincent Mailhol <mailhol.vincent@wanadoo.fr>
> ---
> drivers/net/can/dev/netlink.c | 2 +-
> 1 file changed, 1 insertion(+), 1 deletion(-)
>
> diff --git a/drivers/net/can/dev/netlink.c b/drivers/net/can/dev/netlink.c
> index 036d85ef07f5..dfdc039d92a6 100644
> --- a/drivers/net/can/dev/netlink.c
> +++ b/drivers/net/can/dev/netlink.c
> @@ -346,7 +346,7 @@ static int can_changelink(struct net_device *dev, struct nlattr *tb[],
> /* Neither of TDC parameters nor TDC flags are
> * provided: do calculation
> */
> - can_calc_tdco(&priv->tdc, priv->tdc_const, &priv->data_bittiming,
> + can_calc_tdco(&priv->tdc, priv->tdc_const, &dbt,
> &priv->ctrlmode, priv->ctrlmode_supported);
> } /* else: both CAN_CTRLMODE_TDC_{AUTO,MANUAL} are explicitly
> * turned off. TDC is disabled: do nothing
> --
> 2.34.1
>
^ permalink raw reply
* [PATCH v1] net/tcp: use kfree_sensitive() instend of kfree() in tcp_md5_twsk_free_rcu()
From: Minjie Du @ 2023-11-07 2:34 UTC (permalink / raw)
To: Eric Dumazet, David S. Miller, David Ahern, Jakub Kicinski,
Paolo Abeni, open list:NETWORKING [TCP], open list
Cc: opensource.kernel, Minjie Du
key might contain private information, so better use
kfree_sensitive to free it.
In tcp_md5_twsk_free_rcu() use kfree_sensitive().
Signed-off-by: Minjie Du <duminjie@vivo.com>
---
net/ipv4/tcp_minisocks.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/net/ipv4/tcp_minisocks.c b/net/ipv4/tcp_minisocks.c
index a9807eeb311c..a7be78096783 100644
--- a/net/ipv4/tcp_minisocks.c
+++ b/net/ipv4/tcp_minisocks.c
@@ -368,7 +368,7 @@ static void tcp_md5_twsk_free_rcu(struct rcu_head *head)
struct tcp_md5sig_key *key;
key = container_of(head, struct tcp_md5sig_key, rcu);
- kfree(key);
+ kfree_sensitive(key);
static_branch_slow_dec_deferred(&tcp_md5_needed);
tcp_md5_release_sigpool();
}
--
2.39.0
^ permalink raw reply related
* [PATCH] crypto: ahash - Set using_shash for cloned ahash wrapper over shash
From: Dmitry Safonov @ 2023-11-07 2:37 UTC (permalink / raw)
To: Herbert Xu
Cc: linux-kernel, Dmitry Safonov, David Ahern, David S. Miller,
Dmitry Safonov, Eric Biggers, Eric Dumazet, Francesco Ruggeri,
Jakub Kicinski, Paolo Abeni, Salam Noureddine, netdev,
linux-crypto
The cloned child of ahash that uses shash under the hood should use
shash helpers (like crypto_shash_setkey()).
The following panic may be observed on TCP-AO selftests:
> ==================================================================
> BUG: KASAN: wild-memory-access in crypto_mod_get+0x1b/0x60
> Write of size 4 at addr 5d5be0ff5c415e14 by task connect_ipv4/1397
>
> CPU: 0 PID: 1397 Comm: connect_ipv4 Tainted: G W 6.6.0+ #47
> Call Trace:
> <TASK>
> dump_stack_lvl+0x46/0x70
> kasan_report+0xc3/0xf0
> kasan_check_range+0xec/0x190
> crypto_mod_get+0x1b/0x60
> crypto_spawn_alg+0x53/0x140
> crypto_spawn_tfm2+0x13/0x60
> hmac_init_tfm+0x25/0x60
> crypto_ahash_setkey+0x8b/0x100
> tcp_ao_add_cmd+0xe7a/0x1120
> do_tcp_setsockopt+0x5ed/0x12a0
> do_sock_setsockopt+0x82/0x100
> __sys_setsockopt+0xe9/0x160
> __x64_sys_setsockopt+0x60/0x70
> do_syscall_64+0x3c/0xe0
> entry_SYSCALL_64_after_hwframe+0x46/0x4e
> ==================================================================
> general protection fault, probably for non-canonical address 0x5d5be0ff5c415e14: 0000 [#1] PREEMPT SMP KASAN
> CPU: 0 PID: 1397 Comm: connect_ipv4 Tainted: G B W 6.6.0+ #47
> Call Trace:
> <TASK>
> ? die_addr+0x3c/0xa0
> ? exc_general_protection+0x144/0x210
> ? asm_exc_general_protection+0x22/0x30
> ? add_taint+0x26/0x90
> ? crypto_mod_get+0x20/0x60
> ? crypto_mod_get+0x1b/0x60
> ? ahash_def_finup_done1+0x58/0x80
> crypto_spawn_alg+0x53/0x140
> crypto_spawn_tfm2+0x13/0x60
> hmac_init_tfm+0x25/0x60
> crypto_ahash_setkey+0x8b/0x100
> tcp_ao_add_cmd+0xe7a/0x1120
> do_tcp_setsockopt+0x5ed/0x12a0
> do_sock_setsockopt+0x82/0x100
> __sys_setsockopt+0xe9/0x160
> __x64_sys_setsockopt+0x60/0x70
> do_syscall_64+0x3c/0xe0
> entry_SYSCALL_64_after_hwframe+0x46/0x4e
> </TASK>
> RIP: 0010:crypto_mod_get+0x20/0x60
Make sure that the child/clone has using_shash set when parent is
an shash user.
Fixes: 2f1f34c1bf7b ("crypto: ahash - optimize performance when wrapping shash")
Cc: David Ahern <dsahern@kernel.org>
Cc: "David S. Miller" <davem@davemloft.net>
Cc: Dmitry Safonov <0x7f454c46@gmail.com>
Cc: Eric Biggers <ebiggers@google.com>
Cc: Eric Dumazet <edumazet@google.com>
Cc: Francesco Ruggeri <fruggeri05@gmail.com>
To: Herbert Xu <herbert@gondor.apana.org.au>
Cc: Jakub Kicinski <kuba@kernel.org>
Cc: Paolo Abeni <pabeni@redhat.com>
Cc: Salam Noureddine <noureddine@arista.com>
Cc: netdev@vger.kernel.org
Cc: linux-crypto@vger.kernel.org
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Dmitry Safonov <dima@arista.com>
---
crypto/ahash.c | 1 +
1 file changed, 1 insertion(+)
diff --git a/crypto/ahash.c b/crypto/ahash.c
index deee55f939dc..80c3e5354711 100644
--- a/crypto/ahash.c
+++ b/crypto/ahash.c
@@ -651,6 +651,7 @@ struct crypto_ahash *crypto_clone_ahash(struct crypto_ahash *hash)
err = PTR_ERR(shash);
goto out_free_nhash;
}
+ nhash->using_shash = true;
*nctx = shash;
return nhash;
}
base-commit: be3ca57cfb777ad820c6659d52e60bbdd36bf5ff
--
2.42.0
^ permalink raw reply related
* [PATCH net-next v2 00/21] virtio-net: support AF_XDP zero copy
From: Xuan Zhuo @ 2023-11-07 3:12 UTC (permalink / raw)
To: netdev
Cc: David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Michael S. Tsirkin, Jason Wang, Xuan Zhuo, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
## AF_XDP
XDP socket(AF_XDP) is an excellent bypass kernel network framework. The zero
copy feature of xsk (XDP socket) needs to be supported by the driver. The
performance of zero copy is very good. mlx5 and intel ixgbe already support
this feature, This patch set allows virtio-net to support xsk's zerocopy xmit
feature.
At present, we have completed some preparation:
1. vq-reset (virtio spec and kernel code)
2. virtio-core premapped dma
3. virtio-net xdp refactor
So it is time for Virtio-Net to complete the support for the XDP Socket
Zerocopy.
Virtio-net can not increase the queue num at will, so xsk shares the queue with
kernel.
On the other hand, Virtio-Net does not support generate interrupt from driver
manually, so when we wakeup tx xmit, we used some tips. If the CPU run by TX
NAPI last time is other CPUs, use IPI to wake up NAPI on the remote CPU. If it
is also the local CPU, then we wake up napi directly.
This patch set includes some refactor to the virtio-net to let that to support
AF_XDP.
## performance
ENV: Qemu with vhost-user(polling mode).
Host CPU: Intel(R) Xeon(R) Platinum 8163 CPU @ 2.50GHz
### virtio PMD in guest with testpmd
testpmd> show port stats all
######################## NIC statistics for port 0 ########################
RX-packets: 19531092064 RX-missed: 0 RX-bytes: 1093741155584
RX-errors: 0
RX-nombuf: 0
TX-packets: 5959955552 TX-errors: 0 TX-bytes: 371030645664
Throughput (since last show)
Rx-pps: 8861574 Rx-bps: 3969985208
Tx-pps: 8861493 Tx-bps: 3969962736
############################################################################
### AF_XDP PMD in guest with testpmd
testpmd> show port stats all
######################## NIC statistics for port 0 ########################
RX-packets: 68152727 RX-missed: 0 RX-bytes: 3816552712
RX-errors: 0
RX-nombuf: 0
TX-packets: 68114967 TX-errors: 33216 TX-bytes: 3814438152
Throughput (since last show)
Rx-pps: 6333196 Rx-bps: 2837272088
Tx-pps: 6333227 Tx-bps: 2837285936
############################################################################
But AF_XDP consumes more CPU for tx and rx napi(100% and 86%).
## maintain
I am currently a reviewer for virtio-net. I commit to maintain AF_XDP support in
virtio-net.
Please review.
Thanks.
v2
1. wakeup uses the way of GVE. No send ipi to wakeup napi on remote cpu.
2. remove rcu. Because we synchronize all operat, so the rcu is not needed.
3. split the commit "move to virtio_net.h" in last patch set. Just move the
struct/api to header when we use them.
4. add comments for some code
v1:
1. remove two virtio commits. Push this patchset to net-next
2. squash "virtio_net: virtnet_poll_tx support rescheduled" to xsk: support tx
3. fix some warnings
Xuan Zhuo (21):
virtio_net: rename free_old_xmit_skbs to free_old_xmit
virtio_net: unify the code for recycling the xmit ptr
virtio_net: independent directory
virtio_net: move core structures to virtio_net.h
virtio_net: add prefix virtnet to all struct inside virtio_net.h
virtio_net: separate virtnet_rx_resize()
virtio_net: separate virtnet_tx_resize()
virtio_net: sq support premapped mode
virtio_net: xsk: bind/unbind xsk
virtio_net: xsk: prevent disable tx napi
virtio_net: move some api to header
virtio_net: xsk: tx: support tx
virtio_net: xsk: tx: support wakeup
virtio_net: xsk: tx: virtnet_free_old_xmit() distinguishes xsk buffer
virtio_net: xsk: tx: virtnet_sq_free_unused_buf() check xsk buffer
virtio_net: xsk: rx: introduce add_recvbuf_xsk()
virtio_net: xsk: rx: skip dma unmap when rq is bind with AF_XDP
virtio_net: xsk: rx: introduce receive_xsk() to recv xsk buffer
virtio_net: xsk: rx: virtnet_rq_free_unused_buf() check xsk buffer
virtio_net: update tx timeout record
virtio_net: xdp_features add NETDEV_XDP_ACT_XSK_ZEROCOPY
MAINTAINERS | 2 +-
drivers/net/Kconfig | 8 +-
drivers/net/Makefile | 2 +-
drivers/net/virtio/Kconfig | 13 +
drivers/net/virtio/Makefile | 8 +
drivers/net/{virtio_net.c => virtio/main.c} | 630 +++++++++-----------
drivers/net/virtio/virtio_net.h | 346 +++++++++++
drivers/net/virtio/xsk.c | 498 ++++++++++++++++
drivers/net/virtio/xsk.h | 32 +
9 files changed, 1174 insertions(+), 365 deletions(-)
create mode 100644 drivers/net/virtio/Kconfig
create mode 100644 drivers/net/virtio/Makefile
rename drivers/net/{virtio_net.c => virtio/main.c} (92%)
create mode 100644 drivers/net/virtio/virtio_net.h
create mode 100644 drivers/net/virtio/xsk.c
create mode 100644 drivers/net/virtio/xsk.h
--
2.32.0.3.g01195cf9f
^ permalink raw reply
* [PATCH net-next v2 01/21] virtio_net: rename free_old_xmit_skbs to free_old_xmit
From: Xuan Zhuo @ 2023-11-07 3:12 UTC (permalink / raw)
To: netdev
Cc: David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Michael S. Tsirkin, Jason Wang, Xuan Zhuo, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
In-Reply-To: <20231107031227.100015-1-xuanzhuo@linux.alibaba.com>
Since free_old_xmit_skbs not only deals with skb, but also xdp frame and
subsequent added xsk, so change the name of this function to
free_old_xmit.
Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
Acked-by: Jason Wang <jasowang@redhat.com>
---
drivers/net/virtio_net.c | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index cf67c618b07b..e7dd131234b5 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -744,7 +744,7 @@ static void virtnet_rq_set_premapped(struct virtnet_info *vi)
}
}
-static void free_old_xmit_skbs(struct send_queue *sq, bool in_napi)
+static void free_old_xmit(struct send_queue *sq, bool in_napi)
{
unsigned int len;
unsigned int packets = 0;
@@ -816,7 +816,7 @@ static void check_sq_full_and_disable(struct virtnet_info *vi,
virtqueue_napi_schedule(&sq->napi, sq->vq);
} else if (unlikely(!virtqueue_enable_cb_delayed(sq->vq))) {
/* More just got used, free them then recheck. */
- free_old_xmit_skbs(sq, false);
+ free_old_xmit(sq, false);
if (sq->vq->num_free >= 2+MAX_SKB_FRAGS) {
netif_start_subqueue(dev, qnum);
virtqueue_disable_cb(sq->vq);
@@ -2127,7 +2127,7 @@ static void virtnet_poll_cleantx(struct receive_queue *rq)
do {
virtqueue_disable_cb(sq->vq);
- free_old_xmit_skbs(sq, true);
+ free_old_xmit(sq, true);
} while (unlikely(!virtqueue_enable_cb_delayed(sq->vq)));
if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
@@ -2249,7 +2249,7 @@ static int virtnet_poll_tx(struct napi_struct *napi, int budget)
txq = netdev_get_tx_queue(vi->dev, index);
__netif_tx_lock(txq, raw_smp_processor_id());
virtqueue_disable_cb(sq->vq);
- free_old_xmit_skbs(sq, true);
+ free_old_xmit(sq, true);
if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
netif_tx_wake_queue(txq);
@@ -2339,7 +2339,7 @@ static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
if (use_napi)
virtqueue_disable_cb(sq->vq);
- free_old_xmit_skbs(sq, false);
+ free_old_xmit(sq, false);
} while (use_napi && kick &&
unlikely(!virtqueue_enable_cb_delayed(sq->vq)));
--
2.32.0.3.g01195cf9f
^ permalink raw reply related
* [PATCH net-next v2 02/21] virtio_net: unify the code for recycling the xmit ptr
From: Xuan Zhuo @ 2023-11-07 3:12 UTC (permalink / raw)
To: netdev
Cc: David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Michael S. Tsirkin, Jason Wang, Xuan Zhuo, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
In-Reply-To: <20231107031227.100015-1-xuanzhuo@linux.alibaba.com>
There are two completely similar and independent implementations. This
is inconvenient for the subsequent addition of new types. So extract a
function from this piece of code and call this function uniformly to
recover old xmit ptr.
Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
Acked-by: Jason Wang <jasowang@redhat.com>
---
drivers/net/virtio_net.c | 66 +++++++++++++++++-----------------------
1 file changed, 28 insertions(+), 38 deletions(-)
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index e7dd131234b5..3fe26f48025f 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -352,6 +352,30 @@ static struct xdp_frame *ptr_to_xdp(void *ptr)
return (struct xdp_frame *)((unsigned long)ptr & ~VIRTIO_XDP_FLAG);
}
+static void __free_old_xmit(struct send_queue *sq, bool in_napi,
+ u64 *bytes, u64 *packets)
+{
+ unsigned int len;
+ void *ptr;
+
+ while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {
+ if (!is_xdp_frame(ptr)) {
+ struct sk_buff *skb = ptr;
+
+ pr_debug("Sent skb %p\n", skb);
+
+ *bytes += skb->len;
+ napi_consume_skb(skb, in_napi);
+ } else {
+ struct xdp_frame *frame = ptr_to_xdp(ptr);
+
+ *bytes += xdp_get_frame_len(frame);
+ xdp_return_frame(frame);
+ }
+ (*packets)++;
+ }
+}
+
/* Converting between virtqueue no. and kernel tx/rx queue no.
* 0:rx0 1:tx0 2:rx1 3:tx1 ... 2N:rxN 2N+1:txN 2N+2:cvq
*/
@@ -746,27 +770,9 @@ static void virtnet_rq_set_premapped(struct virtnet_info *vi)
static void free_old_xmit(struct send_queue *sq, bool in_napi)
{
- unsigned int len;
- unsigned int packets = 0;
- unsigned int bytes = 0;
- void *ptr;
-
- while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {
- if (likely(!is_xdp_frame(ptr))) {
- struct sk_buff *skb = ptr;
-
- pr_debug("Sent skb %p\n", skb);
+ u64 bytes, packets = 0;
- bytes += skb->len;
- napi_consume_skb(skb, in_napi);
- } else {
- struct xdp_frame *frame = ptr_to_xdp(ptr);
-
- bytes += xdp_get_frame_len(frame);
- xdp_return_frame(frame);
- }
- packets++;
- }
+ __free_old_xmit(sq, in_napi, &bytes, &packets);
/* Avoid overhead when no packets have been processed
* happens when called speculatively from start_xmit.
@@ -916,14 +922,11 @@ static int virtnet_xdp_xmit(struct net_device *dev,
{
struct virtnet_info *vi = netdev_priv(dev);
struct receive_queue *rq = vi->rq;
+ u64 bytes = 0, packets = 0;
struct bpf_prog *xdp_prog;
struct send_queue *sq;
- unsigned int len;
- int packets = 0;
- int bytes = 0;
int nxmit = 0;
int kicks = 0;
- void *ptr;
int ret;
int i;
@@ -942,20 +945,7 @@ static int virtnet_xdp_xmit(struct net_device *dev,
}
/* Free up any pending old buffers before queueing new ones. */
- while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {
- if (likely(is_xdp_frame(ptr))) {
- struct xdp_frame *frame = ptr_to_xdp(ptr);
-
- bytes += xdp_get_frame_len(frame);
- xdp_return_frame(frame);
- } else {
- struct sk_buff *skb = ptr;
-
- bytes += skb->len;
- napi_consume_skb(skb, false);
- }
- packets++;
- }
+ __free_old_xmit(sq, false, &bytes, &packets);
for (i = 0; i < n; i++) {
struct xdp_frame *xdpf = frames[i];
--
2.32.0.3.g01195cf9f
^ permalink raw reply related
* [PATCH net-next v2 03/21] virtio_net: independent directory
From: Xuan Zhuo @ 2023-11-07 3:12 UTC (permalink / raw)
To: netdev
Cc: David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Michael S. Tsirkin, Jason Wang, Xuan Zhuo, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
In-Reply-To: <20231107031227.100015-1-xuanzhuo@linux.alibaba.com>
Create a separate directory for virtio-net. AF_XDP support will be added
later, then a separate xsk.c file will be added, so we should create a
directory for virtio-net.
Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
Acked-by: Jason Wang <jasowang@redhat.com>
---
MAINTAINERS | 2 +-
drivers/net/Kconfig | 8 +-------
drivers/net/Makefile | 2 +-
drivers/net/virtio/Kconfig | 13 +++++++++++++
drivers/net/virtio/Makefile | 8 ++++++++
drivers/net/{virtio_net.c => virtio/main.c} | 0
6 files changed, 24 insertions(+), 9 deletions(-)
create mode 100644 drivers/net/virtio/Kconfig
create mode 100644 drivers/net/virtio/Makefile
rename drivers/net/{virtio_net.c => virtio/main.c} (100%)
diff --git a/MAINTAINERS b/MAINTAINERS
index 14e1194faa4b..81e7d31f6cc9 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -22905,7 +22905,7 @@ F: Documentation/devicetree/bindings/virtio/
F: Documentation/driver-api/virtio/
F: drivers/block/virtio_blk.c
F: drivers/crypto/virtio/
-F: drivers/net/virtio_net.c
+F: drivers/net/virtio/
F: drivers/vdpa/
F: drivers/virtio/
F: include/linux/vdpa.h
diff --git a/drivers/net/Kconfig b/drivers/net/Kconfig
index af0da4bb429b..a14ef645aa01 100644
--- a/drivers/net/Kconfig
+++ b/drivers/net/Kconfig
@@ -430,13 +430,7 @@ config VETH
When one end receives the packet it appears on its pair and vice
versa.
-config VIRTIO_NET
- tristate "Virtio network driver"
- depends on VIRTIO
- select NET_FAILOVER
- help
- This is the virtual network driver for virtio. It can be used with
- QEMU based VMMs (like KVM or Xen). Say Y or M.
+source "drivers/net/virtio/Kconfig"
config NLMON
tristate "Virtual netlink monitoring device"
diff --git a/drivers/net/Makefile b/drivers/net/Makefile
index 7cab36f94782..a205dd2be77e 100644
--- a/drivers/net/Makefile
+++ b/drivers/net/Makefile
@@ -32,7 +32,7 @@ obj-$(CONFIG_NET_TEAM) += team/
obj-$(CONFIG_TUN) += tun.o
obj-$(CONFIG_TAP) += tap.o
obj-$(CONFIG_VETH) += veth.o
-obj-$(CONFIG_VIRTIO_NET) += virtio_net.o
+obj-$(CONFIG_VIRTIO_NET) += virtio/
obj-$(CONFIG_VXLAN) += vxlan/
obj-$(CONFIG_GENEVE) += geneve.o
obj-$(CONFIG_BAREUDP) += bareudp.o
diff --git a/drivers/net/virtio/Kconfig b/drivers/net/virtio/Kconfig
new file mode 100644
index 000000000000..d8ccb3ac49df
--- /dev/null
+++ b/drivers/net/virtio/Kconfig
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# virtio-net device configuration
+#
+config VIRTIO_NET
+ tristate "Virtio network driver"
+ depends on VIRTIO
+ select NET_FAILOVER
+ help
+ This is the virtual network driver for virtio. It can be used with
+ QEMU based VMMs (like KVM or Xen).
+
+ Say Y or M.
diff --git a/drivers/net/virtio/Makefile b/drivers/net/virtio/Makefile
new file mode 100644
index 000000000000..15ed7c97fd4f
--- /dev/null
+++ b/drivers/net/virtio/Makefile
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Makefile for the virtio network device drivers.
+#
+
+obj-$(CONFIG_VIRTIO_NET) += virtio_net.o
+
+virtio_net-y := main.o
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio/main.c
similarity index 100%
rename from drivers/net/virtio_net.c
rename to drivers/net/virtio/main.c
--
2.32.0.3.g01195cf9f
^ permalink raw reply related
* [PATCH net-next v2 04/21] virtio_net: move core structures to virtio_net.h
From: Xuan Zhuo @ 2023-11-07 3:12 UTC (permalink / raw)
To: netdev
Cc: David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Michael S. Tsirkin, Jason Wang, Xuan Zhuo, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
In-Reply-To: <20231107031227.100015-1-xuanzhuo@linux.alibaba.com>
Move some core structures (send_queue, receive_queue, virtnet_info)
definitions and the relative structures definitions into the
virtio_net.h file.
That will be used by the other c code files.
Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
---
drivers/net/virtio/main.c | 189 +------------------------------
drivers/net/virtio/virtio_net.h | 193 ++++++++++++++++++++++++++++++++
2 files changed, 195 insertions(+), 187 deletions(-)
create mode 100644 drivers/net/virtio/virtio_net.h
diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
index 3fe26f48025f..66060cff9387 100644
--- a/drivers/net/virtio/main.c
+++ b/drivers/net/virtio/main.c
@@ -6,7 +6,6 @@
//#define DEBUG
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
-#include <linux/ethtool.h>
#include <linux/module.h>
#include <linux/virtio.h>
#include <linux/virtio_net.h>
@@ -16,7 +15,6 @@
#include <linux/if_vlan.h>
#include <linux/slab.h>
#include <linux/cpu.h>
-#include <linux/average.h>
#include <linux/filter.h>
#include <linux/kernel.h>
#include <net/route.h>
@@ -24,6 +22,8 @@
#include <net/net_failover.h>
#include <net/netdev_rx_queue.h>
+#include "virtio_net.h"
+
static int napi_weight = NAPI_POLL_WEIGHT;
module_param(napi_weight, int, 0444);
@@ -47,13 +47,6 @@ module_param(napi_tx, bool, 0644);
#define VIRTIO_XDP_FLAG BIT(0)
-/* RX packet size EWMA. The average packet size is used to determine the packet
- * buffer size when refilling RX rings. As the entire RX ring may be refilled
- * at once, the weight is chosen so that the EWMA will be insensitive to short-
- * term, transient changes in packet size.
- */
-DECLARE_EWMA(pkt_len, 0, 64)
-
#define VIRTNET_DRIVER_VERSION "1.0.0"
static const unsigned long guest_offloads[] = {
@@ -79,28 +72,6 @@ struct virtnet_stat_desc {
size_t offset;
};
-struct virtnet_sq_stats {
- struct u64_stats_sync syncp;
- u64_stats_t packets;
- u64_stats_t bytes;
- u64_stats_t xdp_tx;
- u64_stats_t xdp_tx_drops;
- u64_stats_t kicks;
- u64_stats_t tx_timeouts;
-};
-
-struct virtnet_rq_stats {
- struct u64_stats_sync syncp;
- u64_stats_t packets;
- u64_stats_t bytes;
- u64_stats_t drops;
- u64_stats_t xdp_packets;
- u64_stats_t xdp_tx;
- u64_stats_t xdp_redirects;
- u64_stats_t xdp_drops;
- u64_stats_t kicks;
-};
-
#define VIRTNET_SQ_STAT(m) offsetof(struct virtnet_sq_stats, m)
#define VIRTNET_RQ_STAT(m) offsetof(struct virtnet_rq_stats, m)
@@ -127,80 +98,6 @@ static const struct virtnet_stat_desc virtnet_rq_stats_desc[] = {
#define VIRTNET_SQ_STATS_LEN ARRAY_SIZE(virtnet_sq_stats_desc)
#define VIRTNET_RQ_STATS_LEN ARRAY_SIZE(virtnet_rq_stats_desc)
-struct virtnet_interrupt_coalesce {
- u32 max_packets;
- u32 max_usecs;
-};
-
-/* The dma information of pages allocated at a time. */
-struct virtnet_rq_dma {
- dma_addr_t addr;
- u32 ref;
- u16 len;
- u16 need_sync;
-};
-
-/* Internal representation of a send virtqueue */
-struct send_queue {
- /* Virtqueue associated with this send _queue */
- struct virtqueue *vq;
-
- /* TX: fragments + linear part + virtio header */
- struct scatterlist sg[MAX_SKB_FRAGS + 2];
-
- /* Name of the send queue: output.$index */
- char name[16];
-
- struct virtnet_sq_stats stats;
-
- struct virtnet_interrupt_coalesce intr_coal;
-
- struct napi_struct napi;
-
- /* Record whether sq is in reset state. */
- bool reset;
-};
-
-/* Internal representation of a receive virtqueue */
-struct receive_queue {
- /* Virtqueue associated with this receive_queue */
- struct virtqueue *vq;
-
- struct napi_struct napi;
-
- struct bpf_prog __rcu *xdp_prog;
-
- struct virtnet_rq_stats stats;
-
- struct virtnet_interrupt_coalesce intr_coal;
-
- /* Chain pages by the private ptr. */
- struct page *pages;
-
- /* Average packet length for mergeable receive buffers. */
- struct ewma_pkt_len mrg_avg_pkt_len;
-
- /* Page frag for packet buffer allocation. */
- struct page_frag alloc_frag;
-
- /* RX: fragments + linear part + virtio header */
- struct scatterlist sg[MAX_SKB_FRAGS + 2];
-
- /* Min single buffer size for mergeable buffers case. */
- unsigned int min_buf_len;
-
- /* Name of this receive queue: input.$index */
- char name[16];
-
- struct xdp_rxq_info xdp_rxq;
-
- /* Record the last dma info to free after new pages is allocated. */
- struct virtnet_rq_dma *last_dma;
-
- /* Do dma by self */
- bool do_dma;
-};
-
/* This structure can contain rss message with maximum settings for indirection table and keysize
* Note, that default structure that describes RSS configuration virtio_net_rss_config
* contains same info but can't handle table values.
@@ -234,88 +131,6 @@ struct control_buf {
struct virtio_net_ctrl_coal_vq coal_vq;
};
-struct virtnet_info {
- struct virtio_device *vdev;
- struct virtqueue *cvq;
- struct net_device *dev;
- struct send_queue *sq;
- struct receive_queue *rq;
- unsigned int status;
-
- /* Max # of queue pairs supported by the device */
- u16 max_queue_pairs;
-
- /* # of queue pairs currently used by the driver */
- u16 curr_queue_pairs;
-
- /* # of XDP queue pairs currently used by the driver */
- u16 xdp_queue_pairs;
-
- /* xdp_queue_pairs may be 0, when xdp is already loaded. So add this. */
- bool xdp_enabled;
-
- /* I like... big packets and I cannot lie! */
- bool big_packets;
-
- /* number of sg entries allocated for big packets */
- unsigned int big_packets_num_skbfrags;
-
- /* Host will merge rx buffers for big packets (shake it! shake it!) */
- bool mergeable_rx_bufs;
-
- /* Host supports rss and/or hash report */
- bool has_rss;
- bool has_rss_hash_report;
- u8 rss_key_size;
- u16 rss_indir_table_size;
- u32 rss_hash_types_supported;
- u32 rss_hash_types_saved;
-
- /* Has control virtqueue */
- bool has_cvq;
-
- /* Host can handle any s/g split between our header and packet data */
- bool any_header_sg;
-
- /* Packet virtio header size */
- u8 hdr_len;
-
- /* Work struct for delayed refilling if we run low on memory. */
- struct delayed_work refill;
-
- /* Is delayed refill enabled? */
- bool refill_enabled;
-
- /* The lock to synchronize the access to refill_enabled */
- spinlock_t refill_lock;
-
- /* Work struct for config space updates */
- struct work_struct config_work;
-
- /* Does the affinity hint is set for virtqueues? */
- bool affinity_hint_set;
-
- /* CPU hotplug instances for online & dead */
- struct hlist_node node;
- struct hlist_node node_dead;
-
- struct control_buf *ctrl;
-
- /* Ethtool settings */
- u8 duplex;
- u32 speed;
-
- /* Interrupt coalescing settings */
- struct virtnet_interrupt_coalesce intr_coal_tx;
- struct virtnet_interrupt_coalesce intr_coal_rx;
-
- unsigned long guest_offloads;
- unsigned long guest_offloads_capable;
-
- /* failover when STANDBY feature enabled */
- struct failover *failover;
-};
-
struct padded_vnet_hdr {
struct virtio_net_hdr_v1_hash hdr;
/*
diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
new file mode 100644
index 000000000000..38061e15d494
--- /dev/null
+++ b/drivers/net/virtio/virtio_net.h
@@ -0,0 +1,193 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+
+#ifndef __VIRTIO_NET_H__
+#define __VIRTIO_NET_H__
+
+#include <linux/ethtool.h>
+#include <linux/average.h>
+
+/* RX packet size EWMA. The average packet size is used to determine the packet
+ * buffer size when refilling RX rings. As the entire RX ring may be refilled
+ * at once, the weight is chosen so that the EWMA will be insensitive to short-
+ * term, transient changes in packet size.
+ */
+DECLARE_EWMA(pkt_len, 0, 64)
+
+struct virtnet_sq_stats {
+ struct u64_stats_sync syncp;
+ u64_stats_t packets;
+ u64_stats_t bytes;
+ u64_stats_t xdp_tx;
+ u64_stats_t xdp_tx_drops;
+ u64_stats_t kicks;
+ u64_stats_t tx_timeouts;
+};
+
+struct virtnet_rq_stats {
+ struct u64_stats_sync syncp;
+ u64_stats_t packets;
+ u64_stats_t bytes;
+ u64_stats_t drops;
+ u64_stats_t xdp_packets;
+ u64_stats_t xdp_tx;
+ u64_stats_t xdp_redirects;
+ u64_stats_t xdp_drops;
+ u64_stats_t kicks;
+};
+
+struct virtnet_interrupt_coalesce {
+ u32 max_packets;
+ u32 max_usecs;
+};
+
+/* The dma information of pages allocated at a time. */
+struct virtnet_rq_dma {
+ dma_addr_t addr;
+ u32 ref;
+ u16 len;
+ u16 need_sync;
+};
+
+/* Internal representation of a send virtqueue */
+struct send_queue {
+ /* Virtqueue associated with this send _queue */
+ struct virtqueue *vq;
+
+ /* TX: fragments + linear part + virtio header */
+ struct scatterlist sg[MAX_SKB_FRAGS + 2];
+
+ /* Name of the send queue: output.$index */
+ char name[16];
+
+ struct virtnet_sq_stats stats;
+
+ struct virtnet_interrupt_coalesce intr_coal;
+
+ struct napi_struct napi;
+
+ /* Record whether sq is in reset state. */
+ bool reset;
+};
+
+/* Internal representation of a receive virtqueue */
+struct receive_queue {
+ /* Virtqueue associated with this receive_queue */
+ struct virtqueue *vq;
+
+ struct napi_struct napi;
+
+ struct bpf_prog __rcu *xdp_prog;
+
+ struct virtnet_rq_stats stats;
+
+ struct virtnet_interrupt_coalesce intr_coal;
+
+ /* Chain pages by the private ptr. */
+ struct page *pages;
+
+ /* Average packet length for mergeable receive buffers. */
+ struct ewma_pkt_len mrg_avg_pkt_len;
+
+ /* Page frag for packet buffer allocation. */
+ struct page_frag alloc_frag;
+
+ /* RX: fragments + linear part + virtio header */
+ struct scatterlist sg[MAX_SKB_FRAGS + 2];
+
+ /* Min single buffer size for mergeable buffers case. */
+ unsigned int min_buf_len;
+
+ /* Name of this receive queue: input.$index */
+ char name[16];
+
+ struct xdp_rxq_info xdp_rxq;
+
+ /* Record the last dma info to free after new pages is allocated. */
+ struct virtnet_rq_dma *last_dma;
+
+ /* Do dma by self */
+ bool do_dma;
+};
+
+struct virtnet_info {
+ struct virtio_device *vdev;
+ struct virtqueue *cvq;
+ struct net_device *dev;
+ struct send_queue *sq;
+ struct receive_queue *rq;
+ unsigned int status;
+
+ /* Max # of queue pairs supported by the device */
+ u16 max_queue_pairs;
+
+ /* # of queue pairs currently used by the driver */
+ u16 curr_queue_pairs;
+
+ /* # of XDP queue pairs currently used by the driver */
+ u16 xdp_queue_pairs;
+
+ /* xdp_queue_pairs may be 0, when xdp is already loaded. So add this. */
+ bool xdp_enabled;
+
+ /* I like... big packets and I cannot lie! */
+ bool big_packets;
+
+ /* number of sg entries allocated for big packets */
+ unsigned int big_packets_num_skbfrags;
+
+ /* Host will merge rx buffers for big packets (shake it! shake it!) */
+ bool mergeable_rx_bufs;
+
+ /* Host supports rss and/or hash report */
+ bool has_rss;
+ bool has_rss_hash_report;
+ u8 rss_key_size;
+ u16 rss_indir_table_size;
+ u32 rss_hash_types_supported;
+ u32 rss_hash_types_saved;
+
+ /* Has control virtqueue */
+ bool has_cvq;
+
+ /* Host can handle any s/g split between our header and packet data */
+ bool any_header_sg;
+
+ /* Packet virtio header size */
+ u8 hdr_len;
+
+ /* Work struct for delayed refilling if we run low on memory. */
+ struct delayed_work refill;
+
+ /* Is delayed refill enabled? */
+ bool refill_enabled;
+
+ /* The lock to synchronize the access to refill_enabled */
+ spinlock_t refill_lock;
+
+ /* Work struct for config space updates */
+ struct work_struct config_work;
+
+ /* Does the affinity hint is set for virtqueues? */
+ bool affinity_hint_set;
+
+ /* CPU hotplug instances for online & dead */
+ struct hlist_node node;
+ struct hlist_node node_dead;
+
+ struct control_buf *ctrl;
+
+ /* Ethtool settings */
+ u8 duplex;
+ u32 speed;
+
+ /* Interrupt coalescing settings */
+ struct virtnet_interrupt_coalesce intr_coal_tx;
+ struct virtnet_interrupt_coalesce intr_coal_rx;
+
+ unsigned long guest_offloads;
+ unsigned long guest_offloads_capable;
+
+ /* failover when STANDBY feature enabled */
+ struct failover *failover;
+};
+#endif
--
2.32.0.3.g01195cf9f
^ permalink raw reply related
* [PATCH net-next v2 06/21] virtio_net: separate virtnet_rx_resize()
From: Xuan Zhuo @ 2023-11-07 3:12 UTC (permalink / raw)
To: netdev
Cc: David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Michael S. Tsirkin, Jason Wang, Xuan Zhuo, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
In-Reply-To: <20231107031227.100015-1-xuanzhuo@linux.alibaba.com>
This patch separates two sub-functions from virtnet_rx_resize():
* virtnet_rx_pause
* virtnet_rx_resume
Then the subsequent reset rx for xsk can share these two functions.
Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
Acked-by: Jason Wang <jasowang@redhat.com>
---
drivers/net/virtio/main.c | 29 +++++++++++++++++++++--------
drivers/net/virtio/virtio_net.h | 3 +++
2 files changed, 24 insertions(+), 8 deletions(-)
diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
index 46a8be0c609c..c9f8294153e2 100644
--- a/drivers/net/virtio/main.c
+++ b/drivers/net/virtio/main.c
@@ -2186,26 +2186,39 @@ static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
return NETDEV_TX_OK;
}
-static int virtnet_rx_resize(struct virtnet_info *vi,
- struct virtnet_rq *rq, u32 ring_num)
+void virtnet_rx_pause(struct virtnet_info *vi, struct virtnet_rq *rq)
{
bool running = netif_running(vi->dev);
- int err, qindex;
-
- qindex = rq - vi->rq;
if (running)
napi_disable(&rq->napi);
+}
- err = virtqueue_resize(rq->vq, ring_num, virtnet_rq_free_unused_buf);
- if (err)
- netdev_err(vi->dev, "resize rx fail: rx queue index: %d err: %d\n", qindex, err);
+void virtnet_rx_resume(struct virtnet_info *vi, struct virtnet_rq *rq)
+{
+ bool running = netif_running(vi->dev);
if (!try_fill_recv(vi, rq, GFP_KERNEL))
schedule_delayed_work(&vi->refill, 0);
if (running)
virtnet_napi_enable(rq->vq, &rq->napi);
+}
+
+static int virtnet_rx_resize(struct virtnet_info *vi,
+ struct virtnet_rq *rq, u32 ring_num)
+{
+ int err, qindex;
+
+ qindex = rq - vi->rq;
+
+ virtnet_rx_pause(vi, rq);
+
+ err = virtqueue_resize(rq->vq, ring_num, virtnet_rq_free_unused_buf);
+ if (err)
+ netdev_err(vi->dev, "resize rx fail: rx queue index: %d err: %d\n", qindex, err);
+
+ virtnet_rx_resume(vi, rq);
return err;
}
diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
index ebf9f344648a..693ca166fc93 100644
--- a/drivers/net/virtio/virtio_net.h
+++ b/drivers/net/virtio/virtio_net.h
@@ -190,4 +190,7 @@ struct virtnet_info {
/* failover when STANDBY feature enabled */
struct failover *failover;
};
+
+void virtnet_rx_pause(struct virtnet_info *vi, struct virtnet_rq *rq);
+void virtnet_rx_resume(struct virtnet_info *vi, struct virtnet_rq *rq);
#endif
--
2.32.0.3.g01195cf9f
^ permalink raw reply related
* [PATCH net-next v2 05/21] virtio_net: add prefix virtnet to all struct inside virtio_net.h
From: Xuan Zhuo @ 2023-11-07 3:12 UTC (permalink / raw)
To: netdev
Cc: David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Michael S. Tsirkin, Jason Wang, Xuan Zhuo, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
In-Reply-To: <20231107031227.100015-1-xuanzhuo@linux.alibaba.com>
We move some structures to the header file, but these structures do not
prefixed with virtnet. This patch adds virtnet for these.
Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
---
drivers/net/virtio/main.c | 100 ++++++++++++++++----------------
drivers/net/virtio/virtio_net.h | 12 ++--
2 files changed, 56 insertions(+), 56 deletions(-)
diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
index 66060cff9387..46a8be0c609c 100644
--- a/drivers/net/virtio/main.c
+++ b/drivers/net/virtio/main.c
@@ -167,7 +167,7 @@ static struct xdp_frame *ptr_to_xdp(void *ptr)
return (struct xdp_frame *)((unsigned long)ptr & ~VIRTIO_XDP_FLAG);
}
-static void __free_old_xmit(struct send_queue *sq, bool in_napi,
+static void __free_old_xmit(struct virtnet_sq *sq, bool in_napi,
u64 *bytes, u64 *packets)
{
unsigned int len;
@@ -224,7 +224,7 @@ skb_vnet_common_hdr(struct sk_buff *skb)
* private is used to chain pages for big packets, put the whole
* most recent used list in the beginning for reuse
*/
-static void give_pages(struct receive_queue *rq, struct page *page)
+static void give_pages(struct virtnet_rq *rq, struct page *page)
{
struct page *end;
@@ -234,7 +234,7 @@ static void give_pages(struct receive_queue *rq, struct page *page)
rq->pages = page;
}
-static struct page *get_a_page(struct receive_queue *rq, gfp_t gfp_mask)
+static struct page *get_a_page(struct virtnet_rq *rq, gfp_t gfp_mask)
{
struct page *p = rq->pages;
@@ -248,7 +248,7 @@ static struct page *get_a_page(struct receive_queue *rq, gfp_t gfp_mask)
}
static void virtnet_rq_free_buf(struct virtnet_info *vi,
- struct receive_queue *rq, void *buf)
+ struct virtnet_rq *rq, void *buf)
{
if (vi->mergeable_rx_bufs)
put_page(virt_to_head_page(buf));
@@ -345,7 +345,7 @@ static struct sk_buff *virtnet_build_skb(void *buf, unsigned int buflen,
/* Called from bottom half context */
static struct sk_buff *page_to_skb(struct virtnet_info *vi,
- struct receive_queue *rq,
+ struct virtnet_rq *rq,
struct page *page, unsigned int offset,
unsigned int len, unsigned int truesize,
unsigned int headroom)
@@ -444,7 +444,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,
return skb;
}
-static void virtnet_rq_unmap(struct receive_queue *rq, void *buf, u32 len)
+static void virtnet_rq_unmap(struct virtnet_rq *rq, void *buf, u32 len)
{
struct page *page = virt_to_head_page(buf);
struct virtnet_rq_dma *dma;
@@ -473,7 +473,7 @@ static void virtnet_rq_unmap(struct receive_queue *rq, void *buf, u32 len)
put_page(page);
}
-static void *virtnet_rq_get_buf(struct receive_queue *rq, u32 *len, void **ctx)
+static void *virtnet_rq_get_buf(struct virtnet_rq *rq, u32 *len, void **ctx)
{
void *buf;
@@ -484,7 +484,7 @@ static void *virtnet_rq_get_buf(struct receive_queue *rq, u32 *len, void **ctx)
return buf;
}
-static void virtnet_rq_init_one_sg(struct receive_queue *rq, void *buf, u32 len)
+static void virtnet_rq_init_one_sg(struct virtnet_rq *rq, void *buf, u32 len)
{
struct virtnet_rq_dma *dma;
dma_addr_t addr;
@@ -509,7 +509,7 @@ static void virtnet_rq_init_one_sg(struct receive_queue *rq, void *buf, u32 len)
rq->sg[0].length = len;
}
-static void *virtnet_rq_alloc(struct receive_queue *rq, u32 size, gfp_t gfp)
+static void *virtnet_rq_alloc(struct virtnet_rq *rq, u32 size, gfp_t gfp)
{
struct page_frag *alloc_frag = &rq->alloc_frag;
struct virtnet_rq_dma *dma;
@@ -583,7 +583,7 @@ static void virtnet_rq_set_premapped(struct virtnet_info *vi)
}
}
-static void free_old_xmit(struct send_queue *sq, bool in_napi)
+static void free_old_xmit(struct virtnet_sq *sq, bool in_napi)
{
u64 bytes, packets = 0;
@@ -613,7 +613,7 @@ static bool is_xdp_raw_buffer_queue(struct virtnet_info *vi, int q)
static void check_sq_full_and_disable(struct virtnet_info *vi,
struct net_device *dev,
- struct send_queue *sq)
+ struct virtnet_sq *sq)
{
bool use_napi = sq->napi.weight;
int qnum;
@@ -647,7 +647,7 @@ static void check_sq_full_and_disable(struct virtnet_info *vi,
}
static int __virtnet_xdp_xmit_one(struct virtnet_info *vi,
- struct send_queue *sq,
+ struct virtnet_sq *sq,
struct xdp_frame *xdpf)
{
struct virtio_net_hdr_mrg_rxbuf *hdr;
@@ -736,10 +736,10 @@ static int virtnet_xdp_xmit(struct net_device *dev,
int n, struct xdp_frame **frames, u32 flags)
{
struct virtnet_info *vi = netdev_priv(dev);
- struct receive_queue *rq = vi->rq;
+ struct virtnet_rq *rq = vi->rq;
u64 bytes = 0, packets = 0;
struct bpf_prog *xdp_prog;
- struct send_queue *sq;
+ struct virtnet_sq *sq;
int nxmit = 0;
int kicks = 0;
int ret;
@@ -879,7 +879,7 @@ static unsigned int virtnet_get_headroom(struct virtnet_info *vi)
* across multiple buffers (num_buf > 1), and we make sure buffers
* have enough headroom.
*/
-static struct page *xdp_linearize_page(struct receive_queue *rq,
+static struct page *xdp_linearize_page(struct virtnet_rq *rq,
int *num_buf,
struct page *p,
int offset,
@@ -960,7 +960,7 @@ static struct sk_buff *receive_small_build_skb(struct virtnet_info *vi,
static struct sk_buff *receive_small_xdp(struct net_device *dev,
struct virtnet_info *vi,
- struct receive_queue *rq,
+ struct virtnet_rq *rq,
struct bpf_prog *xdp_prog,
void *buf,
unsigned int xdp_headroom,
@@ -1047,7 +1047,7 @@ static struct sk_buff *receive_small_xdp(struct net_device *dev,
static struct sk_buff *receive_small(struct net_device *dev,
struct virtnet_info *vi,
- struct receive_queue *rq,
+ struct virtnet_rq *rq,
void *buf, void *ctx,
unsigned int len,
unsigned int *xdp_xmit,
@@ -1094,7 +1094,7 @@ static struct sk_buff *receive_small(struct net_device *dev,
static struct sk_buff *receive_big(struct net_device *dev,
struct virtnet_info *vi,
- struct receive_queue *rq,
+ struct virtnet_rq *rq,
void *buf,
unsigned int len,
struct virtnet_rq_stats *stats)
@@ -1115,7 +1115,7 @@ static struct sk_buff *receive_big(struct net_device *dev,
return NULL;
}
-static void mergeable_buf_free(struct receive_queue *rq, int num_buf,
+static void mergeable_buf_free(struct virtnet_rq *rq, int num_buf,
struct net_device *dev,
struct virtnet_rq_stats *stats)
{
@@ -1189,7 +1189,7 @@ static struct sk_buff *build_skb_from_xdp_buff(struct net_device *dev,
/* TODO: build xdp in big mode */
static int virtnet_build_xdp_buff_mrg(struct net_device *dev,
struct virtnet_info *vi,
- struct receive_queue *rq,
+ struct virtnet_rq *rq,
struct xdp_buff *xdp,
void *buf,
unsigned int len,
@@ -1277,7 +1277,7 @@ static int virtnet_build_xdp_buff_mrg(struct net_device *dev,
}
static void *mergeable_xdp_get_buf(struct virtnet_info *vi,
- struct receive_queue *rq,
+ struct virtnet_rq *rq,
struct bpf_prog *xdp_prog,
void *ctx,
unsigned int *frame_sz,
@@ -1352,7 +1352,7 @@ static void *mergeable_xdp_get_buf(struct virtnet_info *vi,
static struct sk_buff *receive_mergeable_xdp(struct net_device *dev,
struct virtnet_info *vi,
- struct receive_queue *rq,
+ struct virtnet_rq *rq,
struct bpf_prog *xdp_prog,
void *buf,
void *ctx,
@@ -1412,7 +1412,7 @@ static struct sk_buff *receive_mergeable_xdp(struct net_device *dev,
static struct sk_buff *receive_mergeable(struct net_device *dev,
struct virtnet_info *vi,
- struct receive_queue *rq,
+ struct virtnet_rq *rq,
void *buf,
void *ctx,
unsigned int len,
@@ -1557,7 +1557,7 @@ static void virtio_skb_set_hash(const struct virtio_net_hdr_v1_hash *hdr_hash,
skb_set_hash(skb, __le32_to_cpu(hdr_hash->hash_value), rss_hash_type);
}
-static void receive_buf(struct virtnet_info *vi, struct receive_queue *rq,
+static void receive_buf(struct virtnet_info *vi, struct virtnet_rq *rq,
void *buf, unsigned int len, void **ctx,
unsigned int *xdp_xmit,
struct virtnet_rq_stats *stats)
@@ -1617,7 +1617,7 @@ static void receive_buf(struct virtnet_info *vi, struct receive_queue *rq,
* not need to use mergeable_len_to_ctx here - it is enough
* to store the headroom as the context ignoring the truesize.
*/
-static int add_recvbuf_small(struct virtnet_info *vi, struct receive_queue *rq,
+static int add_recvbuf_small(struct virtnet_info *vi, struct virtnet_rq *rq,
gfp_t gfp)
{
char *buf;
@@ -1646,7 +1646,7 @@ static int add_recvbuf_small(struct virtnet_info *vi, struct receive_queue *rq,
return err;
}
-static int add_recvbuf_big(struct virtnet_info *vi, struct receive_queue *rq,
+static int add_recvbuf_big(struct virtnet_info *vi, struct virtnet_rq *rq,
gfp_t gfp)
{
struct page *first, *list = NULL;
@@ -1695,7 +1695,7 @@ static int add_recvbuf_big(struct virtnet_info *vi, struct receive_queue *rq,
return err;
}
-static unsigned int get_mergeable_buf_len(struct receive_queue *rq,
+static unsigned int get_mergeable_buf_len(struct virtnet_rq *rq,
struct ewma_pkt_len *avg_pkt_len,
unsigned int room)
{
@@ -1713,7 +1713,7 @@ static unsigned int get_mergeable_buf_len(struct receive_queue *rq,
}
static int add_recvbuf_mergeable(struct virtnet_info *vi,
- struct receive_queue *rq, gfp_t gfp)
+ struct virtnet_rq *rq, gfp_t gfp)
{
struct page_frag *alloc_frag = &rq->alloc_frag;
unsigned int headroom = virtnet_get_headroom(vi);
@@ -1768,7 +1768,7 @@ static int add_recvbuf_mergeable(struct virtnet_info *vi,
* before we're receiving packets, or from refill_work which is
* careful to disable receiving (using napi_disable).
*/
-static bool try_fill_recv(struct virtnet_info *vi, struct receive_queue *rq,
+static bool try_fill_recv(struct virtnet_info *vi, struct virtnet_rq *rq,
gfp_t gfp)
{
int err;
@@ -1800,7 +1800,7 @@ static bool try_fill_recv(struct virtnet_info *vi, struct receive_queue *rq,
static void skb_recv_done(struct virtqueue *rvq)
{
struct virtnet_info *vi = rvq->vdev->priv;
- struct receive_queue *rq = &vi->rq[vq2rxq(rvq)];
+ struct virtnet_rq *rq = &vi->rq[vq2rxq(rvq)];
virtqueue_napi_schedule(&rq->napi, rvq);
}
@@ -1850,7 +1850,7 @@ static void refill_work(struct work_struct *work)
int i;
for (i = 0; i < vi->curr_queue_pairs; i++) {
- struct receive_queue *rq = &vi->rq[i];
+ struct virtnet_rq *rq = &vi->rq[i];
napi_disable(&rq->napi);
still_empty = !try_fill_recv(vi, rq, GFP_KERNEL);
@@ -1864,7 +1864,7 @@ static void refill_work(struct work_struct *work)
}
}
-static int virtnet_receive(struct receive_queue *rq, int budget,
+static int virtnet_receive(struct virtnet_rq *rq, int budget,
unsigned int *xdp_xmit)
{
struct virtnet_info *vi = rq->vq->vdev->priv;
@@ -1914,11 +1914,11 @@ static int virtnet_receive(struct receive_queue *rq, int budget,
return packets;
}
-static void virtnet_poll_cleantx(struct receive_queue *rq)
+static void virtnet_poll_cleantx(struct virtnet_rq *rq)
{
struct virtnet_info *vi = rq->vq->vdev->priv;
unsigned int index = vq2rxq(rq->vq);
- struct send_queue *sq = &vi->sq[index];
+ struct virtnet_sq *sq = &vi->sq[index];
struct netdev_queue *txq = netdev_get_tx_queue(vi->dev, index);
if (!sq->napi.weight || is_xdp_raw_buffer_queue(vi, index))
@@ -1944,10 +1944,10 @@ static void virtnet_poll_cleantx(struct receive_queue *rq)
static int virtnet_poll(struct napi_struct *napi, int budget)
{
- struct receive_queue *rq =
- container_of(napi, struct receive_queue, napi);
+ struct virtnet_rq *rq =
+ container_of(napi, struct virtnet_rq, napi);
struct virtnet_info *vi = rq->vq->vdev->priv;
- struct send_queue *sq;
+ struct virtnet_sq *sq;
unsigned int received;
unsigned int xdp_xmit = 0;
@@ -2038,7 +2038,7 @@ static int virtnet_open(struct net_device *dev)
static int virtnet_poll_tx(struct napi_struct *napi, int budget)
{
- struct send_queue *sq = container_of(napi, struct send_queue, napi);
+ struct virtnet_sq *sq = container_of(napi, struct virtnet_sq, napi);
struct virtnet_info *vi = sq->vq->vdev->priv;
unsigned int index = vq2txq(sq->vq);
struct netdev_queue *txq;
@@ -2082,7 +2082,7 @@ static int virtnet_poll_tx(struct napi_struct *napi, int budget)
return 0;
}
-static int xmit_skb(struct send_queue *sq, struct sk_buff *skb)
+static int xmit_skb(struct virtnet_sq *sq, struct sk_buff *skb)
{
struct virtio_net_hdr_mrg_rxbuf *hdr;
const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest;
@@ -2133,7 +2133,7 @@ static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
{
struct virtnet_info *vi = netdev_priv(dev);
int qnum = skb_get_queue_mapping(skb);
- struct send_queue *sq = &vi->sq[qnum];
+ struct virtnet_sq *sq = &vi->sq[qnum];
int err;
struct netdev_queue *txq = netdev_get_tx_queue(dev, qnum);
bool kick = !netdev_xmit_more();
@@ -2187,7 +2187,7 @@ static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
}
static int virtnet_rx_resize(struct virtnet_info *vi,
- struct receive_queue *rq, u32 ring_num)
+ struct virtnet_rq *rq, u32 ring_num)
{
bool running = netif_running(vi->dev);
int err, qindex;
@@ -2210,7 +2210,7 @@ static int virtnet_rx_resize(struct virtnet_info *vi,
}
static int virtnet_tx_resize(struct virtnet_info *vi,
- struct send_queue *sq, u32 ring_num)
+ struct virtnet_sq *sq, u32 ring_num)
{
bool running = netif_running(vi->dev);
struct netdev_queue *txq;
@@ -2356,8 +2356,8 @@ static void virtnet_stats(struct net_device *dev,
for (i = 0; i < vi->max_queue_pairs; i++) {
u64 tpackets, tbytes, terrors, rpackets, rbytes, rdrops;
- struct receive_queue *rq = &vi->rq[i];
- struct send_queue *sq = &vi->sq[i];
+ struct virtnet_rq *rq = &vi->rq[i];
+ struct virtnet_sq *sq = &vi->sq[i];
do {
start = u64_stats_fetch_begin(&sq->stats.syncp);
@@ -2673,8 +2673,8 @@ static int virtnet_set_ringparam(struct net_device *dev,
{
struct virtnet_info *vi = netdev_priv(dev);
u32 rx_pending, tx_pending;
- struct receive_queue *rq;
- struct send_queue *sq;
+ struct virtnet_rq *rq;
+ struct virtnet_sq *sq;
int i, err;
if (ring->rx_mini_pending || ring->rx_jumbo_pending)
@@ -3003,7 +3003,7 @@ static void virtnet_get_ethtool_stats(struct net_device *dev,
size_t offset;
for (i = 0; i < vi->curr_queue_pairs; i++) {
- struct receive_queue *rq = &vi->rq[i];
+ struct virtnet_rq *rq = &vi->rq[i];
stats_base = (const u8 *)&rq->stats;
do {
@@ -3018,7 +3018,7 @@ static void virtnet_get_ethtool_stats(struct net_device *dev,
}
for (i = 0; i < vi->curr_queue_pairs; i++) {
- struct send_queue *sq = &vi->sq[i];
+ struct virtnet_sq *sq = &vi->sq[i];
stats_base = (const u8 *)&sq->stats;
do {
@@ -3705,7 +3705,7 @@ static int virtnet_set_features(struct net_device *dev,
static void virtnet_tx_timeout(struct net_device *dev, unsigned int txqueue)
{
struct virtnet_info *priv = netdev_priv(dev);
- struct send_queue *sq = &priv->sq[txqueue];
+ struct virtnet_sq *sq = &priv->sq[txqueue];
struct netdev_queue *txq = netdev_get_tx_queue(dev, txqueue);
u64_stats_update_begin(&sq->stats.syncp);
@@ -3839,7 +3839,7 @@ static void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf)
static void virtnet_rq_free_unused_buf(struct virtqueue *vq, void *buf)
{
struct virtnet_info *vi = vq->vdev->priv;
- struct receive_queue *rq;
+ struct virtnet_rq *rq;
int i = vq2rxq(vq);
rq = &vi->rq[i];
diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
index 38061e15d494..ebf9f344648a 100644
--- a/drivers/net/virtio/virtio_net.h
+++ b/drivers/net/virtio/virtio_net.h
@@ -49,8 +49,8 @@ struct virtnet_rq_dma {
};
/* Internal representation of a send virtqueue */
-struct send_queue {
- /* Virtqueue associated with this send _queue */
+struct virtnet_sq {
+ /* Virtqueue associated with this virtnet_sq */
struct virtqueue *vq;
/* TX: fragments + linear part + virtio header */
@@ -70,8 +70,8 @@ struct send_queue {
};
/* Internal representation of a receive virtqueue */
-struct receive_queue {
- /* Virtqueue associated with this receive_queue */
+struct virtnet_rq {
+ /* Virtqueue associated with this virtnet_rq */
struct virtqueue *vq;
struct napi_struct napi;
@@ -113,8 +113,8 @@ struct virtnet_info {
struct virtio_device *vdev;
struct virtqueue *cvq;
struct net_device *dev;
- struct send_queue *sq;
- struct receive_queue *rq;
+ struct virtnet_sq *sq;
+ struct virtnet_rq *rq;
unsigned int status;
/* Max # of queue pairs supported by the device */
--
2.32.0.3.g01195cf9f
^ permalink raw reply related
* [PATCH net-next v2 07/21] virtio_net: separate virtnet_tx_resize()
From: Xuan Zhuo @ 2023-11-07 3:12 UTC (permalink / raw)
To: netdev
Cc: David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Michael S. Tsirkin, Jason Wang, Xuan Zhuo, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
In-Reply-To: <20231107031227.100015-1-xuanzhuo@linux.alibaba.com>
This patch separates two sub-functions from virtnet_tx_resize():
* virtnet_tx_pause
* virtnet_tx_resume
Then the subsequent virtnet_tx_reset() can share these two functions.
Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
Acked-by: Jason Wang <jasowang@redhat.com>
---
drivers/net/virtio/main.c | 35 +++++++++++++++++++++++++++------
drivers/net/virtio/virtio_net.h | 2 ++
2 files changed, 31 insertions(+), 6 deletions(-)
diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
index c9f8294153e2..16e75c08639e 100644
--- a/drivers/net/virtio/main.c
+++ b/drivers/net/virtio/main.c
@@ -2222,12 +2222,11 @@ static int virtnet_rx_resize(struct virtnet_info *vi,
return err;
}
-static int virtnet_tx_resize(struct virtnet_info *vi,
- struct virtnet_sq *sq, u32 ring_num)
+void virtnet_tx_pause(struct virtnet_info *vi, struct virtnet_sq *sq)
{
bool running = netif_running(vi->dev);
struct netdev_queue *txq;
- int err, qindex;
+ int qindex;
qindex = sq - vi->sq;
@@ -2248,10 +2247,17 @@ static int virtnet_tx_resize(struct virtnet_info *vi,
netif_stop_subqueue(vi->dev, qindex);
__netif_tx_unlock_bh(txq);
+}
- err = virtqueue_resize(sq->vq, ring_num, virtnet_sq_free_unused_buf);
- if (err)
- netdev_err(vi->dev, "resize tx fail: tx queue index: %d err: %d\n", qindex, err);
+void virtnet_tx_resume(struct virtnet_info *vi, struct virtnet_sq *sq)
+{
+ bool running = netif_running(vi->dev);
+ struct netdev_queue *txq;
+ int qindex;
+
+ qindex = sq - vi->sq;
+
+ txq = netdev_get_tx_queue(vi->dev, qindex);
__netif_tx_lock_bh(txq);
sq->reset = false;
@@ -2260,6 +2266,23 @@ static int virtnet_tx_resize(struct virtnet_info *vi,
if (running)
virtnet_napi_tx_enable(vi, sq->vq, &sq->napi);
+}
+
+static int virtnet_tx_resize(struct virtnet_info *vi, struct virtnet_sq *sq,
+ u32 ring_num)
+{
+ int qindex, err;
+
+ qindex = sq - vi->sq;
+
+ virtnet_tx_pause(vi, sq);
+
+ err = virtqueue_resize(sq->vq, ring_num, virtnet_sq_free_unused_buf);
+ if (err)
+ netdev_err(vi->dev, "resize tx fail: tx queue index: %d err: %d\n", qindex, err);
+
+ virtnet_tx_resume(vi, sq);
+
return err;
}
diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
index 693ca166fc93..d814341d9f97 100644
--- a/drivers/net/virtio/virtio_net.h
+++ b/drivers/net/virtio/virtio_net.h
@@ -193,4 +193,6 @@ struct virtnet_info {
void virtnet_rx_pause(struct virtnet_info *vi, struct virtnet_rq *rq);
void virtnet_rx_resume(struct virtnet_info *vi, struct virtnet_rq *rq);
+void virtnet_tx_pause(struct virtnet_info *vi, struct virtnet_sq *sq);
+void virtnet_tx_resume(struct virtnet_info *vi, struct virtnet_sq *sq);
#endif
--
2.32.0.3.g01195cf9f
^ permalink raw reply related
* [PATCH net-next v2 08/21] virtio_net: sq support premapped mode
From: Xuan Zhuo @ 2023-11-07 3:12 UTC (permalink / raw)
To: netdev
Cc: David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Michael S. Tsirkin, Jason Wang, Xuan Zhuo, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
In-Reply-To: <20231107031227.100015-1-xuanzhuo@linux.alibaba.com>
If the xsk is enabling, the xsk tx will share the send queue.
But the xsk requires that the send queue use the premapped mode.
So the send queue must support premapped mode.
Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
---
drivers/net/virtio/main.c | 163 ++++++++++++++++++++++++++++----
drivers/net/virtio/virtio_net.h | 16 ++++
2 files changed, 163 insertions(+), 16 deletions(-)
diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
index 16e75c08639e..f052db459156 100644
--- a/drivers/net/virtio/main.c
+++ b/drivers/net/virtio/main.c
@@ -46,6 +46,7 @@ module_param(napi_tx, bool, 0644);
#define VIRTIO_XDP_REDIR BIT(1)
#define VIRTIO_XDP_FLAG BIT(0)
+#define VIRTIO_XMIT_DATA_MASK (VIRTIO_XDP_FLAG)
#define VIRTNET_DRIVER_VERSION "1.0.0"
@@ -167,6 +168,29 @@ static struct xdp_frame *ptr_to_xdp(void *ptr)
return (struct xdp_frame *)((unsigned long)ptr & ~VIRTIO_XDP_FLAG);
}
+static inline void *virtnet_sq_unmap(struct virtnet_sq *sq, void *data)
+{
+ struct virtnet_sq_dma *next, *head;
+
+ head = (void *)((unsigned long)data & ~VIRTIO_XMIT_DATA_MASK);
+
+ data = head->data;
+
+ while (head) {
+ virtqueue_dma_unmap_single_attrs(sq->vq, head->addr, head->len,
+ DMA_TO_DEVICE, 0);
+
+ next = head->next;
+
+ head->next = sq->dmainfo.free;
+ sq->dmainfo.free = head;
+
+ head = next;
+ }
+
+ return data;
+}
+
static void __free_old_xmit(struct virtnet_sq *sq, bool in_napi,
u64 *bytes, u64 *packets)
{
@@ -175,14 +199,24 @@ static void __free_old_xmit(struct virtnet_sq *sq, bool in_napi,
while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {
if (!is_xdp_frame(ptr)) {
- struct sk_buff *skb = ptr;
+ struct sk_buff *skb;
+
+ if (sq->do_dma)
+ ptr = virtnet_sq_unmap(sq, ptr);
+
+ skb = ptr;
pr_debug("Sent skb %p\n", skb);
*bytes += skb->len;
napi_consume_skb(skb, in_napi);
} else {
- struct xdp_frame *frame = ptr_to_xdp(ptr);
+ struct xdp_frame *frame;
+
+ if (sq->do_dma)
+ ptr = virtnet_sq_unmap(sq, ptr);
+
+ frame = ptr_to_xdp(ptr);
*bytes += xdp_get_frame_len(frame);
xdp_return_frame(frame);
@@ -567,22 +601,104 @@ static void *virtnet_rq_alloc(struct virtnet_rq *rq, u32 size, gfp_t gfp)
return buf;
}
-static void virtnet_rq_set_premapped(struct virtnet_info *vi)
+static int virtnet_sq_set_premapped(struct virtnet_sq *sq)
{
- int i;
+ struct virtnet_sq_dma *d;
+ int err, size, i;
- /* disable for big mode */
- if (!vi->mergeable_rx_bufs && vi->big_packets)
- return;
+ size = virtqueue_get_vring_size(sq->vq);
+
+ size += MAX_SKB_FRAGS + 2;
+
+ sq->dmainfo.head = kcalloc(size, sizeof(*sq->dmainfo.head), GFP_KERNEL);
+ if (!sq->dmainfo.head)
+ return -ENOMEM;
+
+ err = virtqueue_set_dma_premapped(sq->vq);
+ if (err) {
+ kfree(sq->dmainfo.head);
+ return err;
+ }
+
+ sq->dmainfo.free = NULL;
+
+ sq->do_dma = true;
+
+ for (i = 0; i < size; ++i) {
+ d = &sq->dmainfo.head[i];
+
+ d->next = sq->dmainfo.free;
+ sq->dmainfo.free = d;
+ }
+
+ return 0;
+}
+
+static void virtnet_set_premapped(struct virtnet_info *vi)
+{
+ int i;
for (i = 0; i < vi->max_queue_pairs; i++) {
- if (virtqueue_set_dma_premapped(vi->rq[i].vq))
- continue;
+ virtnet_sq_set_premapped(&vi->sq[i]);
- vi->rq[i].do_dma = true;
+ /* disable for big mode */
+ if (vi->mergeable_rx_bufs || !vi->big_packets) {
+ if (!virtqueue_set_dma_premapped(vi->rq[i].vq))
+ vi->rq[i].do_dma = true;
+ }
}
}
+static struct virtnet_sq_dma *virtnet_sq_map_sg(struct virtnet_sq *sq, int nents, void *data)
+{
+ struct virtnet_sq_dma *d, *head;
+ struct scatterlist *sg;
+ int i;
+
+ head = NULL;
+
+ for_each_sg(sq->sg, sg, nents, i) {
+ sg->dma_address = virtqueue_dma_map_single_attrs(sq->vq, sg_virt(sg),
+ sg->length,
+ DMA_TO_DEVICE, 0);
+ if (virtqueue_dma_mapping_error(sq->vq, sg->dma_address))
+ goto err;
+
+ d = sq->dmainfo.free;
+ sq->dmainfo.free = d->next;
+
+ d->addr = sg->dma_address;
+ d->len = sg->length;
+
+ d->next = head;
+ head = d;
+ }
+
+ head->data = data;
+
+ return (void *)((unsigned long)head | ((unsigned long)data & VIRTIO_XMIT_DATA_MASK));
+err:
+ virtnet_sq_unmap(sq, head);
+ return NULL;
+}
+
+static int virtnet_add_outbuf(struct virtnet_sq *sq, u32 num, void *data)
+{
+ int ret;
+
+ if (sq->do_dma) {
+ data = virtnet_sq_map_sg(sq, num, data);
+ if (!data)
+ return -ENOMEM;
+ }
+
+ ret = virtqueue_add_outbuf(sq->vq, sq->sg, num, data, GFP_ATOMIC);
+ if (ret && sq->do_dma)
+ virtnet_sq_unmap(sq, data);
+
+ return ret;
+}
+
static void free_old_xmit(struct virtnet_sq *sq, bool in_napi)
{
u64 bytes, packets = 0;
@@ -686,8 +802,7 @@ static int __virtnet_xdp_xmit_one(struct virtnet_info *vi,
skb_frag_size(frag), skb_frag_off(frag));
}
- err = virtqueue_add_outbuf(sq->vq, sq->sg, nr_frags + 1,
- xdp_to_ptr(xdpf), GFP_ATOMIC);
+ err = virtnet_add_outbuf(sq, nr_frags + 1, xdp_to_ptr(xdpf));
if (unlikely(err))
return -ENOSPC; /* Caller handle free/refcnt */
@@ -2126,7 +2241,8 @@ static int xmit_skb(struct virtnet_sq *sq, struct sk_buff *skb)
return num_sg;
num_sg++;
}
- return virtqueue_add_outbuf(sq->vq, sq->sg, num_sg, skb, GFP_ATOMIC);
+
+ return virtnet_add_outbuf(sq, num_sg, skb);
}
static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
@@ -3818,6 +3934,8 @@ static void virtnet_free_queues(struct virtnet_info *vi)
for (i = 0; i < vi->max_queue_pairs; i++) {
__netif_napi_del(&vi->rq[i].napi);
__netif_napi_del(&vi->sq[i].napi);
+
+ kfree(vi->sq[i].dmainfo.head);
}
/* We called __netif_napi_del(),
@@ -3866,10 +3984,23 @@ static void free_receive_page_frags(struct virtnet_info *vi)
static void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf)
{
- if (!is_xdp_frame(buf))
+ struct virtnet_info *vi = vq->vdev->priv;
+ struct virtnet_sq *sq;
+ int i = vq2rxq(vq);
+
+ sq = &vi->sq[i];
+
+ if (!is_xdp_frame(buf)) {
+ if (sq->do_dma)
+ buf = virtnet_sq_unmap(sq, buf);
+
dev_kfree_skb(buf);
- else
+ } else {
+ if (sq->do_dma)
+ buf = virtnet_sq_unmap(sq, buf);
+
xdp_return_frame(ptr_to_xdp(buf));
+ }
}
static void virtnet_rq_free_unused_buf(struct virtqueue *vq, void *buf)
@@ -4075,7 +4206,7 @@ static int init_vqs(struct virtnet_info *vi)
if (ret)
goto err_free;
- virtnet_rq_set_premapped(vi);
+ virtnet_set_premapped(vi);
cpus_read_lock();
virtnet_set_affinity(vi);
diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
index d814341d9f97..ce806afb6d64 100644
--- a/drivers/net/virtio/virtio_net.h
+++ b/drivers/net/virtio/virtio_net.h
@@ -48,6 +48,18 @@ struct virtnet_rq_dma {
u16 need_sync;
};
+struct virtnet_sq_dma {
+ struct virtnet_sq_dma *next;
+ dma_addr_t addr;
+ u32 len;
+ void *data;
+};
+
+struct virtnet_sq_dma_head {
+ struct virtnet_sq_dma *free;
+ struct virtnet_sq_dma *head;
+};
+
/* Internal representation of a send virtqueue */
struct virtnet_sq {
/* Virtqueue associated with this virtnet_sq */
@@ -67,6 +79,10 @@ struct virtnet_sq {
/* Record whether sq is in reset state. */
bool reset;
+
+ bool do_dma;
+
+ struct virtnet_sq_dma_head dmainfo;
};
/* Internal representation of a receive virtqueue */
--
2.32.0.3.g01195cf9f
^ permalink raw reply related
* [PATCH net-next v2 09/21] virtio_net: xsk: bind/unbind xsk
From: Xuan Zhuo @ 2023-11-07 3:12 UTC (permalink / raw)
To: netdev
Cc: David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Michael S. Tsirkin, Jason Wang, Xuan Zhuo, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
In-Reply-To: <20231107031227.100015-1-xuanzhuo@linux.alibaba.com>
This patch implement the logic of bind/unbind xsk pool to sq and rq.
Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
---
drivers/net/virtio/Makefile | 2 +-
drivers/net/virtio/main.c | 11 +-
drivers/net/virtio/virtio_net.h | 16 +++
drivers/net/virtio/xsk.c | 187 ++++++++++++++++++++++++++++++++
drivers/net/virtio/xsk.h | 7 ++
5 files changed, 216 insertions(+), 7 deletions(-)
create mode 100644 drivers/net/virtio/xsk.c
create mode 100644 drivers/net/virtio/xsk.h
diff --git a/drivers/net/virtio/Makefile b/drivers/net/virtio/Makefile
index 15ed7c97fd4f..8c2a884d2dba 100644
--- a/drivers/net/virtio/Makefile
+++ b/drivers/net/virtio/Makefile
@@ -5,4 +5,4 @@
obj-$(CONFIG_VIRTIO_NET) += virtio_net.o
-virtio_net-y := main.o
+virtio_net-y := main.o xsk.o
diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
index f052db459156..c4601784b6d1 100644
--- a/drivers/net/virtio/main.c
+++ b/drivers/net/virtio/main.c
@@ -8,7 +8,6 @@
#include <linux/etherdevice.h>
#include <linux/module.h>
#include <linux/virtio.h>
-#include <linux/virtio_net.h>
#include <linux/bpf.h>
#include <linux/bpf_trace.h>
#include <linux/scatterlist.h>
@@ -23,6 +22,7 @@
#include <net/netdev_rx_queue.h>
#include "virtio_net.h"
+#include "xsk.h"
static int napi_weight = NAPI_POLL_WEIGHT;
module_param(napi_weight, int, 0444);
@@ -150,9 +150,6 @@ struct virtio_net_common_hdr {
};
};
-static void virtnet_rq_free_unused_buf(struct virtqueue *vq, void *buf);
-static void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf);
-
static bool is_xdp_frame(void *ptr)
{
return (unsigned long)ptr & VIRTIO_XDP_FLAG;
@@ -3797,6 +3794,8 @@ static int virtnet_xdp(struct net_device *dev, struct netdev_bpf *xdp)
switch (xdp->command) {
case XDP_SETUP_PROG:
return virtnet_xdp_set(dev, xdp->prog, xdp->extack);
+ case XDP_SETUP_XSK_POOL:
+ return virtnet_xsk_pool_setup(dev, xdp);
default:
return -EINVAL;
}
@@ -3982,7 +3981,7 @@ static void free_receive_page_frags(struct virtnet_info *vi)
}
}
-static void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf)
+void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf)
{
struct virtnet_info *vi = vq->vdev->priv;
struct virtnet_sq *sq;
@@ -4003,7 +4002,7 @@ static void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf)
}
}
-static void virtnet_rq_free_unused_buf(struct virtqueue *vq, void *buf)
+void virtnet_rq_free_unused_buf(struct virtqueue *vq, void *buf)
{
struct virtnet_info *vi = vq->vdev->priv;
struct virtnet_rq *rq;
diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
index ce806afb6d64..98ba23cfdb20 100644
--- a/drivers/net/virtio/virtio_net.h
+++ b/drivers/net/virtio/virtio_net.h
@@ -5,6 +5,8 @@
#include <linux/ethtool.h>
#include <linux/average.h>
+#include <linux/virtio_net.h>
+#include <net/xdp_sock_drv.h>
/* RX packet size EWMA. The average packet size is used to determine the packet
* buffer size when refilling RX rings. As the entire RX ring may be refilled
@@ -83,6 +85,11 @@ struct virtnet_sq {
bool do_dma;
struct virtnet_sq_dma_head dmainfo;
+ struct {
+ struct xsk_buff_pool *pool;
+
+ dma_addr_t hdr_dma_address;
+ } xsk;
};
/* Internal representation of a receive virtqueue */
@@ -123,6 +130,13 @@ struct virtnet_rq {
/* Do dma by self */
bool do_dma;
+
+ struct {
+ struct xsk_buff_pool *pool;
+
+ /* xdp rxq used by xsk */
+ struct xdp_rxq_info xdp_rxq;
+ } xsk;
};
struct virtnet_info {
@@ -211,4 +225,6 @@ void virtnet_rx_pause(struct virtnet_info *vi, struct virtnet_rq *rq);
void virtnet_rx_resume(struct virtnet_info *vi, struct virtnet_rq *rq);
void virtnet_tx_pause(struct virtnet_info *vi, struct virtnet_sq *sq);
void virtnet_tx_resume(struct virtnet_info *vi, struct virtnet_sq *sq);
+void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf);
+void virtnet_rq_free_unused_buf(struct virtqueue *vq, void *buf);
#endif
diff --git a/drivers/net/virtio/xsk.c b/drivers/net/virtio/xsk.c
new file mode 100644
index 000000000000..8b397787603f
--- /dev/null
+++ b/drivers/net/virtio/xsk.c
@@ -0,0 +1,187 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * virtio-net xsk
+ */
+
+#include "virtio_net.h"
+
+static struct virtio_net_hdr_mrg_rxbuf xsk_hdr;
+
+static int virtnet_rq_bind_xsk_pool(struct virtnet_info *vi, struct virtnet_rq *rq,
+ struct xsk_buff_pool *pool)
+{
+ int err, qindex;
+
+ qindex = rq - vi->rq;
+
+ if (pool) {
+ err = xdp_rxq_info_reg(&rq->xsk.xdp_rxq, vi->dev, qindex, rq->napi.napi_id);
+ if (err < 0)
+ return err;
+
+ err = xdp_rxq_info_reg_mem_model(&rq->xsk.xdp_rxq,
+ MEM_TYPE_XSK_BUFF_POOL, NULL);
+ if (err < 0) {
+ xdp_rxq_info_unreg(&rq->xsk.xdp_rxq);
+ return err;
+ }
+
+ xsk_pool_set_rxq_info(pool, &rq->xsk.xdp_rxq);
+ }
+
+ virtnet_rx_pause(vi, rq);
+
+ err = virtqueue_reset(rq->vq, virtnet_rq_free_unused_buf);
+ if (err) {
+ netdev_err(vi->dev, "reset rx fail: rx queue index: %d err: %d\n", qindex, err);
+
+ pool = NULL;
+ }
+
+ if (!pool)
+ xdp_rxq_info_unreg(&rq->xsk.xdp_rxq);
+
+ rq->xsk.pool = pool;
+
+ virtnet_rx_resume(vi, rq);
+
+ return err;
+}
+
+static int virtnet_sq_bind_xsk_pool(struct virtnet_info *vi,
+ struct virtnet_sq *sq,
+ struct xsk_buff_pool *pool)
+{
+ int err, qindex;
+
+ qindex = sq - vi->sq;
+
+ virtnet_tx_pause(vi, sq);
+
+ err = virtqueue_reset(sq->vq, virtnet_sq_free_unused_buf);
+ if (err) {
+ pool = NULL;
+ netdev_err(vi->dev, "reset tx fail: tx queue index: %d err: %d\n", qindex, err);
+ }
+
+ sq->xsk.pool = pool;
+
+ virtnet_tx_resume(vi, sq);
+
+ return err;
+}
+
+static int virtnet_xsk_pool_enable(struct net_device *dev,
+ struct xsk_buff_pool *pool,
+ u16 qid)
+{
+ struct virtnet_info *vi = netdev_priv(dev);
+ struct virtnet_rq *rq;
+ struct virtnet_sq *sq;
+ struct device *dma_dev;
+ dma_addr_t hdr_dma;
+ int err;
+
+ /* In big_packets mode, xdp cannot work, so there is no need to
+ * initialize xsk of rq.
+ */
+ if (vi->big_packets && !vi->mergeable_rx_bufs)
+ return -ENOENT;
+
+ if (qid >= vi->curr_queue_pairs)
+ return -EINVAL;
+
+ sq = &vi->sq[qid];
+ rq = &vi->rq[qid];
+
+ /* xsk tx zerocopy depend on the tx napi.
+ *
+ * All xsk packets are actually consumed and sent out from the xsk tx
+ * queue under the tx napi mechanism.
+ */
+ if (!sq->napi.weight)
+ return -EPERM;
+
+ if (!rq->do_dma || !sq->do_dma)
+ return -EPERM;
+
+ /* For the xsk, the tx and rx should have the same device. But
+ * vq->dma_dev allows every vq has the respective dma dev. So I check
+ * the dma dev of vq and sq is the same dev.
+ */
+ if (virtqueue_dma_dev(rq->vq) != virtqueue_dma_dev(sq->vq))
+ return -EPERM;
+
+ dma_dev = virtqueue_dma_dev(rq->vq);
+ if (!dma_dev)
+ return -EPERM;
+
+ hdr_dma = dma_map_single(dma_dev, &xsk_hdr, vi->hdr_len, DMA_TO_DEVICE);
+ if (dma_mapping_error(dma_dev, hdr_dma))
+ return -ENOMEM;
+
+ err = xsk_pool_dma_map(pool, dma_dev, 0);
+ if (err)
+ goto err_xsk_map;
+
+ err = virtnet_rq_bind_xsk_pool(vi, rq, pool);
+ if (err)
+ goto err_rq;
+
+ err = virtnet_sq_bind_xsk_pool(vi, sq, pool);
+ if (err)
+ goto err_sq;
+
+ /* Now, we do not support tx offset, so all the tx virtnet hdr is zero.
+ * So all the tx packets can share a single hdr.
+ */
+ sq->xsk.hdr_dma_address = hdr_dma;
+
+ return 0;
+
+err_sq:
+ virtnet_rq_bind_xsk_pool(vi, rq, NULL);
+err_rq:
+ xsk_pool_dma_unmap(pool, 0);
+err_xsk_map:
+ dma_unmap_single(dma_dev, hdr_dma, vi->hdr_len, DMA_TO_DEVICE);
+ return err;
+}
+
+static int virtnet_xsk_pool_disable(struct net_device *dev, u16 qid)
+{
+ struct virtnet_info *vi = netdev_priv(dev);
+ struct xsk_buff_pool *pool;
+ struct device *dma_dev;
+ struct virtnet_rq *rq;
+ struct virtnet_sq *sq;
+ int err1, err2;
+
+ if (qid >= vi->curr_queue_pairs)
+ return -EINVAL;
+
+ sq = &vi->sq[qid];
+ rq = &vi->rq[qid];
+
+ pool = sq->xsk.pool;
+
+ err1 = virtnet_sq_bind_xsk_pool(vi, sq, NULL);
+ err2 = virtnet_rq_bind_xsk_pool(vi, rq, NULL);
+
+ xsk_pool_dma_unmap(pool, 0);
+
+ dma_dev = virtqueue_dma_dev(rq->vq);
+
+ dma_unmap_single(dma_dev, sq->xsk.hdr_dma_address, vi->hdr_len, DMA_TO_DEVICE);
+
+ return err1 | err2;
+}
+
+int virtnet_xsk_pool_setup(struct net_device *dev, struct netdev_bpf *xdp)
+{
+ if (xdp->xsk.pool)
+ return virtnet_xsk_pool_enable(dev, xdp->xsk.pool,
+ xdp->xsk.queue_id);
+ else
+ return virtnet_xsk_pool_disable(dev, xdp->xsk.queue_id);
+}
diff --git a/drivers/net/virtio/xsk.h b/drivers/net/virtio/xsk.h
new file mode 100644
index 000000000000..1918285c310c
--- /dev/null
+++ b/drivers/net/virtio/xsk.h
@@ -0,0 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+
+#ifndef __XSK_H__
+#define __XSK_H__
+
+int virtnet_xsk_pool_setup(struct net_device *dev, struct netdev_bpf *xdp);
+#endif
--
2.32.0.3.g01195cf9f
^ permalink raw reply related
* [PATCH net-next v2 10/21] virtio_net: xsk: prevent disable tx napi
From: Xuan Zhuo @ 2023-11-07 3:12 UTC (permalink / raw)
To: netdev
Cc: David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Michael S. Tsirkin, Jason Wang, Xuan Zhuo, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
In-Reply-To: <20231107031227.100015-1-xuanzhuo@linux.alibaba.com>
Since xsk's TX queue is consumed by TX NAPI, if sq is bound to xsk, then
we must stop tx napi from being disabled.
Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
Acked-by: Jason Wang <jasowang@redhat.com>
---
drivers/net/virtio/main.c | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
index c4601784b6d1..02e054fd217c 100644
--- a/drivers/net/virtio/main.c
+++ b/drivers/net/virtio/main.c
@@ -3337,7 +3337,7 @@ static int virtnet_set_coalesce(struct net_device *dev,
struct netlink_ext_ack *extack)
{
struct virtnet_info *vi = netdev_priv(dev);
- int ret, queue_number, napi_weight;
+ int ret, queue_number, napi_weight, i;
bool update_napi = false;
/* Can't change NAPI weight if the link is up */
@@ -3366,6 +3366,14 @@ static int virtnet_set_coalesce(struct net_device *dev,
return ret;
if (update_napi) {
+ /* xsk xmit depends on the tx napi. So if xsk is active,
+ * prevent modifications to tx napi.
+ */
+ for (i = queue_number; i < vi->max_queue_pairs; i++) {
+ if (vi->sq[i].xsk.pool)
+ return -EBUSY;
+ }
+
for (; queue_number < vi->max_queue_pairs; queue_number++)
vi->sq[queue_number].napi.weight = napi_weight;
}
--
2.32.0.3.g01195cf9f
^ permalink raw reply related
* [PATCH net-next v2 11/21] virtio_net: move some api to header
From: Xuan Zhuo @ 2023-11-07 3:12 UTC (permalink / raw)
To: netdev
Cc: David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Michael S. Tsirkin, Jason Wang, Xuan Zhuo, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
In-Reply-To: <20231107031227.100015-1-xuanzhuo@linux.alibaba.com>
__free_old_xmit
is_xdp_raw_buffer_queue
These two APIs are needed by the xsk part.
So this commit move theses to the header. And add prefix "virtnet_".
Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
---
drivers/net/virtio/main.c | 94 +++------------------------------
drivers/net/virtio/virtio_net.h | 80 ++++++++++++++++++++++++++++
2 files changed, 87 insertions(+), 87 deletions(-)
diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
index 02e054fd217c..6c608b3ce27d 100644
--- a/drivers/net/virtio/main.c
+++ b/drivers/net/virtio/main.c
@@ -45,9 +45,6 @@ module_param(napi_tx, bool, 0644);
#define VIRTIO_XDP_TX BIT(0)
#define VIRTIO_XDP_REDIR BIT(1)
-#define VIRTIO_XDP_FLAG BIT(0)
-#define VIRTIO_XMIT_DATA_MASK (VIRTIO_XDP_FLAG)
-
#define VIRTNET_DRIVER_VERSION "1.0.0"
static const unsigned long guest_offloads[] = {
@@ -150,78 +147,11 @@ struct virtio_net_common_hdr {
};
};
-static bool is_xdp_frame(void *ptr)
-{
- return (unsigned long)ptr & VIRTIO_XDP_FLAG;
-}
-
static void *xdp_to_ptr(struct xdp_frame *ptr)
{
return (void *)((unsigned long)ptr | VIRTIO_XDP_FLAG);
}
-static struct xdp_frame *ptr_to_xdp(void *ptr)
-{
- return (struct xdp_frame *)((unsigned long)ptr & ~VIRTIO_XDP_FLAG);
-}
-
-static inline void *virtnet_sq_unmap(struct virtnet_sq *sq, void *data)
-{
- struct virtnet_sq_dma *next, *head;
-
- head = (void *)((unsigned long)data & ~VIRTIO_XMIT_DATA_MASK);
-
- data = head->data;
-
- while (head) {
- virtqueue_dma_unmap_single_attrs(sq->vq, head->addr, head->len,
- DMA_TO_DEVICE, 0);
-
- next = head->next;
-
- head->next = sq->dmainfo.free;
- sq->dmainfo.free = head;
-
- head = next;
- }
-
- return data;
-}
-
-static void __free_old_xmit(struct virtnet_sq *sq, bool in_napi,
- u64 *bytes, u64 *packets)
-{
- unsigned int len;
- void *ptr;
-
- while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {
- if (!is_xdp_frame(ptr)) {
- struct sk_buff *skb;
-
- if (sq->do_dma)
- ptr = virtnet_sq_unmap(sq, ptr);
-
- skb = ptr;
-
- pr_debug("Sent skb %p\n", skb);
-
- *bytes += skb->len;
- napi_consume_skb(skb, in_napi);
- } else {
- struct xdp_frame *frame;
-
- if (sq->do_dma)
- ptr = virtnet_sq_unmap(sq, ptr);
-
- frame = ptr_to_xdp(ptr);
-
- *bytes += xdp_get_frame_len(frame);
- xdp_return_frame(frame);
- }
- (*packets)++;
- }
-}
-
/* Converting between virtqueue no. and kernel tx/rx queue no.
* 0:rx0 1:tx0 2:rx1 3:tx1 ... 2N:rxN 2N+1:txN 2N+2:cvq
*/
@@ -700,7 +630,7 @@ static void free_old_xmit(struct virtnet_sq *sq, bool in_napi)
{
u64 bytes, packets = 0;
- __free_old_xmit(sq, in_napi, &bytes, &packets);
+ virtnet_free_old_xmit(sq, in_napi, &bytes, &packets);
/* Avoid overhead when no packets have been processed
* happens when called speculatively from start_xmit.
@@ -714,16 +644,6 @@ static void free_old_xmit(struct virtnet_sq *sq, bool in_napi)
u64_stats_update_end(&sq->stats.syncp);
}
-static bool is_xdp_raw_buffer_queue(struct virtnet_info *vi, int q)
-{
- if (q < (vi->curr_queue_pairs - vi->xdp_queue_pairs))
- return false;
- else if (q < vi->curr_queue_pairs)
- return true;
- else
- return false;
-}
-
static void check_sq_full_and_disable(struct virtnet_info *vi,
struct net_device *dev,
struct virtnet_sq *sq)
@@ -872,7 +792,7 @@ static int virtnet_xdp_xmit(struct net_device *dev,
}
/* Free up any pending old buffers before queueing new ones. */
- __free_old_xmit(sq, false, &bytes, &packets);
+ virtnet_free_old_xmit(sq, false, &bytes, &packets);
for (i = 0; i < n; i++) {
struct xdp_frame *xdpf = frames[i];
@@ -883,7 +803,7 @@ static int virtnet_xdp_xmit(struct net_device *dev,
}
ret = nxmit;
- if (!is_xdp_raw_buffer_queue(vi, sq - vi->sq))
+ if (!virtnet_is_xdp_raw_buffer_queue(vi, sq - vi->sq))
check_sq_full_and_disable(vi, dev, sq);
if (flags & XDP_XMIT_FLUSH) {
@@ -2033,7 +1953,7 @@ static void virtnet_poll_cleantx(struct virtnet_rq *rq)
struct virtnet_sq *sq = &vi->sq[index];
struct netdev_queue *txq = netdev_get_tx_queue(vi->dev, index);
- if (!sq->napi.weight || is_xdp_raw_buffer_queue(vi, index))
+ if (!sq->napi.weight || virtnet_is_xdp_raw_buffer_queue(vi, index))
return;
if (__netif_tx_trylock(txq)) {
@@ -2157,7 +2077,7 @@ static int virtnet_poll_tx(struct napi_struct *napi, int budget)
int opaque;
bool done;
- if (unlikely(is_xdp_raw_buffer_queue(vi, index))) {
+ if (unlikely(virtnet_is_xdp_raw_buffer_queue(vi, index))) {
/* We don't need to enable cb for XDP */
napi_complete_done(napi, 0);
return 0;
@@ -3997,7 +3917,7 @@ void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf)
sq = &vi->sq[i];
- if (!is_xdp_frame(buf)) {
+ if (!virtnet_is_xdp_frame(buf)) {
if (sq->do_dma)
buf = virtnet_sq_unmap(sq, buf);
@@ -4006,7 +3926,7 @@ void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf)
if (sq->do_dma)
buf = virtnet_sq_unmap(sq, buf);
- xdp_return_frame(ptr_to_xdp(buf));
+ xdp_return_frame(virtnet_ptr_to_xdp(buf));
}
}
diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
index 98ba23cfdb20..442af4673bf8 100644
--- a/drivers/net/virtio/virtio_net.h
+++ b/drivers/net/virtio/virtio_net.h
@@ -8,6 +8,9 @@
#include <linux/virtio_net.h>
#include <net/xdp_sock_drv.h>
+#define VIRTIO_XDP_FLAG BIT(0)
+#define VIRTIO_XMIT_DATA_MASK (VIRTIO_XDP_FLAG)
+
/* RX packet size EWMA. The average packet size is used to determine the packet
* buffer size when refilling RX rings. As the entire RX ring may be refilled
* at once, the weight is chosen so that the EWMA will be insensitive to short-
@@ -221,6 +224,83 @@ struct virtnet_info {
struct failover *failover;
};
+static inline bool virtnet_is_xdp_frame(void *ptr)
+{
+ return (unsigned long)ptr & VIRTIO_XDP_FLAG;
+}
+
+static inline struct xdp_frame *virtnet_ptr_to_xdp(void *ptr)
+{
+ return (struct xdp_frame *)((unsigned long)ptr & ~VIRTIO_XDP_FLAG);
+}
+
+static inline void *virtnet_sq_unmap(struct virtnet_sq *sq, void *data)
+{
+ struct virtnet_sq_dma *next, *head;
+
+ head = (void *)((unsigned long)data & ~VIRTIO_XMIT_DATA_MASK);
+
+ data = head->data;
+
+ while (head) {
+ virtqueue_dma_unmap_single_attrs(sq->vq, head->addr, head->len,
+ DMA_TO_DEVICE, 0);
+
+ next = head->next;
+
+ head->next = sq->dmainfo.free;
+ sq->dmainfo.free = head;
+
+ head = next;
+ }
+
+ return data;
+}
+
+static inline void virtnet_free_old_xmit(struct virtnet_sq *sq, bool in_napi,
+ u64 *bytes, u64 *packets)
+{
+ unsigned int len;
+ void *ptr;
+
+ while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {
+ if (!virtnet_is_xdp_frame(ptr)) {
+ struct sk_buff *skb;
+
+ if (sq->do_dma)
+ ptr = virtnet_sq_unmap(sq, ptr);
+
+ skb = ptr;
+
+ pr_debug("Sent skb %p\n", skb);
+
+ *bytes += skb->len;
+ napi_consume_skb(skb, in_napi);
+ } else {
+ struct xdp_frame *frame;
+
+ if (sq->do_dma)
+ ptr = virtnet_sq_unmap(sq, ptr);
+
+ frame = virtnet_ptr_to_xdp(ptr);
+
+ *bytes += xdp_get_frame_len(frame);
+ xdp_return_frame(frame);
+ }
+ (*packets)++;
+ }
+}
+
+static inline bool virtnet_is_xdp_raw_buffer_queue(struct virtnet_info *vi, int q)
+{
+ if (q < (vi->curr_queue_pairs - vi->xdp_queue_pairs))
+ return false;
+ else if (q < vi->curr_queue_pairs)
+ return true;
+ else
+ return false;
+}
+
void virtnet_rx_pause(struct virtnet_info *vi, struct virtnet_rq *rq);
void virtnet_rx_resume(struct virtnet_info *vi, struct virtnet_rq *rq);
void virtnet_tx_pause(struct virtnet_info *vi, struct virtnet_sq *sq);
--
2.32.0.3.g01195cf9f
^ permalink raw reply related
* [PATCH net-next v2 12/21] virtio_net: xsk: tx: support tx
From: Xuan Zhuo @ 2023-11-07 3:12 UTC (permalink / raw)
To: netdev
Cc: David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Michael S. Tsirkin, Jason Wang, Xuan Zhuo, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
In-Reply-To: <20231107031227.100015-1-xuanzhuo@linux.alibaba.com>
The driver's tx napi is very important for XSK. It is responsible for
obtaining data from the XSK queue and sending it out.
At the beginning, we need to trigger tx napi.
Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
---
drivers/net/virtio/main.c | 12 +++-
drivers/net/virtio/virtio_net.h | 3 +-
drivers/net/virtio/xsk.c | 110 ++++++++++++++++++++++++++++++++
drivers/net/virtio/xsk.h | 13 ++++
4 files changed, 136 insertions(+), 2 deletions(-)
diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
index 6c608b3ce27d..ff6bc764089d 100644
--- a/drivers/net/virtio/main.c
+++ b/drivers/net/virtio/main.c
@@ -2074,6 +2074,7 @@ static int virtnet_poll_tx(struct napi_struct *napi, int budget)
struct virtnet_info *vi = sq->vq->vdev->priv;
unsigned int index = vq2txq(sq->vq);
struct netdev_queue *txq;
+ int busy = 0;
int opaque;
bool done;
@@ -2086,11 +2087,20 @@ static int virtnet_poll_tx(struct napi_struct *napi, int budget)
txq = netdev_get_tx_queue(vi->dev, index);
__netif_tx_lock(txq, raw_smp_processor_id());
virtqueue_disable_cb(sq->vq);
- free_old_xmit(sq, true);
+
+ if (sq->xsk.pool)
+ busy |= virtnet_xsk_xmit(sq, sq->xsk.pool, budget);
+ else
+ free_old_xmit(sq, true);
if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
netif_tx_wake_queue(txq);
+ if (busy) {
+ __netif_tx_unlock(txq);
+ return budget;
+ }
+
opaque = virtqueue_enable_cb_prepare(sq->vq);
done = napi_complete_done(napi, 0);
diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
index 442af4673bf8..1c21af47e13c 100644
--- a/drivers/net/virtio/virtio_net.h
+++ b/drivers/net/virtio/virtio_net.h
@@ -9,7 +9,8 @@
#include <net/xdp_sock_drv.h>
#define VIRTIO_XDP_FLAG BIT(0)
-#define VIRTIO_XMIT_DATA_MASK (VIRTIO_XDP_FLAG)
+#define VIRTIO_XSK_FLAG BIT(1)
+#define VIRTIO_XMIT_DATA_MASK (VIRTIO_XDP_FLAG | VIRTIO_XSK_FLAG)
/* RX packet size EWMA. The average packet size is used to determine the packet
* buffer size when refilling RX rings. As the entire RX ring may be refilled
diff --git a/drivers/net/virtio/xsk.c b/drivers/net/virtio/xsk.c
index 8b397787603f..caa448308232 100644
--- a/drivers/net/virtio/xsk.c
+++ b/drivers/net/virtio/xsk.c
@@ -4,9 +4,119 @@
*/
#include "virtio_net.h"
+#include "xsk.h"
static struct virtio_net_hdr_mrg_rxbuf xsk_hdr;
+static void sg_fill_dma(struct scatterlist *sg, dma_addr_t addr, u32 len)
+{
+ sg->dma_address = addr;
+ sg->length = len;
+}
+
+static void virtnet_xsk_check_queue(struct virtnet_sq *sq)
+{
+ struct virtnet_info *vi = sq->vq->vdev->priv;
+ struct net_device *dev = vi->dev;
+ int qnum = sq - vi->sq;
+
+ /* If it is a raw buffer queue, it does not check whether the status
+ * of the queue is stopped when sending. So there is no need to check
+ * the situation of the raw buffer queue.
+ */
+ if (virtnet_is_xdp_raw_buffer_queue(vi, qnum))
+ return;
+
+ /* If this sq is not the exclusive queue of the current cpu,
+ * then it may be called by start_xmit, so check it running out
+ * of space.
+ *
+ * Stop the queue to avoid getting packets that we are
+ * then unable to transmit. Then wait the tx interrupt.
+ */
+ if (sq->vq->num_free < 2 + MAX_SKB_FRAGS)
+ netif_stop_subqueue(dev, qnum);
+}
+
+static int virtnet_xsk_xmit_one(struct virtnet_sq *sq,
+ struct xsk_buff_pool *pool,
+ struct xdp_desc *desc)
+{
+ struct virtnet_info *vi;
+ dma_addr_t addr;
+
+ vi = sq->vq->vdev->priv;
+
+ addr = xsk_buff_raw_get_dma(pool, desc->addr);
+ xsk_buff_raw_dma_sync_for_device(pool, addr, desc->len);
+
+ sg_init_table(sq->sg, 2);
+
+ sg_fill_dma(sq->sg, sq->xsk.hdr_dma_address, vi->hdr_len);
+ sg_fill_dma(sq->sg + 1, addr, desc->len);
+
+ return virtqueue_add_outbuf(sq->vq, sq->sg, 2,
+ virtnet_xsk_to_ptr(desc->len), GFP_ATOMIC);
+}
+
+static int virtnet_xsk_xmit_batch(struct virtnet_sq *sq,
+ struct xsk_buff_pool *pool,
+ unsigned int budget,
+ u64 *kicks)
+{
+ struct xdp_desc *descs = pool->tx_descs;
+ u32 nb_pkts, max_pkts, i;
+ bool kick = false;
+ int err;
+
+ /* Every xsk tx packet needs two desc(virtnet header and packet). So we
+ * use sq->vq->num_free / 2 as the limitation.
+ */
+ max_pkts = min_t(u32, budget, sq->vq->num_free / 2);
+
+ nb_pkts = xsk_tx_peek_release_desc_batch(pool, max_pkts);
+ if (!nb_pkts)
+ return 0;
+
+ for (i = 0; i < nb_pkts; i++) {
+ err = virtnet_xsk_xmit_one(sq, pool, &descs[i]);
+ if (unlikely(err))
+ break;
+
+ kick = true;
+ }
+
+ if (kick && virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq))
+ (*kicks)++;
+
+ return i;
+}
+
+bool virtnet_xsk_xmit(struct virtnet_sq *sq, struct xsk_buff_pool *pool,
+ int budget)
+{
+ u64 bytes = 0, packets = 0, kicks = 0;
+ int sent;
+
+ virtnet_free_old_xmit(sq, true, &bytes, &packets);
+
+ sent = virtnet_xsk_xmit_batch(sq, pool, budget, &kicks);
+
+ virtnet_xsk_check_queue(sq);
+
+ u64_stats_update_begin(&sq->stats.syncp);
+ u64_stats_add(&sq->stats.packets, packets);
+ u64_stats_add(&sq->stats.bytes, bytes);
+ u64_stats_add(&sq->stats.kicks, kicks);
+ u64_stats_add(&sq->stats.xdp_tx, sent);
+ u64_stats_update_end(&sq->stats.syncp);
+
+ if (xsk_uses_need_wakeup(pool))
+ xsk_set_tx_need_wakeup(pool);
+
+ return sent == budget;
+}
+
static int virtnet_rq_bind_xsk_pool(struct virtnet_info *vi, struct virtnet_rq *rq,
struct xsk_buff_pool *pool)
{
diff --git a/drivers/net/virtio/xsk.h b/drivers/net/virtio/xsk.h
index 1918285c310c..73ca8cd5308b 100644
--- a/drivers/net/virtio/xsk.h
+++ b/drivers/net/virtio/xsk.h
@@ -3,5 +3,18 @@
#ifndef __XSK_H__
#define __XSK_H__
+#define VIRTIO_XSK_FLAG_OFFSET 4
+
+static inline void *virtnet_xsk_to_ptr(u32 len)
+{
+ unsigned long p;
+
+ p = len << VIRTIO_XSK_FLAG_OFFSET;
+
+ return (void *)(p | VIRTIO_XSK_FLAG);
+}
+
int virtnet_xsk_pool_setup(struct net_device *dev, struct netdev_bpf *xdp);
+bool virtnet_xsk_xmit(struct virtnet_sq *sq, struct xsk_buff_pool *pool,
+ int budget);
#endif
--
2.32.0.3.g01195cf9f
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox