* Re: [RFC PATCH net-next v0.1 0/1] add GeoNetworking protocol
From: Andrew Lunn @ 2026-07-19 16:21 UTC (permalink / raw)
To: Simon Dietz
Cc: netdev, andrew+netdev, davem, edumazet, johannes, kuniyu,
linux-wireless, dietz23838
In-Reply-To: <20260718210046.2357882-1-simon.dietz@plantwatch.de>
On Sat, Jul 18, 2026 at 11:00:32PM +0200, Simon Dietz wrote:
> Implement the GeoNetworking / ETSI ITS-G5 ('net/gn') protocol which
> is based on 802.11p wifi and used for vehicle2x applications. It is
> standardized by the ETSI and used by some car manufacturers
> (especially in europe). It enables ad-hoc, multi-hop geographical
> communication and routing among vehicles (and road- or railside
> infrastructure).
>
> Most work of this implementation has been done by the bachelor
> project 2018/2019 of the operating systems and middleware group of
> the Hasso Plattner Institute, University of Potsdam, which the author
> was part of.
Hi Simon
Is there are architecture documentation somewhere?
One of my comments was about routing tables. Should there be a user
space component determining the routes, and the kernel just has a
static routing table? That would be typical for IP.
There also seems to be a need for location information. How does that
get into the kernel? Is there a daemon for that? Patches to gpsd?
Andrew
^ permalink raw reply
* [PATCH net-next v4] selftests/net/openvswitch: add SCTP flow key test
From: Minxi Hou @ 2026-07-19 16:26 UTC (permalink / raw)
To: netdev; +Cc: davem, edumazet, kuba, pabeni, aconole, Minxi Hou
Add test_sctp_connect_v4() to verify OVS can match on SCTP flow keys
(sctp src/dst port).
The test sets up client and server namespaces connected through an
OVS bridge, installs port-keyed flows, and verifies:
- sctp(dst=4443) matches client-to-server INIT
- sctp(src=4443) matches server-to-client INIT-ACK
- removing flows drops the connection
- reinstalling flows restores connectivity
Signed-off-by: Minxi Hou <houminxi@gmail.com>
---
.../selftests/net/openvswitch/openvswitch.sh | 105 ++++++++++++++++++
.../selftests/net/openvswitch/ovs-dpctl.py | 5 +
2 files changed, 110 insertions(+)
v3 -> v4: rebase onto latest net-next (2026-07-19), resolve test list
conflict from merged trunc test
diff --git a/tools/testing/selftests/net/openvswitch/openvswitch.sh b/tools/testing/selftests/net/openvswitch/openvswitch.sh
index f75ee723415a..af9cf0888316 100755
--- a/tools/testing/selftests/net/openvswitch/openvswitch.sh
+++ b/tools/testing/selftests/net/openvswitch/openvswitch.sh
@@ -33,6 +33,7 @@ tests="
flow_set flow-set: Flow modify
action_set set: SET action rewrites fields
trunc trunc: output truncation
+ sctp_connect_v4 sctp: SCTP flow key matching
psample psample: Sampling packets with psample"
info() {
@@ -530,6 +531,110 @@ test_trunc() {
return 0
}
+# sctp_connect_v4 test
+# - sctp(dst=4443) matches client-to-server INIT
+# - sctp(src=4443) matches server-to-client INIT-ACK
+# - remove flows and verify connection fails, reinstall and recover
+test_sctp_connect_v4() {
+ local t="test_sctp_connect_v4"
+
+ which nc >/dev/null 2>&1 || return $ksft_skip
+ nc --sctp -z 127.0.0.1 1 </dev/null 2>/dev/null || return $ksft_skip
+ modprobe -q sctp 2>/dev/null || return $ksft_skip
+
+ sbx_add "$t" || return $?
+ ovs_add_dp "$t" sctp4 || return 1
+
+ info "create namespaces"
+ for ns in client server; do
+ ovs_add_netns_and_veths "$t" "sctp4" "$ns" \
+ "${ns:0:1}0" "${ns:0:1}1" || return 1
+ done
+
+ ip netns exec client ip addr add 172.31.110.10/24 dev c1
+ ip netns exec client ip link set c1 up
+ ip netns exec server ip addr add 172.31.110.20/24 dev s1
+ ip netns exec server ip link set s1 up
+
+ # ARP forwarding
+ ovs_add_flow "$t" sctp4 \
+ 'in_port(1),eth(),eth_type(0x0806),arp()' \
+ '2' || return 1
+ ovs_add_flow "$t" sctp4 \
+ 'in_port(2),eth(),eth_type(0x0806),arp()' \
+ '1' || return 1
+
+ # SCTP port matching: dst for request, src for reply
+ ovs_add_flow "$t" sctp4 \
+ 'in_port(1),eth(),eth_type(0x0800),ipv4(proto=132),sctp(dst=4443)' \
+ '2' || return 1
+ ovs_add_flow "$t" sctp4 \
+ 'in_port(2),eth(),eth_type(0x0800),ipv4(proto=132),sctp(src=4443)' \
+ '1' || return 1
+
+ echo "server" | \
+ ovs_netns_spawn_daemon "$t" "server" \
+ nc --sctp -l 172.31.110.20 -vn 4443
+ local server_pid=$pid
+ ovs_wait ip netns exec server \
+ ss -lnH sport = :4443 \| grep -q . \
+ || return 1
+
+ info "verify SCTP association with port-keyed flows"
+ ovs_sbx "$t" ip netns exec client \
+ nc --sctp -i 1 -zv 172.31.110.20 4443 \
+ || return 1
+
+ ovs_del_flows "$t" sctp4
+
+ info "verify connection fails without flows"
+ ovs_add_flow "$t" sctp4 \
+ 'in_port(1),eth(),eth_type(0x0806),arp()' \
+ '2' || return 1
+ ovs_add_flow "$t" sctp4 \
+ 'in_port(2),eth(),eth_type(0x0806),arp()' \
+ '1' || return 1
+
+ kill -TERM $server_pid 2>/dev/null
+ wait $server_pid 2>/dev/null
+ echo "server2" | \
+ ovs_netns_spawn_daemon "$t" "server" \
+ nc --sctp -l 172.31.110.20 -vn 4443
+ server_pid=$pid
+ ovs_wait ip netns exec server \
+ ss -lnH sport = :4443 \| grep -q . \
+ || return 1
+
+ ovs_sbx "$t" ip netns exec client \
+ nc --sctp -w 2 -zv 172.31.110.20 4443 \
+ >/dev/null 2>&1 \
+ && { info "FAIL: connection should fail without flows"
+ return 1; }
+
+ info "reinstall flows and verify recovery"
+ ovs_add_flow "$t" sctp4 \
+ 'in_port(1),eth(),eth_type(0x0800),ipv4(proto=132),sctp(dst=4443)' \
+ '2' || return 1
+ ovs_add_flow "$t" sctp4 \
+ 'in_port(2),eth(),eth_type(0x0800),ipv4(proto=132),sctp(src=4443)' \
+ '1' || return 1
+
+ kill -TERM $server_pid 2>/dev/null
+ wait $server_pid 2>/dev/null
+ echo "server3" | \
+ ovs_netns_spawn_daemon "$t" "server" \
+ nc --sctp -l 172.31.110.20 -vn 4443
+ ovs_wait ip netns exec server \
+ ss -lnH sport = :4443 \| grep -q . \
+ || return 1
+
+ ovs_sbx "$t" ip netns exec client \
+ nc --sctp -i 1 -zv 172.31.110.20 4443 \
+ || return 1
+
+ return 0
+}
+
# psample test
# - use psample to observe packets
test_psample() {
diff --git a/tools/testing/selftests/net/openvswitch/ovs-dpctl.py b/tools/testing/selftests/net/openvswitch/ovs-dpctl.py
index e1ecfad2c03e..7cfc29ec7e59 100644
--- a/tools/testing/selftests/net/openvswitch/ovs-dpctl.py
+++ b/tools/testing/selftests/net/openvswitch/ovs-dpctl.py
@@ -1982,6 +1982,11 @@ class ovskey(nla):
"icmp",
ovskey.ovs_key_icmp,
),
+ (
+ "OVS_KEY_ATTR_SCTP",
+ "sctp",
+ ovskey.ovs_key_sctp,
+ ),
(
"OVS_KEY_ATTR_TCP_FLAGS",
"tcp_flags",
--
2.55.0
^ permalink raw reply related
* Re: [PATCH net-next,v3] xfrm: allow to enable udp encapsulation without userspace socket
From: Eyal Birger @ 2026-07-19 16:49 UTC (permalink / raw)
To: Pablo Neira Ayuso; +Cc: devel, netdev, tobias, antony, steffen.klassert
In-Reply-To: <20260717110118.251866-1-pablo@netfilter.org>
Hi,
On Fri, Jul 17, 2026 at 4:01 AM Pablo Neira Ayuso <pablo@netfilter.org> wrote:
>
> It is currently not possible to enable UDP encapsulation in xfrm without
> a userspace process that listens on the specified UDP listener port in
> the SA.
>
> People have work around this by creating dummy userspace daemons such as
> the one in the smallish perl program (see the script at the bottom of
> this link):
>
> http://techblog.newsnow.co.uk/2011/11/simple-udp-esp-encapsulation-nat-t-for.html
>
> This patch adds XFRM_SA_XFLAG_UDP_ENCAP_SOCK to create the UDP socket
> from the kernel.
>
> Use a hole in net->xfrm to place the new encap_socket list.
>
> The following example shows how to enable the standalone UDP
> encapsulation:
>
> ip xfrm state add src 192.168.10.10 dst 192.168.10.11 proto esp spi 1 \
> encap espinudp 9999 9999 0.0.0.0 \
> if_id 0x1 reqid 1 replay-window 0 mode tunnel aead 'rfc4106(gcm(aes))' \
> 0x1111111111111111111111111111111111111111 96 \
> sel src 10.141.10.0/24 dst 10.141.11.0/24 dir out
>
> and the receiving side uses 'extra-flag udp-encap-sock':
>
> ip xfrm state add src 192.168.10.11 dst 192.168.10.10 proto esp spi 2 \
> encap espinudp 9999 0 0.0.0.0 extra-flag udp-encap-sock \
> if_id 0x1 reqid 2 replay-window 10 mode tunnel aead 'rfc4106(gcm(aes))' \
> 0x2222222222222222222222222222222222222222 96 dir in
>
> This allows for multiple SAs using the same listener udp port.
>
> This is useful for testing scenarios where UDP encapsulation is
> required.
>
> Note this patch exports xfrm6_udp_encap_rcv() just like
> xfrm4_udp_encap_rcv() otherwise linker complains due to unreachable
> symbol.
>
> Signed-off-by: Pablo Neira Ayuso <pablo@netfilter.org>
LGTM. Thanks!
Reviewed-by: Eyal Birger <eyal.birger@gmail.com>
> ---
> v3: - Use x->id.daddr.a{4,6} instead of encap_oa, as suggested by Eyal and Tobias.
> - Fix several SA using same udp encapsulation configuration.
>
> This is a follow up to:
> https://lists.linux-ipsec.org/archives/list/devel@lists.linux-ipsec.org/thread/F4IOY2DKFDX3E45UUFQHUZKTHV7AD6G4/
>
> NOTE for netdev maintainers:
> Targetting net-next so sashiko kicks in for review, ipsec-devel still has no such service yet.
>
> include/net/netns/xfrm.h | 1 +
> include/net/xfrm.h | 11 ++++
> include/uapi/linux/xfrm.h | 1 +
> net/ipv6/xfrm6_input.c | 1 +
> net/xfrm/Kconfig | 2 +
> net/xfrm/xfrm_state.c | 16 +++++
> net/xfrm/xfrm_user.c | 122 +++++++++++++++++++++++++++++++++++++-
> 7 files changed, 152 insertions(+), 2 deletions(-)
>
> diff --git a/include/net/netns/xfrm.h b/include/net/netns/xfrm.h
> index b73983a17e08..5091c07b1e46 100644
> --- a/include/net/netns/xfrm.h
> +++ b/include/net/netns/xfrm.h
> @@ -56,6 +56,7 @@ struct netns_xfrm {
> unsigned int policy_count[XFRM_POLICY_MAX * 2];
> struct work_struct policy_hash_work;
> struct xfrm_policy_hthresh policy_hthresh;
> + struct hlist_head encap_socket;
> struct list_head inexact_bins;
>
>
> diff --git a/include/net/xfrm.h b/include/net/xfrm.h
> index a6d69aaa6cd2..99730a2fc8b9 100644
> --- a/include/net/xfrm.h
> +++ b/include/net/xfrm.h
> @@ -25,6 +25,7 @@
> #include <net/ipv6.h>
> #include <net/ip6_fib.h>
> #include <net/flow.h>
> +#include <net/udp_tunnel.h>
> #include <net/gro_cells.h>
>
> #include <linux/interrupt.h>
> @@ -169,6 +170,13 @@ struct xfrm_dev_offload {
> u8 flags : 2;
> };
>
> +struct xfrm_encap_sock {
> + struct hlist_node list;
> + struct udp_port_cfg cfg;
> + struct sock *sk;
> + refcount_t refcnt;
> +};
> +
> struct xfrm_mode {
> u8 encap;
> u8 family;
> @@ -249,6 +257,7 @@ struct xfrm_state {
>
> /* Data for encapsulator */
> struct xfrm_encap_tmpl *encap;
> + struct xfrm_encap_sock *encap_sock;
>
> /* NAT keepalive */
> u32 nat_keepalive_interval; /* seconds */
> @@ -2343,6 +2352,8 @@ static inline bool xfrm6_local_dontfrag(const struct sock *sk)
> }
> #endif
>
> +int xfrm4_udp_encap_rcv(struct sock *sk, struct sk_buff *skb);
> +
> #if (IS_BUILTIN(CONFIG_XFRM_INTERFACE) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) || \
> (IS_MODULE(CONFIG_XFRM_INTERFACE) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF_MODULES))
>
> diff --git a/include/uapi/linux/xfrm.h b/include/uapi/linux/xfrm.h
> index 051f8066efd1..28aa5c8ca317 100644
> --- a/include/uapi/linux/xfrm.h
> +++ b/include/uapi/linux/xfrm.h
> @@ -413,6 +413,7 @@ struct xfrm_usersa_info {
>
> #define XFRM_SA_XFLAG_DONT_ENCAP_DSCP 1
> #define XFRM_SA_XFLAG_OSEQ_MAY_WRAP 2
> +#define XFRM_SA_XFLAG_UDP_ENCAP_SOCK 4
>
> struct xfrm_usersa_id {
> xfrm_address_t daddr;
> diff --git a/net/ipv6/xfrm6_input.c b/net/ipv6/xfrm6_input.c
> index 89d0443b5307..2e3f7b9e5a4f 100644
> --- a/net/ipv6/xfrm6_input.c
> +++ b/net/ipv6/xfrm6_input.c
> @@ -173,6 +173,7 @@ int xfrm6_udp_encap_rcv(struct sock *sk, struct sk_buff *skb)
>
> return ret;
> }
> +EXPORT_SYMBOL(xfrm6_udp_encap_rcv);
>
> struct sk_buff *xfrm6_gro_udp_encap_rcv(struct sock *sk, struct list_head *head,
> struct sk_buff *skb)
> diff --git a/net/xfrm/Kconfig b/net/xfrm/Kconfig
> index 4a62817a88f8..fe42cfca9423 100644
> --- a/net/xfrm/Kconfig
> +++ b/net/xfrm/Kconfig
> @@ -7,6 +7,7 @@ config XFRM
> depends on INET
> select GRO_CELLS
> select SKB_EXTENSIONS
> + select NET_UDP_TUNNEL
>
> config XFRM_OFFLOAD
> bool
> @@ -23,6 +24,7 @@ if INET
> config XFRM_USER
> tristate "Transformation user configuration interface"
> select XFRM_ALGO
> + select NET_UDP_TUNNEL
> help
> Support for Transformation(XFRM) user configuration interface
> like IPsec used by native Linux tools.
> diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c
> index 36a4f6793ede..c849e7ac750a 100644
> --- a/net/xfrm/xfrm_state.c
> +++ b/net/xfrm/xfrm_state.c
> @@ -27,6 +27,7 @@
> #include <linux/slab.h>
> #include <linux/interrupt.h>
> #include <linux/kernel.h>
> +#include <net/udp_tunnel.h>
>
> #include <crypto/aead.h>
>
> @@ -586,6 +587,16 @@ static const struct xfrm_mode_cbs *xfrm_get_mode_cbs(u8 mode)
> return cbs;
> }
>
> +static bool xfrm_socket_put(struct xfrm_encap_sock *encap_sock)
> +{
> + if (refcount_dec_and_test(&encap_sock->refcnt)) {
> + udp_tunnel_sock_release(encap_sock->sk);
> + return true;
> + }
> +
> + return false;
> +}
> +
> void xfrm_state_free(struct xfrm_state *x)
> {
> kmem_cache_free(xfrm_state_cache, x);
> @@ -597,6 +608,10 @@ static void xfrm_state_gc_destroy(struct xfrm_state *x)
> {
> if (x->mode_cbs && x->mode_cbs->destroy_state)
> x->mode_cbs->destroy_state(x);
> +
> + if (x->encap_sock && xfrm_socket_put(x->encap_sock))
> + kfree(x->encap_sock);
> +
> hrtimer_cancel(&x->mtimer);
> timer_delete_sync(&x->rtimer);
> kfree_sensitive(x->aead);
> @@ -3332,6 +3347,7 @@ int __net_init xfrm_state_init(struct net *net)
> SLAB_HWCACHE_ALIGN | SLAB_PANIC);
>
> INIT_LIST_HEAD(&net->xfrm.state_all);
> + INIT_HLIST_HEAD(&net->xfrm.encap_socket);
>
> sz = sizeof(struct hlist_head) * 8;
>
> diff --git a/net/xfrm/xfrm_user.c b/net/xfrm/xfrm_user.c
> index d6db63304ba6..bc88581e51e8 100644
> --- a/net/xfrm/xfrm_user.c
> +++ b/net/xfrm/xfrm_user.c
> @@ -29,6 +29,7 @@
> #include <net/xfrm.h>
> #include <net/netlink.h>
> #include <net/ah.h>
> +#include <net/udp_tunnel.h>
> #include <linux/uaccess.h>
> #if IS_ENABLED(CONFIG_IPV6)
> #include <linux/in6.h>
> @@ -484,6 +485,17 @@ static int verify_newsa_info(struct xfrm_usersa_info *p,
> goto out;
> }
>
> + if (attrs[XFRMA_SA_EXTRA_FLAGS]) {
> + u32 xflags = nla_get_u32(attrs[XFRMA_SA_EXTRA_FLAGS]);
> +
> + if (xflags & XFRM_SA_XFLAG_UDP_ENCAP_SOCK &&
> + (!sa_dir || sa_dir == XFRM_SA_DIR_OUT)) {
> + NL_SET_ERR_MSG(extack, "Flag UDP_ENCAP_SOCK can only be set on input SA");
> + err = -EINVAL;
> + goto out;
> + }
> + }
> +
> if (sa_dir == XFRM_SA_DIR_OUT) {
> if (p->flags & XFRM_STATE_DECAP_DSCP) {
> NL_SET_ERR_MSG(extack, "Flag DECAP_DSCP should not be set for output SA");
> @@ -556,7 +568,6 @@ static int verify_newsa_info(struct xfrm_usersa_info *p,
> err = -EINVAL;
> goto out;
> }
> -
> }
>
> if (attrs[XFRMA_IPTFS_DONT_FRAG]) {
> @@ -932,9 +943,17 @@ static struct xfrm_state *xfrm_state_construct(struct net *net,
> goto error;
> }
>
> - if (attrs[XFRMA_SA_EXTRA_FLAGS])
> + if (attrs[XFRMA_SA_EXTRA_FLAGS]) {
> x->props.extra_flags = nla_get_u32(attrs[XFRMA_SA_EXTRA_FLAGS]);
>
> + if (x->props.extra_flags & XFRM_SA_XFLAG_UDP_ENCAP_SOCK &&
> + x->encap && x->encap->encap_type != UDP_ENCAP_ESPINUDP) {
> + NL_SET_ERR_MSG(extack, "XFRM_SA_XFLAG_UDP_ENCAP_SOCK can only be set on UDP_ENCAP_ESPINUDP type");
> + err = -EOPNOTSUPP;
> + goto error;
> + }
> + }
> +
> if ((err = attach_aead(x, attrs[XFRMA_ALG_AEAD], extack)))
> goto error;
> if ((err = attach_auth_trunc(&x->aalg, &x->props.aalgo,
> @@ -1036,6 +1055,97 @@ static struct xfrm_state *xfrm_state_construct(struct net *net,
> return NULL;
> }
>
> +static struct xfrm_encap_sock *
> +xfrm_socket_find_get(struct net *net, const struct udp_port_cfg *udp_conf)
> +{
> + struct xfrm_encap_sock *listener;
> +
> + hlist_for_each_entry(listener, &net->xfrm.encap_socket, list) {
> + if (!memcmp(&listener->cfg, udp_conf, sizeof(*udp_conf))) {
> + refcount_inc(&listener->refcnt);
> + return listener;
> + }
> + }
> +
> + return NULL;
> +}
> +
> +static int xfrm_socket_encap_create(struct net *net, struct xfrm_state *x,
> + struct udp_port_cfg *udp_conf,
> + struct udp_tunnel_sock_cfg *tuncfg)
> +{
> + struct xfrm_encap_sock *listener;
> + struct socket *sock;
> + int err;
> +
> + listener = kzalloc_obj(*listener);
> + if (!listener)
> + return -ENOMEM;
> +
> + err = udp_sock_create(net, udp_conf, &sock);
> + if (err) {
> + kfree(listener);
> + return err;
> + }
> + setup_udp_tunnel_sock(net, sock->sk, tuncfg);
> +
> + listener->sk = sock->sk;
> + listener->cfg = *udp_conf;
> + refcount_set(&listener->refcnt, 1);
> + hlist_add_head(&listener->list, &net->xfrm.encap_socket);
> +
> + x->encap_sock = listener;
> +
> + return 0;
> +}
> +
> +static int xfrm_socket_setup(struct net *net, struct xfrm_state *x,
> + struct netlink_ext_ack *extack)
> +{
> + struct udp_tunnel_sock_cfg tuncfg = {};
> + struct xfrm_encap_sock *listener;
> + struct udp_port_cfg udp_conf;
> + int err;
> +
> + if (!x->encap)
> + return -EOPNOTSUPP;
> +
> + memset(&udp_conf, 0, sizeof(udp_conf));
> + udp_conf.family = x->props.family;
> +
> + switch (x->props.family) {
> + case AF_INET:
> + udp_conf.local_ip.s_addr = x->id.daddr.a4;
> + tuncfg.encap_rcv = xfrm4_udp_encap_rcv;
> + break;
> +#if IS_ENABLED(CONFIG_IPV6)
> + case AF_INET6:
> + udp_conf.local_ip6 = x->id.daddr.in6;
> + tuncfg.encap_rcv = xfrm6_udp_encap_rcv;
> + break;
> +#endif
> + default:
> + return -EOPNOTSUPP;
> + }
> + udp_conf.local_udp_port = x->encap->encap_sport;
> +
> + listener = xfrm_socket_find_get(net, &udp_conf);
> + if (listener) {
> + x->encap_sock = listener;
> + return 0;
> + }
> +
> + tuncfg.encap_type = UDP_ENCAP_ESPINUDP;
> +
> + err = xfrm_socket_encap_create(net, x, &udp_conf, &tuncfg);
> + if (err < 0) {
> + NL_SET_ERR_MSG(extack, "Cannot initialize kernel UDP socket");
> + return err;
> + }
> +
> + return 0;
> +}
> +
> static int xfrm_add_sa(struct sk_buff *skb, struct nlmsghdr *nlh,
> struct nlattr **attrs, struct netlink_ext_ack *extack)
> {
> @@ -1053,6 +1163,14 @@ static int xfrm_add_sa(struct sk_buff *skb, struct nlmsghdr *nlh,
> if (!x)
> return err;
>
> + if (x->props.extra_flags & XFRM_SA_XFLAG_UDP_ENCAP_SOCK) {
> + err = xfrm_socket_setup(net, x, extack);
> + if (err < 0) {
> + x->km.state = XFRM_STATE_DEAD;
> + goto out;
> + }
> + }
> +
> xfrm_state_hold(x);
> if (nlh->nlmsg_type == XFRM_MSG_NEWSA)
> err = xfrm_state_add(x);
> --
> 2.47.3
>
^ permalink raw reply
* Re: [RFC PATCH net-next v0.1 1/1] net: add GeoNetworking protocol
From: Andrew Lunn @ 2026-07-19 17:16 UTC (permalink / raw)
To: Simon Dietz
Cc: netdev, andrew+netdev, davem, edumazet, johannes, kuniyu,
linux-wireless, dietz23838
In-Reply-To: <20260718210046.2357882-2-simon.dietz@plantwatch.de>
Can this:
> +#define GN_MAX_HEADER_SZ 88
Be defined in terms of:
> +#define GN_GUC_HLEN sizeof(struct gn_guc_header)
> +#define GN_GXC_HLEN sizeof(struct gn_gxc_header)
> +#define GN_SHB_HLEN sizeof(struct gn_shb_header)
> +#define GN_TSB_HLEN sizeof(struct gn_tsb_header)
> +#define GN_BEACON_HLEN sizeof(struct gn_beacon_header)
> +#define GN_LS_REQUEST_HLEN sizeof(struct gn_ls_request_header)
> +#define GN_LS_REPLY_HLEN sizeof(struct gn_ls_reply_header)
> +#define BTP_HLEN sizeof(struct btp_header)
#define GN_MAX_HEADER_SZ (GN_GUC_HLEN + GN_GXC_HLEN + GN_SHB_HLEN + GN_TSB_HLEN + ...)
Doing it like this makes it also documentation, the assumption is, the
biggest header contains these sub headers.
> +/**
> + * struct gn_iface - GeoNetworking interface
> + * @dev - Network device associated with this interface
> + * @address - Our address
> + * @local_sn - Current sequence number
> + */
> +struct gn_iface {
> + struct net_device *dev;
> + gn_address_t address;
> + struct gn_position pos;
> + atomic_t local_sn;
> + struct hlist_node hnode;
> + struct rcu_head rcu;
> +};
It does not look like all members have the same indentation here. I
would suggest they are all indented to the same level, or none are
indented. This is a general comment for the code as a whole.
> +enum ITS_TYPE {
> + UNKNOWN,
It is typical to use
UNKNOWN = 0,
> + PEDESTRIAN,
> + CYCLIST,
and then the rest without explicit values. I don't know what the C
standard says, but maybe without the = 0, a language lawyer might
argue it is allowed to start at 42 and count upwards from there?
> + MOPED,
> + MOTORCYCLE,
Are these from the standard? Maybe reference the clause in the
standard? The point being, we reviewers want to know if random
developer can add HOT_AIR_BALLOON, and HORSE_AND_TRAP at the end, or
if additions must first be accepted at the standards level and
assigned numbers.
Andrew
^ permalink raw reply
* Re: [RFC PATCH net-next v0.1 1/1] net: add GeoNetworking protocol
From: Andrew Lunn @ 2026-07-19 17:28 UTC (permalink / raw)
To: Simon Dietz
Cc: netdev, andrew+netdev, davem, edumazet, johannes, kuniyu,
linux-wireless, dietz23838
In-Reply-To: <20260718210046.2357882-2-simon.dietz@plantwatch.de>
> +#define GN_EPOCH_UNIX_MS 1072915232000LLU
> +
> +/* time until location table entry becomes invalid */
> +#define GN_LOC_TE_LIFETIME 20000
> +/* time between two beacons */
> +#define GN_BEACON_RETRANSMIT_TIME 3000
> +/* time between retransmits of an unanswered location service request */
> +#define GN_LS_RETRANSMIT_TIME 1000
It would be nice to have some units in these #defines. Classical bugs
are forgetting to convert from milliseconds to jiffies, or
milliseconds to microseconds. Having the unit in the name helpers
avoid that.
> +/* sizes in KiB */
> +#define GN_UC_BUF_SIZE 256
> +#define GN_BC_BUF_SIZE 1024
> +#define GN_CBF_BUF_SIZE 256
Again, units would be useful, the default would be to these are bytes,
not KB.
There is also linux/sizes.h.
Andrew
^ permalink raw reply
* [PATCH net-next v4 0/8] net: dsa: mt7530: modernise register access and add two DSA ops
From: Daniel Golle @ 2026-07-19 17:33 UTC (permalink / raw)
To: Chester A. Unal, Daniel Golle, Andrew Lunn, Vladimir Oltean,
David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Matthias Brugger, AngeloGioacchino Del Regno, Russell King,
netdev, linux-kernel, linux-arm-kernel, linux-mediatek
The mt7530 driver carries its own register accessors that predate the
regmap conversion and now largely duplicate what regmap already
provides, including locking. Most of this series removes that layer.
It first moves the MDIO bus locking into the switch regmap via
.lock/.unlock callbacks, matching the PCS regmaps, so any path reaching
the regmap is serialised automatically. With the wrappers no longer
adding locking, the thin mt7530_mii_* indirection is folded away and the
remaining accessors are replaced mechanically with the plain regmap API,
using the coccinelle semantic patches included in the commit messages.
Open-coded register fields are then converted to FIELD_GET/FIELD_PREP.
None of this is intended to change behaviour.
The last two patches implement .port_fast_age, which flushes dynamically
learned MAC entries on topology changes, and .port_change_conduit, which
moves a user port's CPU-port affinity at runtime.
v3 of this series arrived too close to the end of the last development
cycle and was deferred, hence this respin on top of current net-next.
---
v4:
* rebase onto current net-next
* 5/8: zero the read-back buffer in mt7530_regmap_read() instead of
initialising the result variables at every call site, as suggested
by Jakub
* 6/8: keep MT753X_CTRL_PHY_ADDR() as plain address arithmetic rather
than converting it to FIELD_PREP() as suggested in review: the macro
computes the MDIO bus address of the switch PHYs, used as the addr
argument of bus->read/write and as an iterator base in
mt7530_setup_mdio(), not a hardware register field
v3:
* 5/8: initialise register read-back variables to 0 so a failed
regmap_read keeps the previous read-as-zero behaviour, and use u32
for the value in mt7530_setup_port5
* 6/8: name the age timer field AGE_TIMER_MASK and document the
corrected ATC_HASH/VTCR_VID field macros
* 7/8: serialise the port_fast_age ATC flush under reg_mutex and log
on timeout, align a define, and correct the commit message which
wrongly claimed per-port parity with b53/realtek
* 8/8: populate the netlink extack on rejection and refuse a conduit
that lives on a different switch
v2:
* fix stray 'static void' left-over in 4/8 which had a fix accidentally
folded into 5/8 (byte-identical state at 8/8, but bisectability is
restored)
* extend port_change_conduit op commit message
Daniel Golle (8):
net: dsa: mt7530: move MDIO bus locking into regmap
net: dsa: mt7530: fold mt7530_mii_write/read into mt7530_write/read
net: dsa: mt7530: replace mt7530_write with regmap_write
net: dsa: mt7530: replace mt7530_rmw/set/clear with regmap API
net: dsa: mt7530: replace mt7530_read with regmap_read
net: dsa: mt7530: convert to use field accessor macros
net: dsa: mt7530: implement port_fast_age
net: dsa: mt7530: implement port_change_conduit op
drivers/net/dsa/mt7530-mdio.c | 12 +-
drivers/net/dsa/mt7530.c | 809 +++++++++++++++++-----------------
drivers/net/dsa/mt7530.h | 209 +++++----
3 files changed, 537 insertions(+), 493 deletions(-)
base-commit: ce6b4d3216b63f902bb8e9695ee6c10c83415f65
--
2.55.0
^ permalink raw reply
* [PATCH net] netdevsim: tc: serialize access to nsim_block_cb_list
From: Weiming Shi @ 2026-07-19 17:50 UTC (permalink / raw)
To: Jakub Kicinski, Andrew Lunn, David S . Miller, Eric Dumazet,
Paolo Abeni
Cc: Pablo Neira Ayuso, netdev, linux-kernel, Xiang Mei, Weiming Shi,
stable
nsim_setup_tc() passes a single global nsim_block_cb_list, shared by every
netdevsim device, to flow_block_cb_setup_simple(). That helper does
list_add_tail()/list_del() on the list on block bind/unbind and walks it
in flow_block_cb_is_busy(); it takes no lock of its own and relies on the
caller for serialization.
The tc control path calls ndo_setup_tc(TC_SETUP_BLOCK) under rtnl, but the
nf_tables hardware offload path reaches it under the per-netns nftables
commit_mutex only. Two nft transactions committing offload chains from
different netns thus run flow_block_cb_setup_simple() on the same list
concurrently, corrupting it and freeing a flow_block_cb that the other CPU
still walks:
list_del corruption. prev->next should be ffff88801f18ff00, but was ffff88801de82b00.
kernel BUG at lib/list_debug.c:62!
RIP: 0010:__list_del_entry_valid_or_report (lib/list_debug.c:62)
flow_block_cb_setup_simple (net/core/flow_offload.c:369)
nsim_setup_tc (drivers/net/netdevsim/tc.c)
nft_block_offload_cmd (net/netfilter/nf_tables_offload.c:394)
nft_flow_rule_offload_commit (net/netfilter/nf_tables_offload.c:585)
nf_tables_commit (net/netfilter/nf_tables_api.c:10490)
nfnetlink_rcv_batch (net/netfilter/nfnetlink.c:577)
nfnetlink_rcv (net/netfilter/nfnetlink.c:649)
netlink_unicast (net/netlink/af_netlink.c:1314)
netlink_sendmsg (net/netlink/af_netlink.c:1889)
__sys_sendto (net/socket.c:729)
With KASAN the same race is reported as a slab-use-after-free read of the
freed flow_block_cb (kmalloc-192) in flow_block_cb_setup_simple(). The
trace above is from a 6.12.y reproduction.
Serialize the list with a mutex. ndo_setup_tc(TC_SETUP_BLOCK) always runs
in process context, so sleeping on the mutex is fine.
Fixes: 955bcb6ea0df ("drivers: net: use flow block API")
Reported-by: Xiang Mei <xmei5@asu.edu>
Cc: stable@vger.kernel.org
Assisted-by: Claude:claude-opus-4-8
Signed-off-by: Weiming Shi <bestswngs@gmail.com>
---
drivers/net/netdevsim/tc.c | 13 +++++++++----
1 file changed, 9 insertions(+), 4 deletions(-)
diff --git a/drivers/net/netdevsim/tc.c b/drivers/net/netdevsim/tc.c
index a415e02a6df1..30dd7f924b71 100644
--- a/drivers/net/netdevsim/tc.c
+++ b/drivers/net/netdevsim/tc.c
@@ -72,11 +72,13 @@ static int nsim_setup_tc_ets(struct net_device *dev,
}
static LIST_HEAD(nsim_block_cb_list);
+static DEFINE_MUTEX(nsim_block_cb_lock);
int
nsim_setup_tc(struct net_device *dev, enum tc_setup_type type, void *type_data)
{
struct netdevsim *ns = netdev_priv(dev);
+ int err;
switch (type) {
case TC_SETUP_QDISC_TAPRIO:
@@ -84,10 +86,13 @@ nsim_setup_tc(struct net_device *dev, enum tc_setup_type type, void *type_data)
case TC_SETUP_QDISC_ETS:
return nsim_setup_tc_ets(dev, type_data);
case TC_SETUP_BLOCK:
- return flow_block_cb_setup_simple(type_data,
- &nsim_block_cb_list,
- nsim_setup_tc_block_cb,
- ns, ns, true);
+ mutex_lock(&nsim_block_cb_lock);
+ err = flow_block_cb_setup_simple(type_data,
+ &nsim_block_cb_list,
+ nsim_setup_tc_block_cb,
+ ns, ns, true);
+ mutex_unlock(&nsim_block_cb_lock);
+ return err;
case TC_SETUP_FT:
return 0;
default:
--
2.43.0
^ permalink raw reply related
* [PATCH net-next v4 1/8] net: dsa: mt7530: move MDIO bus locking into regmap
From: Daniel Golle @ 2026-07-19 17:53 UTC (permalink / raw)
To: Chester A. Unal, Daniel Golle, Andrew Lunn, Vladimir Oltean,
David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Matthias Brugger, AngeloGioacchino Del Regno, Russell King,
netdev, linux-kernel, linux-arm-kernel, linux-mediatek
In-Reply-To: <cover.1784481922.git.daniel@makrotopia.org>
The switch register regmap was created with .disable_locking = true,
relying on callers to manually lock the MDIO bus. Move the locking
into the regmap using .lock/.unlock callbacks, matching the PCS
regmaps that already do this. This allows any code path reaching the
regmap to be automatically protected.
With regmap handling bus locking, the manual mt7530_mutex_lock/unlock
wrappers in mt7530_write(), _mt7530_read(), mt7530_rmw() and
mt7530_port_change_mtu() become redundant and are removed.
The MT7531 indirect PHY access functions need serialization of their
multi-step register sequences, but no longer need to hold bus->mdio_lock
across the whole operation. Switch them to reg_mutex.
core_write()/core_rmw() are the only remaining callers of
mt7530_mutex_lock(). They access TRGMII core PHY registers via the
clause 22 MMD indirect protocol -- a separate register space that
bypasses regmap and needs manual bus->mdio_lock protection.
Generated using the following semantic patch:
// Remove mt7530_mutex_lock/unlock around single regmap-based calls.
@@
expression priv, reg, val;
@@
{
- mt7530_mutex_lock(priv);
-
mt7530_mii_write(priv, reg, val);
-
- mt7530_mutex_unlock(priv);
}
@@
expression priv, reg, mask, set;
@@
{
- mt7530_mutex_lock(priv);
-
regmap_update_bits(priv->regmap, reg, mask, set);
-
- mt7530_mutex_unlock(priv);
}
@@
expression p;
identifier val;
@@
{
- u32 val;
- mt7530_mutex_lock(p->priv);
- val = mt7530_mii_read(p->priv, p->reg);
- mt7530_mutex_unlock(p->priv);
- return val;
+ return mt7530_mii_read(p->priv, p->reg);
}
@@
expression priv;
@@
- mt7530_mutex_lock(priv);
val = mt7530_mii_read(priv, MT7530_GMACCR);
...
mt7530_mii_write(priv, MT7530_GMACCR, val);
- mt7530_mutex_unlock(priv);
@@
expression priv, port;
@@
INIT_MT7530_DUMMY_POLL(&p, priv, MT7531_PHY_IAC);
- mt7530_mutex_lock(priv);
+ mutex_lock(&priv->reg_mutex);
@@
expression priv;
@@
out:
- mt7530_mutex_unlock(priv);
+ mutex_unlock(&priv->reg_mutex);
Signed-off-by: Daniel Golle <daniel@makrotopia.org>
---
drivers/net/dsa/mt7530-mdio.c | 9 ++++++---
drivers/net/dsa/mt7530.c | 38 +++++++++--------------------------
2 files changed, 15 insertions(+), 32 deletions(-)
diff --git a/drivers/net/dsa/mt7530-mdio.c b/drivers/net/dsa/mt7530-mdio.c
index 11ea924a9f35..f7c8eeb27211 100644
--- a/drivers/net/dsa/mt7530-mdio.c
+++ b/drivers/net/dsa/mt7530-mdio.c
@@ -141,12 +141,14 @@ static const struct regmap_config regmap_config = {
.val_bits = 32,
.reg_stride = 4,
.max_register = MT7530_CREV,
- .disable_locking = true,
+ .lock = mt7530_mdio_regmap_lock,
+ .unlock = mt7530_mdio_regmap_unlock,
};
static int
mt7530_probe(struct mdio_device *mdiodev)
{
+ struct regmap_config rc = regmap_config;
struct mt7530_priv *priv;
struct device_node *dn;
int ret;
@@ -200,8 +202,9 @@ mt7530_probe(struct mdio_device *mdiodev)
return PTR_ERR(priv->io_pwr);
}
- priv->regmap = devm_regmap_init(priv->dev, &mt7530_regmap_bus, priv,
- ®map_config);
+ rc.lock_arg = &priv->bus->mdio_lock;
+ priv->regmap = devm_regmap_init(priv->dev, &mt7530_regmap_bus,
+ priv, &rc);
if (IS_ERR(priv->regmap))
return PTR_ERR(priv->regmap);
diff --git a/drivers/net/dsa/mt7530.c b/drivers/net/dsa/mt7530.c
index 3c2a3029b10c..5f56a423b147 100644
--- a/drivers/net/dsa/mt7530.c
+++ b/drivers/net/dsa/mt7530.c
@@ -184,11 +184,7 @@ mt7530_mii_read(struct mt7530_priv *priv, u32 reg)
static void
mt7530_write(struct mt7530_priv *priv, u32 reg, u32 val)
{
- mt7530_mutex_lock(priv);
-
mt7530_mii_write(priv, reg, val);
-
- mt7530_mutex_unlock(priv);
}
static u32
@@ -200,15 +196,7 @@ _mt7530_unlocked_read(struct mt7530_dummy_poll *p)
static u32
_mt7530_read(struct mt7530_dummy_poll *p)
{
- u32 val;
-
- mt7530_mutex_lock(p->priv);
-
- val = mt7530_mii_read(p->priv, p->reg);
-
- mt7530_mutex_unlock(p->priv);
-
- return val;
+ return mt7530_mii_read(p->priv, p->reg);
}
static u32
@@ -224,11 +212,7 @@ static void
mt7530_rmw(struct mt7530_priv *priv, u32 reg,
u32 mask, u32 set)
{
- mt7530_mutex_lock(priv);
-
regmap_update_bits(priv->regmap, reg, mask, set);
-
- mt7530_mutex_unlock(priv);
}
static void
@@ -555,7 +539,7 @@ mt7531_ind_c45_phy_read(struct mt7530_priv *priv, int port, int devad,
INIT_MT7530_DUMMY_POLL(&p, priv, MT7531_PHY_IAC);
- mt7530_mutex_lock(priv);
+ mutex_lock(&priv->reg_mutex);
ret = readx_poll_timeout(_mt7530_unlocked_read, &p, val,
!(val & MT7531_PHY_ACS_ST), 20, 100000);
@@ -588,7 +572,7 @@ mt7531_ind_c45_phy_read(struct mt7530_priv *priv, int port, int devad,
ret = val & MT7531_MDIO_RW_DATA_MASK;
out:
- mt7530_mutex_unlock(priv);
+ mutex_unlock(&priv->reg_mutex);
return ret;
}
@@ -603,7 +587,7 @@ mt7531_ind_c45_phy_write(struct mt7530_priv *priv, int port, int devad,
INIT_MT7530_DUMMY_POLL(&p, priv, MT7531_PHY_IAC);
- mt7530_mutex_lock(priv);
+ mutex_lock(&priv->reg_mutex);
ret = readx_poll_timeout(_mt7530_unlocked_read, &p, val,
!(val & MT7531_PHY_ACS_ST), 20, 100000);
@@ -635,7 +619,7 @@ mt7531_ind_c45_phy_write(struct mt7530_priv *priv, int port, int devad,
}
out:
- mt7530_mutex_unlock(priv);
+ mutex_unlock(&priv->reg_mutex);
return ret;
}
@@ -649,7 +633,7 @@ mt7531_ind_c22_phy_read(struct mt7530_priv *priv, int port, int regnum)
INIT_MT7530_DUMMY_POLL(&p, priv, MT7531_PHY_IAC);
- mt7530_mutex_lock(priv);
+ mutex_lock(&priv->reg_mutex);
ret = readx_poll_timeout(_mt7530_unlocked_read, &p, val,
!(val & MT7531_PHY_ACS_ST), 20, 100000);
@@ -672,7 +656,7 @@ mt7531_ind_c22_phy_read(struct mt7530_priv *priv, int port, int regnum)
ret = val & MT7531_MDIO_RW_DATA_MASK;
out:
- mt7530_mutex_unlock(priv);
+ mutex_unlock(&priv->reg_mutex);
return ret;
}
@@ -687,7 +671,7 @@ mt7531_ind_c22_phy_write(struct mt7530_priv *priv, int port, int regnum,
INIT_MT7530_DUMMY_POLL(&p, priv, MT7531_PHY_IAC);
- mt7530_mutex_lock(priv);
+ mutex_lock(&priv->reg_mutex);
ret = readx_poll_timeout(_mt7530_unlocked_read, &p, reg,
!(reg & MT7531_PHY_ACS_ST), 20, 100000);
@@ -709,7 +693,7 @@ mt7531_ind_c22_phy_write(struct mt7530_priv *priv, int port, int regnum,
}
out:
- mt7530_mutex_unlock(priv);
+ mutex_unlock(&priv->reg_mutex);
return ret;
}
@@ -1444,8 +1428,6 @@ mt7530_port_change_mtu(struct dsa_switch *ds, int port, int new_mtu)
if (!dsa_is_cpu_port(ds, port))
return 0;
- mt7530_mutex_lock(priv);
-
val = mt7530_mii_read(priv, MT7530_GMACCR);
val &= ~MAX_RX_PKT_LEN_MASK;
@@ -1465,8 +1447,6 @@ mt7530_port_change_mtu(struct dsa_switch *ds, int port, int new_mtu)
mt7530_mii_write(priv, MT7530_GMACCR, val);
- mt7530_mutex_unlock(priv);
-
return 0;
}
--
2.55.0
^ permalink raw reply related
* [PATCH net-next v4 2/8] net: dsa: mt7530: fold mt7530_mii_write/read into mt7530_write/read
From: Daniel Golle @ 2026-07-19 17:53 UTC (permalink / raw)
To: Chester A. Unal, Daniel Golle, Andrew Lunn, Vladimir Oltean,
David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Matthias Brugger, AngeloGioacchino Del Regno, Russell King,
netdev, linux-kernel, linux-arm-kernel, linux-mediatek
In-Reply-To: <cover.1784481922.git.daniel@makrotopia.org>
With the lock wrappers removed in the previous commit, mt7530_write()
was a trivial wrapper around mt7530_mii_write(), and mt7530_read()
around mt7530_mii_read() via _mt7530_read(). Fold the function bodies
and eliminate the intermediate functions.
The _mt7530_unlocked_read() and _mt7530_read() poll helpers, which
existed as locked/unlocked variants for readx_poll_timeout(), are
consolidated into a single mt7530_mii_poll() that calls mt7530_read().
Callers are updated using the following semantic patch:
@@
expression E1, E2, E3;
@@
-mt7530_mii_write(E1, E2, E3)
+mt7530_write(E1, E2, E3)
@@
expression E1, E2;
@@
-mt7530_mii_read(E1, E2)
+mt7530_read(E1, E2)
Signed-off-by: Daniel Golle <daniel@makrotopia.org>
---
drivers/net/dsa/mt7530.c | 78 ++++++++++++++--------------------------
1 file changed, 27 insertions(+), 51 deletions(-)
diff --git a/drivers/net/dsa/mt7530.c b/drivers/net/dsa/mt7530.c
index 5f56a423b147..9ccc848195cf 100644
--- a/drivers/net/dsa/mt7530.c
+++ b/drivers/net/dsa/mt7530.c
@@ -150,22 +150,19 @@ core_clear(struct mt7530_priv *priv, u32 reg, u32 val)
core_rmw(priv, reg, val, 0);
}
-static int
-mt7530_mii_write(struct mt7530_priv *priv, u32 reg, u32 val)
+static void
+mt7530_write(struct mt7530_priv *priv, u32 reg, u32 val)
{
int ret;
ret = regmap_write(priv->regmap, reg, val);
-
if (ret < 0)
dev_err(priv->dev,
"failed to write mt7530 register\n");
-
- return ret;
}
static u32
-mt7530_mii_read(struct mt7530_priv *priv, u32 reg)
+mt7530_read(struct mt7530_priv *priv, u32 reg)
{
int ret;
u32 val;
@@ -181,31 +178,10 @@ mt7530_mii_read(struct mt7530_priv *priv, u32 reg)
return val;
}
-static void
-mt7530_write(struct mt7530_priv *priv, u32 reg, u32 val)
-{
- mt7530_mii_write(priv, reg, val);
-}
-
static u32
-_mt7530_unlocked_read(struct mt7530_dummy_poll *p)
+mt7530_mii_poll(struct mt7530_dummy_poll *p)
{
- return mt7530_mii_read(p->priv, p->reg);
-}
-
-static u32
-_mt7530_read(struct mt7530_dummy_poll *p)
-{
- return mt7530_mii_read(p->priv, p->reg);
-}
-
-static u32
-mt7530_read(struct mt7530_priv *priv, u32 reg)
-{
- struct mt7530_dummy_poll p;
-
- INIT_MT7530_DUMMY_POLL(&p, priv, reg);
- return _mt7530_read(&p);
+ return mt7530_read(p->priv, p->reg);
}
static void
@@ -239,7 +215,7 @@ mt7530_fdb_cmd(struct mt7530_priv *priv, enum mt7530_fdb_cmd cmd, u32 *rsp)
mt7530_write(priv, MT7530_ATC, val);
INIT_MT7530_DUMMY_POLL(&p, priv, MT7530_ATC);
- ret = readx_poll_timeout(_mt7530_read, &p, val,
+ ret = readx_poll_timeout(mt7530_mii_poll, &p, val,
!(val & ATC_BUSY), 20, 20000);
if (ret < 0) {
dev_err(priv->dev, "reset timeout\n");
@@ -541,7 +517,7 @@ mt7531_ind_c45_phy_read(struct mt7530_priv *priv, int port, int devad,
mutex_lock(&priv->reg_mutex);
- ret = readx_poll_timeout(_mt7530_unlocked_read, &p, val,
+ ret = readx_poll_timeout(mt7530_mii_poll, &p, val,
!(val & MT7531_PHY_ACS_ST), 20, 100000);
if (ret < 0) {
dev_err(priv->dev, "poll timeout\n");
@@ -550,9 +526,9 @@ mt7531_ind_c45_phy_read(struct mt7530_priv *priv, int port, int devad,
reg = MT7531_MDIO_CL45_ADDR | MT7531_MDIO_PHY_ADDR(port) |
MT7531_MDIO_DEV_ADDR(devad) | regnum;
- mt7530_mii_write(priv, MT7531_PHY_IAC, reg | MT7531_PHY_ACS_ST);
+ mt7530_write(priv, MT7531_PHY_IAC, reg | MT7531_PHY_ACS_ST);
- ret = readx_poll_timeout(_mt7530_unlocked_read, &p, val,
+ ret = readx_poll_timeout(mt7530_mii_poll, &p, val,
!(val & MT7531_PHY_ACS_ST), 20, 100000);
if (ret < 0) {
dev_err(priv->dev, "poll timeout\n");
@@ -561,9 +537,9 @@ mt7531_ind_c45_phy_read(struct mt7530_priv *priv, int port, int devad,
reg = MT7531_MDIO_CL45_READ | MT7531_MDIO_PHY_ADDR(port) |
MT7531_MDIO_DEV_ADDR(devad);
- mt7530_mii_write(priv, MT7531_PHY_IAC, reg | MT7531_PHY_ACS_ST);
+ mt7530_write(priv, MT7531_PHY_IAC, reg | MT7531_PHY_ACS_ST);
- ret = readx_poll_timeout(_mt7530_unlocked_read, &p, val,
+ ret = readx_poll_timeout(mt7530_mii_poll, &p, val,
!(val & MT7531_PHY_ACS_ST), 20, 100000);
if (ret < 0) {
dev_err(priv->dev, "poll timeout\n");
@@ -589,7 +565,7 @@ mt7531_ind_c45_phy_write(struct mt7530_priv *priv, int port, int devad,
mutex_lock(&priv->reg_mutex);
- ret = readx_poll_timeout(_mt7530_unlocked_read, &p, val,
+ ret = readx_poll_timeout(mt7530_mii_poll, &p, val,
!(val & MT7531_PHY_ACS_ST), 20, 100000);
if (ret < 0) {
dev_err(priv->dev, "poll timeout\n");
@@ -598,9 +574,9 @@ mt7531_ind_c45_phy_write(struct mt7530_priv *priv, int port, int devad,
reg = MT7531_MDIO_CL45_ADDR | MT7531_MDIO_PHY_ADDR(port) |
MT7531_MDIO_DEV_ADDR(devad) | regnum;
- mt7530_mii_write(priv, MT7531_PHY_IAC, reg | MT7531_PHY_ACS_ST);
+ mt7530_write(priv, MT7531_PHY_IAC, reg | MT7531_PHY_ACS_ST);
- ret = readx_poll_timeout(_mt7530_unlocked_read, &p, val,
+ ret = readx_poll_timeout(mt7530_mii_poll, &p, val,
!(val & MT7531_PHY_ACS_ST), 20, 100000);
if (ret < 0) {
dev_err(priv->dev, "poll timeout\n");
@@ -609,9 +585,9 @@ mt7531_ind_c45_phy_write(struct mt7530_priv *priv, int port, int devad,
reg = MT7531_MDIO_CL45_WRITE | MT7531_MDIO_PHY_ADDR(port) |
MT7531_MDIO_DEV_ADDR(devad) | data;
- mt7530_mii_write(priv, MT7531_PHY_IAC, reg | MT7531_PHY_ACS_ST);
+ mt7530_write(priv, MT7531_PHY_IAC, reg | MT7531_PHY_ACS_ST);
- ret = readx_poll_timeout(_mt7530_unlocked_read, &p, val,
+ ret = readx_poll_timeout(mt7530_mii_poll, &p, val,
!(val & MT7531_PHY_ACS_ST), 20, 100000);
if (ret < 0) {
dev_err(priv->dev, "poll timeout\n");
@@ -635,7 +611,7 @@ mt7531_ind_c22_phy_read(struct mt7530_priv *priv, int port, int regnum)
mutex_lock(&priv->reg_mutex);
- ret = readx_poll_timeout(_mt7530_unlocked_read, &p, val,
+ ret = readx_poll_timeout(mt7530_mii_poll, &p, val,
!(val & MT7531_PHY_ACS_ST), 20, 100000);
if (ret < 0) {
dev_err(priv->dev, "poll timeout\n");
@@ -645,9 +621,9 @@ mt7531_ind_c22_phy_read(struct mt7530_priv *priv, int port, int regnum)
val = MT7531_MDIO_CL22_READ | MT7531_MDIO_PHY_ADDR(port) |
MT7531_MDIO_REG_ADDR(regnum);
- mt7530_mii_write(priv, MT7531_PHY_IAC, val | MT7531_PHY_ACS_ST);
+ mt7530_write(priv, MT7531_PHY_IAC, val | MT7531_PHY_ACS_ST);
- ret = readx_poll_timeout(_mt7530_unlocked_read, &p, val,
+ ret = readx_poll_timeout(mt7530_mii_poll, &p, val,
!(val & MT7531_PHY_ACS_ST), 20, 100000);
if (ret < 0) {
dev_err(priv->dev, "poll timeout\n");
@@ -673,7 +649,7 @@ mt7531_ind_c22_phy_write(struct mt7530_priv *priv, int port, int regnum,
mutex_lock(&priv->reg_mutex);
- ret = readx_poll_timeout(_mt7530_unlocked_read, &p, reg,
+ ret = readx_poll_timeout(mt7530_mii_poll, &p, reg,
!(reg & MT7531_PHY_ACS_ST), 20, 100000);
if (ret < 0) {
dev_err(priv->dev, "poll timeout\n");
@@ -683,9 +659,9 @@ mt7531_ind_c22_phy_write(struct mt7530_priv *priv, int port, int regnum,
reg = MT7531_MDIO_CL22_WRITE | MT7531_MDIO_PHY_ADDR(port) |
MT7531_MDIO_REG_ADDR(regnum) | data;
- mt7530_mii_write(priv, MT7531_PHY_IAC, reg | MT7531_PHY_ACS_ST);
+ mt7530_write(priv, MT7531_PHY_IAC, reg | MT7531_PHY_ACS_ST);
- ret = readx_poll_timeout(_mt7530_unlocked_read, &p, reg,
+ ret = readx_poll_timeout(mt7530_mii_poll, &p, reg,
!(reg & MT7531_PHY_ACS_ST), 20, 100000);
if (ret < 0) {
dev_err(priv->dev, "poll timeout\n");
@@ -1428,7 +1404,7 @@ mt7530_port_change_mtu(struct dsa_switch *ds, int port, int new_mtu)
if (!dsa_is_cpu_port(ds, port))
return 0;
- val = mt7530_mii_read(priv, MT7530_GMACCR);
+ val = mt7530_read(priv, MT7530_GMACCR);
val &= ~MAX_RX_PKT_LEN_MASK;
/* RX length also includes Ethernet header, MTK tag, and FCS length */
@@ -1445,7 +1421,7 @@ mt7530_port_change_mtu(struct dsa_switch *ds, int port, int new_mtu)
val |= MAX_RX_PKT_LEN_JUMBO;
}
- mt7530_mii_write(priv, MT7530_GMACCR, val);
+ mt7530_write(priv, MT7530_GMACCR, val);
return 0;
}
@@ -1614,7 +1590,7 @@ mt7530_vlan_cmd(struct mt7530_priv *priv, enum mt7530_vlan_cmd cmd, u16 vid)
mt7530_write(priv, MT7530_VTCR, val);
INIT_MT7530_DUMMY_POLL(&p, priv, MT7530_VTCR);
- ret = readx_poll_timeout(_mt7530_read, &p, val,
+ ret = readx_poll_timeout(mt7530_mii_poll, &p, val,
!(val & VTCR_BUSY), 20, 20000);
if (ret < 0) {
dev_err(priv->dev, "poll timeout\n");
@@ -2465,7 +2441,7 @@ mt7530_setup(struct dsa_switch *ds)
/* Waiting for MT7530 got to stable */
INIT_MT7530_DUMMY_POLL(&p, priv, MT753X_TRAP);
- ret = readx_poll_timeout(_mt7530_read, &p, val, val != 0,
+ ret = readx_poll_timeout(mt7530_mii_poll, &p, val, val != 0,
20, 1000000);
if (ret < 0) {
dev_err(priv->dev, "reset timeout\n");
@@ -2706,7 +2682,7 @@ mt7531_setup(struct dsa_switch *ds)
/* Waiting for MT7530 got to stable */
INIT_MT7530_DUMMY_POLL(&p, priv, MT753X_TRAP);
- ret = readx_poll_timeout(_mt7530_read, &p, val, val != 0,
+ ret = readx_poll_timeout(mt7530_mii_poll, &p, val, val != 0,
20, 1000000);
if (ret < 0) {
dev_err(priv->dev, "reset timeout\n");
--
2.55.0
^ permalink raw reply related
* [PATCH net-next v4 3/8] net: dsa: mt7530: replace mt7530_write with regmap_write
From: Daniel Golle @ 2026-07-19 17:53 UTC (permalink / raw)
To: Chester A. Unal, Daniel Golle, Andrew Lunn, Vladimir Oltean,
David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Matthias Brugger, AngeloGioacchino Del Regno, Russell King,
netdev, linux-kernel, linux-arm-kernel, linux-mediatek
In-Reply-To: <cover.1784481922.git.daniel@makrotopia.org>
Replace all mt7530_write() calls with direct regmap_write() calls
and remove the wrapper function. The per-call error logging is
dropped -- regmap has its own tracing infrastructure.
Generated using the following semantic patch:
@@
expression priv, reg, val;
@@
-mt7530_write(priv, reg, val)
+regmap_write(priv->regmap, reg, val)
Signed-off-by: Daniel Golle <daniel@makrotopia.org>
---
drivers/net/dsa/mt7530.c | 126 ++++++++++++++++++---------------------
1 file changed, 59 insertions(+), 67 deletions(-)
diff --git a/drivers/net/dsa/mt7530.c b/drivers/net/dsa/mt7530.c
index 9ccc848195cf..ce4efcf1b3e6 100644
--- a/drivers/net/dsa/mt7530.c
+++ b/drivers/net/dsa/mt7530.c
@@ -150,16 +150,6 @@ core_clear(struct mt7530_priv *priv, u32 reg, u32 val)
core_rmw(priv, reg, val, 0);
}
-static void
-mt7530_write(struct mt7530_priv *priv, u32 reg, u32 val)
-{
- int ret;
-
- ret = regmap_write(priv->regmap, reg, val);
- if (ret < 0)
- dev_err(priv->dev,
- "failed to write mt7530 register\n");
-}
static u32
mt7530_read(struct mt7530_priv *priv, u32 reg)
@@ -212,7 +202,7 @@ mt7530_fdb_cmd(struct mt7530_priv *priv, enum mt7530_fdb_cmd cmd, u32 *rsp)
/* Set the command operating upon the MAC address entries */
val = ATC_BUSY | ATC_MAT(0) | cmd;
- mt7530_write(priv, MT7530_ATC, val);
+ regmap_write(priv->regmap, MT7530_ATC, val);
INIT_MT7530_DUMMY_POLL(&p, priv, MT7530_ATC);
ret = readx_poll_timeout(mt7530_mii_poll, &p, val,
@@ -288,7 +278,7 @@ mt7530_fdb_write(struct mt7530_priv *priv, u16 vid,
/* Write array into the ARL table */
for (i = 0; i < 3; i++)
- mt7530_write(priv, MT7530_ATA1 + (i * 4), reg[i]);
+ regmap_write(priv->regmap, MT7530_ATA1 + (i * 4), reg[i]);
}
/* Set up switch core clock for MT7530 */
@@ -406,27 +396,27 @@ mt7531_pll_setup(struct mt7530_priv *priv)
/* Step 1 : Disable MT7531 COREPLL */
val = mt7530_read(priv, MT7531_PLLGP_EN);
val &= ~EN_COREPLL;
- mt7530_write(priv, MT7531_PLLGP_EN, val);
+ regmap_write(priv->regmap, MT7531_PLLGP_EN, val);
/* Step 2: switch to XTAL output */
val = mt7530_read(priv, MT7531_PLLGP_EN);
val |= SW_CLKSW;
- mt7530_write(priv, MT7531_PLLGP_EN, val);
+ regmap_write(priv->regmap, MT7531_PLLGP_EN, val);
val = mt7530_read(priv, MT7531_PLLGP_CR0);
val &= ~RG_COREPLL_EN;
- mt7530_write(priv, MT7531_PLLGP_CR0, val);
+ regmap_write(priv->regmap, MT7531_PLLGP_CR0, val);
/* Step 3: disable PLLGP and enable program PLLGP */
val = mt7530_read(priv, MT7531_PLLGP_EN);
val |= SW_PLLGP;
- mt7530_write(priv, MT7531_PLLGP_EN, val);
+ regmap_write(priv->regmap, MT7531_PLLGP_EN, val);
/* Step 4: program COREPLL output frequency to 500MHz */
val = mt7530_read(priv, MT7531_PLLGP_CR0);
val &= ~RG_COREPLL_POSDIV_M;
val |= 2 << RG_COREPLL_POSDIV_S;
- mt7530_write(priv, MT7531_PLLGP_CR0, val);
+ regmap_write(priv->regmap, MT7531_PLLGP_CR0, val);
usleep_range(25, 35);
switch (xtal) {
@@ -434,42 +424,42 @@ mt7531_pll_setup(struct mt7530_priv *priv)
val = mt7530_read(priv, MT7531_PLLGP_CR0);
val &= ~RG_COREPLL_SDM_PCW_M;
val |= 0x140000 << RG_COREPLL_SDM_PCW_S;
- mt7530_write(priv, MT7531_PLLGP_CR0, val);
+ regmap_write(priv->regmap, MT7531_PLLGP_CR0, val);
break;
case MT7531_XTAL_FSEL_40MHZ:
val = mt7530_read(priv, MT7531_PLLGP_CR0);
val &= ~RG_COREPLL_SDM_PCW_M;
val |= 0x190000 << RG_COREPLL_SDM_PCW_S;
- mt7530_write(priv, MT7531_PLLGP_CR0, val);
+ regmap_write(priv->regmap, MT7531_PLLGP_CR0, val);
break;
}
/* Set feedback divide ratio update signal to high */
val = mt7530_read(priv, MT7531_PLLGP_CR0);
val |= RG_COREPLL_SDM_PCW_CHG;
- mt7530_write(priv, MT7531_PLLGP_CR0, val);
+ regmap_write(priv->regmap, MT7531_PLLGP_CR0, val);
/* Wait for at least 16 XTAL clocks */
usleep_range(10, 20);
/* Step 5: set feedback divide ratio update signal to low */
val = mt7530_read(priv, MT7531_PLLGP_CR0);
val &= ~RG_COREPLL_SDM_PCW_CHG;
- mt7530_write(priv, MT7531_PLLGP_CR0, val);
+ regmap_write(priv->regmap, MT7531_PLLGP_CR0, val);
/* Enable 325M clock for SGMII */
- mt7530_write(priv, MT7531_ANA_PLLGP_CR5, 0xad0000);
+ regmap_write(priv->regmap, MT7531_ANA_PLLGP_CR5, 0xad0000);
/* Enable 250SSC clock for RGMII */
- mt7530_write(priv, MT7531_ANA_PLLGP_CR2, 0x4f40000);
+ regmap_write(priv->regmap, MT7531_ANA_PLLGP_CR2, 0x4f40000);
/* Step 6: Enable MT7531 PLL */
val = mt7530_read(priv, MT7531_PLLGP_CR0);
val |= RG_COREPLL_EN;
- mt7530_write(priv, MT7531_PLLGP_CR0, val);
+ regmap_write(priv->regmap, MT7531_PLLGP_CR0, val);
val = mt7530_read(priv, MT7531_PLLGP_EN);
val |= EN_COREPLL;
- mt7530_write(priv, MT7531_PLLGP_EN, val);
+ regmap_write(priv->regmap, MT7531_PLLGP_EN, val);
usleep_range(25, 35);
}
@@ -478,8 +468,8 @@ mt7530_mib_reset(struct dsa_switch *ds)
{
struct mt7530_priv *priv = ds->priv;
- mt7530_write(priv, MT7530_MIB_CCR, CCR_MIB_FLUSH);
- mt7530_write(priv, MT7530_MIB_CCR, CCR_MIB_ACTIVATE);
+ regmap_write(priv->regmap, MT7530_MIB_CCR, CCR_MIB_FLUSH);
+ regmap_write(priv->regmap, MT7530_MIB_CCR, CCR_MIB_ACTIVATE);
}
static int mt7530_phy_read_c22(struct mt7530_priv *priv, int port, int regnum)
@@ -526,7 +516,7 @@ mt7531_ind_c45_phy_read(struct mt7530_priv *priv, int port, int devad,
reg = MT7531_MDIO_CL45_ADDR | MT7531_MDIO_PHY_ADDR(port) |
MT7531_MDIO_DEV_ADDR(devad) | regnum;
- mt7530_write(priv, MT7531_PHY_IAC, reg | MT7531_PHY_ACS_ST);
+ regmap_write(priv->regmap, MT7531_PHY_IAC, reg | MT7531_PHY_ACS_ST);
ret = readx_poll_timeout(mt7530_mii_poll, &p, val,
!(val & MT7531_PHY_ACS_ST), 20, 100000);
@@ -537,7 +527,7 @@ mt7531_ind_c45_phy_read(struct mt7530_priv *priv, int port, int devad,
reg = MT7531_MDIO_CL45_READ | MT7531_MDIO_PHY_ADDR(port) |
MT7531_MDIO_DEV_ADDR(devad);
- mt7530_write(priv, MT7531_PHY_IAC, reg | MT7531_PHY_ACS_ST);
+ regmap_write(priv->regmap, MT7531_PHY_IAC, reg | MT7531_PHY_ACS_ST);
ret = readx_poll_timeout(mt7530_mii_poll, &p, val,
!(val & MT7531_PHY_ACS_ST), 20, 100000);
@@ -574,7 +564,7 @@ mt7531_ind_c45_phy_write(struct mt7530_priv *priv, int port, int devad,
reg = MT7531_MDIO_CL45_ADDR | MT7531_MDIO_PHY_ADDR(port) |
MT7531_MDIO_DEV_ADDR(devad) | regnum;
- mt7530_write(priv, MT7531_PHY_IAC, reg | MT7531_PHY_ACS_ST);
+ regmap_write(priv->regmap, MT7531_PHY_IAC, reg | MT7531_PHY_ACS_ST);
ret = readx_poll_timeout(mt7530_mii_poll, &p, val,
!(val & MT7531_PHY_ACS_ST), 20, 100000);
@@ -585,7 +575,7 @@ mt7531_ind_c45_phy_write(struct mt7530_priv *priv, int port, int devad,
reg = MT7531_MDIO_CL45_WRITE | MT7531_MDIO_PHY_ADDR(port) |
MT7531_MDIO_DEV_ADDR(devad) | data;
- mt7530_write(priv, MT7531_PHY_IAC, reg | MT7531_PHY_ACS_ST);
+ regmap_write(priv->regmap, MT7531_PHY_IAC, reg | MT7531_PHY_ACS_ST);
ret = readx_poll_timeout(mt7530_mii_poll, &p, val,
!(val & MT7531_PHY_ACS_ST), 20, 100000);
@@ -621,7 +611,7 @@ mt7531_ind_c22_phy_read(struct mt7530_priv *priv, int port, int regnum)
val = MT7531_MDIO_CL22_READ | MT7531_MDIO_PHY_ADDR(port) |
MT7531_MDIO_REG_ADDR(regnum);
- mt7530_write(priv, MT7531_PHY_IAC, val | MT7531_PHY_ACS_ST);
+ regmap_write(priv->regmap, MT7531_PHY_IAC, val | MT7531_PHY_ACS_ST);
ret = readx_poll_timeout(mt7530_mii_poll, &p, val,
!(val & MT7531_PHY_ACS_ST), 20, 100000);
@@ -659,7 +649,7 @@ mt7531_ind_c22_phy_write(struct mt7530_priv *priv, int port, int regnum,
reg = MT7531_MDIO_CL22_WRITE | MT7531_MDIO_PHY_ADDR(port) |
MT7531_MDIO_REG_ADDR(regnum) | data;
- mt7530_write(priv, MT7531_PHY_IAC, reg | MT7531_PHY_ACS_ST);
+ regmap_write(priv->regmap, MT7531_PHY_IAC, reg | MT7531_PHY_ACS_ST);
ret = readx_poll_timeout(mt7530_mii_poll, &p, reg,
!(reg & MT7531_PHY_ACS_ST), 20, 100000);
@@ -1012,7 +1002,8 @@ mt7530_set_ageing_time(struct dsa_switch *ds, unsigned int msecs)
}
}
- mt7530_write(priv, MT7530_AAC, AGE_CNT(age_count) | AGE_UNIT(age_unit));
+ regmap_write(priv->regmap, MT7530_AAC,
+ AGE_CNT(age_count) | AGE_UNIT(age_unit));
return 0;
}
@@ -1050,7 +1041,7 @@ static void mt7530_setup_port5(struct dsa_switch *ds, phy_interface_t interface)
/* MUX_PHY_P4: P4 -> P5 -> SoC MAC */
case MUX_PHY_P4:
/* Setup the MAC by default for the cpu port */
- mt7530_write(priv, MT753X_PMCR_P(5), 0x56300);
+ regmap_write(priv->regmap, MT753X_PMCR_P(5), 0x56300);
break;
/* GMAC5: P5 -> SoC MAC or external PHY */
@@ -1064,7 +1055,8 @@ static void mt7530_setup_port5(struct dsa_switch *ds, phy_interface_t interface)
val |= MT7530_P5_RGMII_MODE;
/* P5 RGMII RX Clock Control: delay setting for 1000M */
- mt7530_write(priv, MT7530_P5RGMIIRXCR, CSR_RGMII_EDGE_ALIGN);
+ regmap_write(priv->regmap, MT7530_P5RGMIIRXCR,
+ CSR_RGMII_EDGE_ALIGN);
/* Don't set delay in DSA mode */
if (!dsa_is_dsa_port(priv->ds, 5) &&
@@ -1073,15 +1065,15 @@ static void mt7530_setup_port5(struct dsa_switch *ds, phy_interface_t interface)
tx_delay = 4; /* n * 0.5 ns */
/* P5 RGMII TX Clock Control: delay x */
- mt7530_write(priv, MT7530_P5RGMIITXCR,
+ regmap_write(priv->regmap, MT7530_P5RGMIITXCR,
CSR_RGMII_TXC_CFG(0x10 + tx_delay));
/* reduce P5 RGMII Tx driving, 8mA */
- mt7530_write(priv, MT7530_IO_DRV_CR,
+ regmap_write(priv->regmap, MT7530_IO_DRV_CR,
P5_IO_CLK_DRV(1) | P5_IO_DATA_DRV(1));
}
- mt7530_write(priv, MT753X_MTRAP, val);
+ regmap_write(priv->regmap, MT753X_MTRAP, val);
dev_dbg(ds->dev, "Setup P5, HWTRAP=0x%x, mode=%s, phy-mode=%s\n", val,
mt7530_p5_mode_str(priv->p5_mode), phy_modes(interface));
@@ -1303,8 +1295,7 @@ mt753x_cpu_port_enable(struct dsa_switch *ds, int port)
struct mt7530_priv *priv = ds->priv;
/* Enable Mediatek header mode on the cpu port */
- mt7530_write(priv, MT7530_PVC_P(port),
- PORT_SPEC_TAG);
+ regmap_write(priv->regmap, MT7530_PVC_P(port), PORT_SPEC_TAG);
/* Enable flooding on the CPU port */
mt7530_set(priv, MT753X_MFC, BC_FFP(BIT(port)) | UNM_FFP(BIT(port)) |
@@ -1321,7 +1312,7 @@ mt753x_cpu_port_enable(struct dsa_switch *ds, int port)
/* CPU port gets connected to all user ports of
* the switch.
*/
- mt7530_write(priv, MT7530_PCR_P(port),
+ regmap_write(priv->regmap, MT7530_PCR_P(port),
PCR_MATRIX(dsa_user_ports(priv->ds)));
/* Set to fallback mode for independent VLAN learning */
@@ -1421,7 +1412,7 @@ mt7530_port_change_mtu(struct dsa_switch *ds, int port, int new_mtu)
val |= MAX_RX_PKT_LEN_JUMBO;
}
- mt7530_write(priv, MT7530_GMACCR, val);
+ regmap_write(priv->regmap, MT7530_GMACCR, val);
return 0;
}
@@ -1587,7 +1578,7 @@ mt7530_vlan_cmd(struct mt7530_priv *priv, enum mt7530_vlan_cmd cmd, u16 vid)
int ret;
val = VTCR_BUSY | VTCR_FUNC(cmd) | vid;
- mt7530_write(priv, MT7530_VTCR, val);
+ regmap_write(priv->regmap, MT7530_VTCR, val);
INIT_MT7530_DUMMY_POLL(&p, priv, MT7530_VTCR);
ret = readx_poll_timeout(mt7530_mii_poll, &p, val,
@@ -1616,8 +1607,8 @@ mt7530_setup_vlan0(struct mt7530_priv *priv)
*/
val = IVL_MAC | EG_CON | PORT_MEM(MT7530_ALL_MEMBERS) | FID(FID_BRIDGED) |
VLAN_VALID;
- mt7530_write(priv, MT7530_VAWD1, val);
- mt7530_write(priv, MT7530_VAWD2, 0);
+ regmap_write(priv->regmap, MT7530_VAWD1, val);
+ regmap_write(priv->regmap, MT7530_VAWD2, 0);
return mt7530_vlan_cmd(priv, MT7530_VTCR_WR_VID, 0);
}
@@ -1887,7 +1878,7 @@ mt7530_hw_vlan_add(struct mt7530_priv *priv,
*/
val = IVL_MAC | VTAG_EN | PORT_MEM(new_members) | FID(FID_BRIDGED) |
VLAN_VALID;
- mt7530_write(priv, MT7530_VAWD1, val);
+ regmap_write(priv->regmap, MT7530_VAWD1, val);
/* Decide whether adding tag or not for those outgoing packets from the
* port inside the VLAN.
@@ -1926,10 +1917,10 @@ mt7530_hw_vlan_del(struct mt7530_priv *priv,
if (new_members) {
val = IVL_MAC | VTAG_EN | PORT_MEM(new_members) |
VLAN_VALID;
- mt7530_write(priv, MT7530_VAWD1, val);
+ regmap_write(priv->regmap, MT7530_VAWD1, val);
} else {
- mt7530_write(priv, MT7530_VAWD1, 0);
- mt7530_write(priv, MT7530_VAWD2, 0);
+ regmap_write(priv->regmap, MT7530_VAWD1, 0);
+ regmap_write(priv->regmap, MT7530_VAWD2, 0);
}
}
@@ -2070,7 +2061,7 @@ static int mt753x_port_mirror_add(struct dsa_switch *ds, int port,
val |= MT753X_MIRROR_EN(priv->id);
val &= ~MT753X_MIRROR_PORT_MASK(priv->id);
val |= MT753X_MIRROR_PORT_SET(priv->id, mirror->to_local_port);
- mt7530_write(priv, MT753X_MIRROR_REG(priv->id), val);
+ regmap_write(priv->regmap, MT753X_MIRROR_REG(priv->id), val);
val = mt7530_read(priv, MT7530_PCR_P(port));
if (ingress) {
@@ -2080,7 +2071,7 @@ static int mt753x_port_mirror_add(struct dsa_switch *ds, int port,
val |= PORT_TX_MIR;
priv->mirror_tx |= BIT(port);
}
- mt7530_write(priv, MT7530_PCR_P(port), val);
+ regmap_write(priv->regmap, MT7530_PCR_P(port), val);
return 0;
}
@@ -2099,12 +2090,12 @@ static void mt753x_port_mirror_del(struct dsa_switch *ds, int port,
val &= ~PORT_TX_MIR;
priv->mirror_tx &= ~BIT(port);
}
- mt7530_write(priv, MT7530_PCR_P(port), val);
+ regmap_write(priv->regmap, MT7530_PCR_P(port), val);
if (!priv->mirror_rx && !priv->mirror_tx) {
val = mt7530_read(priv, MT753X_MIRROR_REG(priv->id));
val &= ~MT753X_MIRROR_EN(priv->id);
- mt7530_write(priv, MT753X_MIRROR_REG(priv->id), val);
+ regmap_write(priv->regmap, MT753X_MIRROR_REG(priv->id), val);
}
}
@@ -2202,9 +2193,9 @@ mt7530_setup_gpio(struct mt7530_priv *priv)
if (!gc)
return -ENOMEM;
- mt7530_write(priv, MT7530_LED_GPIO_OE, 0);
- mt7530_write(priv, MT7530_LED_GPIO_DIR, 0);
- mt7530_write(priv, MT7530_LED_IO_MODE, 0);
+ regmap_write(priv->regmap, MT7530_LED_GPIO_OE, 0);
+ regmap_write(priv->regmap, MT7530_LED_GPIO_DIR, 0);
+ regmap_write(priv->regmap, MT7530_LED_IO_MODE, 0);
gc->label = "mt7530";
gc->parent = dev;
@@ -2462,13 +2453,12 @@ mt7530_setup(struct dsa_switch *ds)
}
/* Reset the switch through internal reset */
- mt7530_write(priv, MT7530_SYS_CTRL,
- SYS_CTRL_PHY_RST | SYS_CTRL_SW_RST |
- SYS_CTRL_REG_RST);
+ regmap_write(priv->regmap, MT7530_SYS_CTRL,
+ SYS_CTRL_PHY_RST | SYS_CTRL_SW_RST | SYS_CTRL_REG_RST);
/* Lower Tx driving for TRGMII path */
for (i = 0; i < NUM_TRGMII_CTRL; i++)
- mt7530_write(priv, MT7530_TRGMII_TD_ODT(i),
+ regmap_write(priv->regmap, MT7530_TRGMII_TD_ODT(i),
TD_DM_DRVP(8) | TD_DM_DRVN(8));
for (i = 0; i < NUM_TRGMII_CTRL; i++)
@@ -2647,7 +2637,7 @@ mt7531_setup_common(struct dsa_switch *ds)
/* Enable Special Tag for rx frames */
if (priv->id == ID_EN7581 || priv->id == ID_AN7583)
- mt7530_write(priv, MT753X_CPORT_SPTAG_CFG,
+ regmap_write(priv->regmap, MT753X_CPORT_SPTAG_CFG,
CPORT_SW2FE_STAG_EN | CPORT_FE2SW_STAG_EN);
/* Flush the FDB table */
@@ -2705,10 +2695,12 @@ mt7531_setup(struct dsa_switch *ds)
/* Force link down on all ports before internal reset */
for (i = 0; i < priv->ds->num_ports; i++)
- mt7530_write(priv, MT753X_PMCR_P(i), MT7531_FORCE_MODE_LNK);
+ regmap_write(priv->regmap, MT753X_PMCR_P(i),
+ MT7531_FORCE_MODE_LNK);
/* Reset the switch through internal reset */
- mt7530_write(priv, MT7530_SYS_CTRL, SYS_CTRL_SW_RST | SYS_CTRL_REG_RST);
+ regmap_write(priv->regmap, MT7530_SYS_CTRL,
+ SYS_CTRL_SW_RST | SYS_CTRL_REG_RST);
if (!priv->p5_sgmii) {
mt7531_pll_setup(priv);
@@ -2917,7 +2909,7 @@ static void mt7531_rgmii_setup(struct mt7530_priv *priv,
}
}
- mt7530_write(priv, MT7531_CLKGEN_CTRL, val);
+ regmap_write(priv->regmap, MT7531_CLKGEN_CTRL, val);
}
static void
@@ -3254,7 +3246,7 @@ static int mt753x_tc_setup_qdisc_tbf(struct dsa_switch *ds, int port,
FIELD_PREP(ERLCR_EXP_MASK, tick) |
ERLCR_TBF_MODE_MASK |
FIELD_PREP(ERLCR_MANT_MASK, 0xf);
- mt7530_write(priv, MT753X_ERLCR_P(port), val);
+ regmap_write(priv->regmap, MT753X_ERLCR_P(port), val);
break;
}
default:
@@ -3296,7 +3288,7 @@ static int mt7988_setup(struct dsa_switch *ds)
FIELD_PREP(AN7583_CSR_ETHER_AFE_PWD, 0));
/* Reset the switch PHYs */
- mt7530_write(priv, MT7530_SYS_CTRL, SYS_CTRL_PHY_RST);
+ regmap_write(priv->regmap, MT7530_SYS_CTRL, SYS_CTRL_PHY_RST);
return mt7531_setup_common(ds);
}
--
2.55.0
^ permalink raw reply related
* [PATCH net-next v4 4/8] net: dsa: mt7530: replace mt7530_rmw/set/clear with regmap API
From: Daniel Golle @ 2026-07-19 17:53 UTC (permalink / raw)
To: Chester A. Unal, Daniel Golle, Andrew Lunn, Vladimir Oltean,
David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Matthias Brugger, AngeloGioacchino Del Regno, Russell King,
netdev, linux-kernel, linux-arm-kernel, linux-mediatek
In-Reply-To: <cover.1784481922.git.daniel@makrotopia.org>
Replace all mt7530_rmw() calls with regmap_update_bits(), mt7530_set()
with regmap_set_bits(), and mt7530_clear() with regmap_clear_bits().
Remove the wrapper function definitions.
Generated using the following semantic patch:
@@
expression priv, reg, mask, set;
@@
-mt7530_rmw(priv, reg, mask, set)
+regmap_update_bits(priv->regmap, reg, mask, set)
@@
expression priv, reg, val;
@@
-mt7530_set(priv, reg, val)
+regmap_set_bits(priv->regmap, reg, val)
@@
expression priv, reg, val;
@@
-mt7530_clear(priv, reg, val)
+regmap_clear_bits(priv->regmap, reg, val)
Signed-off-by: Daniel Golle <daniel@makrotopia.org>
---
drivers/net/dsa/mt7530.c | 359 ++++++++++++++++++++-------------------
1 file changed, 182 insertions(+), 177 deletions(-)
diff --git a/drivers/net/dsa/mt7530.c b/drivers/net/dsa/mt7530.c
index ce4efcf1b3e6..fe7e4ab5ae9c 100644
--- a/drivers/net/dsa/mt7530.c
+++ b/drivers/net/dsa/mt7530.c
@@ -174,25 +174,6 @@ mt7530_mii_poll(struct mt7530_dummy_poll *p)
return mt7530_read(p->priv, p->reg);
}
-static void
-mt7530_rmw(struct mt7530_priv *priv, u32 reg,
- u32 mask, u32 set)
-{
- regmap_update_bits(priv->regmap, reg, mask, set);
-}
-
-static void
-mt7530_set(struct mt7530_priv *priv, u32 reg, u32 val)
-{
- mt7530_rmw(priv, reg, val, val);
-}
-
-static void
-mt7530_clear(struct mt7530_priv *priv, u32 reg, u32 val)
-{
- mt7530_rmw(priv, reg, val, 0);
-}
-
static int
mt7530_fdb_cmd(struct mt7530_priv *priv, enum mt7530_fdb_cmd cmd, u32 *rsp)
{
@@ -332,12 +313,13 @@ mt7530_setup_port6(struct dsa_switch *ds, phy_interface_t interface)
core_clear(priv, CORE_TRGMII_GSW_CLK_CG, REG_TRGMIICK_EN);
if (interface == PHY_INTERFACE_MODE_RGMII) {
- mt7530_rmw(priv, MT7530_P6ECR, P6_INTF_MODE_MASK,
- P6_INTF_MODE(0));
+ regmap_update_bits(priv->regmap, MT7530_P6ECR,
+ P6_INTF_MODE_MASK, P6_INTF_MODE(0));
return;
}
- mt7530_rmw(priv, MT7530_P6ECR, P6_INTF_MODE_MASK, P6_INTF_MODE(1));
+ regmap_update_bits(priv->regmap, MT7530_P6ECR, P6_INTF_MODE_MASK,
+ P6_INTF_MODE(1));
xtal = mt7530_read(priv, MT753X_MTRAP) & MT7530_XTAL_MASK;
@@ -1258,35 +1240,35 @@ mt753x_trap_frames(struct mt7530_priv *priv)
* switch egress VLAN tag processing. This preserves VLAN tags
* for reception on VLAN sub-interfaces.
*/
- mt7530_rmw(priv, MT753X_BPC,
- PAE_BPDU_FR | PAE_EG_TAG_MASK | PAE_PORT_FW_MASK |
- BPDU_EG_TAG_MASK | BPDU_PORT_FW_MASK,
- PAE_BPDU_FR | PAE_EG_TAG(MT7530_VLAN_EG_DISABLED) |
- PAE_PORT_FW(TO_CPU_FW_CPU_ONLY) |
- BPDU_EG_TAG(MT7530_VLAN_EG_DISABLED) |
- TO_CPU_FW_CPU_ONLY);
+ regmap_update_bits(priv->regmap, MT753X_BPC,
+ PAE_BPDU_FR | PAE_EG_TAG_MASK | PAE_PORT_FW_MASK |
+ BPDU_EG_TAG_MASK | BPDU_PORT_FW_MASK,
+ PAE_BPDU_FR | PAE_EG_TAG(MT7530_VLAN_EG_DISABLED) |
+ PAE_PORT_FW(TO_CPU_FW_CPU_ONLY) |
+ BPDU_EG_TAG(MT7530_VLAN_EG_DISABLED) |
+ TO_CPU_FW_CPU_ONLY);
/* Trap frames with :01 and :02 MAC DAs to the CPU port(s) and
* egress them with EG_TAG disabled.
*/
- mt7530_rmw(priv, MT753X_RGAC1,
- R02_BPDU_FR | R02_EG_TAG_MASK | R02_PORT_FW_MASK |
- R01_BPDU_FR | R01_EG_TAG_MASK | R01_PORT_FW_MASK,
- R02_BPDU_FR | R02_EG_TAG(MT7530_VLAN_EG_DISABLED) |
- R02_PORT_FW(TO_CPU_FW_CPU_ONLY) | R01_BPDU_FR |
- R01_EG_TAG(MT7530_VLAN_EG_DISABLED) |
- TO_CPU_FW_CPU_ONLY);
+ regmap_update_bits(priv->regmap, MT753X_RGAC1,
+ R02_BPDU_FR | R02_EG_TAG_MASK | R02_PORT_FW_MASK |
+ R01_BPDU_FR | R01_EG_TAG_MASK | R01_PORT_FW_MASK,
+ R02_BPDU_FR | R02_EG_TAG(MT7530_VLAN_EG_DISABLED) |
+ R02_PORT_FW(TO_CPU_FW_CPU_ONLY) | R01_BPDU_FR |
+ R01_EG_TAG(MT7530_VLAN_EG_DISABLED) |
+ TO_CPU_FW_CPU_ONLY);
/* Trap frames with :03 and :0E MAC DAs to the CPU port(s) and
* egress them with EG_TAG disabled.
*/
- mt7530_rmw(priv, MT753X_RGAC2,
- R0E_BPDU_FR | R0E_EG_TAG_MASK | R0E_PORT_FW_MASK |
- R03_BPDU_FR | R03_EG_TAG_MASK | R03_PORT_FW_MASK,
- R0E_BPDU_FR | R0E_EG_TAG(MT7530_VLAN_EG_DISABLED) |
- R0E_PORT_FW(TO_CPU_FW_CPU_ONLY) | R03_BPDU_FR |
- R03_EG_TAG(MT7530_VLAN_EG_DISABLED) |
- TO_CPU_FW_CPU_ONLY);
+ regmap_update_bits(priv->regmap, MT753X_RGAC2,
+ R0E_BPDU_FR | R0E_EG_TAG_MASK | R0E_PORT_FW_MASK |
+ R03_BPDU_FR | R03_EG_TAG_MASK | R03_PORT_FW_MASK,
+ R0E_BPDU_FR | R0E_EG_TAG(MT7530_VLAN_EG_DISABLED) |
+ R0E_PORT_FW(TO_CPU_FW_CPU_ONLY) | R03_BPDU_FR |
+ R03_EG_TAG(MT7530_VLAN_EG_DISABLED) |
+ TO_CPU_FW_CPU_ONLY);
}
static void
@@ -1298,8 +1280,8 @@ mt753x_cpu_port_enable(struct dsa_switch *ds, int port)
regmap_write(priv->regmap, MT7530_PVC_P(port), PORT_SPEC_TAG);
/* Enable flooding on the CPU port */
- mt7530_set(priv, MT753X_MFC, BC_FFP(BIT(port)) | UNM_FFP(BIT(port)) |
- UNU_FFP(BIT(port)));
+ regmap_set_bits(priv->regmap, MT753X_MFC,
+ BC_FFP(BIT(port)) | UNM_FFP(BIT(port)) | UNU_FFP(BIT(port)));
/* Add the CPU port to the CPU port bitmap for MT7531 and the switch on
* the MT7988 SoC. Trapped frames will be forwarded to the CPU port that
@@ -1307,7 +1289,8 @@ mt753x_cpu_port_enable(struct dsa_switch *ds, int port)
*/
if (priv->id == ID_MT7531 || priv->id == ID_MT7988 ||
priv->id == ID_EN7581 || priv->id == ID_AN7583)
- mt7530_set(priv, MT7531_CFC, MT7531_CPU_PMAP(BIT(port)));
+ regmap_set_bits(priv->regmap, MT7531_CFC,
+ MT7531_CPU_PMAP(BIT(port)));
/* CPU port gets connected to all user ports of
* the switch.
@@ -1316,8 +1299,8 @@ mt753x_cpu_port_enable(struct dsa_switch *ds, int port)
PCR_MATRIX(dsa_user_ports(priv->ds)));
/* Set to fallback mode for independent VLAN learning */
- mt7530_rmw(priv, MT7530_PCR_P(port), PCR_PORT_VLAN_MASK,
- MT7530_PORT_FALLBACK_MODE);
+ regmap_update_bits(priv->regmap, MT7530_PCR_P(port),
+ PCR_PORT_VLAN_MASK, MT7530_PORT_FALLBACK_MODE);
}
static int
@@ -1339,8 +1322,8 @@ mt7530_port_enable(struct dsa_switch *ds, int port,
priv->ports[port].pm |= PCR_MATRIX(BIT(cpu_dp->index));
}
priv->ports[port].enable = true;
- mt7530_rmw(priv, MT7530_PCR_P(port), PCR_MATRIX_MASK,
- priv->ports[port].pm);
+ regmap_update_bits(priv->regmap, MT7530_PCR_P(port), PCR_MATRIX_MASK,
+ priv->ports[port].pm);
mutex_unlock(&priv->reg_mutex);
@@ -1348,9 +1331,9 @@ mt7530_port_enable(struct dsa_switch *ds, int port,
return 0;
if (port == 5)
- mt7530_clear(priv, MT753X_MTRAP, MT7530_P5_DIS);
+ regmap_clear_bits(priv->regmap, MT753X_MTRAP, MT7530_P5_DIS);
else if (port == 6)
- mt7530_clear(priv, MT753X_MTRAP, MT7530_P6_DIS);
+ regmap_clear_bits(priv->regmap, MT753X_MTRAP, MT7530_P6_DIS);
return 0;
}
@@ -1366,8 +1349,8 @@ mt7530_port_disable(struct dsa_switch *ds, int port)
* enablement for the port.
*/
priv->ports[port].enable = false;
- mt7530_rmw(priv, MT7530_PCR_P(port), PCR_MATRIX_MASK,
- PCR_MATRIX_CLR);
+ regmap_update_bits(priv->regmap, MT7530_PCR_P(port), PCR_MATRIX_MASK,
+ PCR_MATRIX_CLR);
mutex_unlock(&priv->reg_mutex);
@@ -1376,9 +1359,9 @@ mt7530_port_disable(struct dsa_switch *ds, int port)
/* Do not set MT7530_P5_DIS when port 5 is being used for PHY muxing. */
if (port == 5 && priv->p5_mode == GMAC5)
- mt7530_set(priv, MT753X_MTRAP, MT7530_P5_DIS);
+ regmap_set_bits(priv->regmap, MT753X_MTRAP, MT7530_P5_DIS);
else if (port == 6)
- mt7530_set(priv, MT753X_MTRAP, MT7530_P6_DIS);
+ regmap_set_bits(priv->regmap, MT753X_MTRAP, MT7530_P6_DIS);
}
static int
@@ -1448,8 +1431,9 @@ mt7530_stp_state_set(struct dsa_switch *ds, int port, u8 state)
break;
}
- mt7530_rmw(priv, MT7530_SSP_P(port), FID_PST_MASK(FID_BRIDGED),
- FID_PST(FID_BRIDGED, stp_state));
+ regmap_update_bits(priv->regmap, MT7530_SSP_P(port),
+ FID_PST_MASK(FID_BRIDGED),
+ FID_PST(FID_BRIDGED, stp_state));
}
static void mt7530_update_port_member(struct mt7530_priv *priv, int port,
@@ -1488,8 +1472,9 @@ static void mt7530_update_port_member(struct mt7530_priv *priv, int port,
}
if (other_p->enable)
- mt7530_rmw(priv, MT7530_PCR_P(other_port),
- PCR_MATRIX_MASK, other_p->pm);
+ regmap_update_bits(priv->regmap,
+ MT7530_PCR_P(other_port),
+ PCR_MATRIX_MASK, other_p->pm);
}
/* Add/remove the all other ports to this port matrix. For !join
@@ -1498,7 +1483,8 @@ static void mt7530_update_port_member(struct mt7530_priv *priv, int port,
*/
p->pm = PCR_MATRIX(port_bitmap);
if (priv->ports[port].enable)
- mt7530_rmw(priv, MT7530_PCR_P(port), PCR_MATRIX_MASK, p->pm);
+ regmap_update_bits(priv->regmap, MT7530_PCR_P(port),
+ PCR_MATRIX_MASK, p->pm);
}
static int
@@ -1521,20 +1507,23 @@ mt7530_port_bridge_flags(struct dsa_switch *ds, int port,
struct mt7530_priv *priv = ds->priv;
if (flags.mask & BR_LEARNING)
- mt7530_rmw(priv, MT7530_PSC_P(port), SA_DIS,
- flags.val & BR_LEARNING ? 0 : SA_DIS);
+ regmap_update_bits(priv->regmap, MT7530_PSC_P(port), SA_DIS,
+ flags.val & BR_LEARNING ? 0 : SA_DIS);
if (flags.mask & BR_FLOOD)
- mt7530_rmw(priv, MT753X_MFC, UNU_FFP(BIT(port)),
- flags.val & BR_FLOOD ? UNU_FFP(BIT(port)) : 0);
+ regmap_update_bits(priv->regmap, MT753X_MFC,
+ UNU_FFP(BIT(port)),
+ flags.val & BR_FLOOD ? UNU_FFP(BIT(port)) : 0);
if (flags.mask & BR_MCAST_FLOOD)
- mt7530_rmw(priv, MT753X_MFC, UNM_FFP(BIT(port)),
- flags.val & BR_MCAST_FLOOD ? UNM_FFP(BIT(port)) : 0);
+ regmap_update_bits(priv->regmap, MT753X_MFC,
+ UNM_FFP(BIT(port)),
+ flags.val & BR_MCAST_FLOOD ? UNM_FFP(BIT(port)) : 0);
if (flags.mask & BR_BCAST_FLOOD)
- mt7530_rmw(priv, MT753X_MFC, BC_FFP(BIT(port)),
- flags.val & BR_BCAST_FLOOD ? BC_FFP(BIT(port)) : 0);
+ regmap_update_bits(priv->regmap, MT753X_MFC,
+ BC_FFP(BIT(port)),
+ flags.val & BR_BCAST_FLOOD ? BC_FFP(BIT(port)) : 0);
if (flags.mask & BR_ISOLATED) {
struct dsa_port *dp = dsa_to_port(ds, port);
@@ -1562,8 +1551,8 @@ mt7530_port_bridge_join(struct dsa_switch *ds, int port,
mt7530_update_port_member(priv, port, bridge.dev, true);
/* Set to fallback mode for independent VLAN learning */
- mt7530_rmw(priv, MT7530_PCR_P(port), PCR_PORT_VLAN_MASK,
- MT7530_PORT_FALLBACK_MODE);
+ regmap_update_bits(priv->regmap, MT7530_PCR_P(port),
+ PCR_PORT_VLAN_MASK, MT7530_PORT_FALLBACK_MODE);
mutex_unlock(&priv->reg_mutex);
@@ -1624,18 +1613,19 @@ mt7530_port_set_vlan_unaware(struct dsa_switch *ds, int port)
* bridge. Don't set standalone ports to fallback mode.
*/
if (dsa_port_bridge_dev_get(dsa_to_port(ds, port)))
- mt7530_rmw(priv, MT7530_PCR_P(port), PCR_PORT_VLAN_MASK,
- MT7530_PORT_FALLBACK_MODE);
-
- mt7530_rmw(priv, MT7530_PVC_P(port),
- VLAN_ATTR_MASK | PVC_EG_TAG_MASK | ACC_FRM_MASK,
- VLAN_ATTR(MT7530_VLAN_TRANSPARENT) |
- PVC_EG_TAG(MT7530_VLAN_EG_CONSISTENT) |
- MT7530_VLAN_ACC_ALL);
+ regmap_update_bits(priv->regmap, MT7530_PCR_P(port),
+ PCR_PORT_VLAN_MASK,
+ MT7530_PORT_FALLBACK_MODE);
+
+ regmap_update_bits(priv->regmap, MT7530_PVC_P(port),
+ VLAN_ATTR_MASK | PVC_EG_TAG_MASK | ACC_FRM_MASK,
+ VLAN_ATTR(MT7530_VLAN_TRANSPARENT) |
+ PVC_EG_TAG(MT7530_VLAN_EG_CONSISTENT) |
+ MT7530_VLAN_ACC_ALL);
/* Set PVID to 0 */
- mt7530_rmw(priv, MT7530_PPBV1_P(port), G0_PORT_VID_MASK,
- G0_PORT_VID_DEF);
+ regmap_update_bits(priv->regmap, MT7530_PPBV1_P(port),
+ G0_PORT_VID_MASK, G0_PORT_VID_DEF);
for (i = 0; i < priv->ds->num_ports; i++) {
if (i == port)
@@ -1666,24 +1656,27 @@ mt7530_port_set_vlan_aware(struct dsa_switch *ds, int port)
* table lookup.
*/
if (dsa_is_user_port(ds, port)) {
- mt7530_rmw(priv, MT7530_PCR_P(port), PCR_PORT_VLAN_MASK,
- MT7530_PORT_SECURITY_MODE);
- mt7530_rmw(priv, MT7530_PPBV1_P(port), G0_PORT_VID_MASK,
- G0_PORT_VID(priv->ports[port].pvid));
+ regmap_update_bits(priv->regmap, MT7530_PCR_P(port),
+ PCR_PORT_VLAN_MASK,
+ MT7530_PORT_SECURITY_MODE);
+ regmap_update_bits(priv->regmap, MT7530_PPBV1_P(port),
+ G0_PORT_VID_MASK,
+ G0_PORT_VID(priv->ports[port].pvid));
/* Only accept tagged frames if PVID is not set */
if (!priv->ports[port].pvid)
- mt7530_rmw(priv, MT7530_PVC_P(port), ACC_FRM_MASK,
- MT7530_VLAN_ACC_TAGGED);
+ regmap_update_bits(priv->regmap, MT7530_PVC_P(port),
+ ACC_FRM_MASK,
+ MT7530_VLAN_ACC_TAGGED);
/* Set the port as a user port which is to be able to recognize
* VID from incoming packets before fetching entry within the
* VLAN table.
*/
- mt7530_rmw(priv, MT7530_PVC_P(port),
- VLAN_ATTR_MASK | PVC_EG_TAG_MASK,
- VLAN_ATTR(MT7530_VLAN_USER) |
- PVC_EG_TAG(MT7530_VLAN_EG_DISABLED));
+ regmap_update_bits(priv->regmap, MT7530_PVC_P(port),
+ VLAN_ATTR_MASK | PVC_EG_TAG_MASK,
+ VLAN_ATTR(MT7530_VLAN_USER) |
+ PVC_EG_TAG(MT7530_VLAN_EG_DISABLED));
} else {
/* Also set CPU ports to the "user" VLAN port attribute, to
* allow VLAN classification, but keep the EG_TAG attribute as
@@ -1692,8 +1685,9 @@ mt7530_port_set_vlan_aware(struct dsa_switch *ds, int port)
* are forwarded to user ports as tagged, and untagged as
* untagged.
*/
- mt7530_rmw(priv, MT7530_PVC_P(port), VLAN_ATTR_MASK,
- VLAN_ATTR(MT7530_VLAN_USER));
+ regmap_update_bits(priv->regmap, MT7530_PVC_P(port),
+ VLAN_ATTR_MASK,
+ VLAN_ATTR(MT7530_VLAN_USER));
}
}
@@ -1711,8 +1705,8 @@ mt7530_port_bridge_leave(struct dsa_switch *ds, int port,
* back to the default as is at initial boot which is a VLAN-unaware
* port.
*/
- mt7530_rmw(priv, MT7530_PCR_P(port), PCR_PORT_VLAN_MASK,
- MT7530_PORT_MATRIX_MODE);
+ regmap_update_bits(priv->regmap, MT7530_PCR_P(port),
+ PCR_PORT_VLAN_MASK, MT7530_PORT_MATRIX_MODE);
mutex_unlock(&priv->reg_mutex);
}
@@ -1893,9 +1887,9 @@ mt7530_hw_vlan_add(struct mt7530_priv *priv,
val = MT7530_VLAN_EGRESS_UNTAG;
else
val = MT7530_VLAN_EGRESS_TAG;
- mt7530_rmw(priv, MT7530_VAWD2,
- ETAG_CTRL_P_MASK(entry->port),
- ETAG_CTRL_P(entry->port, val));
+ regmap_update_bits(priv->regmap, MT7530_VAWD2,
+ ETAG_CTRL_P_MASK(entry->port),
+ ETAG_CTRL_P(entry->port, val));
}
static void
@@ -1973,25 +1967,26 @@ mt7530_port_vlan_add(struct dsa_switch *ds, int port,
priv->ports[port].pvid = vlan->vid;
/* Accept all frames if PVID is set */
- mt7530_rmw(priv, MT7530_PVC_P(port), ACC_FRM_MASK,
- MT7530_VLAN_ACC_ALL);
+ regmap_update_bits(priv->regmap, MT7530_PVC_P(port),
+ ACC_FRM_MASK, MT7530_VLAN_ACC_ALL);
/* Only configure PVID if VLAN filtering is enabled */
if (dsa_port_is_vlan_filtering(dsa_to_port(ds, port)))
- mt7530_rmw(priv, MT7530_PPBV1_P(port),
- G0_PORT_VID_MASK,
- G0_PORT_VID(vlan->vid));
+ regmap_update_bits(priv->regmap, MT7530_PPBV1_P(port),
+ G0_PORT_VID_MASK,
+ G0_PORT_VID(vlan->vid));
} else if (vlan->vid && priv->ports[port].pvid == vlan->vid) {
/* This VLAN is overwritten without PVID, so unset it */
priv->ports[port].pvid = G0_PORT_VID_DEF;
/* Only accept tagged frames if the port is VLAN-aware */
if (dsa_port_is_vlan_filtering(dsa_to_port(ds, port)))
- mt7530_rmw(priv, MT7530_PVC_P(port), ACC_FRM_MASK,
- MT7530_VLAN_ACC_TAGGED);
+ regmap_update_bits(priv->regmap, MT7530_PVC_P(port),
+ ACC_FRM_MASK,
+ MT7530_VLAN_ACC_TAGGED);
- mt7530_rmw(priv, MT7530_PPBV1_P(port), G0_PORT_VID_MASK,
- G0_PORT_VID_DEF);
+ regmap_update_bits(priv->regmap, MT7530_PPBV1_P(port),
+ G0_PORT_VID_MASK, G0_PORT_VID_DEF);
}
mutex_unlock(&priv->reg_mutex);
@@ -2025,11 +2020,12 @@ mt7530_port_vlan_del(struct dsa_switch *ds, int port,
/* Only accept tagged frames if the port is VLAN-aware */
if (dsa_port_is_vlan_filtering(dsa_to_port(ds, port)))
- mt7530_rmw(priv, MT7530_PVC_P(port), ACC_FRM_MASK,
- MT7530_VLAN_ACC_TAGGED);
+ regmap_update_bits(priv->regmap, MT7530_PVC_P(port),
+ ACC_FRM_MASK,
+ MT7530_VLAN_ACC_TAGGED);
- mt7530_rmw(priv, MT7530_PPBV1_P(port), G0_PORT_VID_MASK,
- G0_PORT_VID_DEF);
+ regmap_update_bits(priv->regmap, MT7530_PPBV1_P(port),
+ G0_PORT_VID_MASK, G0_PORT_VID_DEF);
}
@@ -2136,9 +2132,9 @@ mt7530_gpio_set(struct gpio_chip *gc, unsigned int offset, int value)
u32 bit = mt7530_gpio_to_bit(offset);
if (value)
- mt7530_set(priv, MT7530_LED_GPIO_DATA, bit);
+ regmap_set_bits(priv->regmap, MT7530_LED_GPIO_DATA, bit);
else
- mt7530_clear(priv, MT7530_LED_GPIO_DATA, bit);
+ regmap_clear_bits(priv->regmap, MT7530_LED_GPIO_DATA, bit);
return 0;
}
@@ -2159,8 +2155,8 @@ mt7530_gpio_direction_input(struct gpio_chip *gc, unsigned int offset)
struct mt7530_priv *priv = gpiochip_get_data(gc);
u32 bit = mt7530_gpio_to_bit(offset);
- mt7530_clear(priv, MT7530_LED_GPIO_OE, bit);
- mt7530_clear(priv, MT7530_LED_GPIO_DIR, bit);
+ regmap_clear_bits(priv->regmap, MT7530_LED_GPIO_OE, bit);
+ regmap_clear_bits(priv->regmap, MT7530_LED_GPIO_DIR, bit);
return 0;
}
@@ -2171,14 +2167,14 @@ mt7530_gpio_direction_output(struct gpio_chip *gc, unsigned int offset, int valu
struct mt7530_priv *priv = gpiochip_get_data(gc);
u32 bit = mt7530_gpio_to_bit(offset);
- mt7530_set(priv, MT7530_LED_GPIO_DIR, bit);
+ regmap_set_bits(priv->regmap, MT7530_LED_GPIO_DIR, bit);
if (value)
- mt7530_set(priv, MT7530_LED_GPIO_DATA, bit);
+ regmap_set_bits(priv->regmap, MT7530_LED_GPIO_DATA, bit);
else
- mt7530_clear(priv, MT7530_LED_GPIO_DATA, bit);
+ regmap_clear_bits(priv->regmap, MT7530_LED_GPIO_DATA, bit);
- mt7530_set(priv, MT7530_LED_GPIO_OE, bit);
+ regmap_set_bits(priv->regmap, MT7530_LED_GPIO_OE, bit);
return 0;
}
@@ -2284,7 +2280,8 @@ mt7530_setup_irq(struct mt7530_priv *priv)
/* This register must be set for MT7530 to properly fire interrupts */
if (priv->id == ID_MT7530 || priv->id == ID_MT7621)
- mt7530_set(priv, MT7530_TOP_SIG_CTRL, TOP_SIG_CTRL_NORMAL);
+ regmap_set_bits(priv->regmap, MT7530_TOP_SIG_CTRL,
+ TOP_SIG_CTRL_NORMAL);
ret = devm_regmap_add_irq_chip_fwnode(dev, dev_fwnode(dev),
priv->regmap, irq,
@@ -2462,14 +2459,15 @@ mt7530_setup(struct dsa_switch *ds)
TD_DM_DRVP(8) | TD_DM_DRVN(8));
for (i = 0; i < NUM_TRGMII_CTRL; i++)
- mt7530_rmw(priv, MT7530_TRGMII_RD(i),
- RD_TAP_MASK, RD_TAP(16));
+ regmap_update_bits(priv->regmap, MT7530_TRGMII_RD(i),
+ RD_TAP_MASK, RD_TAP(16));
/* Allow modifying the trap and directly access PHY registers via the
* MDIO bus the switch is on.
*/
- mt7530_rmw(priv, MT753X_MTRAP, MT7530_CHG_TRAP |
- MT7530_PHY_INDIRECT_ACCESS, MT7530_CHG_TRAP);
+ regmap_update_bits(priv->regmap, MT753X_MTRAP,
+ MT7530_CHG_TRAP | MT7530_PHY_INDIRECT_ACCESS,
+ MT7530_CHG_TRAP);
if ((val & MT7530_XTAL_MASK) == MT7530_XTAL_40MHZ)
mt7530_pll_setup(priv);
@@ -2483,17 +2481,16 @@ mt7530_setup(struct dsa_switch *ds)
/* Clear link settings and enable force mode to force link down
* on all ports until they're enabled later.
*/
- mt7530_rmw(priv, MT753X_PMCR_P(i),
- PMCR_LINK_SETTINGS_MASK |
- MT753X_FORCE_MODE(priv->id),
- MT753X_FORCE_MODE(priv->id));
+ regmap_update_bits(priv->regmap, MT753X_PMCR_P(i),
+ PMCR_LINK_SETTINGS_MASK | MT753X_FORCE_MODE(priv->id),
+ MT753X_FORCE_MODE(priv->id));
/* Disable forwarding by default on all ports */
- mt7530_rmw(priv, MT7530_PCR_P(i), PCR_MATRIX_MASK,
- PCR_MATRIX_CLR);
+ regmap_update_bits(priv->regmap, MT7530_PCR_P(i),
+ PCR_MATRIX_MASK, PCR_MATRIX_CLR);
/* Disable learning by default on all ports */
- mt7530_set(priv, MT7530_PSC_P(i), SA_DIS);
+ regmap_set_bits(priv->regmap, MT7530_PSC_P(i), SA_DIS);
if (dsa_is_cpu_port(ds, i)) {
mt753x_cpu_port_enable(ds, i);
@@ -2501,16 +2498,17 @@ mt7530_setup(struct dsa_switch *ds)
mt7530_port_disable(ds, i);
/* Set default PVID to 0 on all user ports */
- mt7530_rmw(priv, MT7530_PPBV1_P(i), G0_PORT_VID_MASK,
- G0_PORT_VID_DEF);
+ regmap_update_bits(priv->regmap, MT7530_PPBV1_P(i),
+ G0_PORT_VID_MASK, G0_PORT_VID_DEF);
}
/* Enable consistent egress tag */
- mt7530_rmw(priv, MT7530_PVC_P(i), PVC_EG_TAG_MASK,
- PVC_EG_TAG(MT7530_VLAN_EG_CONSISTENT));
+ regmap_update_bits(priv->regmap, MT7530_PVC_P(i),
+ PVC_EG_TAG_MASK,
+ PVC_EG_TAG(MT7530_VLAN_EG_CONSISTENT));
}
/* Allow mirroring frames received on the local port (monitor port). */
- mt7530_set(priv, MT753X_AGC, LOCAL_EN);
+ regmap_set_bits(priv->regmap, MT753X_AGC, LOCAL_EN);
/* Setup VLAN ID 0 for VLAN-unaware bridges */
ret = mt7530_setup_vlan0(priv);
@@ -2557,7 +2555,8 @@ mt7530_setup(struct dsa_switch *ds)
if (priv->p5_mode == MUX_PHY_P0 ||
priv->p5_mode == MUX_PHY_P4) {
- mt7530_clear(priv, MT753X_MTRAP, MT7530_P5_DIS);
+ regmap_clear_bits(priv->regmap, MT753X_MTRAP,
+ MT7530_P5_DIS);
mt7530_setup_port5(ds, interface);
}
}
@@ -2596,26 +2595,26 @@ mt7531_setup_common(struct dsa_switch *ds)
mt7530_mib_reset(ds);
/* Disable flooding on all ports */
- mt7530_clear(priv, MT753X_MFC, BC_FFP_MASK | UNM_FFP_MASK |
- UNU_FFP_MASK);
+ regmap_clear_bits(priv->regmap, MT753X_MFC,
+ BC_FFP_MASK | UNM_FFP_MASK | UNU_FFP_MASK);
for (i = 0; i < priv->ds->num_ports; i++) {
/* Clear link settings and enable force mode to force link down
* on all ports until they're enabled later.
*/
- mt7530_rmw(priv, MT753X_PMCR_P(i),
- PMCR_LINK_SETTINGS_MASK |
- MT753X_FORCE_MODE(priv->id),
- MT753X_FORCE_MODE(priv->id));
+ regmap_update_bits(priv->regmap, MT753X_PMCR_P(i),
+ PMCR_LINK_SETTINGS_MASK | MT753X_FORCE_MODE(priv->id),
+ MT753X_FORCE_MODE(priv->id));
/* Disable forwarding by default on all ports */
- mt7530_rmw(priv, MT7530_PCR_P(i), PCR_MATRIX_MASK,
- PCR_MATRIX_CLR);
+ regmap_update_bits(priv->regmap, MT7530_PCR_P(i),
+ PCR_MATRIX_MASK, PCR_MATRIX_CLR);
/* Disable learning by default on all ports */
- mt7530_set(priv, MT7530_PSC_P(i), SA_DIS);
+ regmap_set_bits(priv->regmap, MT7530_PSC_P(i), SA_DIS);
- mt7530_set(priv, MT7531_DBG_CNT(i), MT7531_DIS_CLR);
+ regmap_set_bits(priv->regmap, MT7531_DBG_CNT(i),
+ MT7531_DIS_CLR);
if (dsa_is_cpu_port(ds, i)) {
mt753x_cpu_port_enable(ds, i);
@@ -2623,17 +2622,18 @@ mt7531_setup_common(struct dsa_switch *ds)
mt7530_port_disable(ds, i);
/* Set default PVID to 0 on all user ports */
- mt7530_rmw(priv, MT7530_PPBV1_P(i), G0_PORT_VID_MASK,
- G0_PORT_VID_DEF);
+ regmap_update_bits(priv->regmap, MT7530_PPBV1_P(i),
+ G0_PORT_VID_MASK, G0_PORT_VID_DEF);
}
/* Enable consistent egress tag */
- mt7530_rmw(priv, MT7530_PVC_P(i), PVC_EG_TAG_MASK,
- PVC_EG_TAG(MT7530_VLAN_EG_CONSISTENT));
+ regmap_update_bits(priv->regmap, MT7530_PVC_P(i),
+ PVC_EG_TAG_MASK,
+ PVC_EG_TAG(MT7530_VLAN_EG_CONSISTENT));
}
/* Allow mirroring frames received on the local port (monitor port). */
- mt7530_set(priv, MT753X_AGC, LOCAL_EN);
+ regmap_set_bits(priv->regmap, MT753X_AGC, LOCAL_EN);
/* Enable Special Tag for rx frames */
if (priv->id == ID_EN7581 || priv->id == ID_AN7583)
@@ -2709,14 +2709,16 @@ mt7531_setup(struct dsa_switch *ds)
* MT7531AE. Set the GPIO 11-12 pins to function as MDC and MDIO
* to expose the MDIO bus of the switch.
*/
- mt7530_rmw(priv, MT7531_GPIO_MODE1, MT7531_GPIO11_RG_RXD2_MASK,
- MT7531_EXT_P_MDC_11);
- mt7530_rmw(priv, MT7531_GPIO_MODE1, MT7531_GPIO12_RG_RXD3_MASK,
- MT7531_EXT_P_MDIO_12);
+ regmap_update_bits(priv->regmap, MT7531_GPIO_MODE1,
+ MT7531_GPIO11_RG_RXD2_MASK,
+ MT7531_EXT_P_MDC_11);
+ regmap_update_bits(priv->regmap, MT7531_GPIO_MODE1,
+ MT7531_GPIO12_RG_RXD3_MASK,
+ MT7531_EXT_P_MDIO_12);
}
- mt7530_rmw(priv, MT7531_GPIO_MODE0, MT7531_GPIO0_MASK,
- MT7531_GPIO0_INTERRUPT);
+ regmap_update_bits(priv->regmap, MT7531_GPIO_MODE0, MT7531_GPIO0_MASK,
+ MT7531_GPIO0_INTERRUPT);
/* Enable Energy-Efficient Ethernet (EEE) and PHY core PLL, since
* phy_device has not yet been created provided for
@@ -2962,7 +2964,8 @@ mt753x_phylink_mac_config(struct phylink_config *config, unsigned int mode,
/* Are we connected to external phy */
if (port == 5 && dsa_is_user_port(ds, 5))
- mt7530_set(priv, MT753X_PMCR_P(port), PMCR_EXT_PHY);
+ regmap_set_bits(priv->regmap, MT753X_PMCR_P(port),
+ PMCR_EXT_PHY);
}
static void mt753x_phylink_mac_link_down(struct phylink_config *config,
@@ -2972,7 +2975,8 @@ static void mt753x_phylink_mac_link_down(struct phylink_config *config,
struct dsa_port *dp = dsa_phylink_to_port(config);
struct mt7530_priv *priv = dp->ds->priv;
- mt7530_clear(priv, MT753X_PMCR_P(dp->index), PMCR_LINK_SETTINGS_MASK);
+ regmap_clear_bits(priv->regmap, MT753X_PMCR_P(dp->index),
+ PMCR_LINK_SETTINGS_MASK);
}
static void mt753x_phylink_mac_link_up(struct phylink_config *config,
@@ -3006,7 +3010,7 @@ static void mt753x_phylink_mac_link_up(struct phylink_config *config,
mcr |= PMCR_FORCE_RX_FC_EN;
}
- mt7530_set(priv, MT753X_PMCR_P(dp->index), mcr);
+ regmap_set_bits(priv->regmap, MT753X_PMCR_P(dp->index), mcr);
}
static void mt753x_phylink_mac_disable_tx_lpi(struct phylink_config *config)
@@ -3014,8 +3018,8 @@ static void mt753x_phylink_mac_disable_tx_lpi(struct phylink_config *config)
struct dsa_port *dp = dsa_phylink_to_port(config);
struct mt7530_priv *priv = dp->ds->priv;
- mt7530_clear(priv, MT753X_PMCR_P(dp->index),
- PMCR_FORCE_EEE1G | PMCR_FORCE_EEE100);
+ regmap_clear_bits(priv->regmap, MT753X_PMCR_P(dp->index),
+ PMCR_FORCE_EEE1G | PMCR_FORCE_EEE100);
}
static int mt753x_phylink_mac_enable_tx_lpi(struct phylink_config *config,
@@ -3036,11 +3040,11 @@ static int mt753x_phylink_mac_enable_tx_lpi(struct phylink_config *config,
else
val = LPI_THRESH_MASK;
- mt7530_rmw(priv, MT753X_PMEEECR_P(dp->index),
- LPI_THRESH_MASK | LPI_MODE_EN, val);
+ regmap_update_bits(priv->regmap, MT753X_PMEEECR_P(dp->index),
+ LPI_THRESH_MASK | LPI_MODE_EN, val);
- mt7530_set(priv, MT753X_PMCR_P(dp->index),
- PMCR_FORCE_EEE1G | PMCR_FORCE_EEE100);
+ regmap_set_bits(priv->regmap, MT753X_PMCR_P(dp->index),
+ PMCR_FORCE_EEE1G | PMCR_FORCE_EEE100);
return 0;
}
@@ -3217,7 +3221,8 @@ mt753x_conduit_state_change(struct dsa_switch *ds,
MT7530_CPU_PORT(__ffs(priv->active_cpu_ports));
}
- mt7530_rmw(priv, MT753X_MFC, MT7530_CPU_EN | MT7530_CPU_PORT_MASK, val);
+ regmap_update_bits(priv->regmap, MT753X_MFC,
+ MT7530_CPU_EN | MT7530_CPU_PORT_MASK, val);
}
static int mt753x_tc_setup_qdisc_tbf(struct dsa_switch *ds, int port,
@@ -3234,8 +3239,8 @@ static int mt753x_tc_setup_qdisc_tbf(struct dsa_switch *ds, int port,
case TC_TBF_DESTROY: {
u32 val, tick;
- mt7530_rmw(priv, MT753X_GERLCR, EGR_BC_MASK,
- EGR_BC_CRC_IPG_PREAMBLE);
+ regmap_update_bits(priv->regmap, MT753X_GERLCR, EGR_BC_MASK,
+ EGR_BC_CRC_IPG_PREAMBLE);
/* if rate is greater than 10Mbps tick is 1/32 ms,
* 1ms otherwise
@@ -3279,13 +3284,13 @@ static int mt7988_setup(struct dsa_switch *ds)
/* AN7583 require additional tweak to CONN_CFG */
if (priv->id == ID_AN7583)
- mt7530_rmw(priv, AN7583_GEPHY_CONN_CFG,
- AN7583_CSR_DPHY_CKIN_SEL |
- AN7583_CSR_PHY_CORE_REG_CLK_SEL |
- AN7583_CSR_ETHER_AFE_PWD,
- AN7583_CSR_DPHY_CKIN_SEL |
- AN7583_CSR_PHY_CORE_REG_CLK_SEL |
- FIELD_PREP(AN7583_CSR_ETHER_AFE_PWD, 0));
+ regmap_update_bits(priv->regmap, AN7583_GEPHY_CONN_CFG,
+ AN7583_CSR_DPHY_CKIN_SEL |
+ AN7583_CSR_PHY_CORE_REG_CLK_SEL |
+ AN7583_CSR_ETHER_AFE_PWD,
+ AN7583_CSR_DPHY_CKIN_SEL |
+ AN7583_CSR_PHY_CORE_REG_CLK_SEL |
+ FIELD_PREP(AN7583_CSR_ETHER_AFE_PWD, 0));
/* Reset the switch PHYs */
regmap_write(priv->regmap, MT7530_SYS_CTRL, SYS_CTRL_PHY_RST);
--
2.55.0
^ permalink raw reply related
* [PATCH net-next v4 5/8] net: dsa: mt7530: replace mt7530_read with regmap_read
From: Daniel Golle @ 2026-07-19 17:53 UTC (permalink / raw)
To: Chester A. Unal, Daniel Golle, Andrew Lunn, Vladimir Oltean,
David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Matthias Brugger, AngeloGioacchino Del Regno, Russell King,
netdev, linux-kernel, linux-arm-kernel, linux-mediatek
In-Reply-To: <cover.1784481922.git.daniel@makrotopia.org>
Replace all mt7530_read() calls with direct regmap_read() calls and
remove the wrapper function. The wrapper returned 0 on failed reads;
preserve that behaviour by zeroing the read-back buffer at the start of
mt7530_regmap_read(), so callers that do not check the return code keep
seeing a deterministic value. The WARN_ON_ONCE error logging is dropped.
Most callsites follow the val = mt7530_read(priv, reg) pattern and are
converted mechanically using the following semantic patch:
@@
expression priv, reg;
identifier val;
@@
-val = mt7530_read(priv, reg);
+regmap_read(priv->regmap, reg, &val);
Remaining inline uses are converted by hand.
Signed-off-by: Daniel Golle <daniel@makrotopia.org>
---
v4:
* zero the read-back buffer in mt7530_regmap_read() instead of
initializing the result variables at every call site
v3:
* init read-back variables to 0 to preserve the old read-failure
behaviour
* use u32 for val in mt7530_setup_port5
---
drivers/net/dsa/mt7530-mdio.c | 3 +
drivers/net/dsa/mt7530.c | 115 +++++++++++++++++-----------------
2 files changed, 60 insertions(+), 58 deletions(-)
diff --git a/drivers/net/dsa/mt7530-mdio.c b/drivers/net/dsa/mt7530-mdio.c
index f7c8eeb27211..b1178924eb06 100644
--- a/drivers/net/dsa/mt7530-mdio.c
+++ b/drivers/net/dsa/mt7530-mdio.c
@@ -48,6 +48,9 @@ mt7530_regmap_read(void *context, unsigned int reg, unsigned int *val)
u16 page, r, lo, hi;
int ret;
+ /* Callers do not check for errors, keep the value deterministic */
+ *val = 0;
+
page = (reg >> 6) & 0x3ff;
r = (reg >> 2) & 0xf;
diff --git a/drivers/net/dsa/mt7530.c b/drivers/net/dsa/mt7530.c
index fe7e4ab5ae9c..26c7f3de4e1a 100644
--- a/drivers/net/dsa/mt7530.c
+++ b/drivers/net/dsa/mt7530.c
@@ -152,28 +152,15 @@ core_clear(struct mt7530_priv *priv, u32 reg, u32 val)
static u32
-mt7530_read(struct mt7530_priv *priv, u32 reg)
+mt7530_mii_poll(struct mt7530_dummy_poll *p)
{
- int ret;
u32 val;
- ret = regmap_read(priv->regmap, reg, &val);
- if (ret) {
- WARN_ON_ONCE(1);
- dev_err(priv->dev,
- "failed to read mt7530 register\n");
- return 0;
- }
+ regmap_read(p->priv->regmap, p->reg, &val);
return val;
}
-static u32
-mt7530_mii_poll(struct mt7530_dummy_poll *p)
-{
- return mt7530_read(p->priv, p->reg);
-}
-
static int
mt7530_fdb_cmd(struct mt7530_priv *priv, enum mt7530_fdb_cmd cmd, u32 *rsp)
{
@@ -196,7 +183,7 @@ mt7530_fdb_cmd(struct mt7530_priv *priv, enum mt7530_fdb_cmd cmd, u32 *rsp)
/* Additional sanity for read command if the specified
* entry is invalid
*/
- val = mt7530_read(priv, MT7530_ATC);
+ regmap_read(priv->regmap, MT7530_ATC, &val);
if ((cmd == MT7530_FDB_READ) && (val & ATC_INVALID))
return -EINVAL;
@@ -214,7 +201,8 @@ mt7530_fdb_read(struct mt7530_priv *priv, struct mt7530_fdb *fdb)
/* Read from ARL table into an array */
for (i = 0; i < 3; i++) {
- reg[i] = mt7530_read(priv, MT7530_TSRA1 + (i * 4));
+ regmap_read(priv->regmap, MT7530_TSRA1 + (i * 4),
+ ®[i]);
dev_dbg(priv->dev, "%s(%d) reg[%d]=0x%x\n",
__func__, __LINE__, i, reg[i]);
@@ -321,7 +309,8 @@ mt7530_setup_port6(struct dsa_switch *ds, phy_interface_t interface)
regmap_update_bits(priv->regmap, MT7530_P6ECR, P6_INTF_MODE_MASK,
P6_INTF_MODE(1));
- xtal = mt7530_read(priv, MT753X_MTRAP) & MT7530_XTAL_MASK;
+ regmap_read(priv->regmap, MT753X_MTRAP, &xtal);
+ xtal &= MT7530_XTAL_MASK;
if (xtal == MT7530_XTAL_25MHZ)
ssc_delta = 0x57;
@@ -365,9 +354,9 @@ mt7531_pll_setup(struct mt7530_priv *priv)
u32 hwstrap;
u32 val;
- val = mt7530_read(priv, MT7531_CREV);
- top_sig = mt7530_read(priv, MT7531_TOP_SIG_SR);
- hwstrap = mt7530_read(priv, MT753X_TRAP);
+ regmap_read(priv->regmap, MT7531_CREV, &val);
+ regmap_read(priv->regmap, MT7531_TOP_SIG_SR, &top_sig);
+ regmap_read(priv->regmap, MT753X_TRAP, &hwstrap);
if ((val & CHIP_REV_M) > 0)
xtal = (top_sig & PAD_MCM_SMI_EN) ? MT7531_XTAL_FSEL_40MHZ :
MT7531_XTAL_FSEL_25MHZ;
@@ -376,26 +365,26 @@ mt7531_pll_setup(struct mt7530_priv *priv)
MT7531_XTAL_FSEL_40MHZ;
/* Step 1 : Disable MT7531 COREPLL */
- val = mt7530_read(priv, MT7531_PLLGP_EN);
+ regmap_read(priv->regmap, MT7531_PLLGP_EN, &val);
val &= ~EN_COREPLL;
regmap_write(priv->regmap, MT7531_PLLGP_EN, val);
/* Step 2: switch to XTAL output */
- val = mt7530_read(priv, MT7531_PLLGP_EN);
+ regmap_read(priv->regmap, MT7531_PLLGP_EN, &val);
val |= SW_CLKSW;
regmap_write(priv->regmap, MT7531_PLLGP_EN, val);
- val = mt7530_read(priv, MT7531_PLLGP_CR0);
+ regmap_read(priv->regmap, MT7531_PLLGP_CR0, &val);
val &= ~RG_COREPLL_EN;
regmap_write(priv->regmap, MT7531_PLLGP_CR0, val);
/* Step 3: disable PLLGP and enable program PLLGP */
- val = mt7530_read(priv, MT7531_PLLGP_EN);
+ regmap_read(priv->regmap, MT7531_PLLGP_EN, &val);
val |= SW_PLLGP;
regmap_write(priv->regmap, MT7531_PLLGP_EN, val);
/* Step 4: program COREPLL output frequency to 500MHz */
- val = mt7530_read(priv, MT7531_PLLGP_CR0);
+ regmap_read(priv->regmap, MT7531_PLLGP_CR0, &val);
val &= ~RG_COREPLL_POSDIV_M;
val |= 2 << RG_COREPLL_POSDIV_S;
regmap_write(priv->regmap, MT7531_PLLGP_CR0, val);
@@ -403,13 +392,13 @@ mt7531_pll_setup(struct mt7530_priv *priv)
switch (xtal) {
case MT7531_XTAL_FSEL_25MHZ:
- val = mt7530_read(priv, MT7531_PLLGP_CR0);
+ regmap_read(priv->regmap, MT7531_PLLGP_CR0, &val);
val &= ~RG_COREPLL_SDM_PCW_M;
val |= 0x140000 << RG_COREPLL_SDM_PCW_S;
regmap_write(priv->regmap, MT7531_PLLGP_CR0, val);
break;
case MT7531_XTAL_FSEL_40MHZ:
- val = mt7530_read(priv, MT7531_PLLGP_CR0);
+ regmap_read(priv->regmap, MT7531_PLLGP_CR0, &val);
val &= ~RG_COREPLL_SDM_PCW_M;
val |= 0x190000 << RG_COREPLL_SDM_PCW_S;
regmap_write(priv->regmap, MT7531_PLLGP_CR0, val);
@@ -417,14 +406,14 @@ mt7531_pll_setup(struct mt7530_priv *priv)
}
/* Set feedback divide ratio update signal to high */
- val = mt7530_read(priv, MT7531_PLLGP_CR0);
+ regmap_read(priv->regmap, MT7531_PLLGP_CR0, &val);
val |= RG_COREPLL_SDM_PCW_CHG;
regmap_write(priv->regmap, MT7531_PLLGP_CR0, val);
/* Wait for at least 16 XTAL clocks */
usleep_range(10, 20);
/* Step 5: set feedback divide ratio update signal to low */
- val = mt7530_read(priv, MT7531_PLLGP_CR0);
+ regmap_read(priv->regmap, MT7531_PLLGP_CR0, &val);
val &= ~RG_COREPLL_SDM_PCW_CHG;
regmap_write(priv->regmap, MT7531_PLLGP_CR0, val);
@@ -435,11 +424,11 @@ mt7531_pll_setup(struct mt7530_priv *priv)
regmap_write(priv->regmap, MT7531_ANA_PLLGP_CR2, 0x4f40000);
/* Step 6: Enable MT7531 PLL */
- val = mt7530_read(priv, MT7531_PLLGP_CR0);
+ regmap_read(priv->regmap, MT7531_PLLGP_CR0, &val);
val |= RG_COREPLL_EN;
regmap_write(priv->regmap, MT7531_PLLGP_CR0, val);
- val = mt7530_read(priv, MT7531_PLLGP_EN);
+ regmap_read(priv->regmap, MT7531_PLLGP_EN, &val);
val |= EN_COREPLL;
regmap_write(priv->regmap, MT7531_PLLGP_EN, val);
usleep_range(25, 35);
@@ -698,11 +687,11 @@ mt7530_read_port_stats(struct mt7530_priv *priv, int port,
{
u32 val, reg = MT7530_PORT_MIB_COUNTER(port) + offset;
- val = mt7530_read(priv, reg);
+ regmap_read(priv->regmap, reg, &val);
*data = val;
if (size == 2) {
- val = mt7530_read(priv, reg + 4);
+ regmap_read(priv->regmap, reg + 4, &val);
*data |= (u64)val << 32;
}
}
@@ -1006,11 +995,11 @@ static void mt7530_setup_port5(struct dsa_switch *ds, phy_interface_t interface)
{
struct mt7530_priv *priv = ds->priv;
u8 tx_delay = 0;
- int val;
+ u32 val;
mutex_lock(&priv->reg_mutex);
- val = mt7530_read(priv, MT753X_MTRAP);
+ regmap_read(priv->regmap, MT753X_MTRAP, &val);
val &= ~MT7530_P5_PHY0_SEL & ~MT7530_P5_MAC_SEL & ~MT7530_P5_RGMII_MODE;
@@ -1378,7 +1367,7 @@ mt7530_port_change_mtu(struct dsa_switch *ds, int port, int new_mtu)
if (!dsa_is_cpu_port(ds, port))
return 0;
- val = mt7530_read(priv, MT7530_GMACCR);
+ regmap_read(priv->regmap, MT7530_GMACCR, &val);
val &= ~MAX_RX_PKT_LEN_MASK;
/* RX length also includes Ethernet header, MTK tag, and FCS length */
@@ -1577,7 +1566,7 @@ mt7530_vlan_cmd(struct mt7530_priv *priv, enum mt7530_vlan_cmd cmd, u16 vid)
return ret;
}
- val = mt7530_read(priv, MT7530_VTCR);
+ regmap_read(priv->regmap, MT7530_VTCR, &val);
if (val & VTCR_INVALID) {
dev_err(priv->dev, "read VTCR invalid\n");
return -EINVAL;
@@ -1789,14 +1778,16 @@ mt7530_port_mdb_add(struct dsa_switch *ds, int port,
const u8 *addr = mdb->addr;
u16 vid = mdb->vid;
u8 port_mask = 0;
+ u32 val;
int ret;
mutex_lock(&priv->reg_mutex);
mt7530_fdb_write(priv, vid, 0, addr, 0, STATIC_EMP);
- if (!mt7530_fdb_cmd(priv, MT7530_FDB_READ, NULL))
- port_mask = (mt7530_read(priv, MT7530_ATRD) >> PORT_MAP)
- & PORT_MAP_MASK;
+ if (!mt7530_fdb_cmd(priv, MT7530_FDB_READ, NULL)) {
+ regmap_read(priv->regmap, MT7530_ATRD, &val);
+ port_mask = (val >> PORT_MAP) & PORT_MAP_MASK;
+ }
port_mask |= BIT(port);
mt7530_fdb_write(priv, vid, port_mask, addr, -1, STATIC_ENT);
@@ -1816,14 +1807,16 @@ mt7530_port_mdb_del(struct dsa_switch *ds, int port,
const u8 *addr = mdb->addr;
u16 vid = mdb->vid;
u8 port_mask = 0;
+ u32 val;
int ret;
mutex_lock(&priv->reg_mutex);
mt7530_fdb_write(priv, vid, 0, addr, 0, STATIC_EMP);
- if (!mt7530_fdb_cmd(priv, MT7530_FDB_READ, NULL))
- port_mask = (mt7530_read(priv, MT7530_ATRD) >> PORT_MAP)
- & PORT_MAP_MASK;
+ if (!mt7530_fdb_cmd(priv, MT7530_FDB_READ, NULL)) {
+ regmap_read(priv->regmap, MT7530_ATRD, &val);
+ port_mask = (val >> PORT_MAP) & PORT_MAP_MASK;
+ }
port_mask &= ~BIT(port);
mt7530_fdb_write(priv, vid, port_mask, addr, -1,
@@ -1901,7 +1894,7 @@ mt7530_hw_vlan_del(struct mt7530_priv *priv,
new_members = entry->old_members & ~BIT(entry->port);
- val = mt7530_read(priv, MT7530_VAWD1);
+ regmap_read(priv->regmap, MT7530_VAWD1, &val);
if (!(val & VLAN_VALID)) {
dev_err(priv->dev,
"Cannot be deleted due to invalid entry\n");
@@ -1928,7 +1921,7 @@ mt7530_hw_vlan_update(struct mt7530_priv *priv, u16 vid,
/* Fetch entry */
mt7530_vlan_cmd(priv, MT7530_VTCR_RD_VID, vid);
- val = mt7530_read(priv, MT7530_VAWD1);
+ regmap_read(priv->regmap, MT7530_VAWD1, &val);
entry->old_members = (val >> PORT_MEM_SHFT) & PORT_MEM_MASK;
@@ -2046,7 +2039,7 @@ static int mt753x_port_mirror_add(struct dsa_switch *ds, int port,
if ((ingress ? priv->mirror_rx : priv->mirror_tx) & BIT(port))
return -EEXIST;
- val = mt7530_read(priv, MT753X_MIRROR_REG(priv->id));
+ regmap_read(priv->regmap, MT753X_MIRROR_REG(priv->id), &val);
/* MT7530 only supports one monitor port */
monitor_port = MT753X_MIRROR_PORT_GET(priv->id, val);
@@ -2059,7 +2052,7 @@ static int mt753x_port_mirror_add(struct dsa_switch *ds, int port,
val |= MT753X_MIRROR_PORT_SET(priv->id, mirror->to_local_port);
regmap_write(priv->regmap, MT753X_MIRROR_REG(priv->id), val);
- val = mt7530_read(priv, MT7530_PCR_P(port));
+ regmap_read(priv->regmap, MT7530_PCR_P(port), &val);
if (ingress) {
val |= PORT_RX_MIR;
priv->mirror_rx |= BIT(port);
@@ -2078,7 +2071,7 @@ static void mt753x_port_mirror_del(struct dsa_switch *ds, int port,
struct mt7530_priv *priv = ds->priv;
u32 val;
- val = mt7530_read(priv, MT7530_PCR_P(port));
+ regmap_read(priv->regmap, MT7530_PCR_P(port), &val);
if (mirror->ingress) {
val &= ~PORT_RX_MIR;
priv->mirror_rx &= ~BIT(port);
@@ -2089,7 +2082,7 @@ static void mt753x_port_mirror_del(struct dsa_switch *ds, int port,
regmap_write(priv->regmap, MT7530_PCR_P(port), val);
if (!priv->mirror_rx && !priv->mirror_tx) {
- val = mt7530_read(priv, MT753X_MIRROR_REG(priv->id));
+ regmap_read(priv->regmap, MT753X_MIRROR_REG(priv->id), &val);
val &= ~MT753X_MIRROR_EN(priv->id);
regmap_write(priv->regmap, MT753X_MIRROR_REG(priv->id), val);
}
@@ -2121,8 +2114,11 @@ mt7530_gpio_get(struct gpio_chip *gc, unsigned int offset)
{
struct mt7530_priv *priv = gpiochip_get_data(gc);
u32 bit = mt7530_gpio_to_bit(offset);
+ u32 val;
+
+ regmap_read(priv->regmap, MT7530_LED_GPIO_DATA, &val);
- return !!(mt7530_read(priv, MT7530_LED_GPIO_DATA) & bit);
+ return !!(val & bit);
}
static int
@@ -2144,8 +2140,11 @@ mt7530_gpio_get_direction(struct gpio_chip *gc, unsigned int offset)
{
struct mt7530_priv *priv = gpiochip_get_data(gc);
u32 bit = mt7530_gpio_to_bit(offset);
+ u32 val;
+
+ regmap_read(priv->regmap, MT7530_LED_GPIO_DIR, &val);
- return (mt7530_read(priv, MT7530_LED_GPIO_DIR) & bit) ?
+ return (val & bit) ?
GPIO_LINE_DIRECTION_OUT : GPIO_LINE_DIRECTION_IN;
}
@@ -2436,7 +2435,7 @@ mt7530_setup(struct dsa_switch *ds)
return ret;
}
- id = mt7530_read(priv, MT7530_CREV);
+ regmap_read(priv->regmap, MT7530_CREV, &id);
id >>= CHIP_NAME_SHIFT;
if (id != MT7530_ID) {
dev_err(priv->dev, "chip %x can't be supported\n", id);
@@ -2679,7 +2678,7 @@ mt7531_setup(struct dsa_switch *ds)
return ret;
}
- id = mt7530_read(priv, MT7531_CREV);
+ regmap_read(priv->regmap, MT7531_CREV, &id);
id >>= CHIP_NAME_SHIFT;
if (id != MT7531_ID) {
@@ -2690,7 +2689,7 @@ mt7531_setup(struct dsa_switch *ds)
/* MT7531AE has got two SGMII units. One for port 5, one for port 6.
* MT7531BE has got only one SGMII unit which is for port 6.
*/
- val = mt7530_read(priv, MT7531_TOP_SIG_SR);
+ regmap_read(priv->regmap, MT7531_TOP_SIG_SR, &val);
priv->p5_sgmii = !!(val & PAD_DUAL_SGMII_EN);
/* Force link down on all ports before internal reset */
@@ -2880,7 +2879,7 @@ static void mt7531_rgmii_setup(struct mt7530_priv *priv,
{
u32 val;
- val = mt7530_read(priv, MT7531_CLKGEN_CTRL);
+ regmap_read(priv->regmap, MT7531_CLKGEN_CTRL, &val);
val |= GP_CLK_EN;
val &= ~GP_MODE_MASK;
val |= GP_MODE(MT7531_GP_MODE_RGMII);
@@ -3059,7 +3058,7 @@ static void mt753x_phylink_get_caps(struct dsa_switch *ds, int port,
config->lpi_capabilities = MAC_100FD | MAC_1000FD | MAC_2500FD;
- eeecr = mt7530_read(priv, MT753X_PMEEECR_P(port));
+ regmap_read(priv->regmap, MT753X_PMEEECR_P(port), &eeecr);
/* tx_lpi_timer should be in microseconds. The time units for
* LPI threshold are unspecified.
*/
@@ -3087,7 +3086,7 @@ static void mt7530_pcs_get_state(struct phylink_pcs *pcs, unsigned int neg_mode,
int port = pcs_to_mt753x_pcs(pcs)->port;
u32 pmsr;
- pmsr = mt7530_read(priv, MT7530_PMSR_P(port));
+ regmap_read(priv->regmap, MT7530_PMSR_P(port), &pmsr);
state->link = (pmsr & PMSR_LINK);
state->an_complete = state->link;
--
2.55.0
^ permalink raw reply related
* [PATCH net-next v4 6/8] net: dsa: mt7530: convert to use field accessor macros
From: Daniel Golle @ 2026-07-19 17:54 UTC (permalink / raw)
To: Chester A. Unal, Daniel Golle, Andrew Lunn, Vladimir Oltean,
David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Matthias Brugger, AngeloGioacchino Del Regno, Russell King,
netdev, linux-kernel, linux-arm-kernel, linux-mediatek
In-Reply-To: <cover.1784481922.git.daniel@makrotopia.org>
Use FIELD_GET and FIELD_PREP instead of open-coding register fields.
Replace 0x1f constant with (PHY_MAX_ADDR - 1).
Some field macros (ATC_HASH and VTCR_VID) were previously defined as
object-like macros referencing an undeclared 'x' and were therefore
unusable; convert them into proper FIELD_PREP() accessors. The masks are
equivalent to the open-coded values they replace, so there is no
functional change.
Signed-off-by: Daniel Golle <daniel@makrotopia.org>
---
v3:
* name the age timer field AGE_TIMER_MASK instead of AGE_TIMER_RD_MASK
* note the corrected ATC_HASH/VTCR_VID macros in the commit message
---
drivers/net/dsa/mt7530.c | 64 ++++++------
drivers/net/dsa/mt7530.h | 208 ++++++++++++++++++++++-----------------
2 files changed, 148 insertions(+), 124 deletions(-)
diff --git a/drivers/net/dsa/mt7530.c b/drivers/net/dsa/mt7530.c
index 26c7f3de4e1a..f7643b86f3c4 100644
--- a/drivers/net/dsa/mt7530.c
+++ b/drivers/net/dsa/mt7530.c
@@ -208,16 +208,16 @@ mt7530_fdb_read(struct mt7530_priv *priv, struct mt7530_fdb *fdb)
__func__, __LINE__, i, reg[i]);
}
- fdb->vid = (reg[1] >> CVID) & CVID_MASK;
- fdb->aging = (reg[2] >> AGE_TIMER) & AGE_TIMER_MASK;
- fdb->port_mask = (reg[2] >> PORT_MAP) & PORT_MAP_MASK;
- fdb->mac[0] = (reg[0] >> MAC_BYTE_0) & MAC_BYTE_MASK;
- fdb->mac[1] = (reg[0] >> MAC_BYTE_1) & MAC_BYTE_MASK;
- fdb->mac[2] = (reg[0] >> MAC_BYTE_2) & MAC_BYTE_MASK;
- fdb->mac[3] = (reg[0] >> MAC_BYTE_3) & MAC_BYTE_MASK;
- fdb->mac[4] = (reg[1] >> MAC_BYTE_4) & MAC_BYTE_MASK;
- fdb->mac[5] = (reg[1] >> MAC_BYTE_5) & MAC_BYTE_MASK;
- fdb->noarp = ((reg[2] >> ENT_STATUS) & ENT_STATUS_MASK) == STATIC_ENT;
+ fdb->vid = FIELD_GET(CVID_MASK, reg[1]);
+ fdb->aging = FIELD_GET(AGE_TIMER_MASK, reg[2]);
+ fdb->port_mask = FIELD_GET(PORT_MAP_MASK, reg[2]);
+ fdb->mac[0] = FIELD_GET(MAC_BYTE_0_MASK, reg[0]);
+ fdb->mac[1] = FIELD_GET(MAC_BYTE_1_MASK, reg[0]);
+ fdb->mac[2] = FIELD_GET(MAC_BYTE_2_MASK, reg[0]);
+ fdb->mac[3] = FIELD_GET(MAC_BYTE_3_MASK, reg[0]);
+ fdb->mac[4] = FIELD_GET(MAC_BYTE_4_MASK, reg[1]);
+ fdb->mac[5] = FIELD_GET(MAC_BYTE_5_MASK, reg[1]);
+ fdb->noarp = FIELD_GET(ENT_STATUS_MASK, reg[2]) == STATIC_ENT;
}
static void
@@ -228,22 +228,22 @@ mt7530_fdb_write(struct mt7530_priv *priv, u16 vid,
u32 reg[3] = { 0 };
int i;
- reg[1] |= vid & CVID_MASK;
+ reg[1] |= FIELD_PREP(CVID_MASK, vid);
reg[1] |= ATA2_IVL;
reg[1] |= ATA2_FID(FID_BRIDGED);
- reg[2] |= (aging & AGE_TIMER_MASK) << AGE_TIMER;
- reg[2] |= (port_mask & PORT_MAP_MASK) << PORT_MAP;
+ reg[2] |= FIELD_PREP(AGE_TIMER_MASK, aging);
+ reg[2] |= FIELD_PREP(PORT_MAP_MASK, port_mask);
/* STATIC_ENT indicate that entry is static wouldn't
* be aged out and STATIC_EMP specified as erasing an
* entry
*/
- reg[2] |= (type & ENT_STATUS_MASK) << ENT_STATUS;
- reg[1] |= mac[5] << MAC_BYTE_5;
- reg[1] |= mac[4] << MAC_BYTE_4;
- reg[0] |= mac[3] << MAC_BYTE_3;
- reg[0] |= mac[2] << MAC_BYTE_2;
- reg[0] |= mac[1] << MAC_BYTE_1;
- reg[0] |= mac[0] << MAC_BYTE_0;
+ reg[2] |= FIELD_PREP(ENT_STATUS_MASK, type);
+ reg[1] |= FIELD_PREP(MAC_BYTE_5_MASK, mac[5]);
+ reg[1] |= FIELD_PREP(MAC_BYTE_4_MASK, mac[4]);
+ reg[0] |= FIELD_PREP(MAC_BYTE_3_MASK, mac[3]);
+ reg[0] |= FIELD_PREP(MAC_BYTE_2_MASK, mac[2]);
+ reg[0] |= FIELD_PREP(MAC_BYTE_1_MASK, mac[1]);
+ reg[0] |= FIELD_PREP(MAC_BYTE_0_MASK, mac[0]);
/* Write array into the ARL table */
for (i = 0; i < 3; i++)
@@ -385,22 +385,22 @@ mt7531_pll_setup(struct mt7530_priv *priv)
/* Step 4: program COREPLL output frequency to 500MHz */
regmap_read(priv->regmap, MT7531_PLLGP_CR0, &val);
- val &= ~RG_COREPLL_POSDIV_M;
- val |= 2 << RG_COREPLL_POSDIV_S;
+ val &= ~RG_COREPLL_POSDIV_MASK;
+ val |= RG_COREPLL_POSDIV(2);
regmap_write(priv->regmap, MT7531_PLLGP_CR0, val);
usleep_range(25, 35);
switch (xtal) {
case MT7531_XTAL_FSEL_25MHZ:
regmap_read(priv->regmap, MT7531_PLLGP_CR0, &val);
- val &= ~RG_COREPLL_SDM_PCW_M;
- val |= 0x140000 << RG_COREPLL_SDM_PCW_S;
+ val &= ~RG_COREPLL_SDM_PCW_MASK;
+ val |= RG_COREPLL_SDM_PCW(0x140000);
regmap_write(priv->regmap, MT7531_PLLGP_CR0, val);
break;
case MT7531_XTAL_FSEL_40MHZ:
regmap_read(priv->regmap, MT7531_PLLGP_CR0, &val);
- val &= ~RG_COREPLL_SDM_PCW_M;
- val |= 0x190000 << RG_COREPLL_SDM_PCW_S;
+ val &= ~RG_COREPLL_SDM_PCW_MASK;
+ val |= RG_COREPLL_SDM_PCW(0x190000);
regmap_write(priv->regmap, MT7531_PLLGP_CR0, val);
break;
}
@@ -1555,7 +1555,7 @@ mt7530_vlan_cmd(struct mt7530_priv *priv, enum mt7530_vlan_cmd cmd, u16 vid)
u32 val;
int ret;
- val = VTCR_BUSY | VTCR_FUNC(cmd) | vid;
+ val = VTCR_BUSY | VTCR_FUNC(cmd) | VTCR_VID(vid);
regmap_write(priv->regmap, MT7530_VTCR, val);
INIT_MT7530_DUMMY_POLL(&p, priv, MT7530_VTCR);
@@ -1786,7 +1786,7 @@ mt7530_port_mdb_add(struct dsa_switch *ds, int port,
mt7530_fdb_write(priv, vid, 0, addr, 0, STATIC_EMP);
if (!mt7530_fdb_cmd(priv, MT7530_FDB_READ, NULL)) {
regmap_read(priv->regmap, MT7530_ATRD, &val);
- port_mask = (val >> PORT_MAP) & PORT_MAP_MASK;
+ port_mask = FIELD_GET(PORT_MAP_MASK, val);
}
port_mask |= BIT(port);
@@ -1815,7 +1815,7 @@ mt7530_port_mdb_del(struct dsa_switch *ds, int port,
mt7530_fdb_write(priv, vid, 0, addr, 0, STATIC_EMP);
if (!mt7530_fdb_cmd(priv, MT7530_FDB_READ, NULL)) {
regmap_read(priv->regmap, MT7530_ATRD, &val);
- port_mask = (val >> PORT_MAP) & PORT_MAP_MASK;
+ port_mask = FIELD_GET(PORT_MAP_MASK, val);
}
port_mask &= ~BIT(port);
@@ -1923,7 +1923,7 @@ mt7530_hw_vlan_update(struct mt7530_priv *priv, u16 vid,
regmap_read(priv->regmap, MT7530_VAWD1, &val);
- entry->old_members = (val >> PORT_MEM_SHFT) & PORT_MEM_MASK;
+ entry->old_members = FIELD_GET(PORT_MEM_MASK, val);
/* Manipulate entry */
vlan_op(priv, entry);
@@ -2436,7 +2436,7 @@ mt7530_setup(struct dsa_switch *ds)
}
regmap_read(priv->regmap, MT7530_CREV, &id);
- id >>= CHIP_NAME_SHIFT;
+ id = FIELD_GET(CHIP_NAME_MASK, id);
if (id != MT7530_ID) {
dev_err(priv->dev, "chip %x can't be supported\n", id);
return -ENODEV;
@@ -2679,7 +2679,7 @@ mt7531_setup(struct dsa_switch *ds)
}
regmap_read(priv->regmap, MT7531_CREV, &id);
- id >>= CHIP_NAME_SHIFT;
+ id = FIELD_GET(CHIP_NAME_MASK, id);
if (id != MT7531_ID) {
dev_err(priv->dev, "chip %x can't be supported\n", id);
diff --git a/drivers/net/dsa/mt7530.h b/drivers/net/dsa/mt7530.h
index dd33b0df3419..804c2e0991a0 100644
--- a/drivers/net/dsa/mt7530.h
+++ b/drivers/net/dsa/mt7530.h
@@ -6,6 +6,8 @@
#ifndef __MT7530_H
#define __MT7530_H
+#include <linux/bitfield.h>
+
#define MT7530_NUM_PORTS 7
#define MT7530_NUM_PHYS 5
#define MT7530_NUM_FDB_RECORDS 2048
@@ -146,19 +148,22 @@ enum mt753x_to_cpu_fw {
#define STATIC_ENT 3
#define MT7530_ATA2 0x78
#define ATA2_IVL BIT(15)
-#define ATA2_FID(x) (((x) & 0x7) << 12)
+#define ATA2_FID_MASK GENMASK(14, 12)
+#define ATA2_FID(x) FIELD_PREP(ATA2_FID_MASK, x)
/* Register for address table write data */
#define MT7530_ATWD 0x7c
/* Register for address table control */
#define MT7530_ATC 0x80
-#define ATC_HASH (((x) & 0xfff) << 16)
+#define ATC_HASH_MASK GENMASK(27, 16)
+#define ATC_HASH(x) FIELD_PREP(ATC_HASH_MASK, x)
#define ATC_BUSY BIT(15)
#define ATC_SRCH_END BIT(14)
#define ATC_SRCH_HIT BIT(13)
#define ATC_INVALID BIT(12)
-#define ATC_MAT(x) (((x) & 0xf) << 8)
+#define ATC_MAT_MASK GENMASK(11, 8)
+#define ATC_MAT(x) FIELD_PREP(ATC_MAT_MASK, x)
#define ATC_MAT_MACTAB ATC_MAT(0)
enum mt7530_fdb_cmd {
@@ -171,32 +176,29 @@ enum mt7530_fdb_cmd {
/* Registers for table search read address */
#define MT7530_TSRA1 0x84
-#define MAC_BYTE_0 24
-#define MAC_BYTE_1 16
-#define MAC_BYTE_2 8
-#define MAC_BYTE_3 0
-#define MAC_BYTE_MASK 0xff
+#define MAC_BYTE_0_MASK GENMASK(31, 24)
+#define MAC_BYTE_1_MASK GENMASK(23, 16)
+#define MAC_BYTE_2_MASK GENMASK(15, 8)
+#define MAC_BYTE_3_MASK GENMASK(7, 0)
#define MT7530_TSRA2 0x88
-#define MAC_BYTE_4 24
-#define MAC_BYTE_5 16
-#define CVID 0
-#define CVID_MASK 0xfff
+#define MAC_BYTE_4_MASK GENMASK(31, 24)
+#define MAC_BYTE_5_MASK GENMASK(23, 16)
+#define CVID_MASK GENMASK(11, 0)
#define MT7530_ATRD 0x8C
-#define AGE_TIMER 24
-#define AGE_TIMER_MASK 0xff
-#define PORT_MAP 4
-#define PORT_MAP_MASK 0xff
-#define ENT_STATUS 2
-#define ENT_STATUS_MASK 0x3
+#define AGE_TIMER_MASK GENMASK(31, 24)
+#define PORT_MAP_MASK GENMASK(11, 4)
+#define ENT_STATUS_MASK GENMASK(3, 2)
/* Register for vlan table control */
#define MT7530_VTCR 0x90
#define VTCR_BUSY BIT(31)
#define VTCR_INVALID BIT(16)
-#define VTCR_FUNC(x) (((x) & 0xf) << 12)
-#define VTCR_VID ((x) & 0xfff)
+#define VTCR_FUNC_MASK GENMASK(15, 12)
+#define VTCR_FUNC(x) FIELD_PREP(VTCR_FUNC_MASK, x)
+#define VTCR_VID_MASK GENMASK(11, 0)
+#define VTCR_VID(x) FIELD_PREP(VTCR_VID_MASK, x)
enum mt7530_vlan_cmd {
/* Read/Write the specified VID entry from VAWD register based
@@ -216,13 +218,13 @@ enum mt7530_vlan_cmd {
/* Per VLAN Egress Tag Control */
#define VTAG_EN BIT(28)
/* VLAN Member Control */
-#define PORT_MEM(x) (((x) & 0xff) << 16)
+#define PORT_MEM_MASK GENMASK(23, 16)
+#define PORT_MEM(x) FIELD_PREP(PORT_MEM_MASK, x)
/* Filter ID */
-#define FID(x) (((x) & 0x7) << 1)
+#define FID_MASK GENMASK(3, 1)
+#define FID(x) FIELD_PREP(FID_MASK, x)
/* VLAN Entry Valid */
#define VLAN_VALID BIT(0)
-#define PORT_MEM_SHFT 16
-#define PORT_MEM_MASK 0xff
enum mt7530_fid {
FID_STANDALONE = 0,
@@ -247,11 +249,11 @@ enum mt7530_vlan_egress_attr {
/* Age count */
#define AGE_CNT_MASK GENMASK(19, 12)
#define AGE_CNT_MAX 0xff
-#define AGE_CNT(x) (AGE_CNT_MASK & ((x) << 12))
+#define AGE_CNT(x) FIELD_PREP(AGE_CNT_MASK, x)
/* Age unit */
#define AGE_UNIT_MASK GENMASK(11, 0)
#define AGE_UNIT_MAX 0xfff
-#define AGE_UNIT(x) (AGE_UNIT_MASK & (x))
+#define AGE_UNIT(x) FIELD_PREP(AGE_UNIT_MASK, x)
#define MT753X_ERLCR_P(x) (0x1040 + ((x) * 0x100))
#define ERLCR_CIR_MASK GENMASK(31, 16)
@@ -282,30 +284,31 @@ enum mt7530_stp_state {
#define MT7530_PCR_P(x) (0x2004 + ((x) * 0x100))
#define PORT_TX_MIR BIT(9)
#define PORT_RX_MIR BIT(8)
-#define PORT_VLAN(x) ((x) & 0x3)
+#define PCR_PORT_VLAN_MASK GENMASK(1, 0)
enum mt7530_port_mode {
/* Port Matrix Mode: Frames are forwarded by the PCR_MATRIX members. */
- MT7530_PORT_MATRIX_MODE = PORT_VLAN(0),
+ MT7530_PORT_MATRIX_MODE = 0,
/* Fallback Mode: Forward received frames with ingress ports that do
* not belong to the VLAN member. Frames whose VID is not listed on
* the VLAN table are forwarded by the PCR_MATRIX members.
*/
- MT7530_PORT_FALLBACK_MODE = PORT_VLAN(1),
+ MT7530_PORT_FALLBACK_MODE = 1,
/* Security Mode: Discard any frame due to ingress membership
* violation or VID missed on the VLAN table.
*/
- MT7530_PORT_SECURITY_MODE = PORT_VLAN(3),
+ MT7530_PORT_SECURITY_MODE = 3,
};
-#define PCR_MATRIX(x) (((x) & 0xff) << 16)
-#define PORT_PRI(x) (((x) & 0x7) << 24)
-#define EG_TAG(x) (((x) & 0x3) << 28)
-#define PCR_MATRIX_MASK PCR_MATRIX(0xff)
+#define PCR_MATRIX_MASK GENMASK(23, 16)
+#define PCR_MATRIX(x) FIELD_PREP(PCR_MATRIX_MASK, x)
+#define PORT_PRI_MASK GENMASK(26, 24)
+#define PORT_PRI(x) FIELD_PREP(PORT_PRI_MASK, x)
+#define EG_TAG_MASK GENMASK(29, 28)
+#define EG_TAG(x) FIELD_PREP(EG_TAG_MASK, x)
#define PCR_MATRIX_CLR PCR_MATRIX(0)
-#define PCR_PORT_VLAN_MASK PORT_VLAN(3)
/* Register for port security control */
#define MT7530_PSC_P(x) (0x200c + ((x) * 0x100))
@@ -314,10 +317,10 @@ enum mt7530_port_mode {
/* Register for port vlan control */
#define MT7530_PVC_P(x) (0x2010 + ((x) * 0x100))
#define PORT_SPEC_TAG BIT(5)
-#define PVC_EG_TAG(x) (((x) & 0x7) << 8)
-#define PVC_EG_TAG_MASK PVC_EG_TAG(7)
-#define VLAN_ATTR(x) (((x) & 0x3) << 6)
-#define VLAN_ATTR_MASK VLAN_ATTR(3)
+#define PVC_EG_TAG_MASK GENMASK(10, 8)
+#define PVC_EG_TAG(x) FIELD_PREP(PVC_EG_TAG_MASK, x)
+#define VLAN_ATTR_MASK GENMASK(7, 6)
+#define VLAN_ATTR(x) FIELD_PREP(VLAN_ATTR_MASK, x)
#define ACC_FRM_MASK GENMASK(1, 0)
enum mt7530_vlan_port_eg_tag {
@@ -337,12 +340,13 @@ enum mt7530_vlan_port_acc_frm {
MT7530_VLAN_ACC_UNTAGGED = 2,
};
-#define STAG_VPID (((x) & 0xffff) << 16)
+#define STAG_VPID_MASK GENMASK(31, 16)
+#define STAG_VPID(x) FIELD_PREP(STAG_VPID_MASK, x)
/* Register for port port-and-protocol based vlan 1 control */
#define MT7530_PPBV1_P(x) (0x2014 + ((x) * 0x100))
-#define G0_PORT_VID(x) (((x) & 0xfff) << 0)
-#define G0_PORT_VID_MASK G0_PORT_VID(0xfff)
+#define G0_PORT_VID_MASK GENMASK(11, 0)
+#define G0_PORT_VID(x) FIELD_PREP(G0_PORT_VID_MASK, x)
#define G0_PORT_VID_DEF G0_PORT_VID(0)
/* Register for port MAC control register */
@@ -418,8 +422,8 @@ enum mt7530_vlan_port_acc_frm {
#define MT7531_DIS_CLR BIT(31)
#define MT7530_GMACCR 0x30e0
-#define MAX_RX_JUMBO(x) ((x) << 2)
#define MAX_RX_JUMBO_MASK GENMASK(5, 2)
+#define MAX_RX_JUMBO(x) FIELD_PREP(MAX_RX_JUMBO_MASK, x)
#define MAX_RX_PKT_LEN_MASK GENMASK(1, 0)
#define MAX_RX_PKT_LEN_1522 0x0
#define MAX_RX_PKT_LEN_1536 0x1
@@ -505,16 +509,16 @@ enum mt7530_vlan_port_acc_frm {
/* Register for PHY Indirect Access Control */
#define MT7531_PHY_IAC 0x701C
#define MT7531_PHY_ACS_ST BIT(31)
-#define MT7531_MDIO_REG_ADDR_MASK (0x1f << 25)
-#define MT7531_MDIO_PHY_ADDR_MASK (0x1f << 20)
-#define MT7531_MDIO_CMD_MASK (0x3 << 18)
-#define MT7531_MDIO_ST_MASK (0x3 << 16)
-#define MT7531_MDIO_RW_DATA_MASK (0xffff)
-#define MT7531_MDIO_REG_ADDR(x) (((x) & 0x1f) << 25)
-#define MT7531_MDIO_DEV_ADDR(x) (((x) & 0x1f) << 25)
-#define MT7531_MDIO_PHY_ADDR(x) (((x) & 0x1f) << 20)
-#define MT7531_MDIO_CMD(x) (((x) & 0x3) << 18)
-#define MT7531_MDIO_ST(x) (((x) & 0x3) << 16)
+#define MT7531_MDIO_REG_ADDR_MASK GENMASK(29, 25)
+#define MT7531_MDIO_PHY_ADDR_MASK GENMASK(24, 20)
+#define MT7531_MDIO_CMD_MASK GENMASK(19, 18)
+#define MT7531_MDIO_ST_MASK GENMASK(17, 16)
+#define MT7531_MDIO_RW_DATA_MASK GENMASK(15, 0)
+#define MT7531_MDIO_REG_ADDR(x) FIELD_PREP(MT7531_MDIO_REG_ADDR_MASK, x)
+#define MT7531_MDIO_DEV_ADDR(x) FIELD_PREP(MT7531_MDIO_REG_ADDR_MASK, x)
+#define MT7531_MDIO_PHY_ADDR(x) FIELD_PREP(MT7531_MDIO_PHY_ADDR_MASK, x)
+#define MT7531_MDIO_CMD(x) FIELD_PREP(MT7531_MDIO_CMD_MASK, x)
+#define MT7531_MDIO_ST(x) FIELD_PREP(MT7531_MDIO_ST_MASK, x)
enum mt7531_phy_iac_cmd {
MT7531_MDIO_ADDR = 0,
@@ -542,14 +546,14 @@ enum mt7531_mdio_st {
/* Register for RGMII clock phase */
#define MT7531_CLKGEN_CTRL 0x7500
-#define CLK_SKEW_OUT(x) (((x) & 0x3) << 8)
#define CLK_SKEW_OUT_MASK GENMASK(9, 8)
-#define CLK_SKEW_IN(x) (((x) & 0x3) << 6)
+#define CLK_SKEW_OUT(x) FIELD_PREP(CLK_SKEW_OUT_MASK, x)
#define CLK_SKEW_IN_MASK GENMASK(7, 6)
+#define CLK_SKEW_IN(x) FIELD_PREP(CLK_SKEW_IN_MASK, x)
#define RXCLK_NO_DELAY BIT(5)
#define TXCLK_NO_REVERSE BIT(4)
-#define GP_MODE(x) (((x) & 0x3) << 1)
#define GP_MODE_MASK GENMASK(2, 1)
+#define GP_MODE(x) FIELD_PREP(GP_MODE_MASK, x)
#define GP_CLK_EN BIT(0)
enum mt7531_gp_mode {
@@ -599,8 +603,10 @@ enum mt7531_xtal_fsel {
#define PAD_MCM_SMI_EN BIT(0)
#define MT7530_IO_DRV_CR 0x7810
-#define P5_IO_CLK_DRV(x) ((x) & 0x3)
-#define P5_IO_DATA_DRV(x) (((x) & 0x3) << 4)
+#define P5_IO_CLK_DRV_MASK GENMASK(1, 0)
+#define P5_IO_CLK_DRV(x) FIELD_PREP(P5_IO_CLK_DRV_MASK, x)
+#define P5_IO_DATA_DRV_MASK GENMASK(5, 4)
+#define P5_IO_DATA_DRV(x) FIELD_PREP(P5_IO_DATA_DRV_MASK, x)
#define MT7531_CHIP_REV 0x781C
@@ -610,15 +616,15 @@ enum mt7531_xtal_fsel {
#define SW_PLLGP BIT(0)
#define MT7530_P6ECR 0x7830
-#define P6_INTF_MODE_MASK 0x3
-#define P6_INTF_MODE(x) ((x) & 0x3)
+#define P6_INTF_MODE_MASK GENMASK(1, 0)
+#define P6_INTF_MODE(x) FIELD_PREP(P6_INTF_MODE_MASK, x)
#define MT7531_PLLGP_CR0 0x78a8
#define RG_COREPLL_EN BIT(22)
-#define RG_COREPLL_POSDIV_S 23
-#define RG_COREPLL_POSDIV_M 0x3800000
-#define RG_COREPLL_SDM_PCW_S 1
-#define RG_COREPLL_SDM_PCW_M 0x3ffffe
+#define RG_COREPLL_POSDIV_MASK GENMASK(25, 23)
+#define RG_COREPLL_POSDIV(x) FIELD_PREP(RG_COREPLL_POSDIV_MASK, x)
+#define RG_COREPLL_SDM_PCW_MASK GENMASK(21, 1)
+#define RG_COREPLL_SDM_PCW(x) FIELD_PREP(RG_COREPLL_SDM_PCW_MASK, x)
#define RG_COREPLL_SDM_PCW_CHG BIT(0)
/* Registers for RGMII and SGMII PLL clock */
@@ -629,10 +635,10 @@ enum mt7531_xtal_fsel {
#define MT7530_TRGMII_RCK_CTRL 0x7a00
#define RX_RST BIT(31)
#define RXC_DQSISEL BIT(30)
-#define DQSI1_TAP_MASK (0x7f << 8)
-#define DQSI0_TAP_MASK 0x7f
-#define DQSI1_TAP(x) (((x) & 0x7f) << 8)
-#define DQSI0_TAP(x) ((x) & 0x7f)
+#define DQSI1_TAP_MASK GENMASK(14, 8)
+#define DQSI0_TAP_MASK GENMASK(6, 0)
+#define DQSI1_TAP(x) FIELD_PREP(DQSI1_TAP_MASK, x)
+#define DQSI0_TAP(x) FIELD_PREP(DQSI0_TAP_MASK, x)
#define MT7530_TRGMII_RCK_RTT 0x7a04
#define DQS1_GATE BIT(31)
@@ -641,8 +647,8 @@ enum mt7531_xtal_fsel {
#define MT7530_TRGMII_RD(x) (0x7a10 + (x) * 8)
#define BSLIP_EN BIT(31)
#define EDGE_CHK BIT(30)
-#define RD_TAP_MASK 0x7f
-#define RD_TAP(x) ((x) & 0x7f)
+#define RD_TAP_MASK GENMASK(6, 0)
+#define RD_TAP(x) FIELD_PREP(RD_TAP_MASK, x)
#define MT7530_TRGMII_TXCTRL 0x7a40
#define TRAIN_TXEN BIT(31)
@@ -650,18 +656,23 @@ enum mt7531_xtal_fsel {
#define TX_RST BIT(28)
#define MT7530_TRGMII_TD_ODT(i) (0x7a54 + 8 * (i))
-#define TD_DM_DRVP(x) ((x) & 0xf)
-#define TD_DM_DRVN(x) (((x) & 0xf) << 4)
+#define TD_DM_DRVP_MASK GENMASK(3, 0)
+#define TD_DM_DRVP(x) FIELD_PREP(TD_DM_DRVP_MASK, x)
+#define TD_DM_DRVN_MASK GENMASK(7, 4)
+#define TD_DM_DRVN(x) FIELD_PREP(TD_DM_DRVN_MASK, x)
#define MT7530_TRGMII_TCK_CTRL 0x7a78
-#define TCK_TAP(x) (((x) & 0xf) << 8)
+#define TCK_TAP_MASK GENMASK(11, 8)
+#define TCK_TAP(x) FIELD_PREP(TCK_TAP_MASK, x)
#define MT7530_P5RGMIIRXCR 0x7b00
#define CSR_RGMII_EDGE_ALIGN BIT(8)
-#define CSR_RGMII_RXC_0DEG_CFG(x) ((x) & 0xf)
+#define CSR_RGMII_RXC_0DEG_CFG_MASK GENMASK(3, 0)
+#define CSR_RGMII_RXC_0DEG_CFG(x) FIELD_PREP(CSR_RGMII_RXC_0DEG_CFG_MASK, x)
#define MT7530_P5RGMIITXCR 0x7b04
-#define CSR_RGMII_TXC_CFG(x) ((x) & 0x1f)
+#define CSR_RGMII_TXC_CFG_MASK GENMASK(4, 0)
+#define CSR_RGMII_TXC_CFG(x) FIELD_PREP(CSR_RGMII_TXC_CFG_MASK, x)
/* Registers for GPIO mode */
#define MT7531_GPIO_MODE0 0x7c0c
@@ -670,9 +681,9 @@ enum mt7531_xtal_fsel {
#define MT7531_GPIO_MODE1 0x7c10
#define MT7531_GPIO11_RG_RXD2_MASK GENMASK(15, 12)
-#define MT7531_EXT_P_MDC_11 (2 << 12)
+#define MT7531_EXT_P_MDC_11 FIELD_PREP(MT7531_GPIO11_RG_RXD2_MASK, 2)
#define MT7531_GPIO12_RG_RXD3_MASK GENMASK(19, 16)
-#define MT7531_EXT_P_MDIO_12 (2 << 16)
+#define MT7531_EXT_P_MDIO_12 FIELD_PREP(MT7531_GPIO12_RG_RXD3_MASK, 2)
#define MT753X_CPORT_SPTAG_CFG 0x7c10
#define CPORT_SW2FE_STAG_EN BIT(1)
@@ -704,7 +715,7 @@ enum mt7531_xtal_fsel {
#define MT7530_LED_GPIO_DATA 0x7d18
#define MT7530_CREV 0x7ffc
-#define CHIP_NAME_SHIFT 16
+#define CHIP_NAME_MASK GENMASK(31, 16)
#define MT7530_ID 0x7530
#define MT7531_CREV 0x781C
@@ -716,10 +727,13 @@ enum mt7531_xtal_fsel {
#define RG_SYSPLL_EN_NORMAL BIT(15)
#define RG_SYSPLL_VODEN BIT(14)
#define RG_SYSPLL_LF BIT(13)
-#define RG_SYSPLL_RST_DLY(x) (((x) & 0x3) << 12)
+#define RG_SYSPLL_RST_DLY_MASK GENMASK(13, 12)
+#define RG_SYSPLL_RST_DLY(x) FIELD_PREP(RG_SYSPLL_RST_DLY_MASK, x)
#define RG_SYSPLL_LVROD_EN BIT(10)
-#define RG_SYSPLL_PREDIV(x) (((x) & 0x3) << 8)
-#define RG_SYSPLL_POSDIV(x) (((x) & 0x3) << 5)
+#define RG_SYSPLL_PREDIV_MASK GENMASK(9, 8)
+#define RG_SYSPLL_PREDIV(x) FIELD_PREP(RG_SYSPLL_PREDIV_MASK, x)
+#define RG_SYSPLL_POSDIV_MASK GENMASK(6, 5)
+#define RG_SYSPLL_POSDIV(x) FIELD_PREP(RG_SYSPLL_POSDIV_MASK, x)
#define RG_SYSPLL_FBKSEL BIT(4)
#define RT_SYSPLL_EN_AFE_OLT BIT(0)
@@ -731,38 +745,48 @@ enum mt7531_xtal_fsel {
#define MT7531_PHY_PLL_OFF BIT(5)
#define MT7531_PHY_PLL_BYPASS_MODE BIT(4)
-#define MT753X_CTRL_PHY_ADDR(addr) ((addr + 1) & 0x1f)
+#define MT753X_CTRL_PHY_ADDR(addr) (((addr) + 1) & (PHY_MAX_ADDR - 1))
#define CORE_PLL_GROUP5 0x404
-#define RG_LCDDS_PCW_NCPO1(x) ((x) & 0xffff)
+#define RG_LCDDS_PCW_NCPO1_MASK GENMASK(15, 0)
+#define RG_LCDDS_PCW_NCPO1(x) FIELD_PREP(RG_LCDDS_PCW_NCPO1_MASK, x)
#define CORE_PLL_GROUP6 0x405
-#define RG_LCDDS_PCW_NCPO0(x) ((x) & 0xffff)
+#define RG_LCDDS_PCW_NCPO0_MASK GENMASK(15, 0)
+#define RG_LCDDS_PCW_NCPO0(x) FIELD_PREP(RG_LCDDS_PCW_NCPO0_MASK, x)
#define CORE_PLL_GROUP7 0x406
#define RG_LCDDS_PWDB BIT(15)
#define RG_LCDDS_ISO_EN BIT(13)
-#define RG_LCCDS_C(x) (((x) & 0x7) << 4)
+#define RG_LCCDS_C_MASK GENMASK(6, 4)
+#define RG_LCCDS_C(x) FIELD_PREP(RG_LCCDS_C_MASK, x)
#define RG_LCDDS_PCW_NCPO_CHG BIT(3)
#define CORE_PLL_GROUP10 0x409
-#define RG_LCDDS_SSC_DELTA(x) ((x) & 0xfff)
+#define RG_LCDDS_SSC_DELTA_MASK GENMASK(11, 0)
+#define RG_LCDDS_SSC_DELTA(x) FIELD_PREP(RG_LCDDS_SSC_DELTA_MASK, x)
#define CORE_PLL_GROUP11 0x40a
-#define RG_LCDDS_SSC_DELTA1(x) ((x) & 0xfff)
+#define RG_LCDDS_SSC_DELTA1_MASK GENMASK(11, 0)
+#define RG_LCDDS_SSC_DELTA1(x) FIELD_PREP(RG_LCDDS_SSC_DELTA1_MASK, x)
#define CORE_GSWPLL_GRP1 0x40d
-#define RG_GSWPLL_PREDIV(x) (((x) & 0x3) << 14)
-#define RG_GSWPLL_POSDIV_200M(x) (((x) & 0x3) << 12)
+#define RG_GSWPLL_PREDIV_MASK GENMASK(15, 14)
+#define RG_GSWPLL_PREDIV(x) FIELD_PREP(RG_GSWPLL_PREDIV_MASK, x)
+#define RG_GSWPLL_POSDIV_200M_MASK GENMASK(13, 12)
+#define RG_GSWPLL_POSDIV_200M(x) FIELD_PREP(RG_GSWPLL_POSDIV_200M_MASK, x)
#define RG_GSWPLL_EN_PRE BIT(11)
#define RG_GSWPLL_FBKSEL BIT(10)
#define RG_GSWPLL_BP BIT(9)
#define RG_GSWPLL_BR BIT(8)
-#define RG_GSWPLL_FBKDIV_200M(x) ((x) & 0xff)
+#define RG_GSWPLL_FBKDIV_200M_MASK GENMASK(7, 0)
+#define RG_GSWPLL_FBKDIV_200M(x) FIELD_PREP(RG_GSWPLL_FBKDIV_200M_MASK, x)
#define CORE_GSWPLL_GRP2 0x40e
-#define RG_GSWPLL_POSDIV_500M(x) (((x) & 0x3) << 8)
-#define RG_GSWPLL_FBKDIV_500M(x) ((x) & 0xff)
+#define RG_GSWPLL_POSDIV_500M_MASK GENMASK(9, 8)
+#define RG_GSWPLL_POSDIV_500M(x) FIELD_PREP(RG_GSWPLL_POSDIV_500M_MASK, x)
+#define RG_GSWPLL_FBKDIV_500M_MASK GENMASK(7, 0)
+#define RG_GSWPLL_FBKDIV_500M(x) FIELD_PREP(RG_GSWPLL_FBKDIV_500M_MASK, x)
#define CORE_TRGMII_GSW_CLK_CG 0x410
#define REG_GSWCK_EN BIT(0)
--
2.55.0
^ permalink raw reply related
* [PATCH net-next v4 7/8] net: dsa: mt7530: implement port_fast_age
From: Daniel Golle @ 2026-07-19 17:54 UTC (permalink / raw)
To: Chester A. Unal, Daniel Golle, Andrew Lunn, Vladimir Oltean,
David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Matthias Brugger, AngeloGioacchino Del Regno, Russell King,
netdev, linux-kernel, linux-arm-kernel, linux-mediatek
In-Reply-To: <cover.1784481922.git.daniel@makrotopia.org>
Implement the .port_fast_age DSA operation by flushing all non-static
(dynamically learned) MAC address entries from the address table.
The switch does not offer a combined "non-static AND per-port" match
mode, so the flush is global and the port argument is not used. Unlike
b53 and realtek, which flush the dynamic entries of the affected port
only, an STP topology change on one port therefore also flushes the
dynamically learned entries of the other ports; they are quickly
relearned.
Access the address table control register under priv->reg_mutex, as done
by all other ATC users (FDB and MDB add/del/dump), to serialise the
write-then-poll command sequence, and log a message should the flush
time out.
Signed-off-by: Daniel Golle <daniel@makrotopia.org>
---
v3:
* take reg_mutex around the ATC flush and log on timeout
* align the ATC_MAT_NON_STATIC_MAC define
* correct the commit message which wrongly claimed per-port parity
with b53/realtek
---
drivers/net/dsa/mt7530.c | 23 +++++++++++++++++++++++
drivers/net/dsa/mt7530.h | 1 +
2 files changed, 24 insertions(+)
diff --git a/drivers/net/dsa/mt7530.c b/drivers/net/dsa/mt7530.c
index f7643b86f3c4..52b549d7ee22 100644
--- a/drivers/net/dsa/mt7530.c
+++ b/drivers/net/dsa/mt7530.c
@@ -193,6 +193,28 @@ mt7530_fdb_cmd(struct mt7530_priv *priv, enum mt7530_fdb_cmd cmd, u32 *rsp)
return 0;
}
+static void mt7530_port_fast_age(struct dsa_switch *ds, int port)
+{
+ struct mt7530_priv *priv = ds->priv;
+ struct mt7530_dummy_poll p;
+ u32 val;
+ int ret;
+
+ mutex_lock(&priv->reg_mutex);
+
+ /* Flush all non-static MAC address entries */
+ val = ATC_BUSY | ATC_MAT_NON_STATIC_MAC | MT7530_FDB_FLUSH;
+ regmap_write(priv->regmap, MT7530_ATC, val);
+
+ INIT_MT7530_DUMMY_POLL(&p, priv, MT7530_ATC);
+ ret = readx_poll_timeout(mt7530_mii_poll, &p, val,
+ !(val & ATC_BUSY), 20, 20000);
+ if (ret < 0)
+ dev_err(priv->dev, "fast age timeout\n");
+
+ mutex_unlock(&priv->reg_mutex);
+}
+
static void
mt7530_fdb_read(struct mt7530_priv *priv, struct mt7530_fdb *fdb)
{
@@ -3319,6 +3341,7 @@ static const struct dsa_switch_ops mt7530_switch_ops = {
.port_bridge_flags = mt7530_port_bridge_flags,
.port_bridge_join = mt7530_port_bridge_join,
.port_bridge_leave = mt7530_port_bridge_leave,
+ .port_fast_age = mt7530_port_fast_age,
.port_fdb_add = mt7530_port_fdb_add,
.port_fdb_del = mt7530_port_fdb_del,
.port_fdb_dump = mt7530_port_fdb_dump,
diff --git a/drivers/net/dsa/mt7530.h b/drivers/net/dsa/mt7530.h
index 804c2e0991a0..241e3d460357 100644
--- a/drivers/net/dsa/mt7530.h
+++ b/drivers/net/dsa/mt7530.h
@@ -165,6 +165,7 @@ enum mt753x_to_cpu_fw {
#define ATC_MAT_MASK GENMASK(11, 8)
#define ATC_MAT(x) FIELD_PREP(ATC_MAT_MASK, x)
#define ATC_MAT_MACTAB ATC_MAT(0)
+#define ATC_MAT_NON_STATIC_MAC ATC_MAT(4)
enum mt7530_fdb_cmd {
MT7530_FDB_READ = 0,
--
2.55.0
^ permalink raw reply related
* [PATCH net-next v4 8/8] net: dsa: mt7530: implement port_change_conduit op
From: Daniel Golle @ 2026-07-19 17:54 UTC (permalink / raw)
To: Chester A. Unal, Daniel Golle, Andrew Lunn, Vladimir Oltean,
David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Matthias Brugger, AngeloGioacchino Del Regno, Russell King,
netdev, linux-kernel, linux-arm-kernel, linux-mediatek
In-Reply-To: <cover.1784481922.git.daniel@makrotopia.org>
Allow changing the CPU port affinity of user ports at runtime via the
IFLA_DSA_CONDUIT netlink attribute. This updates the port matrix to
forward to the new CPU port instead of the old one.
Limit the operation to MT7531. There, trapped link-local frames follow
the per-port affinity, as the MT7531_CPU_PMAP destination mask is
further restricted by the port matrix. A conduit change is hence fully
honoured by the hardware, for regular traffic as well as for trapped
frames.
The MT7530 switch, including the variant embedded in the MT7621 SoC,
instead traps frames to the single CPU port set in the CPU_PORT field
of the MFC register, regardless of the affinity of the inbound user
port. With user ports affine to different CPU ports there is no
correct value for that field, so per-port CPU affinity cannot be fully
implemented for trapped frames. Routing a WAN port via the second SoC
GMAC is conventionally covered by the PHY muxing feature on these
switches, which bypasses the switch fabric and does not involve a CPU
port at all.
The switches on the MT7988, EN7581 and AN7583 SoCs only have a
single CPU port, leaving no other conduit to change to.
As the op lives in the shared mt7530_switch_ops, populate the extack
when rejecting the unsupported variants instead of returning a bare
-EOPNOTSUPP. Also reject a conduit that belongs to a different switch
in the tree, whose port index has no meaning in the local port matrix.
Signed-off-by: Daniel Golle <daniel@makrotopia.org>
Acked-by: Chester A. Unal <chester.a.unal@arinc9.com>
---
v3:
* populate the netlink extack on rejection
* refuse a conduit that lives on a different switch
drivers/net/dsa/mt7530.c | 38 ++++++++++++++++++++++++++++++++++++++
1 file changed, 38 insertions(+)
diff --git a/drivers/net/dsa/mt7530.c b/drivers/net/dsa/mt7530.c
index 52b549d7ee22..6fe6783dfe3f 100644
--- a/drivers/net/dsa/mt7530.c
+++ b/drivers/net/dsa/mt7530.c
@@ -3213,6 +3213,43 @@ static int mt753x_set_mac_eee(struct dsa_switch *ds, int port,
return 0;
}
+static int
+mt753x_port_change_conduit(struct dsa_switch *ds, int port,
+ struct net_device *conduit,
+ struct netlink_ext_ack *extack)
+{
+ struct dsa_port *new_cpu_dp = conduit->dsa_ptr;
+ struct dsa_port *dp = dsa_to_port(ds, port);
+ struct mt7530_priv *priv = ds->priv;
+
+ if (priv->id != ID_MT7531) {
+ NL_SET_ERR_MSG_MOD(extack,
+ "Changing DSA conduit is only supported on MT7531");
+ return -EOPNOTSUPP;
+ }
+
+ if (new_cpu_dp->ds != ds) {
+ NL_SET_ERR_MSG_MOD(extack,
+ "Cannot assign a conduit on a different switch");
+ return -EOPNOTSUPP;
+ }
+
+ mutex_lock(&priv->reg_mutex);
+
+ /* dp->cpu_dp still points to the old CPU port */
+ priv->ports[port].pm &= ~PCR_MATRIX(BIT(dp->cpu_dp->index));
+ priv->ports[port].pm |= PCR_MATRIX(BIT(new_cpu_dp->index));
+ if (priv->ports[port].enable)
+ regmap_update_bits(priv->regmap, MT7530_PCR_P(port),
+ PCR_MATRIX_MASK, priv->ports[port].pm);
+
+ mutex_unlock(&priv->reg_mutex);
+
+ mt7530_port_fast_age(ds, port);
+
+ return 0;
+}
+
static void
mt753x_conduit_state_change(struct dsa_switch *ds,
const struct net_device *conduit,
@@ -3324,6 +3361,7 @@ static const struct dsa_switch_ops mt7530_switch_ops = {
.setup = mt753x_setup,
.teardown = mt753x_teardown,
.preferred_default_local_cpu_port = mt753x_preferred_default_local_cpu_port,
+ .port_change_conduit = mt753x_port_change_conduit,
.get_strings = mt7530_get_strings,
.get_ethtool_stats = mt7530_get_ethtool_stats,
.get_sset_count = mt7530_get_sset_count,
--
2.55.0
^ permalink raw reply related
* [RFC PATCH net-next 00/13] net: knod: in-kernel network offload device
From: Taehee Yoo @ 2026-07-19 17:58 UTC (permalink / raw)
To: Alex Deucher, Alexei Starovoitov, amd-gfx, Andrew Lunn,
Andrii Nakryiko, Bill Wendling, bpf, Christian König,
Daniel Borkmann, David Airlie, David S. Miller, Donald Hunter,
dri-devel, Eduard Zingerman, Emil Tsalapatis, Eric Dumazet,
Felix Kuehling, Hoyeon Lee, Ilias Apalodimas, Jakub Kicinski,
Jesper Dangaard Brouer, Jiri Olsa, John Fastabend, Justin Stitt,
Kees Cook, Kumar Kartikeya Dwivedi, Leon Romanovsky,
linaro-mm-sig, linux-hardening, linux-kernel, linux-kselftest,
linux-media, linux-rdma, llvm, Mark Bloch, Martin KaFai Lau,
Michael Chan, Nathan Chancellor, netdev, Nick Desaulniers,
Paolo Abeni, Pavan Chebbi, Saeed Mahameed, Shuah Khan,
Simona Vetter, Simon Horman, Song Liu, Stanislav Fomichev,
Sumit Semwal, Taehee Yoo, Tariq Toukan, Yonghong Song
knod (in-kernel network offload device) drives a GPU directly from the
Linux kernel - with no userspace GPU runtime such as CUDA or ROCm, and
no userspace component in the data path - to accelerate packet
processing.
The kernel itself allocates the GPU's queues, JIT-compiles the
per-packet program to GPU machine code, and dispatches it; the NIC DMAs
received packets straight into GPU memory and the GPU returns a verdict.
The GPU is programmed like any other in-kernel offload, not through a
userspace framework, so existing XDP programs and IPsec SAs can be
offloaded to it transparently.
The design and motivation were presented at Linux Plumbers Conference
2025:
https://lpc.events/event/19/contributions/2267/
Motivation
==========
Line-rate packet processing that does non-trivial per-packet work - an
XDP program doing L4 load balancing, or IPsec crypto - is bound by the
host CPU: each core handles one packet at a time, so scaling means
spending more cores. A GPU is the opposite shape - thousands of lanes
running the same small program over many packets at once (SIMT) - which
happens to match the per-packet-program model of XDP.
knod moves that per-packet compute off the host CPU and onto a GPU. The
NIC DMAs received packets directly into GPU memory, the GPU runs the
program across a batch of packets in parallel, and only the result
comes back: a verdict for every packet, plus the packet itself for the
ones destined to the host. The CPU no longer pays the per-packet
program cost, and throughput scales with GPU occupancy rather than core
count.
Crucially this happens entirely inside the kernel. GPU packet processing
today generally launches work from a userspace GPU runtime (CUDA and
friends) and keeps that runtime in the data path; knod instead builds
the GPU queues, compiles the program, and dispatches it from the kernel,
so it plugs into existing offload paths (XDP, xfrm) with nothing to
install or keep running in userspace.
Model
=====
knod binds two endpoints through a third object:
- a NIC-side netdev, registered by the NIC driver;
- an accelerator (the GPU), registered by a provider built into the
GPU driver;
- an offload device, created on attach, that connects them and owns
the per-queue data-path state.
The accelerator ops are feature-agnostic, so different per-packet
programs plug in behind one interface. This series ships two features
to show the framework is not tied to a single use case (the accel-type
uAPI also reserves "dpu" for future non-GPU backends):
- BPF: an XDP program attached in offload mode is JIT-compiled from
eBPF to an AMD GCN shader and executed on the GPU.
- IPsec: RX ESP full-packet decrypt on the GPU (proof of concept,
see below).
Data path
=========
For a NIC RX queue bound to an accelerator, a received packet flows as
follows:
1. Attach reconfigures the NIC's page_pool so its pages come from GPU
memory, exported as a dma-buf and plugged in through the devmem
memory provider. The NIC therefore DMAs the packet straight into
GPU memory - there is no host-side copy on RX.
2. Instead of building an skb and entering the stack, the NIC's NAPI
pushes a small descriptor (memory ref + offset + length) onto a
per-queue lock-free SPSC ring shared with the GPU.
3. A persistent GPU worker drains descriptors and runs the active
feature's shader over a batch of packets in parallel - one
workgroup per packet - then writes a per-packet verdict back onto
the ring: PASS, DROP or TX.
4. Back on the NIC NAPI, each verdict is applied:
- DROP: the GPU page is recycled to the pool;
- TX: the packet is sent back out the NIC directly from GPU
memory (XDP_TX), with no host round-trip;
- PASS: the packet is copied out of GPU memory into a host
delivery page by the GPU's DMA engine (async SDMA),
wrapped as an skb, and handed to the normal stack.
So only PASS packets ever touch host memory, and even then the copy is
done by the GPU, not the CPU.
The control plane is a generic netlink family ("knod") for
attach/detach, accelerator/NIC inventory, and per-accelerator feature
selection, with notifications on bind/unbind.
Code layout
===========
knod separates into a subsystem-neutral core and a GPU-specific
provider; nothing in the core knows about GPUs. Roughly:
- core + control plane + NIC drivers net/, kernel/bpf/ (~4.4k)
- GPU provider (queues, JIT, shaders) drivers/gpu/drm/amd/ (~39k)
The provider is the bulk of the diff, but most of it is self-contained
GPU code generation and hand-written shaders, not core logic. It lives
under amdkfd because that is where AMD GPU compute lives; the
load-bearing boundary is the accelerator interface, not amdkfd.
Usage
=====
The control plane is driven with the in-tree YNL CLI. Examples, run from
the kernel tree (ifindex is the NIC's ifindex, e.g. from `ip -j link`):
SPEC=Documentation/netlink/specs/knod.yaml
CLI="tools/net/ynl/pyynl/cli.py --spec $SPEC"
# list accelerators (id, type, supported/enabled features)
$CLI --dump accel-get
# attach a NIC to accelerator 0
$CLI --do attach --json '{"nic-ifindex": <ifindex>, "accel-id": 0}'
# enable a feature on accelerator 0 ("none" | "bpf" | "ipsec")
$CLI --do accel-set --json '{"id": 0, "feature-ena": "bpf"}'
# show current NIC <-> accelerator bindings
$CLI --dump dev-get
# detach the NIC
$CLI --do detach --json '{"nic-ifindex": <ifindex>}'
Attach/detach also emit notifications on the "mgmt" multicast group
(dev-add-ntf / dev-del-ntf), which the CLI can watch with
`--subscribe mgmt`.
Hardware
========
The reference GPUs are GCN (Radeon RX Vega64) and RDNA2 (Radeon RX 6600
and Radeon PRO V620). GCN does not implement the 64-bit atomics the
data path relies on, so active development targets RDNA2; all numbers
below are measured on RDNA2.
Performance
===========
The BPF path is measured with kondor, a katran-based L4 load balancer
running as the offloaded XDP program.
Feature GPU Throughput GPU power
--------- -------- ---------- -------------------
BPF (XDP) RX 6600 32 Mpps 41 W (100 W board)
BPF (XDP) V620 70 Mpps 80 W (300 W board)
IPsec RX RX 6600 80 Gbps 100 W
IPsec RX V620 80 Gbps 100 W
Host CPU utilization drops by roughly the cost of the XDP program or
the IPsec crypto that no longer runs on the CPU, offset by a small knod
bookkeeping overhead (SPSC ring management and packet delivery).
IPsec status
============
The IPsec feature is a functional proof of concept and should be read
as "this is possible on the framework", not as a production-ready
implementation. It is RX only (no TX offload) and supports both tunnel
and transport mode.
Development tree
================
Work continues in a public tree:
https://github.com/TaeheeYoo/knod/tree/knod-7.2
Taehee Yoo (13):
net: knod: add uapi and core headers
net: devmem: extend memory provider for knod
net: core: add XDP_MODE_HW offload hook for knod
net: knod: add offload device core and control plane
bpf: offload: allow PERCPU_ARRAY maps for offloaded programs
drm/amdkfd: prepare kfd core for the knod provider
drm/amdkfd: add knod provider core
drm/amdkfd: add GPU instruction emitter and disassembler
drm/amdkfd: add BPF-to-GPU JIT offload
net/mlx5e: add knod XDP offload support
bnxt_en: add knod XDP offload support
selftests: drivers/net: add knod tests
drm/amdkfd: add IPsec full-packet offload
Documentation/netlink/specs/knod.yaml | 176 +
drivers/gpu/drm/amd/amdgpu/Makefile | 1 +
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 13 +-
.../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 65 +
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 3 +
drivers/gpu/drm/amd/amdkfd/Kconfig | 31 +
drivers/gpu/drm/amd/amdkfd/Makefile | 9 +
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c | 117 +-
drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c | 20 +-
drivers/gpu/drm/amd/amdkfd/kfd_events.c | 112 +-
drivers/gpu/drm/amd/amdkfd/kfd_events.h | 4 +
drivers/gpu/drm/amd/amdkfd/kfd_hsa.h | 451 +
drivers/gpu/drm/amd/amdkfd/kfd_knod.c | 2202 +++
drivers/gpu/drm/amd/amdkfd/kfd_module.c | 1 +
drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 34 +
drivers/gpu/drm/amd/amdkfd/kfd_process.c | 26 +-
.../gpu/drm/amd/amdkfd/knod/aesgcm_shader.h | 984 ++
.../drm/amd/amdkfd/knod/ipsec_fused_gfx10.h | 1796 +++
.../drm/amd/amdkfd/knod/ipsec_fused_gfx9.h | 1349 ++
drivers/gpu/drm/amd/amdkfd/knod/kfd_knod.h | 270 +
drivers/gpu/drm/amd/amdkfd/knod/knod_amdgpu.h | 173 +
.../drm/amd/amdkfd/knod/knod_amdgpu_insn.h | 6362 +++++++++
drivers/gpu/drm/amd/amdkfd/knod/knod_bpf.c | 11554 ++++++++++++++++
drivers/gpu/drm/amd/amdkfd/knod/knod_bpf.h | 597 +
.../gpu/drm/amd/amdkfd/knod/knod_gfx10_insn.h | 3978 ++++++
.../gpu/drm/amd/amdkfd/knod/knod_gfx9_insn.h | 4068 ++++++
drivers/gpu/drm/amd/amdkfd/knod/knod_ipsec.c | 4273 ++++++
drivers/gpu/drm/amd/amdkfd/knod/knod_ipsec.h | 596 +
drivers/net/ethernet/broadcom/bnxt/bnxt.c | 155 +-
drivers/net/ethernet/broadcom/bnxt/bnxt.h | 12 +-
drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c | 274 +-
drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h | 12 +-
drivers/net/ethernet/mellanox/mlx5/core/en.h | 10 +
.../net/ethernet/mellanox/mlx5/core/en/xdp.c | 351 +
.../net/ethernet/mellanox/mlx5/core/en/xdp.h | 19 +-
.../net/ethernet/mellanox/mlx5/core/en_main.c | 38 +-
.../net/ethernet/mellanox/mlx5/core/en_rx.c | 107 +-
.../net/ethernet/mellanox/mlx5/core/en_txrx.c | 45 +
include/net/devmem.h | 58 +
include/net/knod.h | 467 +
include/net/netmem.h | 9 +
include/net/page_pool/memory_provider.h | 4 +
include/net/page_pool/types.h | 23 +-
include/net/spsc_ring.h | 645 +
include/uapi/linux/knod.h | 67 +
kernel/bpf/offload.c | 3 +-
net/Kconfig | 2 +
net/Makefile | 1 +
net/core/dev.c | 2 +-
net/core/devmem.c | 103 +-
net/core/devmem.h | 7 +-
net/core/page_pool.c | 22 +-
net/knod/Kconfig | 5 +
net/knod/Makefile | 8 +
net/knod/knod-nl-gen.c | 121 +
net/knod/knod-nl-gen.h | 31 +
net/knod/knod.h | 26 +
net/knod/knod_core.c | 1231 ++
net/knod/knod_nl.c | 406 +
.../selftests/drivers/net/knod/Makefile | 17 +
.../testing/selftests/drivers/net/knod/config | 8 +
.../selftests/drivers/net/knod/knod_attach.sh | 135 +
.../drivers/net/knod/knod_xdp_ktime.sh | 173 +
.../drivers/net/knod/knod_xdp_loop.sh | 129 +
.../testing/selftests/drivers/net/knod/lib.sh | 181 +
.../drivers/net/knod/xdp_ktime.bpf.c | 32 +
.../selftests/drivers/net/knod/xdp_loop.bpf.c | 42 +
67 files changed, 44131 insertions(+), 115 deletions(-)
create mode 100644 Documentation/netlink/specs/knod.yaml
create mode 100644 drivers/gpu/drm/amd/amdkfd/kfd_hsa.h
create mode 100644 drivers/gpu/drm/amd/amdkfd/kfd_knod.c
create mode 100644 drivers/gpu/drm/amd/amdkfd/knod/aesgcm_shader.h
create mode 100644 drivers/gpu/drm/amd/amdkfd/knod/ipsec_fused_gfx10.h
create mode 100644 drivers/gpu/drm/amd/amdkfd/knod/ipsec_fused_gfx9.h
create mode 100644 drivers/gpu/drm/amd/amdkfd/knod/kfd_knod.h
create mode 100644 drivers/gpu/drm/amd/amdkfd/knod/knod_amdgpu.h
create mode 100644 drivers/gpu/drm/amd/amdkfd/knod/knod_amdgpu_insn.h
create mode 100644 drivers/gpu/drm/amd/amdkfd/knod/knod_bpf.c
create mode 100644 drivers/gpu/drm/amd/amdkfd/knod/knod_bpf.h
create mode 100644 drivers/gpu/drm/amd/amdkfd/knod/knod_gfx10_insn.h
create mode 100644 drivers/gpu/drm/amd/amdkfd/knod/knod_gfx9_insn.h
create mode 100644 drivers/gpu/drm/amd/amdkfd/knod/knod_ipsec.c
create mode 100644 drivers/gpu/drm/amd/amdkfd/knod/knod_ipsec.h
create mode 100644 include/net/devmem.h
create mode 100644 include/net/knod.h
create mode 100644 include/net/spsc_ring.h
create mode 100644 include/uapi/linux/knod.h
create mode 100644 net/knod/Kconfig
create mode 100644 net/knod/Makefile
create mode 100644 net/knod/knod-nl-gen.c
create mode 100644 net/knod/knod-nl-gen.h
create mode 100644 net/knod/knod.h
create mode 100644 net/knod/knod_core.c
create mode 100644 net/knod/knod_nl.c
create mode 100644 tools/testing/selftests/drivers/net/knod/Makefile
create mode 100644 tools/testing/selftests/drivers/net/knod/config
create mode 100755 tools/testing/selftests/drivers/net/knod/knod_attach.sh
create mode 100755 tools/testing/selftests/drivers/net/knod/knod_xdp_ktime.sh
create mode 100755 tools/testing/selftests/drivers/net/knod/knod_xdp_loop.sh
create mode 100755 tools/testing/selftests/drivers/net/knod/lib.sh
create mode 100644 tools/testing/selftests/drivers/net/knod/xdp_ktime.bpf.c
create mode 100644 tools/testing/selftests/drivers/net/knod/xdp_loop.bpf.c
base-commit: ce6b4d3216b63f902bb8e9695ee6c10c83415f65
--
2.43.0
^ permalink raw reply
* [RFC PATCH net-next 01/13] net: knod: add uapi and core headers
From: Taehee Yoo @ 2026-07-19 17:58 UTC (permalink / raw)
To: Alex Deucher, Alexei Starovoitov, amd-gfx, Andrew Lunn,
Andrii Nakryiko, Bill Wendling, bpf, Christian König,
Daniel Borkmann, David Airlie, David S. Miller, Donald Hunter,
dri-devel, Eduard Zingerman, Emil Tsalapatis, Eric Dumazet,
Felix Kuehling, Hoyeon Lee, Ilias Apalodimas, Jakub Kicinski,
Jesper Dangaard Brouer, Jiri Olsa, John Fastabend, Justin Stitt,
Kees Cook, Kumar Kartikeya Dwivedi, Leon Romanovsky,
linaro-mm-sig, linux-hardening, linux-kernel, linux-kselftest,
linux-media, linux-rdma, llvm, Mark Bloch, Martin KaFai Lau,
Michael Chan, Nathan Chancellor, netdev, Nick Desaulniers,
Paolo Abeni, Pavan Chebbi, Saeed Mahameed, Shuah Khan,
Simona Vetter, Simon Horman, Song Liu, Stanislav Fomichev,
Sumit Semwal, Taehee Yoo, Tariq Toukan, Yonghong Song
In-Reply-To: <20260719175857.4071636-1-ap420073@gmail.com>
Add the uAPI and core headers for KNOD, an in-kernel network offload
device that connects a NIC RX path to an accelerator (e.g. a GPU) for
zero-copy packet processing.
- include/uapi/linux/knod.h: genetlink uAPI
- Documentation/netlink/specs/knod.yaml: netlink spec
- include/net/knod.h: core framework types and ops
- include/net/spsc_ring.h: lock-free SPSC ring for the data path
Signed-off-by: Taehee Yoo <ap420073@gmail.com>
(cherry picked from commit d93829acbce14528fcf70e155a0594c0c6151cce)
---
Documentation/netlink/specs/knod.yaml | 176 +++++++
include/net/knod.h | 467 +++++++++++++++++++
include/net/spsc_ring.h | 645 ++++++++++++++++++++++++++
include/uapi/linux/knod.h | 67 +++
4 files changed, 1355 insertions(+)
create mode 100644 Documentation/netlink/specs/knod.yaml
create mode 100644 include/net/knod.h
create mode 100644 include/net/spsc_ring.h
create mode 100644 include/uapi/linux/knod.h
diff --git a/Documentation/netlink/specs/knod.yaml b/Documentation/netlink/specs/knod.yaml
new file mode 100644
index 000000000000..7c64df467634
--- /dev/null
+++ b/Documentation/netlink/specs/knod.yaml
@@ -0,0 +1,176 @@
+# SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause)
+---
+name: knod
+
+doc:
+ KNOD (KFD network offload) control plane.
+
+ Bind a NIC netdevice to a GPU/DPU offload accelerator and select which
+ offload feature (BPF, IPsec) the accelerator runs.
+
+definitions:
+ -
+ type: enum
+ name: feature
+ entries: [none, bpf, ipsec]
+ -
+ type: enum
+ name: accel-type
+ entries: [gpu, dpu]
+
+attribute-sets:
+ -
+ name: accel
+ attributes:
+ -
+ name: id
+ doc: Accelerator id.
+ type: u32
+ checks:
+ min: 1
+ -
+ name: name
+ doc: Accelerator name.
+ type: string
+ -
+ name: type
+ doc: Accelerator type.
+ type: u32
+ enum: accel-type
+ -
+ name: feature-cap
+ doc: Bitmask of offload features the accelerator supports.
+ type: u32
+ enum: feature
+ enum-as-flags: true
+ -
+ name: feature-ena
+ doc: Currently active offload feature on the accelerator.
+ type: u32
+ enum: feature
+ -
+ name: nic
+ attributes:
+ -
+ name: ifindex
+ doc: ifindex of the NIC netdevice registered with KNOD.
+ type: u32
+ checks:
+ min: 1
+ -
+ name: name
+ doc: Name of the NIC netdevice.
+ type: string
+ -
+ name: dev
+ attributes:
+ -
+ name: nic-ifindex
+ doc: ifindex of the bound NIC netdevice.
+ type: u32
+ checks:
+ min: 1
+ -
+ name: accel-id
+ doc: id of the bound accelerator.
+ type: u32
+ checks:
+ min: 1
+
+operations:
+ list:
+ -
+ name: accel-get
+ doc: Get / dump accelerators registered on the system.
+ attribute-set: accel
+ do:
+ request:
+ attributes:
+ - id
+ reply: &accel-all
+ attributes:
+ - id
+ - name
+ - type
+ - feature-cap
+ - feature-ena
+ dump:
+ reply: *accel-all
+ -
+ name: accel-set
+ doc: Select the active offload feature of an accelerator.
+ attribute-set: accel
+ flags: [admin-perm]
+ do:
+ request:
+ attributes:
+ - id
+ - feature-ena
+ reply:
+ attributes: []
+ -
+ name: nic-get
+ doc: Get / dump NICs registered with the KNOD framework.
+ attribute-set: nic
+ do:
+ request:
+ attributes:
+ - ifindex
+ reply: &nic-all
+ attributes:
+ - ifindex
+ - name
+ dump:
+ reply: *nic-all
+ -
+ name: attach
+ doc: Attach a NIC netdevice to an accelerator.
+ attribute-set: dev
+ flags: [admin-perm]
+ do:
+ request:
+ attributes:
+ - nic-ifindex
+ - accel-id
+ reply:
+ attributes: []
+ -
+ name: detach
+ doc: Detach a NIC netdevice from its accelerator.
+ attribute-set: dev
+ flags: [admin-perm]
+ do:
+ request:
+ attributes:
+ - nic-ifindex
+ reply:
+ attributes: []
+ -
+ name: dev-get
+ doc: Get / dump active NIC <-> accelerator bindings.
+ attribute-set: dev
+ do:
+ request:
+ attributes:
+ - nic-ifindex
+ reply: &dev-all
+ attributes:
+ - nic-ifindex
+ - accel-id
+ dump:
+ reply: *dev-all
+ -
+ name: dev-add-ntf
+ doc: Notification about a new NIC <-> accelerator binding.
+ notify: dev-get
+ mcgrp: mgmt
+ -
+ name: dev-del-ntf
+ doc: Notification about a removed NIC <-> accelerator binding.
+ notify: dev-get
+ mcgrp: mgmt
+
+mcast-groups:
+ list:
+ -
+ name: mgmt
diff --git a/include/net/knod.h b/include/net/knod.h
new file mode 100644
index 000000000000..50333b0e028b
--- /dev/null
+++ b/include/net/knod.h
@@ -0,0 +1,467 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __NET_KNOD_H
+#define __NET_KNOD_H
+
+#include <linux/bpf.h>
+#include <linux/dma-buf.h>
+#include <linux/ethtool_netlink.h>
+#include <linux/netdevice.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+#include <linux/rtnetlink.h>
+#include <linux/mutex.h>
+#include <linux/list.h>
+#include <linux/if_link.h>
+#include <linux/workqueue.h>
+#include <linux/completion.h>
+#include <linux/atomic.h>
+#include <net/xdp.h>
+#include <net/spsc_ring.h>
+
+struct knod_dev;
+struct knod_netdev;
+struct knod_accel;
+struct gen_pool;
+struct page_pool;
+struct netlink_ext_ack;
+struct net_devmem_dmabuf_binding;
+
+extern struct mutex knod_lock;
+
+struct spsc_bd {
+ netmem_ref netmem;
+ u64 act;
+ u16 off;
+ u16 len;
+ u32 page_idx;
+ struct page_pool *pp;
+};
+
+/*
+ * KNOD action codes for spsc_bd.act
+ *
+ * Base actions (compatible with XDP constants for BPF/XDP path):
+ */
+#define KNOD_ABORTED XDP_ABORTED /* 0 */
+#define KNOD_DROP XDP_DROP /* 1 */
+#define KNOD_PASS XDP_PASS /* 2 */
+#define KNOD_TX XDP_TX /* 3 */
+#define KNOD_REDIRECT XDP_REDIRECT /* 4 */
+
+/*
+ * Extended actions - accel-specific, must not collide with XDP range [0..7].
+ * The NIC act_handler treats any unknown code as "in-flight to accel":
+ * stop releasing at that entry and wait for the accel to update bd->act.
+ */
+#define KNOD_ACT_INFLIGHT 0x80 /* accel processing in progress */
+#define KNOD_IPSEC_INFLIGHT 0x100 /* GPU IPsec dispatch in progress */
+#define KNOD_IPSEC_PASS 0x101 /* GPU IPsec done, recycle page */
+#define KNOD_IPSEC_DROP 0x102 /* GPU IPsec done, drop + recycle */
+
+/*
+ * PASS hand-off descriptor: the DD (NIC act_handler) fills one per PASS bd
+ * during its single act traversal and hands a batch to accel_ops->pass_copy.
+ * @netmem is the source RX page (GPU memory); @off/@len are post-BPF (may
+ * differ from the bd's original values if the program adjusted head/tail).
+ */
+struct spsc_pass_bd {
+ /* source RX page, recycled after the copy lands */
+ netmem_ref netmem;
+ u32 page_idx; /* page index in the queue's dmabuf RX buffer
+ * (the accel turns this into the GPU src addr;
+ * the netmem dma_addr is the NIC's, not the
+ * GPU's)
+ */
+ u16 off; /* packet offset within the page (post-BPF) */
+ u16 len; /* packet length (post-BPF) */
+};
+
+/*
+ * Host-page page_pool provider context (GPU->host delivery pools). The
+ * framework fills the public fields, points page_pool_params.mp_priv at it and
+ * sets .mp_ops to the NOD-private page_pool_hostmem_ops (knod_dev.c);
+ * ->init() builds @genpool and ->destroy() tears it down. Must outlive the
+ * page_pool; @freed fires once the pool has fully drained.
+ */
+struct page_pool_hostmem {
+ struct page **pages; /* owner-supplied, @count real pages */
+ unsigned int count;
+ dma_addr_t base_addr; /* device addr of pages[0] */
+ void (*freed)(void *arg); /* called once the pool fully drains */
+ void *arg;
+ struct gen_pool *genpool; /* private: managed by the provider */
+};
+
+struct knod_work_priv {
+ struct dma_buf *dmabuf;
+ netmem_ref *netmems;
+ unsigned int *data_lens;
+ int *data_offs;
+ int cnt;
+ int index;
+ struct napi_struct *napi;
+ struct spsc_ring spsc_bds;
+ void *spsc_pool_priv; /* accel driver priv for spsc pool memory */
+ u64 spsc_pool_gaddr; /* device-visible address of spsc pool */
+ /* framework-owned delivery pool */
+ struct page_pool *pass_pool;
+ /* provider ctx (owner storage) */
+ struct page_pool_hostmem pass_hm;
+ /* d2h: SDMA-issued, awaiting drain */
+ struct spsc_ring pass_pending;
+} ____cacheline_aligned_in_smp;
+
+static inline void knod_napi_kick(struct knod_work_priv *wpriv)
+{
+ struct napi_struct *napi;
+
+ rcu_read_lock();
+ napi = READ_ONCE(wpriv->napi);
+ if (napi)
+ napi_schedule(napi);
+ rcu_read_unlock();
+}
+
+#define KNOD_DEFAULT_PASS_SLOTS 64
+
+/*
+ * GPU->host delivery descriptor: a packet at @off (preserved headroom) for
+ * @len bytes within @netmem, a page from the framework delivery pool. The
+ * d2h path fills these into the per-queue pass_pending ring; knod_d2h_drain()
+ * turns each into an skb once its SDMA copy lands and hands it to the stack.
+ */
+struct knod_pass_desc {
+ u16 len; /* head_frag length (ipsec: inner_len) */
+ u16 off; /* head_frag offset (ipsec: inner_off) */
+ netmem_ref netmem; /* dst: framework delivery-pool page */
+ netmem_ref src; /* src: RX page recycled once the copy lands, or
+ * 0 when the producer recycles it elsewhere
+ * (ipsec: NIC act handler recycles via the bd)
+ */
+ /* SDMA fence to await before delivery (async) */
+ u32 fence_val;
+ u8 sdma_idx; /* which accel SDMA queue's fence to await */
+ /* Feature finalisation context, consumed by knod_dev->post_copy. */
+ struct {
+ u32 sa_slot; /* ipsec SA table slot */
+ u32 seq_lo; /* ESP sequence low 32 bits */
+ u32 seq_hi; /* ESN high 32 bits (0 if !ESN) */
+ u8 mode; /* XFRM_MODE_TRANSPORT / _TUNNEL */
+ u8 next_hdr; /* ESP trailer next-header */
+ u8 family; /* AF_INET / AF_INET6 */
+ } feat;
+};
+
+struct knod_accel_xdp_ops {
+ /* init/exit: permanent per-attach setup (attach/detach). */
+ int (*init)(struct knod_dev *knodev);
+ void (*exit)(struct knod_dev *knodev);
+ /* activate/deactivate: feature resource alloc/free (feature select). */
+ int (*activate)(struct knod_dev *knodev);
+ void (*deactivate)(struct knod_dev *knodev);
+ /*
+ * true while user XDP progs/maps are still bound (blocks feature
+ * switch).
+ */
+ bool (*busy)(struct knod_dev *knodev);
+ int (*xdp_offload_init)(struct knod_dev *knodev);
+ void (*xdp_offload_uninit)(struct knod_dev *knodev);
+ int (*xdp_install)(struct knod_dev *knodev,
+ struct netdev_bpf *bpf);
+ int (*rx_netmem)(struct knod_dev *knodev, netmem_ref netmem,
+ unsigned int data_len, int data_offset, int index);
+ int (*rx_netmem_bulk)(struct knod_dev *knodev,
+ struct knod_work_priv *wpriv);
+ /* Direct dispatch */
+ int (*dispatch)(struct knod_dev *knodev, int index);
+ /* Direct finish */
+ int (*finish)(struct knod_dev *knodev, int index);
+ void (*start)(struct knod_dev *knodev);
+ void (*stop)(struct knod_dev *knodev);
+};
+
+struct xfrm_state;
+struct xfrm_policy;
+struct netlink_ext_ack;
+
+struct knod_accel_ipsec_ops {
+ /* init/exit: permanent per-attach setup (attach/detach). */
+ int (*init)(struct knod_dev *knodev);
+ void (*exit)(struct knod_dev *knodev);
+ /* activate/deactivate: feature resource alloc/free (feature select). */
+ int (*activate)(struct knod_dev *knodev);
+ void (*deactivate)(struct knod_dev *knodev);
+ /*
+ * true while offloaded xfrm SAs are still bound (blocks feature
+ * switch).
+ */
+ bool (*busy)(struct knod_dev *knodev);
+ /*
+ * start/stop: worker/dispatcher start + GPU drain (interface
+ * up/down).
+ */
+ void (*start)(struct knod_dev *knodev);
+ void (*stop)(struct knod_dev *knodev);
+ int (*xdo_dev_state_add)(struct knod_dev *knodev,
+ struct xfrm_state *x,
+ struct netlink_ext_ack *extack);
+ void (*xdo_dev_state_delete)(struct knod_dev *knodev,
+ struct xfrm_state *x);
+ void (*xdo_dev_state_free)(struct knod_dev *knodev,
+ struct xfrm_state *x);
+ bool (*xdo_dev_offload_ok)(struct knod_dev *knodev,
+ struct sk_buff *skb,
+ struct xfrm_state *x);
+ void (*xdo_dev_state_advance_esn)(struct knod_dev *knodev,
+ struct xfrm_state *x);
+ void (*xdo_dev_state_update_stats)(struct knod_dev *knodev,
+ struct xfrm_state *x);
+ int (*xdo_dev_policy_add)(struct knod_dev *knodev,
+ struct xfrm_policy *x,
+ struct netlink_ext_ack *extack);
+ void (*xdo_dev_policy_delete)(struct knod_dev *knodev,
+ struct xfrm_policy *x);
+ void (*xdo_dev_policy_free)(struct knod_dev *knodev,
+ struct xfrm_policy *x);
+};
+
+struct knod_accel_ops {
+ int (*attach)(struct knod_dev *knodev);
+ void (*pre_detach)(struct knod_dev *knodev);
+ void (*detach)(struct knod_dev *knodev);
+ /* interface up/down (NIC driver knod_dev_start/stop): worker only. */
+ void (*dev_start)(struct knod_dev *knodev);
+ void (*dev_stop)(struct knod_dev *knodev);
+ void *(*alloc_mem)(struct knod_dev *knodev, size_t size,
+ u64 *gaddr, struct page ***pages, void **priv);
+ void (*free_mem)(struct knod_dev *knodev, void *priv);
+ /*
+ * Map dmabuf RX BOs into the GPU VM; must run after
+ * knod_dmabuf_attach().
+ */
+ int (*mp_map)(struct knod_dev *knodev);
+ /*
+ * Device->host copy primitives, used by the common knod_d2h_copy /
+ * knod_d2h_drain delivery path. The accel owns the SDMA engine (and
+ * its fence/ring); the framework owns the pending ring and dst pool.
+ * d2h_submit: queue one GPU->host copy. Returns a monotonic fence
+ * position to tag the descriptor with, or 0 if the SDMA
+ * ring is full (caller drops -- backpressure).
+ * d2h_kick: publish the batch (fence + doorbell).
+ * d2h_fence: current completed fence position of an SDMA queue
+ * (drain compares the descriptor's tag against this).
+ */
+ u32 (*d2h_submit)(struct knod_dev *knodev, u64 dst, int queue,
+ u32 page_idx, u16 off, u32 len);
+ void (*d2h_kick)(struct knod_dev *knodev);
+ u32 (*d2h_fence)(struct knod_dev *knodev, int sdma_idx);
+ struct knod_accel_xdp_ops *xdp_ops;
+ struct knod_accel_ipsec_ops *ipsec_ops;
+
+ /* control plane (knod genetlink) feature select */
+ int (*feature_get)(struct knod_accel *accel, u32 *ena, u32 *cap);
+ int (*feature_set)(struct knod_accel *accel, u32 feature,
+ struct netlink_ext_ack *extack);
+};
+
+struct knod_nic_ops {
+ int (*attach)(struct knod_dev *knodev);
+ int (*detach)(struct knod_dev *knodev);
+ int (*tx_handler)(struct knod_dev *knodev, struct spsc_bd **bds,
+ int cnt, int napi_index, void *priv);
+ int (*redir_handler)(struct knod_dev *knodev, netmem_ref *netmems,
+ u16 *lens, int cnt, int napi_index,
+ struct net_device *target_dev, void *priv);
+ int (*drop_handler)(struct knod_dev *knodev, netmem_ref *netmems,
+ u16 *lens, int cnt, void *priv);
+};
+
+struct knod_dev_stats {
+ u64_stats_t tx_packets;
+ u64_stats_t tx_bytes;
+ struct u64_stats_sync syncp;
+ u32 tx_dropped;
+ u32 tx_errors;
+};
+
+#define __NOD_FLAGS_XDP 0
+#define __NOD_FLAGS_IPSEC 2
+#define __NOD_FLAGS_KTLS 3
+#define __NOD_FLAGS_MAX (__NOD_FLAGS_KTLS + 1)
+#define KNOD_FLAGS_XDP (1 << __NOD_FLAGS_XDP)
+#define KNOD_FLAGS_IPSEC (1 << __NOD_FLAGS_IPSEC)
+#define KNOD_FLAGS_KTLS (1 << __NOD_FLAGS_KTLS)
+
+#define KNOD_TYPE_GPU 0
+#define KNOD_TYPE_DPU 1
+#define KNOD_TYPE_MAX (KNOD_TYPE_DPU + 1)
+
+#define KNOD_SPSC_MAX 32
+#define KNOD_SPSC_ELEMS_MAX 8192
+
+/* Per-RX-queue GPU->host delivery pages (in-flight cap; sized for the deepest
+ * feature pipeline, independent of any per-feature descriptor ring size).
+ * Must cover the worst case where one RSS-concentrated flow lands every
+ * in-flight work on a single queue: ipsec holds a full batch of delivery
+ * pages per work (alloc precedes the SDMA copy into them), so the cap needs
+ * KNOD_IPSEC_NR_WORK * KNOD_IPSEC_PKT_BATCH (= 4 * 512) plus the pass_pending
+ * ring depth. Sized to absorb a full bd ring (KNOD_SPSC_ELEMS_MAX = 8192)
+ * worth of decrypted-but-undelivered packets plus the in-flight works. The
+ * backing is GTT, sized nqueues * KNOD_PASS_SLOTS * PAGE_SIZE (2 GiB at the
+ * 32-queue cap), which is why the alloc size path is size_t rather than int.
+ */
+#define KNOD_PASS_SLOTS 16384
+
+#define KNOD_STATUS_FREE 0
+#define KNOD_STATUS_USED 1
+
+struct knod_netdev {
+ struct list_head list;
+ struct net_device *dev;
+ struct knod_dev *knodev;
+ struct knod_accel *accel;
+ struct knod_nic_ops *nic_ops;
+ struct module *owner;
+ int flags;
+ int status;
+ void *priv;
+};
+
+struct knod_accel_xdp {
+ struct bpf_offload_dev *bpf_dev;
+ struct xdp_attachment_info xdp;
+ struct xdp_attachment_info xdp_hw;
+ struct bpf_prog *bpf_offloaded;
+ struct list_head bound_maps;
+ void *priv;
+};
+
+struct knod_accel {
+ struct list_head list;
+ struct knod_accel_ops *accel_ops;
+ struct module *owner;
+ struct knod_accel_xdp xdp;
+ struct knod_dev *knodev;
+ struct knod_netdev *knetdev;
+ int status;
+ int type;
+ int flags;
+ int id;
+ void *priv;
+ char name[16];
+};
+
+struct knod_dev {
+ struct list_head list;
+ struct knod_netdev *knetdev;
+ struct knod_accel *accel;
+ struct net_devmem_dmabuf_binding *bindings[KNOD_SPSC_MAX];
+ struct mutex lock;
+ struct net_device *netdev;
+ struct knod_dev_stats *stats __percpu;
+
+ struct knod_nic_ops *nic_ops;
+ struct knod_accel_ops *accel_ops;
+ /* per-queue priv data */
+ struct knod_work_priv *wpriv;
+ bool started;
+
+ /* IPsec proxy: original NIC xfrmdev_ops/feature state saved at attach,
+ * restored at detach so a NIC's native offload is not clobbered.
+ */
+ const struct xfrmdev_ops *ipsec_orig_xfrmdev_ops;
+ bool ipsec_added_hw_esp;
+
+ /* framework-owned GPU->host delivery (default pass): drain barrier */
+ /* accel handle for the delivery buffer */
+ void *pass_priv;
+ atomic_t pp_live; /* live delivery page_pools */
+ struct completion pp_drained; /* all pools drained (teardown) */
+ /*
+ * device->host (d2h) delivery: the accel owns the fence counter (its
+ * SDMA ring position); this lock just serialises the shared SDMA
+ * submit path across the per-queue NAPIs that drive knod_d2h_copy.
+ */
+ spinlock_t d2h_lock;
+
+ /*
+ * Feature delivery hook, set by the active feature on activate (NULL
+ * for bpf/none). knod_d2h_drain calls it after building the head_frag
+ * skb to run feature-specific finalisation (ipsec: SA/replay/secpath);
+ * it returns false to drop the packet.
+ */
+ bool (*post_copy)(struct knod_dev *knodev, struct sk_buff *skb,
+ const struct knod_pass_desc *desc, int queue_idx);
+};
+
+static inline bool knod_dev_active(struct knod_dev *knodev)
+{
+ return !!knodev->accel->xdp.xdp_hw.prog;
+}
+
+static inline struct bpf_prog *
+knod_dev_offloaded(struct knod_dev *knodev)
+{
+ return knodev->accel->xdp.bpf_offloaded;
+}
+
+static inline void knod_dev_offload(struct knod_dev *knodev,
+ struct bpf_prog *bpf_offloaded)
+{
+ knodev->accel->xdp.bpf_offloaded = bpf_offloaded;
+}
+
+static inline bool knod_dev_map_empty(struct knod_dev *knodev)
+{
+ return list_empty(&knodev->accel->xdp.bound_maps);
+}
+
+void knod_netdev_register(struct knod_netdev *netdev);
+void knod_netdev_unregister(struct knod_netdev *netdev);
+void knod_accel_register(struct knod_accel *accel);
+void knod_accel_unregister(struct knod_accel *accel);
+void knod_dev_start(struct knod_dev *knodev);
+void knod_dev_stop(struct knod_dev *knodev);
+int knod_dev_xdp_install(struct knod_dev *knodev,
+ struct netdev_bpf *xdp);
+void knod_dev_get_stats64(struct knod_dev *knodev,
+ struct rtnl_link_stats64 *stats);
+void knod_dev_lock(void);
+void knod_dev_unlock(void);
+struct sk_buff *knod_pass_build_skb(netmem_ref netmem, u16 off, u16 len,
+ struct page_pool *pool, bool napi);
+int knod_d2h_copy(struct knod_dev *knodev, int napi_index,
+ struct spsc_pass_bd *bds, int cnt);
+int knod_d2h_drain(struct knod_dev *knodev, int napi_index,
+ struct napi_struct *napi, int budget);
+
+extern struct list_head knod_dev_list;
+extern struct list_head knod_netdev_list;
+extern struct list_head knod_accel_list;
+
+#define for_each_xdev(d) \
+ list_for_each_entry(d, &knod_dev_list, list)
+#define for_each_xdev_safe(d) \
+ list_for_each_entry_safe(d, n, &knod_dev_list, list)
+#define for_each_nodev(d) \
+ list_for_each_entry(d, &knod_netdev_list, list)
+#define for_each_nodev_safe(d, n) \
+ list_for_each_entry_safe(d, n, &knod_netdev_list, list)
+#define for_each_accel(d) \
+ list_for_each_entry(d, &knod_accel_list, list)
+#define for_each_accel_safe(d, n) \
+ list_for_each_entry_safe(d, n, &knod_accel_list, list)
+
+/* IPsec proxy functions */
+#if IS_ENABLED(CONFIG_XFRM_OFFLOAD)
+int knod_ipsec_attach(struct knod_dev *knodev);
+void knod_ipsec_detach(struct knod_dev *knodev);
+#else
+#endif
+
+/* XDP PASS drain - called from NIC NAPI poll */
+int knod_dev_xdp_drain_pass(struct knod_dev *knodev,
+ struct napi_struct *napi,
+ int queue_idx, int budget);
+
+#endif
diff --git a/include/net/spsc_ring.h b/include/net/spsc_ring.h
new file mode 100644
index 000000000000..1eccadc927d6
--- /dev/null
+++ b/include/net/spsc_ring.h
@@ -0,0 +1,645 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * spsc_ring.h - Lock-free SPSC Ring Buffer with embedded element pool
+ *
+ * Single-Producer Single-Consumer ring buffer. The ring owns a
+ * contiguous page-backed memory pool. Each slot is permanently bound
+ * to a cacheline-aligned element inside that pool.
+ *
+ * Producer calls spsc_produce() -> receives a pointer to a free
+ * element, writes data into it, then calls spsc_produce_commit()
+ * to publish.
+ *
+ * Consumer calls spsc_acquire() / spsc_pop() -> receives a pointer
+ * to a filled element. After spsc_release() the slot becomes
+ * available to the producer again - the element pointer is reused
+ * automatically because it is fixed to the slot.
+ *
+ * Two consumer modes (do NOT mix on the same instance):
+ *
+ * Mode 1 - Sliding window (2-step consumer):
+ * spsc_produce / commit -> peek -> acquire -> release
+ *
+ * Mode 2 - Simple queue:
+ * spsc_push -> pop
+ *
+ * Return convention:
+ * 0 success
+ * -ENOSPC ring full (producer side)
+ * -ENOENT ring empty (consumer side)
+ * -EINVAL bad parameter
+ * -ENOMEM allocation failure
+ *
+ * Memory ordering:
+ * Producer: write data -> smp_store_release(head)
+ * Consumer: smp_load_acquire(head) -> read data
+ * Consumer: done -> smp_store_release(tail)
+ * Producer: smp_load_acquire(tail) -> write data
+ *
+ * Capacity is always a power of two.
+ */
+
+#ifndef _SPSC_RING_H
+#define _SPSC_RING_H
+
+#include <linux/types.h>
+#include <linux/slab.h>
+#include <linux/log2.h>
+#include <linux/compiler.h>
+#include <linux/errno.h>
+#include <linux/minmax.h>
+#include <linux/mm.h>
+#include <linux/cache.h>
+#include <asm/barrier.h>
+
+struct spsc_ring {
+ void **slots; /* pointer-per-slot into pool */
+ unsigned int mask; /* capacity - 1 */
+
+ /* Producer side */
+ unsigned int head; /* next slot to publish */
+
+ /* Consumer side */
+ unsigned int tail; /* oldest unconsumed slot */
+ unsigned int acquired; /* sliding window read cursor
+ * tail <= acquired <= head
+ */
+
+ /* Element pool */
+ struct page *pool_page; /* compound page backing elements */
+ unsigned int pool_order; /* page order */
+ unsigned int elem_stride; /* cacheline-aligned element size */
+} ____cacheline_aligned_in_smp;
+
+/* ================================================================== */
+/* Init / Destroy */
+/* ================================================================== */
+
+/**
+ * __spsc_init - initialize ring with a pre-allocated element pool
+ * @r: pointer to caller-allocated spsc_ring
+ * @elem_size: size of each element (rounded up to cacheline)
+ * @capacity: number of elements (rounded up to power of 2)
+ * @pool: pre-allocated pool memory (must be at least stride * capacity)
+ * @gfp: allocation flags (for slots array only)
+ *
+ * The caller owns the pool memory; spsc_destroy will NOT free it.
+ * Returns 0 on success, negative errno on failure.
+ */
+static inline int __spsc_init(struct spsc_ring *r, unsigned int elem_size,
+ unsigned int capacity, void *pool, gfp_t gfp)
+{
+ unsigned int stride = ALIGN(elem_size, SMP_CACHE_BYTES);
+ unsigned int i;
+ char *base = pool;
+
+ if (elem_size == 0 || capacity == 0 || !pool)
+ return -EINVAL;
+
+ capacity = roundup_pow_of_two(capacity);
+
+ r->slots = kcalloc(capacity, sizeof(void *), gfp);
+ if (!r->slots)
+ return -ENOMEM;
+
+ for (i = 0; i < capacity; i++)
+ r->slots[i] = base + (unsigned long)stride * i;
+
+ r->mask = capacity - 1;
+ r->head = 0;
+ r->tail = 0;
+ r->acquired = 0;
+ r->pool_page = NULL;
+ r->pool_order = 0;
+ r->elem_stride = stride;
+
+ return 0;
+}
+
+/**
+ * spsc_init - allocate ring and element pool
+ * @r: pointer to caller-allocated spsc_ring
+ * @elem_size: size of each element (rounded up to cacheline)
+ * @capacity: number of elements (rounded up to power of 2)
+ * @gfp: allocation flags
+ *
+ * Returns 0 on success, negative errno on failure.
+ */
+static inline int spsc_init(struct spsc_ring *r, unsigned int elem_size,
+ unsigned int capacity, gfp_t gfp)
+{
+ unsigned int stride = ALIGN(elem_size, SMP_CACHE_BYTES);
+ unsigned long pool_bytes;
+ unsigned int order;
+ struct page *page;
+
+ if (elem_size == 0 || capacity == 0)
+ return -EINVAL;
+
+ capacity = roundup_pow_of_two(capacity);
+ pool_bytes = (unsigned long)stride * capacity;
+ order = get_order(pool_bytes);
+
+ if (pool_bytes > (PAGE_SIZE << order))
+ return -EOVERFLOW;
+
+ /* Allocate element pool */
+ page = alloc_pages(gfp | __GFP_COMP | __GFP_ZERO, order);
+ if (!page)
+ return -ENOMEM;
+
+ if (__spsc_init(r, elem_size, capacity, page_address(page), gfp)) {
+ __free_pages(page, order);
+ return -ENOMEM;
+ }
+
+ r->pool_page = page;
+ r->pool_order = order;
+
+ return 0;
+}
+
+/**
+ * spsc_destroy - free ring and element pool
+ */
+static inline void spsc_destroy(struct spsc_ring *r)
+{
+ if (r->pool_page) {
+ __free_pages(r->pool_page, r->pool_order);
+ r->pool_page = NULL;
+ }
+ kfree(r->slots);
+ r->slots = NULL;
+}
+
+/* ================================================================== */
+/* Status helpers */
+/* ================================================================== */
+
+static inline unsigned int spsc_capacity(const struct spsc_ring *r)
+{
+ return r->mask + 1;
+}
+
+static inline unsigned int spsc_elem_size(const struct spsc_ring *r)
+{
+ return r->elem_stride;
+}
+
+/** spsc_count - total unconsumed entries (including acquired) */
+static inline unsigned int spsc_count(const struct spsc_ring *r)
+{
+ /* acquire head so a concurrent producer's slot writes are observed */
+ return smp_load_acquire(&((struct spsc_ring *)r)->head) - r->tail;
+}
+
+static inline bool spsc_empty(const struct spsc_ring *r)
+{
+ return spsc_count(r) == 0;
+}
+
+static inline bool spsc_full(const struct spsc_ring *r)
+{
+ return spsc_count(r) > r->mask;
+}
+
+/** spsc_pending - entries acquired but not yet released */
+static inline unsigned int spsc_pending(const struct spsc_ring *r)
+{
+ return r->acquired - r->tail;
+}
+
+/* ================================================================== */
+/* Producer API (single thread only, shared by both modes) */
+/* */
+/* Two-phase produce: */
+/* 1) spsc_produce() -> get pointer to free element */
+/* 2) caller writes data */
+/* 3) spsc_produce_commit() -> publish to consumer */
+/* */
+/* Or one-shot: spsc_push() for pre-filled elements. */
+/* ================================================================== */
+
+/**
+ * spsc_produce - reserve one slot and return its element pointer
+ * @r: ring buffer
+ * @out: receives pointer to the element to write into
+ *
+ * The slot is NOT yet visible to the consumer. Caller must write
+ * data into *out and then call spsc_produce_commit().
+ *
+ * Returns 0 on success, -ENOSPC if full.
+ */
+static inline int spsc_produce(struct spsc_ring *r, void **out)
+{
+ unsigned int head = r->head;
+ unsigned int tail;
+
+ /* acquire tail to observe the slots the consumer has released */
+ tail = smp_load_acquire(&r->tail);
+
+ if (head - tail > r->mask)
+ return -ENOSPC;
+
+ *out = r->slots[head & r->mask];
+
+ return 0;
+}
+
+/**
+ * spsc_produce_commit - publish the previously reserved slot
+ * @r: ring buffer
+ *
+ * Must be called exactly once after each successful spsc_produce().
+ */
+static inline void spsc_produce_commit(struct spsc_ring *r)
+{
+ /* wmb() (sfence on x86) is needed when the pool backing memory
+ * is mapped Write-Combining (e.g. GPU GTT). WC stores are not
+ * ordered by x86 TSO, so smp_store_release (compiler barrier)
+ * alone cannot guarantee the element writes are visible before
+ * the head update reaches the consumer.
+ */
+ wmb();
+ /* release: publish the reserved slot; pairs with the head acquire */
+ smp_store_release(&r->head, r->head + 1);
+}
+
+/**
+ * spsc_produce_n - reserve up to @n slots
+ * @r: ring buffer
+ * @out: destination array for element pointers
+ * @n: max slots to reserve
+ * @cnt: out - number actually reserved (may be NULL)
+ *
+ * Caller must write data into each out[i] and then call
+ * spsc_produce_commit_n(r, *cnt).
+ *
+ * Returns 0 on success, -ENOSPC if zero could be reserved.
+ */
+static inline int spsc_produce_n(struct spsc_ring *r, void **out,
+ unsigned int n, unsigned int *cnt)
+{
+ /* acquire tail to observe the slots the consumer has released */
+ unsigned int tail = smp_load_acquire(&r->tail);
+ unsigned int head = r->head;
+ unsigned int free;
+ unsigned int i;
+
+ free = (r->mask + 1) - (head - tail);
+
+ n = min(n, free);
+ if (n == 0) {
+ if (cnt)
+ *cnt = 0;
+ return -ENOSPC;
+ }
+
+ for (i = 0; i < n; i++)
+ out[i] = r->slots[(head + i) & r->mask];
+
+ if (cnt)
+ *cnt = n;
+ return 0;
+}
+
+/**
+ * spsc_produce_commit_n - publish @n previously reserved slots
+ * @r: ring buffer
+ * @n: number of slots to publish (must match produce_n count)
+ */
+static inline void spsc_produce_commit_n(struct spsc_ring *r, unsigned int n)
+{
+ /* drain WC element stores before the head update (see commit above) */
+ wmb();
+ /* release: publish the reserved slots; pairs with the head acquire */
+ smp_store_release(&r->head, r->head + n);
+}
+
+/* ================================================================== */
+/* Mode 1: Sliding window consumer (single thread only) */
+/* */
+/* peek -> read from acquired cursor, no cursor movement */
+/* acquire -> advance acquired cursor, return element pointers */
+/* release -> advance tail, slots become reusable by producer */
+/* rewind -> reset acquired back to tail */
+/* ================================================================== */
+
+/**
+ * spsc_peek - read element pointers from acquired cursor (read-only)
+ * @r: ring buffer
+ * @out: destination array for element pointers
+ * @max: max entries to peek
+ * @cnt: out - number of entries peeked (may be NULL)
+ *
+ * Does NOT move any cursor.
+ *
+ * Returns 0 on success, -ENOENT if nothing to peek.
+ */
+static inline int spsc_peek(struct spsc_ring *r, void **out, unsigned int max,
+ unsigned int *cnt)
+{
+ /* acquire the producer's head; slots it published are now visible */
+ unsigned int head = smp_load_acquire(&r->head);
+ unsigned int acq = r->acquired;
+ unsigned int avail;
+ unsigned int i;
+
+ avail = head - acq;
+ avail = min(avail, max);
+
+ if (avail == 0) {
+ if (cnt)
+ *cnt = 0;
+ return -ENOENT;
+ }
+
+ for (i = 0; i < avail; i++)
+ out[i] = r->slots[(acq + i) & r->mask];
+
+ if (cnt)
+ *cnt = avail;
+ return 0;
+}
+
+/**
+ * spsc_peek_at - peek starting at an offset past the acquired cursor
+ * @r: ring buffer
+ * @skip: number of entries to skip past r->acquired
+ * @out: destination array for element pointers
+ * @max: max entries to peek
+ * @cnt: out - number of entries peeked (may be NULL)
+ *
+ * Lets the consumer stage a second batch past entries that have been
+ * read by a previous peek but not yet committed via spsc_acquire.
+ * Does NOT move any cursor. The caller is responsible for tracking
+ * the cumulative skip across staged batches; when those batches are
+ * eventually released to the producer via spsc_acquire, pass the same
+ * count so r->acquired catches up.
+ *
+ * Returns 0 on success, -ENOENT if nothing to peek at that offset.
+ */
+static inline int spsc_peek_at(struct spsc_ring *r, unsigned int skip,
+ void **out, unsigned int max, unsigned int *cnt)
+{
+ /* acquire the producer's head; slots it published are now visible */
+ unsigned int head = smp_load_acquire(&r->head);
+ unsigned int pos = r->acquired + skip;
+ unsigned int avail;
+ unsigned int i;
+
+ if ((int)(head - pos) <= 0) {
+ if (cnt)
+ *cnt = 0;
+ return -ENOENT;
+ }
+
+ avail = min(head - pos, max);
+ for (i = 0; i < avail; i++)
+ out[i] = r->slots[(pos + i) & r->mask];
+
+ if (cnt)
+ *cnt = avail;
+ return 0;
+}
+
+/**
+ * spsc_acquire - advance acquired cursor (step 1)
+ * @r: ring buffer
+ * @out: destination array for element pointers, or NULL to skip
+ * @max: max entries to acquire
+ * @cnt: out - number acquired (may be NULL)
+ *
+ * Returns 0 on success, -ENOENT if nothing to acquire.
+ */
+static inline int spsc_acquire(struct spsc_ring *r, void **out,
+ unsigned int max, unsigned int *cnt)
+{
+ /* acquire the producer's head; slots it published are now visible */
+ unsigned int head = smp_load_acquire(&r->head);
+ unsigned int acq = r->acquired;
+ unsigned int avail;
+ unsigned int i;
+
+ avail = head - acq;
+ avail = min(avail, max);
+
+ if (avail == 0) {
+ if (cnt)
+ *cnt = 0;
+ return -ENOENT;
+ }
+
+ if (out) {
+ for (i = 0; i < avail; i++)
+ out[i] = r->slots[(acq + i) & r->mask];
+ }
+
+ /* Publish the window to the releasing consumer (a different CPU than
+ * this acquirer): pair with the smp_load_acquire() in spsc_release()
+ * so it cannot observe the advanced cursor before the element stores
+ * (e.g. an accel verdict) those slots now point at.
+ */
+ smp_store_release(&r->acquired, acq + avail);
+
+ if (cnt)
+ *cnt = avail;
+ return 0;
+}
+
+/**
+ * spsc_acquire_all - acquire all available entries
+ */
+static inline int spsc_acquire_all(struct spsc_ring *r, void **out,
+ unsigned int *cnt)
+{
+ return spsc_acquire(r, out, UINT_MAX, cnt);
+}
+
+/**
+ * spsc_release - get element pointers of acquired entries (release step 1)
+ * @r: ring buffer
+ * @out: destination array for element pointers, or NULL to skip
+ * @n: number of entries to release (<= pending)
+ * @cnt: out - number of entries prepared for release (may be NULL)
+ *
+ * Returns element pointers for the oldest @n acquired entries but
+ * does NOT advance tail - the producer still cannot reuse these slots.
+ * Caller processes the elements, then calls spsc_release_commit() to
+ * actually free them.
+ *
+ * Returns 0 on success, -ENOENT if nothing to release.
+ */
+static inline int spsc_release(struct spsc_ring *r, void **out, unsigned int n,
+ unsigned int *cnt)
+{
+ unsigned int tail = r->tail;
+ unsigned int pending;
+ unsigned int i;
+
+ /* Pairs with smp_store_release(&r->acquired) in spsc_acquire(): once
+ * we see the advanced cursor we are guaranteed to see the element
+ * stores (e.g. the accel verdict) for the slots it exposes.
+ */
+ pending = smp_load_acquire(&r->acquired) - tail;
+
+ n = min(n, pending);
+ if (n == 0) {
+ if (cnt)
+ *cnt = 0;
+ return -ENOENT;
+ }
+
+ if (out) {
+ for (i = 0; i < n; i++)
+ out[i] = r->slots[(tail + i) & r->mask];
+ }
+
+ if (cnt)
+ *cnt = n;
+ return 0;
+}
+
+/**
+ * spsc_release_commit - advance tail, free slots for producer (release step 2)
+ * @r: ring buffer
+ * @n: number of entries to commit (must match prior spsc_release count)
+ *
+ * After this call the producer may reuse these slots.
+ */
+static inline void spsc_release_commit(struct spsc_ring *r, unsigned int n)
+{
+ /* release: hand the consumed slots back to the producer */
+ smp_store_release(&r->tail, r->tail + n);
+}
+
+/**
+ * spsc_release_all - get all acquired entries' pointers (release step 1)
+ * @r: ring buffer
+ * @out: destination array for element pointers, or NULL to skip
+ * @cnt: out - number of entries prepared for release (may be NULL)
+ *
+ * Convenience for spsc_release(r, out, pending, cnt).
+ * Caller must still call spsc_release_commit(r, *cnt) afterward.
+ *
+ * Returns 0 on success, -ENOENT if nothing pending.
+ */
+static inline int spsc_release_all(struct spsc_ring *r, void **out,
+ unsigned int *cnt)
+{
+ return spsc_release(r, out, r->acquired - r->tail, cnt);
+}
+
+/**
+ * spsc_rewind - undo acquires, reset acquired cursor to tail
+ */
+static inline void spsc_rewind(struct spsc_ring *r)
+{
+ r->acquired = r->tail;
+}
+
+/* ================================================================== */
+/* Mode 2: Simple push / pop (single thread per side) */
+/* */
+/* One-shot convenience wrappers. */
+/* Do NOT mix with Mode 1 acquire/release on the same instance. */
+/* ================================================================== */
+
+/**
+ * spsc_push - reserve, let caller fill, and publish in one shot
+ * @r: ring buffer
+ * @out: receives pointer to the element to write into
+ *
+ * Unlike produce/commit, the slot is published immediately.
+ * Caller must fill *out BEFORE this function returns if another
+ * thread could consume it - but since this is SPSC with push/pop
+ * the typical pattern is:
+ *
+ * spsc_push(&r, &elem);
+ * fill(elem); // safe: consumer hasn't seen it yet?
+ *
+ * NO - push publishes immediately. Use produce/commit if you need
+ * to fill before publishing. push is an alias for produce+commit.
+ *
+ * Returns 0 on success, -ENOSPC if full.
+ */
+static inline int spsc_push(struct spsc_ring *r, void **out)
+{
+ int ret;
+
+ ret = spsc_produce(r, out);
+ if (ret)
+ return ret;
+
+ spsc_produce_commit(r);
+ return 0;
+}
+
+/**
+ * spsc_pop - dequeue one element
+ * @r: ring buffer
+ * @out: receives pointer to the consumed element
+ *
+ * The element pointer remains valid until the next spsc_push() or
+ * spsc_produce() reuses that slot.
+ *
+ * Returns 0 on success, -ENOENT if empty.
+ */
+static inline int spsc_pop(struct spsc_ring *r, void **out)
+{
+ /* acquire the producer's head; slots it published are now visible */
+ unsigned int head = smp_load_acquire(&r->head);
+ unsigned int tail = r->tail;
+
+ if (tail == head)
+ return -ENOENT;
+
+ *out = r->slots[tail & r->mask];
+
+ r->acquired = tail + 1;
+ /* release: hand the consumed slot back to the producer */
+ smp_store_release(&r->tail, tail + 1);
+
+ return 0;
+}
+
+/**
+ * spsc_pop_n - dequeue up to @n elements
+ * @r: ring buffer
+ * @out: destination array for element pointers
+ * @n: max entries to dequeue
+ * @cnt: out - number dequeued (may be NULL)
+ *
+ * Returns 0 on success, -ENOENT if empty.
+ */
+static inline int spsc_pop_n(struct spsc_ring *r, void **out, unsigned int n,
+ unsigned int *cnt)
+{
+ /* acquire the producer's head; slots it published are now visible */
+ unsigned int head = smp_load_acquire(&r->head);
+ unsigned int tail = r->tail;
+ unsigned int avail;
+ unsigned int i;
+
+ avail = head - tail;
+ n = min(n, avail);
+
+ if (n == 0) {
+ if (cnt)
+ *cnt = 0;
+ return -ENOENT;
+ }
+
+ for (i = 0; i < n; i++)
+ out[i] = r->slots[(tail + i) & r->mask];
+
+ r->acquired = tail + n;
+ /* release: hand the consumed slots back to the producer */
+ smp_store_release(&r->tail, tail + n);
+
+ if (cnt)
+ *cnt = n;
+ return 0;
+}
+
+#endif /* _SPSC_RING_H */
diff --git a/include/uapi/linux/knod.h b/include/uapi/linux/knod.h
new file mode 100644
index 000000000000..57602685c14e
--- /dev/null
+++ b/include/uapi/linux/knod.h
@@ -0,0 +1,67 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */
+/* Do not edit directly, auto-generated from: */
+/* Documentation/netlink/specs/knod.yaml */
+/* YNL-GEN uapi header */
+/* To regenerate run: tools/net/ynl/ynl-regen.sh */
+
+#ifndef _UAPI_LINUX_KNOD_H
+#define _UAPI_LINUX_KNOD_H
+
+#define KNOD_FAMILY_NAME "knod"
+#define KNOD_FAMILY_VERSION 1
+
+enum knod_feature {
+ KNOD_FEATURE_NONE,
+ KNOD_FEATURE_BPF,
+ KNOD_FEATURE_IPSEC,
+};
+
+enum knod_accel_type {
+ KNOD_ACCEL_TYPE_GPU,
+ KNOD_ACCEL_TYPE_DPU,
+};
+
+enum {
+ KNOD_A_ACCEL_ID = 1,
+ KNOD_A_ACCEL_NAME,
+ KNOD_A_ACCEL_TYPE,
+ KNOD_A_ACCEL_FEATURE_CAP,
+ KNOD_A_ACCEL_FEATURE_ENA,
+
+ __KNOD_A_ACCEL_MAX,
+ KNOD_A_ACCEL_MAX = (__KNOD_A_ACCEL_MAX - 1)
+};
+
+enum {
+ KNOD_A_NIC_IFINDEX = 1,
+ KNOD_A_NIC_NAME,
+
+ __KNOD_A_NIC_MAX,
+ KNOD_A_NIC_MAX = (__KNOD_A_NIC_MAX - 1)
+};
+
+enum {
+ KNOD_A_DEV_NIC_IFINDEX = 1,
+ KNOD_A_DEV_ACCEL_ID,
+
+ __KNOD_A_DEV_MAX,
+ KNOD_A_DEV_MAX = (__KNOD_A_DEV_MAX - 1)
+};
+
+enum {
+ KNOD_CMD_ACCEL_GET = 1,
+ KNOD_CMD_ACCEL_SET,
+ KNOD_CMD_NIC_GET,
+ KNOD_CMD_ATTACH,
+ KNOD_CMD_DETACH,
+ KNOD_CMD_DEV_GET,
+ KNOD_CMD_DEV_ADD_NTF,
+ KNOD_CMD_DEV_DEL_NTF,
+
+ __KNOD_CMD_MAX,
+ KNOD_CMD_MAX = (__KNOD_CMD_MAX - 1)
+};
+
+#define KNOD_MCGRP_MGMT "mgmt"
+
+#endif /* _UAPI_LINUX_KNOD_H */
--
2.43.0
^ permalink raw reply related
* [RFC PATCH net-next 02/13] net: devmem: extend memory provider for knod
From: Taehee Yoo @ 2026-07-19 17:58 UTC (permalink / raw)
To: Alex Deucher, Alexei Starovoitov, amd-gfx, Andrew Lunn,
Andrii Nakryiko, Bill Wendling, bpf, Christian König,
Daniel Borkmann, David Airlie, David S. Miller, Donald Hunter,
dri-devel, Eduard Zingerman, Emil Tsalapatis, Eric Dumazet,
Felix Kuehling, Hoyeon Lee, Ilias Apalodimas, Jakub Kicinski,
Jesper Dangaard Brouer, Jiri Olsa, John Fastabend, Justin Stitt,
Kees Cook, Kumar Kartikeya Dwivedi, Leon Romanovsky,
linaro-mm-sig, linux-hardening, linux-kernel, linux-kselftest,
linux-media, linux-rdma, llvm, Mark Bloch, Martin KaFai Lau,
Michael Chan, Nathan Chancellor, netdev, Nick Desaulniers,
Paolo Abeni, Pavan Chebbi, Saeed Mahameed, Shuah Khan,
Simona Vetter, Simon Horman, Song Liu, Stanislav Fomichev,
Sumit Semwal, Taehee Yoo, Tariq Toukan, Yonghong Song
In-Reply-To: <20260719175857.4071636-1-ap420073@gmail.com>
Extend the devmem memory-provider path so a knod accelerator can back a
NIC page_pool with accelerator-exported memory (dma-buf), letting the
NIC DMA received packets directly into accelerator memory.
Signed-off-by: Taehee Yoo <ap420073@gmail.com>
(cherry picked from commit d511a8cb3e229f8f5cf060985880d45bd384db87)
---
include/net/devmem.h | 58 +++++++++++++
include/net/netmem.h | 9 +++
include/net/page_pool/memory_provider.h | 4 +
include/net/page_pool/types.h | 23 +++++-
net/core/devmem.c | 103 +++++++++++++++++++-----
net/core/devmem.h | 7 +-
net/core/page_pool.c | 22 ++++-
7 files changed, 198 insertions(+), 28 deletions(-)
create mode 100644 include/net/devmem.h
diff --git a/include/net/devmem.h b/include/net/devmem.h
new file mode 100644
index 000000000000..f1c3895d7833
--- /dev/null
+++ b/include/net/devmem.h
@@ -0,0 +1,58 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Device memory TCP support
+ *
+ * Authors: Mina Almasry <almasrymina@google.com>
+ * Willem de Bruijn <willemb@google.com>
+ * Kaiyuan Zhang <kaiyuanz@google.com>
+ *
+ */
+#ifndef _NET_DEVMEM_H
+#define _NET_DEVMEM_H
+
+#include <linux/dma-direction.h>
+#include <linux/err.h>
+#include <linux/types.h>
+
+struct device;
+struct dma_buf;
+struct dma_buf_attach_ops;
+struct net_device;
+struct net_devmem_dmabuf_binding;
+struct netlink_ext_ack;
+
+#if defined(CONFIG_NET_DEVMEM)
+struct net_devmem_dmabuf_binding *
+__net_devmem_binding_create(struct net_device *dev, struct device *dma_dev,
+ struct dma_buf *dmabuf,
+ enum dma_data_direction direction,
+ const struct dma_buf_attach_ops *importer_ops,
+ struct netlink_ext_ack *extack);
+int net_devmem_bind_dmabuf_to_queue_direct(struct net_device *dev, u32 rxq_idx,
+ struct net_devmem_dmabuf_binding *binding);
+void net_devmem_unbind_dmabuf_direct(struct net_devmem_dmabuf_binding *binding);
+#else
+static inline struct net_devmem_dmabuf_binding *
+__net_devmem_binding_create(struct net_device *dev, struct device *dma_dev,
+ struct dma_buf *dmabuf,
+ enum dma_data_direction direction,
+ const struct dma_buf_attach_ops *importer_ops,
+ struct netlink_ext_ack *extack)
+{
+ return ERR_PTR(-EOPNOTSUPP);
+}
+
+static inline int
+net_devmem_bind_dmabuf_to_queue_direct(struct net_device *dev, u32 rxq_idx,
+ struct net_devmem_dmabuf_binding *binding)
+{
+ return -EOPNOTSUPP;
+}
+
+static inline void
+net_devmem_unbind_dmabuf_direct(struct net_devmem_dmabuf_binding *binding)
+{
+}
+#endif
+
+#endif /* _NET_DEVMEM_H */
diff --git a/include/net/netmem.h b/include/net/netmem.h
index bccacd21b6c3..3ddfbd37500f 100644
--- a/include/net/netmem.h
+++ b/include/net/netmem.h
@@ -127,6 +127,15 @@ static inline void net_iov_init(struct net_iov *niov,
niov->type = type;
}
+/* Global page index within the dma-buf, accounting for multi-chunk
+ * scatter-gather layouts where each chunk owner's niovs start at 0.
+ */
+static inline unsigned int net_iov_binding_idx(const struct net_iov *niov)
+{
+ return (net_iov_owner(niov)->base_virtual >> PAGE_SHIFT) +
+ net_iov_idx(niov);
+}
+
/* netmem */
/**
diff --git a/include/net/page_pool/memory_provider.h b/include/net/page_pool/memory_provider.h
index 255ce4cfd975..4b58a9702fb7 100644
--- a/include/net/page_pool/memory_provider.h
+++ b/include/net/page_pool/memory_provider.h
@@ -23,6 +23,10 @@ bool net_mp_niov_set_dma_addr(struct net_iov *niov, dma_addr_t addr);
void net_mp_niov_set_page_pool(struct page_pool *pool, struct net_iov *niov);
void net_mp_niov_clear_page_pool(struct net_iov *niov);
+void page_pool_provider_set_netmem(struct page_pool *pool, netmem_ref netmem,
+ dma_addr_t addr);
+void page_pool_clear_pp_info(netmem_ref netmem);
+
int netif_mp_open_rxq(struct net_device *dev, unsigned int rxq_idx,
const struct pp_memory_provider_params *p,
struct netlink_ext_ack *extack);
diff --git a/include/net/page_pool/types.h b/include/net/page_pool/types.h
index 03da138722f5..3e866f249768 100644
--- a/include/net/page_pool/types.h
+++ b/include/net/page_pool/types.h
@@ -31,8 +31,16 @@
*/
#define PP_FLAG_ALLOW_UNREADABLE_NETMEM BIT(3)
+/* Driver-managed pool with a directly-supplied memory provider, not bound to a
+ * netdev rx queue. Setting this flag requires page_pool_params.mp_ops and
+ * .mp_priv to both be set.
+ */
+#define PP_FLAG_CUSTOM_MEMORY_PROVIDER BIT(4)
+
#define PP_FLAG_ALL (PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV | \
- PP_FLAG_SYSTEM_POOL | PP_FLAG_ALLOW_UNREADABLE_NETMEM)
+ PP_FLAG_SYSTEM_POOL | \
+ PP_FLAG_ALLOW_UNREADABLE_NETMEM | \
+ PP_FLAG_CUSTOM_MEMORY_PROVIDER)
/* Index limit to stay within PP_DMA_INDEX_BITS for DMA indices */
#define PP_DMA_INDEX_LIMIT XA_LIMIT(1, BIT(PP_DMA_INDEX_BITS) - 1)
@@ -54,11 +62,11 @@
* would have to take a slower code path.
*/
#if PAGE_SIZE >= SZ_64K
-#define PP_ALLOC_CACHE_REFILL 4
+#define PP_ALLOC_CACHE_REFILL 256
#elif PAGE_SIZE >= SZ_16K
-#define PP_ALLOC_CACHE_REFILL 16
+#define PP_ALLOC_CACHE_REFILL 1024
#else
-#define PP_ALLOC_CACHE_REFILL 64
+#define PP_ALLOC_CACHE_REFILL 4096
#endif
#define PP_ALLOC_CACHE_SIZE (PP_ALLOC_CACHE_REFILL * 2)
@@ -67,6 +75,8 @@ struct pp_alloc_cache {
netmem_ref cache[PP_ALLOC_CACHE_SIZE];
};
+struct memory_provider_ops;
+
/**
* struct page_pool_params - page pool parameters
* @fast: params accessed frequently on hotpath
@@ -83,6 +93,9 @@ struct pp_alloc_cache {
* @queue_idx: queue idx this page_pool is being created for.
* @flags: PP_FLAG_DMA_MAP, PP_FLAG_DMA_SYNC_DEV, PP_FLAG_SYSTEM_POOL,
* PP_FLAG_ALLOW_UNREADABLE_NETMEM.
+ * @mp_ops: driver-supplied memory provider for a pool not bound to a
+ * netdev rx queue (NULL to use rxq->mp_params instead)
+ * @mp_priv: context passed to @mp_ops
*/
struct page_pool_params {
struct_group_tagged(page_pool_params_fast, fast,
@@ -99,6 +112,8 @@ struct page_pool_params {
struct net_device *netdev;
unsigned int queue_idx;
unsigned int flags;
+ const struct memory_provider_ops *mp_ops;
+ void *mp_priv;
/* private: used by test code only */
void (*init_callback)(netmem_ref netmem, void *arg);
void *init_arg;
diff --git a/net/core/devmem.c b/net/core/devmem.c
index 957d6b96216b..9e21cffc9643 100644
--- a/net/core/devmem.c
+++ b/net/core/devmem.c
@@ -121,12 +121,9 @@ void net_devmem_free_dmabuf(struct net_iov *niov)
gen_pool_free(binding->chunk_pool, dma_addr, PAGE_SIZE);
}
-void net_devmem_unbind_dmabuf(struct net_devmem_dmabuf_binding *binding)
+static void
+net_devmem_binding_unpublish(struct net_devmem_dmabuf_binding *binding)
{
- struct netdev_rx_queue *rxq;
- unsigned long xa_idx;
- unsigned int rxq_idx;
-
xa_erase(&net_devmem_dmabuf_bindings, binding->id);
/* Ensure no tx net_devmem_lookup_dmabuf() are in flight after the
@@ -136,6 +133,15 @@ void net_devmem_unbind_dmabuf(struct net_devmem_dmabuf_binding *binding)
if (binding->list.next)
list_del(&binding->list);
+}
+
+void net_devmem_unbind_dmabuf(struct net_devmem_dmabuf_binding *binding)
+{
+ struct netdev_rx_queue *rxq;
+ unsigned long xa_idx;
+ unsigned int rxq_idx;
+
+ net_devmem_binding_unpublish(binding);
xa_for_each(&binding->bound_rxqs, xa_idx, rxq) {
const struct pp_memory_provider_params mp_params = {
@@ -151,6 +157,47 @@ void net_devmem_unbind_dmabuf(struct net_devmem_dmabuf_binding *binding)
percpu_ref_kill(&binding->ref);
}
+/* Bind/unbind variants for in-kernel offload importers that drive the rx
+ * queue lifecycle themselves. The mp_params are poked directly, without the
+ * tcp-data-split/XDP guards or the queue reconfigure that the netlink control
+ * plane applies through netif_mp_open_rxq()/netif_mp_close_rxq().
+ */
+int net_devmem_bind_dmabuf_to_queue_direct(struct net_device *dev, u32 rxq_idx,
+ struct net_devmem_dmabuf_binding *binding)
+{
+ struct netdev_rx_queue *rxq;
+ u32 xa_idx;
+ int err;
+
+ rxq = __netif_get_rx_queue(dev, rxq_idx);
+ rxq->mp_params.mp_priv = binding;
+ rxq->mp_params.mp_ops = &dmabuf_devmem_ops;
+
+ err = xa_alloc(&binding->bound_rxqs, &xa_idx, rxq, xa_limit_32b,
+ GFP_KERNEL);
+ if (err) {
+ rxq->mp_params.mp_priv = NULL;
+ rxq->mp_params.mp_ops = NULL;
+ }
+
+ return err;
+}
+
+void net_devmem_unbind_dmabuf_direct(struct net_devmem_dmabuf_binding *binding)
+{
+ struct netdev_rx_queue *rxq;
+ unsigned long xa_idx;
+
+ net_devmem_binding_unpublish(binding);
+
+ xa_for_each(&binding->bound_rxqs, xa_idx, rxq) {
+ rxq->mp_params.mp_priv = NULL;
+ rxq->mp_params.mp_ops = NULL;
+ }
+
+ percpu_ref_kill(&binding->ref);
+}
+
int net_devmem_bind_dmabuf_to_queue(struct net_device *dev, u32 rxq_idx,
struct net_devmem_dmabuf_binding *binding,
struct netlink_ext_ack *extack)
@@ -188,12 +235,7 @@ net_devmem_bind_dmabuf(struct net_device *dev, void *vdev,
struct netlink_ext_ack *extack)
{
struct net_devmem_dmabuf_binding *binding;
- static u32 id_alloc_next;
- struct scatterlist *sg;
struct dma_buf *dmabuf;
- unsigned int sg_idx, i;
- unsigned long virtual;
- int err;
if (!dma_dev) {
NL_SET_ERR_MSG(extack, "Device doesn't support DMA");
@@ -204,15 +246,39 @@ net_devmem_bind_dmabuf(struct net_device *dev, void *vdev,
if (IS_ERR(dmabuf))
return ERR_CAST(dmabuf);
+ binding = __net_devmem_binding_create(dev, dma_dev, dmabuf, direction,
+ NULL, extack);
+ if (IS_ERR(binding)) {
+ dma_buf_put(dmabuf);
+ return binding;
+ }
+
+ binding->vdev = vdev;
+ list_add(&binding->list, &priv->bindings);
+
+ return binding;
+}
+
+struct net_devmem_dmabuf_binding *
+__net_devmem_binding_create(struct net_device *dev, struct device *dma_dev,
+ struct dma_buf *dmabuf,
+ enum dma_data_direction direction,
+ const struct dma_buf_attach_ops *importer_ops,
+ struct netlink_ext_ack *extack)
+{
+ struct net_devmem_dmabuf_binding *binding;
+ static u32 id_alloc_next;
+ struct scatterlist *sg;
+ unsigned int sg_idx, i;
+ unsigned long virtual;
+ int err;
+
binding = kzalloc_node(sizeof(*binding), GFP_KERNEL,
dev_to_node(&dev->dev));
- if (!binding) {
- err = -ENOMEM;
- goto err_put_dmabuf;
- }
+ if (!binding)
+ return ERR_PTR(-ENOMEM);
binding->dev = dev;
- binding->vdev = vdev;
xa_init_flags(&binding->bound_rxqs, XA_FLAGS_ALLOC);
err = percpu_ref_init(&binding->ref,
@@ -226,7 +292,8 @@ net_devmem_bind_dmabuf(struct net_device *dev, void *vdev,
binding->dmabuf = dmabuf;
binding->direction = direction;
- binding->attachment = dma_buf_attach(binding->dmabuf, dma_dev);
+ binding->attachment = dma_buf_dynamic_attach(binding->dmabuf, dma_dev,
+ importer_ops, binding);
if (IS_ERR(binding->attachment)) {
err = PTR_ERR(binding->attachment);
NL_SET_ERR_MSG(extack, "Failed to bind dmabuf to device");
@@ -325,8 +392,6 @@ net_devmem_bind_dmabuf(struct net_device *dev, void *vdev,
if (err < 0)
goto err_free_chunks;
- list_add(&binding->list, &priv->bindings);
-
return binding;
err_free_chunks:
@@ -344,8 +409,6 @@ net_devmem_bind_dmabuf(struct net_device *dev, void *vdev,
percpu_ref_exit(&binding->ref);
err_free_binding:
kfree(binding);
-err_put_dmabuf:
- dma_buf_put(dmabuf);
return ERR_PTR(err);
}
diff --git a/net/core/devmem.h b/net/core/devmem.h
index 3852a56036cb..6b57837ab454 100644
--- a/net/core/devmem.h
+++ b/net/core/devmem.h
@@ -7,9 +7,10 @@
* Kaiyuan Zhang <kaiyuanz@google.com>
*
*/
-#ifndef _NET_DEVMEM_H
-#define _NET_DEVMEM_H
+#ifndef _NET_CORE_DEVMEM_H
+#define _NET_CORE_DEVMEM_H
+#include <net/devmem.h>
#include <net/netmem.h>
#include <net/netdev_netlink.h>
@@ -240,4 +241,4 @@ net_devmem_iov_binding(const struct net_iov *niov)
}
#endif
-#endif /* _NET_DEVMEM_H */
+#endif /* _NET_CORE_DEVMEM_H */
diff --git a/net/core/page_pool.c b/net/core/page_pool.c
index 21dc4a9c8714..fd7444943b9f 100644
--- a/net/core/page_pool.c
+++ b/net/core/page_pool.c
@@ -272,7 +272,17 @@ static int page_pool_init(struct page_pool *pool,
xa_init_flags(&pool->dma_mapped, XA_FLAGS_ALLOC1);
- if (pool->slow.flags & PP_FLAG_ALLOW_UNREADABLE_NETMEM) {
+ if (pool->slow.flags & PP_FLAG_CUSTOM_MEMORY_PROVIDER) {
+ /* Driver-managed pool with a directly-supplied memory
+ * provider, not bound to a netdev rx queue.
+ */
+ if (WARN_ON(!pool->slow.mp_ops || !pool->slow.mp_priv)) {
+ err = -EINVAL;
+ goto free_ptr_ring;
+ }
+ pool->mp_priv = pool->slow.mp_priv;
+ pool->mp_ops = pool->slow.mp_ops;
+ } else if (pool->slow.flags & PP_FLAG_ALLOW_UNREADABLE_NETMEM) {
netdev_assert_locked(pool->slow.netdev);
rxq = __netif_get_rx_queue(pool->slow.netdev,
pool->slow.queue_idx);
@@ -725,6 +735,16 @@ void page_pool_clear_pp_info(netmem_ref netmem)
netmem_set_pp(netmem, NULL);
}
+void page_pool_provider_set_netmem(struct page_pool *pool, netmem_ref netmem,
+ dma_addr_t addr)
+{
+ netmem_to_nmdesc(netmem)->pp_magic = 0;
+ netmem_to_nmdesc(netmem)->pp = NULL;
+ atomic_long_set(&netmem_to_nmdesc(netmem)->pp_ref_count, 0);
+ page_pool_set_pp_info(pool, netmem);
+ page_pool_set_dma_addr_netmem(netmem, addr);
+}
+
static __always_inline void __page_pool_release_netmem_dma(struct page_pool *pool,
netmem_ref netmem)
{
--
2.43.0
^ permalink raw reply related
* [RFC PATCH net-next 03/13] net: core: add XDP_MODE_HW offload hook for knod
From: Taehee Yoo @ 2026-07-19 17:58 UTC (permalink / raw)
To: Alex Deucher, Alexei Starovoitov, amd-gfx, Andrew Lunn,
Andrii Nakryiko, Bill Wendling, bpf, Christian König,
Daniel Borkmann, David Airlie, David S. Miller, Donald Hunter,
dri-devel, Eduard Zingerman, Emil Tsalapatis, Eric Dumazet,
Felix Kuehling, Hoyeon Lee, Ilias Apalodimas, Jakub Kicinski,
Jesper Dangaard Brouer, Jiri Olsa, John Fastabend, Justin Stitt,
Kees Cook, Kumar Kartikeya Dwivedi, Leon Romanovsky,
linaro-mm-sig, linux-hardening, linux-kernel, linux-kselftest,
linux-media, linux-rdma, llvm, Mark Bloch, Martin KaFai Lau,
Michael Chan, Nathan Chancellor, netdev, Nick Desaulniers,
Paolo Abeni, Pavan Chebbi, Saeed Mahameed, Shuah Khan,
Simona Vetter, Simon Horman, Song Liu, Stanislav Fomichev,
Sumit Semwal, Taehee Yoo, Tariq Toukan, Yonghong Song
In-Reply-To: <20260719175857.4071636-1-ap420073@gmail.com>
Route XDP program install/uninstall to the knod accelerator when a
device is attached, so BPF programs run on the accelerator instead of
the host.
Signed-off-by: Taehee Yoo <ap420073@gmail.com>
(cherry picked from commit 9689730acf7d6e4d34cd677e4615be10856c709c)
---
net/core/dev.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/net/core/dev.c b/net/core/dev.c
index 7c21bc0a1e34..5ba1c969029b 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -10334,7 +10334,7 @@ static int dev_xdp_install(struct net_device *dev, enum bpf_xdp_mode mode,
return -EBUSY;
}
- if (dev_get_min_mp_channel_count(dev)) {
+ if (dev_get_min_mp_channel_count(dev) && mode != XDP_MODE_HW) {
NL_SET_ERR_MSG(extack, "unable to install XDP to device using memory provider");
return -EBUSY;
}
--
2.43.0
^ permalink raw reply related
* [RFC PATCH net-next 04/13] net: knod: add offload device core and control plane
From: Taehee Yoo @ 2026-07-19 17:58 UTC (permalink / raw)
To: Alex Deucher, Alexei Starovoitov, amd-gfx, Andrew Lunn,
Andrii Nakryiko, Bill Wendling, bpf, Christian König,
Daniel Borkmann, David Airlie, David S. Miller, Donald Hunter,
dri-devel, Eduard Zingerman, Emil Tsalapatis, Eric Dumazet,
Felix Kuehling, Hoyeon Lee, Ilias Apalodimas, Jakub Kicinski,
Jesper Dangaard Brouer, Jiri Olsa, John Fastabend, Justin Stitt,
Kees Cook, Kumar Kartikeya Dwivedi, Leon Romanovsky,
linaro-mm-sig, linux-hardening, linux-kernel, linux-kselftest,
linux-media, linux-rdma, llvm, Mark Bloch, Martin KaFai Lau,
Michael Chan, Nathan Chancellor, netdev, Nick Desaulniers,
Paolo Abeni, Pavan Chebbi, Saeed Mahameed, Shuah Khan,
Simona Vetter, Simon Horman, Song Liu, Stanislav Fomichev,
Sumit Semwal, Taehee Yoo, Tariq Toukan, Yonghong Song
In-Reply-To: <20260719175857.4071636-1-ap420073@gmail.com>
Add the KNOD core framework and its genetlink control plane.
The core maintains three object types - a NIC-side netdev, an
accelerator, and a dynamically created offload device that binds them -
and drives the RX data path over per-queue SPSC rings backed by a
page_pool of accelerator memory.
The genetlink family ("knod") exposes attach/detach, device/nic/accel
inventory, and per-accelerator feature selection as a stable uAPI.
Signed-off-by: Taehee Yoo <ap420073@gmail.com>
(cherry picked from commit f17f82a39be852aa8c520aea7177ce0086f7e811)
---
net/Kconfig | 2 +
net/Makefile | 1 +
net/knod/Kconfig | 5 +
net/knod/Makefile | 8 +
net/knod/knod-nl-gen.c | 121 ++++
net/knod/knod-nl-gen.h | 31 +
net/knod/knod.h | 26 +
net/knod/knod_core.c | 1231 ++++++++++++++++++++++++++++++++++++++++
net/knod/knod_nl.c | 406 +++++++++++++
9 files changed, 1831 insertions(+)
create mode 100644 net/knod/Kconfig
create mode 100644 net/knod/Makefile
create mode 100644 net/knod/knod-nl-gen.c
create mode 100644 net/knod/knod-nl-gen.h
create mode 100644 net/knod/knod.h
create mode 100644 net/knod/knod_core.c
create mode 100644 net/knod/knod_nl.c
diff --git a/net/Kconfig b/net/Kconfig
index e38477393551..e7a2200e92cc 100644
--- a/net/Kconfig
+++ b/net/Kconfig
@@ -72,6 +72,8 @@ config NET_DEVMEM
depends on DMA_SHARED_BUFFER
depends on PAGE_POOL
+source "net/knod/Kconfig"
+
config NET_SHAPER
bool
diff --git a/net/Makefile b/net/Makefile
index 5b2dd7f07a85..ea4ef5c5906d 100644
--- a/net/Makefile
+++ b/net/Makefile
@@ -19,6 +19,7 @@ obj-$(CONFIG_TLS) += tls/
obj-$(CONFIG_XFRM) += xfrm/
obj-$(CONFIG_UNIX) += unix/
obj-$(CONFIG_INET_PSP) += psp/
+obj-$(CONFIG_KNOD) += knod/
obj-y += ipv6/
obj-$(CONFIG_PACKET) += packet/
obj-$(CONFIG_NET_KEY) += key/
diff --git a/net/knod/Kconfig b/net/knod/Kconfig
new file mode 100644
index 000000000000..d5a6d76d33e3
--- /dev/null
+++ b/net/knod/Kconfig
@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+config KNOD
+ def_bool y
+ depends on NET_DEVMEM
diff --git a/net/knod/Makefile b/net/knod/Makefile
new file mode 100644
index 000000000000..73abe5c99f8c
--- /dev/null
+++ b/net/knod/Makefile
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+#
+# Makefile for the KNOD (KFD network offload) control plane.
+#
+
+obj-$(CONFIG_KNOD) += knod.o
+
+knod-y := knod_core.o knod_nl.o knod-nl-gen.o
diff --git a/net/knod/knod-nl-gen.c b/net/knod/knod-nl-gen.c
new file mode 100644
index 000000000000..96042f0c15c9
--- /dev/null
+++ b/net/knod/knod-nl-gen.c
@@ -0,0 +1,121 @@
+// SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause)
+/* Do not edit directly, auto-generated from: */
+/* Documentation/netlink/specs/knod.yaml */
+/* YNL-GEN kernel source */
+/* To regenerate run: tools/net/ynl/ynl-regen.sh */
+
+#include <net/netlink.h>
+#include <net/genetlink.h>
+
+#include "knod-nl-gen.h"
+
+#include <uapi/linux/knod.h>
+
+/* KNOD_CMD_ACCEL_GET - do */
+static const struct nla_policy knod_accel_get_nl_policy[KNOD_A_ACCEL_ID + 1] = {
+ [KNOD_A_ACCEL_ID] = NLA_POLICY_MIN(NLA_U32, 1),
+};
+
+/* KNOD_CMD_ACCEL_SET - do */
+static const struct nla_policy knod_accel_set_nl_policy[KNOD_A_ACCEL_FEATURE_ENA + 1] = {
+ [KNOD_A_ACCEL_ID] = NLA_POLICY_MIN(NLA_U32, 1),
+ [KNOD_A_ACCEL_FEATURE_ENA] = NLA_POLICY_MAX(NLA_U32, 2),
+};
+
+/* KNOD_CMD_NIC_GET - do */
+static const struct nla_policy knod_nic_get_nl_policy[KNOD_A_NIC_IFINDEX + 1] = {
+ [KNOD_A_NIC_IFINDEX] = NLA_POLICY_MIN(NLA_U32, 1),
+};
+
+/* KNOD_CMD_ATTACH - do */
+static const struct nla_policy knod_attach_nl_policy[KNOD_A_DEV_ACCEL_ID + 1] = {
+ [KNOD_A_DEV_NIC_IFINDEX] = NLA_POLICY_MIN(NLA_U32, 1),
+ [KNOD_A_DEV_ACCEL_ID] = NLA_POLICY_MIN(NLA_U32, 1),
+};
+
+/* KNOD_CMD_DETACH - do */
+static const struct nla_policy knod_detach_nl_policy[KNOD_A_DEV_NIC_IFINDEX + 1] = {
+ [KNOD_A_DEV_NIC_IFINDEX] = NLA_POLICY_MIN(NLA_U32, 1),
+};
+
+/* KNOD_CMD_DEV_GET - do */
+static const struct nla_policy knod_dev_get_nl_policy[KNOD_A_DEV_NIC_IFINDEX + 1] = {
+ [KNOD_A_DEV_NIC_IFINDEX] = NLA_POLICY_MIN(NLA_U32, 1),
+};
+
+/* Ops table for knod */
+static const struct genl_split_ops knod_nl_ops[] = {
+ {
+ .cmd = KNOD_CMD_ACCEL_GET,
+ .doit = knod_nl_accel_get_doit,
+ .policy = knod_accel_get_nl_policy,
+ .maxattr = KNOD_A_ACCEL_ID,
+ .flags = GENL_CMD_CAP_DO,
+ },
+ {
+ .cmd = KNOD_CMD_ACCEL_GET,
+ .dumpit = knod_nl_accel_get_dumpit,
+ .flags = GENL_CMD_CAP_DUMP,
+ },
+ {
+ .cmd = KNOD_CMD_ACCEL_SET,
+ .doit = knod_nl_accel_set_doit,
+ .policy = knod_accel_set_nl_policy,
+ .maxattr = KNOD_A_ACCEL_FEATURE_ENA,
+ .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+ },
+ {
+ .cmd = KNOD_CMD_NIC_GET,
+ .doit = knod_nl_nic_get_doit,
+ .policy = knod_nic_get_nl_policy,
+ .maxattr = KNOD_A_NIC_IFINDEX,
+ .flags = GENL_CMD_CAP_DO,
+ },
+ {
+ .cmd = KNOD_CMD_NIC_GET,
+ .dumpit = knod_nl_nic_get_dumpit,
+ .flags = GENL_CMD_CAP_DUMP,
+ },
+ {
+ .cmd = KNOD_CMD_ATTACH,
+ .doit = knod_nl_attach_doit,
+ .policy = knod_attach_nl_policy,
+ .maxattr = KNOD_A_DEV_ACCEL_ID,
+ .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+ },
+ {
+ .cmd = KNOD_CMD_DETACH,
+ .doit = knod_nl_detach_doit,
+ .policy = knod_detach_nl_policy,
+ .maxattr = KNOD_A_DEV_NIC_IFINDEX,
+ .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+ },
+ {
+ .cmd = KNOD_CMD_DEV_GET,
+ .doit = knod_nl_dev_get_doit,
+ .policy = knod_dev_get_nl_policy,
+ .maxattr = KNOD_A_DEV_NIC_IFINDEX,
+ .flags = GENL_CMD_CAP_DO,
+ },
+ {
+ .cmd = KNOD_CMD_DEV_GET,
+ .dumpit = knod_nl_dev_get_dumpit,
+ .flags = GENL_CMD_CAP_DUMP,
+ },
+};
+
+static const struct genl_multicast_group knod_nl_mcgrps[] = {
+ [KNOD_NLGRP_MGMT] = { "mgmt", },
+};
+
+struct genl_family knod_nl_family __ro_after_init = {
+ .name = KNOD_FAMILY_NAME,
+ .version = KNOD_FAMILY_VERSION,
+ .netnsok = true,
+ .parallel_ops = true,
+ .module = THIS_MODULE,
+ .split_ops = knod_nl_ops,
+ .n_split_ops = ARRAY_SIZE(knod_nl_ops),
+ .mcgrps = knod_nl_mcgrps,
+ .n_mcgrps = ARRAY_SIZE(knod_nl_mcgrps),
+};
diff --git a/net/knod/knod-nl-gen.h b/net/knod/knod-nl-gen.h
new file mode 100644
index 000000000000..a12b6a174bed
--- /dev/null
+++ b/net/knod/knod-nl-gen.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */
+/* Do not edit directly, auto-generated from: */
+/* Documentation/netlink/specs/knod.yaml */
+/* YNL-GEN kernel header */
+/* To regenerate run: tools/net/ynl/ynl-regen.sh */
+
+#ifndef _LINUX_KNOD_GEN_H
+#define _LINUX_KNOD_GEN_H
+
+#include <net/netlink.h>
+#include <net/genetlink.h>
+
+#include <uapi/linux/knod.h>
+
+int knod_nl_accel_get_doit(struct sk_buff *skb, struct genl_info *info);
+int knod_nl_accel_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb);
+int knod_nl_accel_set_doit(struct sk_buff *skb, struct genl_info *info);
+int knod_nl_nic_get_doit(struct sk_buff *skb, struct genl_info *info);
+int knod_nl_nic_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb);
+int knod_nl_attach_doit(struct sk_buff *skb, struct genl_info *info);
+int knod_nl_detach_doit(struct sk_buff *skb, struct genl_info *info);
+int knod_nl_dev_get_doit(struct sk_buff *skb, struct genl_info *info);
+int knod_nl_dev_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb);
+
+enum {
+ KNOD_NLGRP_MGMT,
+};
+
+extern struct genl_family knod_nl_family;
+
+#endif /* _LINUX_KNOD_GEN_H */
diff --git a/net/knod/knod.h b/net/knod/knod.h
new file mode 100644
index 000000000000..eaff0e82ca23
--- /dev/null
+++ b/net/knod/knod.h
@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/* Copyright (c) 2021 Taehee Yoo <ap420073@gmail.com>
+ * Copyright (c) 2021 Hoyeon Lee <hoyeon.rhee@gmail.com>
+ */
+
+#ifndef _NET_KNOD_KNOD_H
+#define _NET_KNOD_KNOD_H
+
+#include <net/knod.h>
+
+#include "knod-nl-gen.h"
+
+struct net_device;
+
+/* knod_core.c */
+struct knod_netdev *knod_netdev_lookup(struct net_device *dev);
+struct knod_dev *knod_dev_lookup(struct net_device *dev);
+struct knod_accel *knod_accel_lookup(int id);
+int knod_dev_attach(struct knod_netdev *knetdev,
+ struct knod_accel *accel);
+int knod_dev_detach(struct knod_dev *knodev);
+
+/* knod_nl.c */
+void knod_nl_notify_dev(struct knod_dev *knodev, u32 cmd);
+
+#endif /* _NET_KNOD_KNOD_H */
diff --git a/net/knod/knod_core.c b/net/knod/knod_core.c
new file mode 100644
index 000000000000..af50a69bca65
--- /dev/null
+++ b/net/knod/knod_core.c
@@ -0,0 +1,1231 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/* Copyright (c) 2021 Taehee Yoo <ap420073@gmail.com>
+ * Copyright (c) 2021 Hoyeon Lee <hoyeon.rhee@gmail.com>
+ */
+
+#include <net/knod.h>
+#include <net/spsc_ring.h>
+#include <net/page_pool/types.h>
+#include <net/page_pool/helpers.h>
+#include <net/page_pool/memory_provider.h>
+#include <net/xfrm.h>
+#include <linux/genalloc.h>
+#include <trace/events/page_pool.h>
+#include <net/devmem.h>
+
+#include "knod.h"
+
+DEFINE_MUTEX(knod_lock);
+LIST_HEAD(knod_dev_list);
+EXPORT_SYMBOL_GPL(knod_dev_list);
+LIST_HEAD(knod_netdev_list);
+EXPORT_SYMBOL_GPL(knod_netdev_list);
+LIST_HEAD(knod_accel_list);
+EXPORT_SYMBOL_GPL(knod_accel_list);
+
+void knod_dev_lock(void)
+{
+ mutex_lock(&knod_lock);
+}
+EXPORT_SYMBOL_GPL(knod_dev_lock);
+
+void knod_dev_unlock(void)
+{
+ mutex_unlock(&knod_lock);
+}
+EXPORT_SYMBOL_GPL(knod_dev_unlock);
+
+void knod_netdev_register(struct knod_netdev *knetdev)
+{
+ mutex_lock(&knod_lock);
+ knetdev->status = KNOD_STATUS_FREE;
+ list_add(&knetdev->list, &knod_netdev_list);
+ mutex_unlock(&knod_lock);
+}
+EXPORT_SYMBOL(knod_netdev_register);
+
+void knod_netdev_unregister(struct knod_netdev *knetdev)
+{
+ mutex_lock(&knod_lock);
+ list_del(&knetdev->list);
+ mutex_unlock(&knod_lock);
+}
+EXPORT_SYMBOL(knod_netdev_unregister);
+
+void knod_accel_register(struct knod_accel *accel)
+{
+ mutex_lock(&knod_lock);
+ accel->status = KNOD_STATUS_FREE;
+ list_add(&accel->list, &knod_accel_list);
+ mutex_unlock(&knod_lock);
+}
+EXPORT_SYMBOL(knod_accel_register);
+
+void knod_accel_unregister(struct knod_accel *accel)
+{
+ mutex_lock(&knod_lock);
+ list_del(&accel->list);
+ mutex_unlock(&knod_lock);
+}
+EXPORT_SYMBOL(knod_accel_unregister);
+
+struct knod_netdev *knod_netdev_lookup(struct net_device *dev)
+{
+ struct knod_netdev *knetdev;
+
+ list_for_each_entry(knetdev, &knod_netdev_list, list)
+ if (knetdev->dev == dev)
+ return knetdev;
+
+ return NULL;
+}
+
+struct knod_dev *knod_dev_lookup(struct net_device *dev)
+{
+ struct knod_dev *knodev;
+
+ list_for_each_entry(knodev, &knod_dev_list, list)
+ if (knodev->netdev == dev)
+ return knodev;
+
+ return NULL;
+}
+
+struct knod_accel *knod_accel_lookup(int id)
+{
+ struct knod_accel *accel;
+
+ list_for_each_entry(accel, &knod_accel_list, list)
+ if (accel->id == id)
+ return accel;
+
+ return NULL;
+}
+
+void knod_dev_start(struct knod_dev *knodev)
+{
+ /* Interface up: start the active feature's worker. */
+ knodev->started = true;
+ if (knodev->accel_ops->dev_start)
+ knodev->accel_ops->dev_start(knodev);
+}
+EXPORT_SYMBOL(knod_dev_start);
+
+void knod_dev_stop(struct knod_dev *knodev)
+{
+ /* Interface down: stop the worker + drain the GPU in-flight. */
+ if (knodev->accel_ops->dev_stop)
+ knodev->accel_ops->dev_stop(knodev);
+ knodev->started = false;
+}
+EXPORT_SYMBOL(knod_dev_stop);
+
+int knod_dev_xdp_install(struct knod_dev *knodev, struct netdev_bpf *xdp)
+{
+ if (!knodev->accel_ops->xdp_ops ||
+ !knodev->accel_ops->xdp_ops->xdp_install)
+ return -EOPNOTSUPP;
+ return knodev->accel_ops->xdp_ops->xdp_install(knodev, xdp);
+}
+EXPORT_SYMBOL(knod_dev_xdp_install);
+
+void knod_dev_get_stats64(struct knod_dev *knodev,
+ struct rtnl_link_stats64 *stats)
+{
+ struct knod_dev_stats *p;
+ u32 tx_dropped = 0, tx_errors = 0;
+ u64 tx_packets, tx_bytes;
+ unsigned int start;
+ int i;
+
+ for_each_possible_cpu(i) {
+ p = per_cpu_ptr(knodev->stats, i);
+ do {
+ start = u64_stats_fetch_begin(&p->syncp);
+ tx_packets = u64_stats_read(&p->tx_packets);
+ tx_bytes = u64_stats_read(&p->tx_bytes);
+ } while (u64_stats_fetch_retry(&p->syncp, start));
+
+ stats->tx_packets += tx_packets;
+ stats->tx_bytes += tx_bytes;
+ tx_dropped += READ_ONCE(p->tx_dropped);
+ tx_errors += READ_ONCE(p->tx_errors);
+ }
+ stats->tx_dropped += tx_dropped;
+ stats->tx_errors += tx_errors;
+}
+EXPORT_SYMBOL(knod_dev_get_stats64);
+
+/*
+ * Wrap a delivery-pool page as a zero-copy head_frag skb: the packet sits at
+ * @off (preserved headroom) for @len bytes. Building it linear keeps
+ * skb->data on the packet so callers can edit headers in place. The page
+ * recycles to @pool when the skb is freed. On oversize or alloc failure the
+ * page is returned to @pool and NULL is returned. @napi selects the NAPI skb
+ * cache; callers outside softirq pass false.
+ */
+struct sk_buff *knod_pass_build_skb(netmem_ref netmem, u16 off, u16 len,
+ struct page_pool *pool, bool napi)
+{
+ struct page *pg = netmem_to_page(netmem);
+ struct sk_buff *skb;
+
+ /* Must fit alongside skb_shared_info at the page tail; MTU is capped
+ * below this, so drop the rare oversized outlier.
+ */
+ if (off + len > SKB_WITH_OVERHEAD(PAGE_SIZE)) {
+ if (pool)
+ page_pool_put_full_netmem(pool, netmem, false);
+ return NULL;
+ }
+
+ if (napi)
+ skb = napi_build_skb(page_address(pg), PAGE_SIZE);
+ else
+ skb = build_skb(page_address(pg), PAGE_SIZE);
+ if (unlikely(!skb)) {
+ if (pool)
+ page_pool_put_full_netmem(pool, netmem, false);
+ return NULL;
+ }
+
+ skb_mark_for_recycle(skb);
+ skb_reserve(skb, off);
+ skb_put(skb, len);
+ return skb;
+}
+EXPORT_SYMBOL(knod_pass_build_skb);
+
+/*
+ * Device->host copy: the NIC DD hands the PASS bds it accumulated during its
+ * single act traversal. Allocate a delivery page per packet, issue the accel
+ * SDMA (GPU -> page) asynchronously, and queue a descriptor on the per-queue
+ * pending ring tagged with this batch's fence; knod_d2h_drain delivers them
+ * once the fence lands and recycles the source. Sources that cannot be queued
+ * (bad len / ring full / pool empty) are recycled here. Returns the count
+ * queued. Producer and the drain consumer run on the same per-queue NAPI, so
+ * the ring is single-threaded; @d2h_lock only guards the shared SDMA submit.
+ */
+int knod_d2h_copy(struct knod_dev *knodev, int napi_index,
+ struct spsc_pass_bd *bds, int cnt)
+{
+ struct knod_accel_ops *ops = knodev->accel_ops;
+ struct knod_work_priv *wpriv;
+ struct page_pool *pool;
+ bool submitted = false;
+ int i, produced = 0;
+
+ if (napi_index < 0 || napi_index >= KNOD_SPSC_MAX || !ops->d2h_submit)
+ goto drop_all;
+ wpriv = &knodev->wpriv[napi_index];
+ pool = READ_ONCE(wpriv->pass_pool);
+ if (!pool || !wpriv->pass_pending.slots)
+ goto drop_all;
+
+ spin_lock(&knodev->d2h_lock);
+
+ for (i = 0; i < cnt; i++) {
+ netmem_ref src = bds[i].netmem;
+ struct knod_pass_desc *desc;
+ netmem_ref dst;
+ void *ptr;
+ u32 fv;
+ u16 off = bds[i].off;
+ u16 len = bds[i].len;
+
+ if (!len || off + len > SKB_WITH_OVERHEAD(PAGE_SIZE))
+ goto drop;
+ if (spsc_produce(&wpriv->pass_pending, &ptr))
+ goto drop;
+ dst = page_pool_dev_alloc_netmems(pool);
+ if (!dst)
+ goto drop; /* slot left uncommitted, reused next */
+
+ fv = ops->d2h_submit(knodev,
+ page_pool_get_dma_addr_netmem(dst) + off,
+ napi_index, bds[i].page_idx, off, len);
+ if (!fv) { /* SDMA ring full: backpressure drop */
+ page_pool_put_full_netmem(pool, dst, false);
+ goto drop;
+ }
+ submitted = true;
+
+ desc = ptr;
+ desc->netmem = dst;
+ desc->src = src;
+ desc->off = off;
+ desc->len = len;
+ desc->fence_val = fv;
+ desc->sdma_idx = 0;
+ spsc_produce_commit(&wpriv->pass_pending);
+ produced++;
+ continue;
+drop:
+ page_pool_recycle_direct_netmem(netmem_get_pp(src), src);
+ }
+
+ if (submitted)
+ ops->d2h_kick(knodev);
+ spin_unlock(&knodev->d2h_lock);
+
+ /* The copies are async; re-arm the NAPI so the drain runs once the
+ * SDMA lands -- at low rate, RX traffic alone may not poll again soon.
+ */
+ if (submitted)
+ knod_napi_kick(wpriv);
+ return produced;
+
+drop_all:
+ for (i = 0; i < cnt; i++)
+ page_pool_recycle_direct_netmem(netmem_get_pp(bds[i].netmem),
+ bds[i].netmem);
+ return 0;
+}
+EXPORT_SYMBOL(knod_d2h_copy);
+
+/*
+ * Drain the per-queue pending ring: deliver every descriptor whose batch
+ * fence has landed (accel_ops->d2h_fence) as a zero-copy head_frag skb from
+ * the delivery page, recycling the source RX page. Stops at the first
+ * not-yet-landed descriptor -- the ring is in fence order. Runs on the NIC
+ * NAPI, the same thread as the knod_d2h_copy producer.
+ */
+int knod_d2h_drain(struct knod_dev *knodev, int napi_index,
+ struct napi_struct *napi, int budget)
+{
+ void *ptrs[KNOD_DEFAULT_PASS_SLOTS];
+ struct knod_work_priv *wpriv;
+ struct knod_pass_desc *d0;
+ struct page_pool *pool;
+ LIST_HEAD(deliver_list);
+ unsigned int got = 0, i, n = 0;
+ int delivered = 0;
+ u32 cur_fence;
+
+ if (napi_index < 0 || napi_index >= KNOD_SPSC_MAX ||
+ !knodev->accel_ops->d2h_fence)
+ return 0;
+ wpriv = &knodev->wpriv[napi_index];
+ if (!wpriv->pass_pending.slots)
+ return 0;
+ pool = READ_ONCE(wpriv->pass_pool);
+
+ if (spsc_peek(&wpriv->pass_pending, ptrs,
+ min_t(unsigned int, budget, KNOD_DEFAULT_PASS_SLOTS),
+ &got) < 0 || got == 0)
+ return 0;
+
+ d0 = ptrs[0];
+ cur_fence = knodev->accel_ops->d2h_fence(knodev, d0->sdma_idx);
+
+ for (i = 0; i < got; i++) {
+ struct knod_pass_desc *desc = ptrs[i];
+ struct sk_buff *skb;
+
+ if ((s32)(cur_fence - desc->fence_val) < 0)
+ break; /* not landed yet; later descs are newer */
+
+ /* bpf carries the RX page as src for post-copy recycle; ipsec
+ * leaves it 0 (the NIC act handler recycles via the bd).
+ */
+ if (desc->src)
+ page_pool_recycle_direct_netmem(
+ netmem_get_pp(desc->src), desc->src);
+ skb = knod_pass_build_skb(desc->netmem, desc->off, desc->len,
+ pool, true);
+ n++;
+ if (!skb)
+ continue;
+ if (knodev->post_copy) {
+ if (!knodev->post_copy(knodev, skb, desc, napi_index)) {
+ kfree_skb(skb);
+ continue;
+ }
+ } else if (likely(skb->len >= ETH_HLEN)) {
+ skb->protocol = eth_type_trans(skb, knodev->netdev);
+ } else {
+ kfree_skb(skb);
+ continue;
+ }
+ list_add_tail(&skb->list, &deliver_list);
+ delivered++;
+ }
+
+ if (n) {
+ spsc_acquire(&wpriv->pass_pending, NULL, n, NULL);
+ spsc_release_commit(&wpriv->pass_pending, n);
+ }
+
+ /* Descriptors whose copy has not landed yet remain queued; re-arm so
+ * we poll again instead of waiting for the next RX event.
+ */
+ if (spsc_count(&wpriv->pass_pending))
+ knod_napi_kick(wpriv);
+
+ if (!list_empty(&deliver_list))
+ netif_receive_skb_list(&deliver_list);
+ return delivered;
+}
+EXPORT_SYMBOL(knod_d2h_drain);
+
+/* ======================================================================
+ * NOD IPsec proxy - bridges standard kernel xfrmdev_ops to
+ * knod_accel_ipsec_ops on the GPU accelerator.
+ * ======================================================================
+ */
+#if IS_ENABLED(CONFIG_XFRM_OFFLOAD)
+
+static struct knod_dev *knod_ipsec_find_xdev(struct net_device *dev)
+{
+ struct knod_dev *knodev;
+
+ list_for_each_entry(knodev, &knod_dev_list, list) {
+ if (knodev->netdev == dev)
+ return knodev;
+ }
+ return NULL;
+}
+
+static int knod_ipsec_xdo_state_add(struct net_device *dev,
+ struct xfrm_state *x,
+ struct netlink_ext_ack *extack)
+{
+ struct knod_dev *knodev = knod_ipsec_find_xdev(dev);
+
+ if (!knodev || !knodev->accel_ops->ipsec_ops ||
+ !knodev->accel_ops->ipsec_ops->xdo_dev_state_add)
+ return -EOPNOTSUPP;
+ return knodev->accel_ops->ipsec_ops->xdo_dev_state_add(knodev, x,
+ extack);
+}
+
+static void knod_ipsec_xdo_state_delete(struct net_device *dev,
+ struct xfrm_state *x)
+{
+ struct knod_dev *knodev = knod_ipsec_find_xdev(dev);
+
+ if (!knodev || !knodev->accel_ops->ipsec_ops ||
+ !knodev->accel_ops->ipsec_ops->xdo_dev_state_delete)
+ return;
+ knodev->accel_ops->ipsec_ops->xdo_dev_state_delete(knodev, x);
+}
+
+static void knod_ipsec_xdo_state_free(struct net_device *dev,
+ struct xfrm_state *x)
+{
+ struct knod_dev *knodev = knod_ipsec_find_xdev(dev);
+
+ if (!knodev || !knodev->accel_ops->ipsec_ops ||
+ !knodev->accel_ops->ipsec_ops->xdo_dev_state_free)
+ return;
+ knodev->accel_ops->ipsec_ops->xdo_dev_state_free(knodev, x);
+}
+
+static bool knod_ipsec_xdo_offload_ok(struct sk_buff *skb,
+ struct xfrm_state *x)
+{
+ struct knod_dev *knodev = knod_ipsec_find_xdev(x->xso.dev);
+
+ if (!knodev || !knodev->accel_ops->ipsec_ops ||
+ !knodev->accel_ops->ipsec_ops->xdo_dev_offload_ok)
+ return false;
+ return knodev->accel_ops->ipsec_ops->xdo_dev_offload_ok(knodev, skb, x);
+}
+
+static void knod_ipsec_xdo_state_advance_esn(struct xfrm_state *x)
+{
+ struct knod_dev *knodev;
+
+ if (!x->xso.dev)
+ return;
+ knodev = knod_ipsec_find_xdev(x->xso.dev);
+ if (!knodev || !knodev->accel_ops->ipsec_ops ||
+ !knodev->accel_ops->ipsec_ops->xdo_dev_state_advance_esn)
+ return;
+ knodev->accel_ops->ipsec_ops->xdo_dev_state_advance_esn(knodev, x);
+}
+
+static void knod_ipsec_xdo_state_update_stats(struct xfrm_state *x)
+{
+ struct knod_dev *knodev;
+
+ if (!x->xso.dev)
+ return;
+ knodev = knod_ipsec_find_xdev(x->xso.dev);
+ if (!knodev || !knodev->accel_ops->ipsec_ops ||
+ !knodev->accel_ops->ipsec_ops->xdo_dev_state_update_stats)
+ return;
+ knodev->accel_ops->ipsec_ops->xdo_dev_state_update_stats(knodev, x);
+}
+
+static int knod_ipsec_xdo_policy_add(struct xfrm_policy *xp,
+ struct netlink_ext_ack *extack)
+{
+ struct knod_dev *knodev;
+
+ if (!xp->xdo.dev)
+ return -EOPNOTSUPP;
+ knodev = knod_ipsec_find_xdev(xp->xdo.dev);
+ if (!knodev || !knodev->accel_ops->ipsec_ops ||
+ !knodev->accel_ops->ipsec_ops->xdo_dev_policy_add)
+ return -EOPNOTSUPP;
+ return knodev->accel_ops->ipsec_ops->xdo_dev_policy_add(knodev, xp,
+ extack);
+}
+
+static void knod_ipsec_xdo_policy_delete(struct xfrm_policy *xp)
+{
+ struct knod_dev *knodev;
+
+ if (!xp->xdo.dev)
+ return;
+ knodev = knod_ipsec_find_xdev(xp->xdo.dev);
+ if (!knodev || !knodev->accel_ops->ipsec_ops ||
+ !knodev->accel_ops->ipsec_ops->xdo_dev_policy_delete)
+ return;
+ knodev->accel_ops->ipsec_ops->xdo_dev_policy_delete(knodev, xp);
+}
+
+static void knod_ipsec_xdo_policy_free(struct xfrm_policy *xp)
+{
+ struct knod_dev *knodev;
+
+ if (!xp->xdo.dev)
+ return;
+ knodev = knod_ipsec_find_xdev(xp->xdo.dev);
+ if (!knodev || !knodev->accel_ops->ipsec_ops ||
+ !knodev->accel_ops->ipsec_ops->xdo_dev_policy_free)
+ return;
+ knodev->accel_ops->ipsec_ops->xdo_dev_policy_free(knodev, xp);
+}
+
+static const struct xfrmdev_ops knod_ipsec_xfrmdev_ops = {
+ .xdo_dev_state_add = knod_ipsec_xdo_state_add,
+ .xdo_dev_state_delete = knod_ipsec_xdo_state_delete,
+ .xdo_dev_state_free = knod_ipsec_xdo_state_free,
+ .xdo_dev_offload_ok = knod_ipsec_xdo_offload_ok,
+ .xdo_dev_state_advance_esn = knod_ipsec_xdo_state_advance_esn,
+ .xdo_dev_state_update_stats = knod_ipsec_xdo_state_update_stats,
+ .xdo_dev_policy_add = knod_ipsec_xdo_policy_add,
+ .xdo_dev_policy_delete = knod_ipsec_xdo_policy_delete,
+ .xdo_dev_policy_free = knod_ipsec_xdo_policy_free,
+};
+
+int knod_dev_xdp_drain_pass(struct knod_dev *knodev, struct napi_struct *napi,
+ int queue_idx, int budget)
+{
+ if (!knodev || !knodev->accel_ops)
+ return 0;
+ /* Common device->host delivery drain. PASS bds were SDMA-copied by
+ * knod_d2h_copy from the NIC act handler; deliver the ones whose copy
+ * has landed.
+ */
+ return knod_d2h_drain(knodev, queue_idx, napi, budget);
+}
+EXPORT_SYMBOL_GPL(knod_dev_xdp_drain_pass);
+
+int knod_ipsec_attach(struct knod_dev *knodev)
+{
+ struct knod_accel_ipsec_ops *ops;
+
+ if (!knodev->accel_ops->ipsec_ops)
+ return 0;
+
+ ops = knodev->accel_ops->ipsec_ops;
+ if (ops->activate) {
+ int err = ops->activate(knodev);
+
+ if (err) {
+ pr_err("nod: IPsec activate failed on %s (%d)\n",
+ netdev_name(knodev->netdev), err);
+ return err;
+ }
+ }
+
+ /* Take over xfrmdev_ops, saving the NIC's original so detach can
+ * restore it (and only clear NETIF_F_HW_ESP if we added it).
+ */
+ knodev->ipsec_orig_xfrmdev_ops = knodev->netdev->xfrmdev_ops;
+ knodev->ipsec_added_hw_esp =
+ !(knodev->netdev->features & NETIF_F_HW_ESP);
+ knodev->netdev->xfrmdev_ops = &knod_ipsec_xfrmdev_ops;
+ knodev->netdev->features |= NETIF_F_HW_ESP;
+ knodev->netdev->hw_enc_features |= NETIF_F_HW_ESP;
+
+ pr_info("nod: IPsec offload enabled on %s\n",
+ netdev_name(knodev->netdev));
+ return 0;
+}
+EXPORT_SYMBOL(knod_ipsec_attach);
+
+void knod_ipsec_detach(struct knod_dev *knodev)
+{
+ struct knod_accel_ipsec_ops *ops = knodev->accel_ops->ipsec_ops;
+
+ if (!ops)
+ return;
+ if (knodev->netdev->xfrmdev_ops != &knod_ipsec_xfrmdev_ops)
+ return;
+
+ if (knodev->ipsec_added_hw_esp) {
+ knodev->netdev->features &= ~NETIF_F_HW_ESP;
+ knodev->netdev->hw_enc_features &= ~NETIF_F_HW_ESP;
+ }
+ knodev->netdev->xfrmdev_ops = knodev->ipsec_orig_xfrmdev_ops;
+
+ if (ops->deactivate)
+ ops->deactivate(knodev);
+
+ pr_info("nod: IPsec offload disabled on %s\n",
+ netdev_name(knodev->netdev));
+}
+EXPORT_SYMBOL(knod_ipsec_detach);
+
+#endif /* CONFIG_XFRM_OFFLOAD */
+
+/*
+ * Host-page page_pool memory provider for GPU->host delivery (NOD-private).
+ * Unlike the devmem/dma-buf providers (which hand out unreadable net_iov), this
+ * returns real host-readable pages (a GPU GTT buffer also mapped into system
+ * memory), so delivered data becomes skbs on the normal receive path. The
+ * pages stay owned by the accel and are never returned to the buddy; the
+ * gen_pool is only the slow-path backing store (page_pool's cache/ring absorb
+ * the per-packet churn), and the device address it hands out doubles as the
+ * netmem dma_addr (the worker's SDMA destination). Selected via
+ * page_pool_params.mp_ops in knod_pass_attach().
+ */
+static int hostmem_pp_init(struct page_pool *pool)
+{
+ struct page_pool_hostmem *hm = pool->mp_priv;
+ int err;
+
+ if (pool->p.order != 0)
+ return -E2BIG;
+ if (!hm || !hm->pages || !hm->count)
+ return -EINVAL;
+
+ hm->genpool = gen_pool_create(PAGE_SHIFT, NUMA_NO_NODE);
+ if (!hm->genpool)
+ return -ENOMEM;
+
+ err = gen_pool_add(hm->genpool, (unsigned long)hm->base_addr,
+ (size_t)hm->count * PAGE_SIZE, NUMA_NO_NODE);
+ if (err) {
+ gen_pool_destroy(hm->genpool);
+ hm->genpool = NULL;
+ return err;
+ }
+
+ /* Device addresses are pre-set; page_pool must not DMA-sync them. */
+ pool->dma_sync = false;
+ pool->dma_sync_for_cpu = false;
+
+ return 0;
+}
+
+static netmem_ref hostmem_pp_alloc_netmems(struct page_pool *pool, gfp_t gfp)
+{
+ struct page_pool_hostmem *hm = pool->mp_priv;
+ unsigned long addr;
+ netmem_ref netmem;
+ unsigned int idx;
+
+ addr = gen_pool_alloc(hm->genpool, PAGE_SIZE);
+ if (!addr)
+ return 0;
+
+ idx = (addr - hm->base_addr) >> PAGE_SHIFT;
+ if (WARN_ON_ONCE(idx >= hm->count)) {
+ gen_pool_free(hm->genpool, addr, PAGE_SIZE);
+ return 0;
+ }
+
+ netmem = page_to_netmem(hm->pages[idx]);
+ page_pool_provider_set_netmem(pool, netmem, addr);
+
+ pool->pages_state_hold_cnt++;
+ trace_page_pool_state_hold(pool, netmem, pool->pages_state_hold_cnt);
+
+ return netmem;
+}
+
+static bool hostmem_pp_release_netmem(struct page_pool *pool, netmem_ref netmem)
+{
+ struct page_pool_hostmem *hm = pool->mp_priv;
+ unsigned long addr = page_pool_get_dma_addr_netmem(netmem);
+
+ page_pool_clear_pp_info(netmem);
+ gen_pool_free(hm->genpool, addr, PAGE_SIZE);
+
+ /* Pages are accel-owned: never put_page() them to the buddy. */
+ return false;
+}
+
+static void hostmem_pp_destroy(struct page_pool *pool)
+{
+ struct page_pool_hostmem *hm = pool->mp_priv;
+
+ gen_pool_destroy(hm->genpool);
+ hm->genpool = NULL;
+
+ /* The pool has fully drained (inflight == 0); tell the owner it may now
+ * release the backing pages. The struct itself is owner-allocated.
+ */
+ if (hm->freed)
+ hm->freed(hm->arg);
+}
+
+static int hostmem_pp_nl_fill(void *mp_priv, struct sk_buff *rsp,
+ struct netdev_rx_queue *rxq)
+{
+ /* Nothing provider-specific to report; page_pool_user.c calls this
+ * unconditionally once mp_ops is set, so it must not be NULL.
+ */
+ return 0;
+}
+
+static const struct memory_provider_ops page_pool_hostmem_ops = {
+ .init = hostmem_pp_init,
+ .alloc_netmems = hostmem_pp_alloc_netmems,
+ .release_netmem = hostmem_pp_release_netmem,
+ .destroy = hostmem_pp_destroy,
+ .nl_fill = hostmem_pp_nl_fill,
+ /*
+ * .uninstall is only invoked for rxq-bound pools and is
+ * NULL-checked.
+ */
+};
+
+/* Provider drain callback: fires once a delivery pool has no inflight pages. */
+static void knod_pass_drained(void *arg)
+{
+ struct knod_dev *knodev = arg;
+
+ if (atomic_dec_and_test(&knodev->pp_live))
+ complete(&knodev->pp_drained);
+}
+
+/*
+ * Create one GPU->host delivery page_pool per RX queue, backed by a single
+ * accel-allocated GTT buffer. Mirrors the spsc-pool setup: alloc_mem() hands
+ * back the buffer (kaddr/gaddr/pages/priv); the framework owns the page_pools
+ * and the drain barrier. The hostmem provider hands out the real GTT pages
+ * with their device address as dma_addr, so the worker's SDMA lands directly
+ * in the page that later becomes the skb frag. page_pool inflight accounting
+ * then keeps the buffer alive until every delivered skb has drained.
+ */
+static int knod_pass_attach(struct knod_dev *knodev)
+{
+ struct page_pool_params pp = {
+ .order = 0,
+ .pool_size = KNOD_PASS_SLOTS,
+ .nid = NUMA_NO_NODE,
+ .dev = knodev->netdev->dev.parent,
+ .dma_dir = DMA_FROM_DEVICE,
+ .max_len = PAGE_SIZE,
+ .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV |
+ PP_FLAG_CUSTOM_MEMORY_PROVIDER,
+ .mp_ops = &page_pool_hostmem_ops,
+ };
+ unsigned int nqueues = min(knodev->netdev->num_rx_queues,
+ KNOD_SPSC_MAX);
+ struct page **pages;
+ void *pass_priv;
+ u64 base_gaddr;
+ size_t total;
+ void *kaddr;
+ int qi;
+
+ /* Accels without GPU memory simply run without zero-copy delivery. */
+ if (!knodev->accel_ops->alloc_mem)
+ return 0;
+
+ total = nqueues * KNOD_PASS_SLOTS * PAGE_SIZE;
+ kaddr = knodev->accel_ops->alloc_mem(knodev, total, &base_gaddr, &pages,
+ &pass_priv);
+ if (!kaddr) {
+ pr_err("%s: delivery alloc_mem failed\n", __func__);
+ return -ENOMEM;
+ }
+ if (!pages) {
+ knodev->accel_ops->free_mem(knodev, pass_priv);
+ return -ENOMEM;
+ }
+
+ init_completion(&knodev->pp_drained);
+ atomic_set(&knodev->pp_live, 0);
+ spin_lock_init(&knodev->d2h_lock);
+
+ for (qi = 0; qi < nqueues; qi++) {
+ struct knod_work_priv *wpriv = &knodev->wpriv[qi];
+ int base = qi * KNOD_PASS_SLOTS;
+
+ if (spsc_init(&wpriv->pass_pending,
+ sizeof(struct knod_pass_desc),
+ KNOD_PASS_SLOTS, GFP_KERNEL))
+ goto err_destroy;
+
+ wpriv->pass_hm.pages = &pages[base];
+ wpriv->pass_hm.count = KNOD_PASS_SLOTS;
+ wpriv->pass_hm.base_addr = base_gaddr + (u64)base * PAGE_SIZE;
+ wpriv->pass_hm.freed = knod_pass_drained;
+ wpriv->pass_hm.arg = knodev;
+ pp.mp_priv = &wpriv->pass_hm;
+
+ wpriv->pass_pool = page_pool_create(&pp);
+ if (IS_ERR(wpriv->pass_pool)) {
+ wpriv->pass_pool = NULL;
+ spsc_destroy(&wpriv->pass_pending);
+ goto err_destroy;
+ }
+ atomic_inc(&knodev->pp_live);
+ }
+
+ knodev->pass_priv = pass_priv;
+ return 0;
+
+err_destroy:
+ while (qi-- > 0) {
+ spsc_destroy(&knodev->wpriv[qi].pass_pending);
+ page_pool_destroy(knodev->wpriv[qi].pass_pool);
+ knodev->wpriv[qi].pass_pool = NULL;
+ }
+ if (atomic_read(&knodev->pp_live))
+ wait_for_completion_timeout(&knodev->pp_drained,
+ msecs_to_jiffies(5000));
+ knodev->accel_ops->free_mem(knodev, pass_priv);
+ return -ENOMEM;
+}
+
+/*
+ * Drain a queue's pass_pending ring on teardown. The interface is down so no
+ * new copies are submitted; wait for each queued copy to land, then return its
+ * pages to the pool instead of delivering. Without this, page_pool_destroy()
+ * stalls on the pages a detach raced against in-flight d2h copies.
+ */
+static void knod_pass_flush(struct knod_dev *knodev, unsigned int qi)
+{
+ struct knod_work_priv *wpriv = &knodev->wpriv[qi];
+ struct page_pool *pool = wpriv->pass_pool;
+ void *ptrs[KNOD_DEFAULT_PASS_SLOTS];
+ unsigned int got, i;
+
+ if (!wpriv->pass_pending.slots || !pool)
+ return;
+
+ while (spsc_peek(&wpriv->pass_pending, ptrs,
+ KNOD_DEFAULT_PASS_SLOTS, &got) >= 0 && got) {
+ for (i = 0; i < got; i++) {
+ struct knod_pass_desc *desc = ptrs[i];
+ int spins = 1000000;
+
+ while (knodev->accel_ops->d2h_fence && spins-- &&
+ (s32)(knodev->accel_ops->d2h_fence(knodev,
+ desc->sdma_idx) - desc->fence_val) < 0)
+ cpu_relax();
+
+ WARN_ONCE(knodev->accel_ops->d2h_fence &&
+ (s32)(knodev->accel_ops->d2h_fence(knodev,
+ desc->sdma_idx) - desc->fence_val) < 0,
+ "knod: d2h fence timeout on pass flush q%u idx%u\n",
+ qi, desc->sdma_idx);
+
+ if (desc->src) {
+ struct page_pool *src_pp =
+ netmem_get_pp(desc->src);
+
+ page_pool_recycle_direct_netmem(src_pp,
+ desc->src);
+ }
+ page_pool_put_full_netmem(pool, desc->netmem, false);
+ }
+ spsc_acquire(&wpriv->pass_pending, NULL, got, NULL);
+ spsc_release_commit(&wpriv->pass_pending, got);
+ }
+}
+
+/*
+ * Tear down the delivery pools and free the backing buffer. page_pool_destroy
+ * is async, so wait for every pool to drain (knod_pass_drained) before handing
+ * the buffer back to the accel -- otherwise an inflight skb frag would outlive
+ * the BO. Idempotent: a no-op for accels that never allocated.
+ */
+static void knod_pass_detach(struct knod_dev *knodev)
+{
+ unsigned int qi;
+
+ if (!knodev->pass_priv)
+ return;
+
+ /* Iterate the full array (like the spsc teardown): destroy every pool
+ * that was created, so the drain barrier is guaranteed to reach zero.
+ */
+ for (qi = 0; qi < KNOD_SPSC_MAX; qi++) {
+ if (!knodev->wpriv[qi].pass_pool)
+ continue;
+ /* Return any queued (now-landed) d2h pages before destroying
+ * the pool, or page_pool_destroy() stalls on the inflight
+ * count a detach raced against in-flight d2h copies.
+ */
+ knod_pass_flush(knodev, qi);
+ spsc_destroy(&knodev->wpriv[qi].pass_pending);
+ page_pool_destroy(knodev->wpriv[qi].pass_pool);
+ knodev->wpriv[qi].pass_pool = NULL;
+ }
+ if (atomic_read(&knodev->pp_live))
+ wait_for_completion_timeout(&knodev->pp_drained,
+ msecs_to_jiffies(5000));
+ knodev->accel_ops->free_mem(knodev, knodev->pass_priv);
+ knodev->pass_priv = NULL;
+}
+
+static void knod_dmabuf_move_notify(struct dma_buf_attachment *attach)
+{
+}
+
+static const struct dma_buf_attach_ops knod_dmabuf_attach_ops = {
+ .allow_peer2peer = true,
+ .invalidate_mappings = knod_dmabuf_move_notify,
+};
+
+static int knod_dmabuf_attach(struct knod_dev *knodev)
+{
+ struct net_device *dev = knodev->netdev;
+ struct net_devmem_dmabuf_binding *binding;
+ struct dma_buf *dmabuf;
+ int i, err;
+
+ for (i = 0; i < min(dev->num_rx_queues, KNOD_SPSC_MAX); i++) {
+ dmabuf = knodev->wpriv[i].dmabuf;
+ if (!dmabuf)
+ continue;
+
+ /* The binding owns a dmabuf reference that
+ * __net_devmem_dmabuf_binding_free() drops on unbind, mirroring
+ * the dma_buf_get(fd) in the fd-based net_devmem_bind_dmabuf().
+ * We hand it the BO's dmabuf directly, so take that reference
+ * here -- otherwise the unbind put races the BO free's put and
+ * underflows the dmabuf file refcount.
+ */
+ get_dma_buf(dmabuf);
+ binding = __net_devmem_binding_create(dev, dev->dev.parent,
+ dmabuf, DMA_BIDIRECTIONAL,
+ &knod_dmabuf_attach_ops,
+ NULL);
+ if (IS_ERR(binding)) {
+ dma_buf_put(dmabuf);
+ err = PTR_ERR(binding);
+ goto err_unwind;
+ }
+
+ err = net_devmem_bind_dmabuf_to_queue_direct(dev, i, binding);
+ if (err) {
+ net_devmem_unbind_dmabuf_direct(binding);
+ goto err_unwind;
+ }
+
+ knodev->bindings[i] = binding;
+ }
+
+ return 0;
+
+err_unwind:
+ while (i-- > 0) {
+ if (knodev->bindings[i]) {
+ net_devmem_unbind_dmabuf_direct(knodev->bindings[i]);
+ knodev->bindings[i] = NULL;
+ }
+ }
+ return err;
+}
+
+static void knod_dmabuf_detach(struct knod_dev *knodev)
+{
+ int i;
+
+ for (i = 0; i < KNOD_SPSC_MAX; i++) {
+ if (!knodev->bindings[i])
+ continue;
+
+ net_devmem_unbind_dmabuf_direct(knodev->bindings[i]);
+ knodev->bindings[i] = NULL;
+ }
+}
+
+int knod_dev_attach(struct knod_netdev *knetdev, struct knod_accel *accel)
+{
+ struct knod_dev *knodev;
+ int err = -EINVAL, i;
+
+ if (knetdev->status == KNOD_STATUS_USED ||
+ accel->status == KNOD_STATUS_USED) {
+ pr_err("knod: %s already attached\n",
+ netdev_name(knetdev->dev));
+ return -EINVAL;
+ }
+
+ knodev = kzalloc(sizeof(struct knod_dev), GFP_KERNEL);
+ if (!knodev)
+ return -ENOMEM;
+
+ if (!try_module_get(knetdev->owner)) {
+ pr_err("knod: NIC driver for %s is unloading\n",
+ netdev_name(knetdev->dev));
+ kfree(knodev);
+ return -ENODEV;
+ }
+ if (!try_module_get(accel->owner)) {
+ pr_err("knod: accelerator driver is unloading\n");
+ module_put(knetdev->owner);
+ kfree(knodev);
+ return -ENODEV;
+ }
+
+ knetdev->accel = accel;
+ knetdev->knodev = knodev;
+ accel->knetdev = knetdev;
+ accel->knodev = knodev;
+ knodev->knetdev = knetdev;
+ knodev->accel = accel;
+ knodev->netdev = knetdev->dev;
+ knodev->accel_ops = accel->accel_ops;
+ knodev->nic_ops = knetdev->nic_ops;
+ mutex_init(&knodev->lock);
+
+ knodev->stats = netdev_alloc_pcpu_stats(struct knod_dev_stats);
+ if (!knodev->stats) {
+ err = -ENOMEM;
+ pr_err("knod: failed to allocate stats for %s\n",
+ netdev_name(knetdev->dev));
+ goto free_xdev;
+ }
+
+ knodev->wpriv = kmalloc_array(KNOD_SPSC_MAX,
+ sizeof(struct knod_work_priv),
+ GFP_KERNEL | __GFP_ZERO);
+ if (!knodev->wpriv) {
+ pr_err("knod: failed to allocate work priv for %s\n",
+ netdev_name(knetdev->dev));
+ err = -ENOMEM;
+ goto free_percpu;
+ }
+
+ netdev_lock(knodev->netdev);
+ err = knodev->nic_ops->attach(knodev);
+ if (err) {
+ err = -ENOMEM;
+ pr_err("knod: NIC attach failed on %s\n",
+ netdev_name(knetdev->dev));
+ goto unlock;
+ }
+
+ err = knodev->accel_ops->attach(knodev);
+ if (err) {
+ err = -ENOMEM;
+ pr_err("knod: accelerator attach failed on %s\n",
+ netdev_name(knetdev->dev));
+ goto nic_detach;
+ }
+
+ {
+ unsigned int nqueues = min(knodev->netdev->num_rx_queues,
+ KNOD_SPSC_MAX);
+ unsigned int stride = ALIGN(sizeof(struct spsc_bd),
+ SMP_CACHE_BYTES);
+ unsigned int cap = roundup_pow_of_two(KNOD_SPSC_ELEMS_MAX);
+ size_t pool_size = (size_t)stride * cap;
+
+ if (knodev->accel_ops->alloc_mem) {
+ size_t total = pool_size * nqueues;
+ u64 base_gaddr;
+ void *base_pool;
+ void *pool_priv;
+
+ base_pool = knodev->accel_ops->alloc_mem(knodev, total,
+ &base_gaddr, NULL, &pool_priv);
+ if (!base_pool) {
+ err = -ENOMEM;
+ pr_err("%s: alloc_mem failed\n", __func__);
+ goto free_spsc;
+ }
+ memset(base_pool, 0, total);
+
+ /* First queue owns the BO, others reference it */
+ knodev->wpriv[0].spsc_pool_priv = pool_priv;
+ for (i = 0; i < nqueues; i++) {
+ void *pool = base_pool +
+ (unsigned long)i * pool_size;
+
+ knodev->wpriv[i].spsc_pool_gaddr =
+ base_gaddr + (u64)i * pool_size;
+ err = __spsc_init(&knodev->wpriv[i].spsc_bds,
+ sizeof(struct spsc_bd),
+ KNOD_SPSC_ELEMS_MAX, pool,
+ GFP_KERNEL);
+ if (err) {
+ pr_err("%s: spsc_init failed q%d\n",
+ __func__, i);
+ goto free_spsc;
+ }
+ }
+ } else {
+ for (i = 0; i < nqueues; i++) {
+ err = spsc_init(&knodev->wpriv[i].spsc_bds,
+ sizeof(struct spsc_bd),
+ KNOD_SPSC_ELEMS_MAX,
+ GFP_KERNEL);
+ if (err) {
+ pr_err("%s: spsc_init failed q%d\n",
+ __func__, i);
+ goto free_spsc;
+ }
+ }
+ }
+ }
+ goto spsc_done;
+
+free_spsc:
+ for (i--; i >= 0; i--)
+ spsc_destroy(&knodev->wpriv[i].spsc_bds);
+ if (knodev->wpriv[0].spsc_pool_priv)
+ knodev->accel_ops->free_mem(knodev,
+ knodev->wpriv[0].spsc_pool_priv);
+ knodev->accel_ops->detach(knodev);
+ goto nic_detach;
+spsc_done:
+
+ err = knod_pass_attach(knodev);
+ if (err) {
+ pr_err("%s: knod_pass_attach failed (%d)\n", __func__, err);
+ goto accel_detach;
+ }
+
+ err = knod_dmabuf_attach(knodev);
+ if (err) {
+ err = -ENOMEM;
+ pr_err("knod: dmabuf attach failed on %s\n",
+ netdev_name(knetdev->dev));
+ goto accel_detach;
+ }
+
+ pr_info("knod: %s attached to accel %d\n",
+ netdev_name(knodev->netdev), accel->id);
+ list_add(&knodev->list, &knod_dev_list);
+ knetdev->status = KNOD_STATUS_USED;
+ accel->status = KNOD_STATUS_USED;
+
+ if (knodev->accel_ops->xdp_ops && knodev->accel_ops->xdp_ops->init) {
+ err = knodev->accel_ops->xdp_ops->init(knodev);
+ if (err) {
+ pr_err("knod: XDP init failed on %s\n",
+ netdev_name(knetdev->dev));
+ goto xdp_err;
+ }
+ }
+
+ /*
+ * Feature offloads (BPF, IPsec) allocate their GPU resources and
+ * advertise their netdev capabilities only when the feature is
+ * selected via knod_accel_feature_set(), not at attach.
+ */
+ netdev_unlock(knodev->netdev);
+
+ if (knodev->accel_ops->mp_map) {
+ err = knodev->accel_ops->mp_map(knodev);
+ if (err) {
+ pr_err("knod: mp_map failed (%d)\n", err);
+ goto dmabuf_detach;
+ }
+ }
+
+ return err;
+
+dmabuf_detach:
+ netdev_lock(knodev->netdev);
+xdp_err:
+ list_del(&knodev->list);
+ knetdev->status = KNOD_STATUS_FREE;
+ accel->status = KNOD_STATUS_FREE;
+ knod_dmabuf_detach(knodev);
+accel_detach:
+ knod_pass_detach(knodev);
+ for (i = 0; i < KNOD_SPSC_MAX; i++)
+ spsc_destroy(&knodev->wpriv[i].spsc_bds);
+ if (knodev->wpriv[0].spsc_pool_priv)
+ knodev->accel_ops->free_mem(knodev,
+ knodev->wpriv[0].spsc_pool_priv);
+ knodev->accel_ops->detach(knodev);
+nic_detach:
+ knodev->nic_ops->detach(knodev);
+unlock:
+ netdev_unlock(knodev->netdev);
+ kfree(knodev->wpriv);
+free_percpu:
+ free_percpu(knodev->stats);
+free_xdev:
+ /* Drop the accel<->knetdev<->knodev links set above before freeing
+ * knodev, or a reader (e.g. knod_default_worker via accel->knodev)
+ * dereferences a dangling pointer after a failed attach.
+ */
+ accel->knodev = NULL;
+ accel->knetdev = NULL;
+ knetdev->knodev = NULL;
+ knetdev->accel = NULL;
+ module_put(accel->owner);
+ module_put(knetdev->owner);
+ kfree(knodev);
+ return err;
+}
+
+int knod_dev_detach(struct knod_dev *knodev)
+{
+ struct knod_accel *accel = knodev->accel;
+ struct knod_netdev *knetdev = knodev->knetdev;
+ int i;
+
+ if (netif_running(knodev->netdev)) {
+ pr_err("knod_dev: interface is up\n");
+ return -EINVAL;
+ }
+
+ netdev_lock(knodev->netdev);
+ /*
+ * pre_detach() runs knod_feature_stop()+knod_feature_deactivate(),
+ * which tears down whatever feature is active and frees its
+ * resources, so no per-feature teardown is needed here.
+ */
+ if (knodev->accel_ops->pre_detach)
+ knodev->accel_ops->pre_detach(knodev);
+ list_del(&knodev->list);
+ knod_dmabuf_detach(knodev);
+ knod_pass_detach(knodev);
+ for (i = 0; i < KNOD_SPSC_MAX; i++)
+ spsc_destroy(&knodev->wpriv[i].spsc_bds);
+ if (knodev->wpriv[0].spsc_pool_priv)
+ knodev->accel_ops->free_mem(knodev,
+ knodev->wpriv[0].spsc_pool_priv);
+ knodev->nic_ops->detach(knodev);
+ knodev->accel_ops->detach(knodev);
+ netdev_unlock(knodev->netdev);
+ knetdev->status = KNOD_STATUS_FREE;
+ knetdev->accel = NULL;
+ knetdev->knodev = NULL;
+ accel->knetdev = NULL;
+ accel->knodev = NULL;
+ accel->status = KNOD_STATUS_FREE;
+ kfree(knodev->wpriv);
+ free_percpu(knodev->stats);
+ kfree(knodev);
+
+ module_put(accel->owner);
+ module_put(knetdev->owner);
+ return 0;
+}
+
+static int __init knod_dev_init(void)
+{
+ return genl_register_family(&knod_nl_family);
+}
+
+core_initcall(knod_dev_init);
+
diff --git a/net/knod/knod_nl.c b/net/knod/knod_nl.c
new file mode 100644
index 000000000000..77799202abcf
--- /dev/null
+++ b/net/knod/knod_nl.c
@@ -0,0 +1,406 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/* Copyright (c) 2021 Taehee Yoo <ap420073@gmail.com>
+ * Copyright (c) 2021 Hoyeon Lee <hoyeon.rhee@gmail.com>
+ */
+
+#include <linux/netdevice.h>
+#include <linux/rtnetlink.h>
+#include <net/genetlink.h>
+#include <net/knod.h>
+#include <net/sock.h>
+
+#include "knod-nl-gen.h"
+#include "knod.h"
+
+/* ---- accel ---- */
+
+static int knod_nl_accel_fill(struct knod_accel *accel,
+ struct sk_buff *rsp, const struct genl_info *info)
+{
+ u32 ena = 0, cap = 0;
+ void *hdr;
+
+ if (accel->accel_ops->feature_get)
+ accel->accel_ops->feature_get(accel, &ena, &cap);
+
+ hdr = genlmsg_iput(rsp, info);
+ if (!hdr)
+ return -EMSGSIZE;
+
+ if (nla_put_u32(rsp, KNOD_A_ACCEL_ID, accel->id) ||
+ nla_put_string(rsp, KNOD_A_ACCEL_NAME, accel->name) ||
+ nla_put_u32(rsp, KNOD_A_ACCEL_TYPE, accel->type) ||
+ nla_put_u32(rsp, KNOD_A_ACCEL_FEATURE_CAP, cap) ||
+ nla_put_u32(rsp, KNOD_A_ACCEL_FEATURE_ENA, ena)) {
+ genlmsg_cancel(rsp, hdr);
+ return -EMSGSIZE;
+ }
+
+ genlmsg_end(rsp, hdr);
+ return 0;
+}
+
+int knod_nl_accel_get_doit(struct sk_buff *skb, struct genl_info *info)
+{
+ struct knod_accel *accel;
+ struct sk_buff *rsp;
+ int err;
+
+ if (GENL_REQ_ATTR_CHECK(info, KNOD_A_ACCEL_ID))
+ return -EINVAL;
+
+ rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
+ if (!rsp)
+ return -ENOMEM;
+
+ mutex_lock(&knod_lock);
+ accel = knod_accel_lookup(nla_get_u32(info->attrs[KNOD_A_ACCEL_ID]));
+ if (!accel) {
+ mutex_unlock(&knod_lock);
+ err = -ENODEV;
+ goto err_free;
+ }
+ err = knod_nl_accel_fill(accel, rsp, info);
+ mutex_unlock(&knod_lock);
+ if (err)
+ goto err_free;
+
+ return genlmsg_reply(rsp, info);
+
+err_free:
+ nlmsg_free(rsp);
+ return err;
+}
+
+int knod_nl_accel_get_dumpit(struct sk_buff *rsp, struct netlink_callback *cb)
+{
+ struct knod_accel *accel;
+ int idx = 0, s_idx = cb->args[0];
+
+ mutex_lock(&knod_lock);
+ list_for_each_entry(accel, &knod_accel_list, list) {
+ if (idx < s_idx)
+ goto cont;
+ if (knod_nl_accel_fill(accel, rsp, genl_info_dump(cb)) < 0)
+ break;
+cont:
+ idx++;
+ }
+ mutex_unlock(&knod_lock);
+
+ cb->args[0] = idx;
+ return rsp->len;
+}
+
+int knod_nl_accel_set_doit(struct sk_buff *skb, struct genl_info *info)
+{
+ struct knod_accel *accel;
+ u32 feature;
+ int err;
+
+ if (GENL_REQ_ATTR_CHECK(info, KNOD_A_ACCEL_ID) ||
+ GENL_REQ_ATTR_CHECK(info, KNOD_A_ACCEL_FEATURE_ENA))
+ return -EINVAL;
+
+ feature = nla_get_u32(info->attrs[KNOD_A_ACCEL_FEATURE_ENA]);
+
+ /*
+ * rtnl serialises the feature switch against attach/detach (which
+ * also take rtnl) and against the NIC driver's interface up/down
+ * (knod_dev_start/stop), and lets feature_set touch netdev->features
+ * and run an expedited synchronize_net(). knod_lock still guards the
+ * accel list walk (lock order: rtnl -> knod_lock).
+ */
+ rtnl_lock();
+ mutex_lock(&knod_lock);
+ accel = knod_accel_lookup(nla_get_u32(info->attrs[KNOD_A_ACCEL_ID]));
+ if (!accel) {
+ err = -ENODEV;
+ goto unlock;
+ }
+ if (!accel->accel_ops->feature_set) {
+ err = -EOPNOTSUPP;
+ goto unlock;
+ }
+ err = accel->accel_ops->feature_set(accel, feature, info->extack);
+unlock:
+ mutex_unlock(&knod_lock);
+ rtnl_unlock();
+ return err;
+}
+
+/* ---- nic ---- */
+
+static int knod_nl_nic_fill(struct knod_netdev *knetdev,
+ struct sk_buff *rsp, const struct genl_info *info)
+{
+ void *hdr;
+
+ hdr = genlmsg_iput(rsp, info);
+ if (!hdr)
+ return -EMSGSIZE;
+
+ if (nla_put_u32(rsp, KNOD_A_NIC_IFINDEX, knetdev->dev->ifindex) ||
+ nla_put_string(rsp, KNOD_A_NIC_NAME, netdev_name(knetdev->dev))) {
+ genlmsg_cancel(rsp, hdr);
+ return -EMSGSIZE;
+ }
+
+ genlmsg_end(rsp, hdr);
+ return 0;
+}
+
+int knod_nl_nic_get_doit(struct sk_buff *skb, struct genl_info *info)
+{
+ struct knod_netdev *knetdev;
+ struct net_device *dev;
+ struct sk_buff *rsp;
+ int err;
+
+ if (GENL_REQ_ATTR_CHECK(info, KNOD_A_NIC_IFINDEX))
+ return -EINVAL;
+
+ rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
+ if (!rsp)
+ return -ENOMEM;
+
+ rtnl_lock();
+ mutex_lock(&knod_lock);
+ dev = __dev_get_by_index(genl_info_net(info),
+ nla_get_u32(info->attrs[KNOD_A_NIC_IFINDEX]));
+ knetdev = dev ? knod_netdev_lookup(dev) : NULL;
+ if (!knetdev) {
+ mutex_unlock(&knod_lock);
+ rtnl_unlock();
+ err = -ENODEV;
+ goto err_free;
+ }
+ err = knod_nl_nic_fill(knetdev, rsp, info);
+ mutex_unlock(&knod_lock);
+ rtnl_unlock();
+ if (err)
+ goto err_free;
+
+ return genlmsg_reply(rsp, info);
+
+err_free:
+ nlmsg_free(rsp);
+ return err;
+}
+
+int knod_nl_nic_get_dumpit(struct sk_buff *rsp, struct netlink_callback *cb)
+{
+ struct net *net = sock_net(rsp->sk);
+ struct knod_netdev *knetdev;
+ int idx = 0, s_idx = cb->args[0];
+
+ rtnl_lock();
+ mutex_lock(&knod_lock);
+ list_for_each_entry(knetdev, &knod_netdev_list, list) {
+ if (idx < s_idx)
+ goto cont;
+ if (dev_net(knetdev->dev) != net)
+ goto cont;
+ if (knod_nl_nic_fill(knetdev, rsp, genl_info_dump(cb)) < 0)
+ break;
+cont:
+ idx++;
+ }
+ mutex_unlock(&knod_lock);
+ rtnl_unlock();
+
+ cb->args[0] = idx;
+ return rsp->len;
+}
+
+/* ---- dev (NIC <-> accel binding) ---- */
+
+static int knod_nl_dev_fill(struct knod_dev *knodev, struct sk_buff *rsp,
+ const struct genl_info *info)
+{
+ void *hdr;
+
+ hdr = genlmsg_iput(rsp, info);
+ if (!hdr)
+ return -EMSGSIZE;
+
+ if (nla_put_u32(rsp, KNOD_A_DEV_NIC_IFINDEX, knodev->netdev->ifindex) ||
+ nla_put_u32(rsp, KNOD_A_DEV_ACCEL_ID, knodev->accel->id)) {
+ genlmsg_cancel(rsp, hdr);
+ return -EMSGSIZE;
+ }
+
+ genlmsg_end(rsp, hdr);
+ return 0;
+}
+
+void knod_nl_notify_dev(struct knod_dev *knodev, u32 cmd)
+{
+ struct net *net = dev_net(knodev->netdev);
+ struct genl_info info;
+ struct sk_buff *ntf;
+
+ if (!genl_has_listeners(&knod_nl_family, net, KNOD_NLGRP_MGMT))
+ return;
+
+ ntf = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
+ if (!ntf)
+ return;
+
+ genl_info_init_ntf(&info, &knod_nl_family, cmd);
+ if (knod_nl_dev_fill(knodev, ntf, &info)) {
+ nlmsg_free(ntf);
+ return;
+ }
+
+ genlmsg_multicast_netns(&knod_nl_family, net, ntf, 0,
+ KNOD_NLGRP_MGMT, GFP_KERNEL);
+}
+
+int knod_nl_attach_doit(struct sk_buff *skb, struct genl_info *info)
+{
+ struct knod_netdev *knetdev;
+ struct knod_accel *accel;
+ struct net_device *dev;
+ int err;
+
+ if (GENL_REQ_ATTR_CHECK(info, KNOD_A_DEV_NIC_IFINDEX) ||
+ GENL_REQ_ATTR_CHECK(info, KNOD_A_DEV_ACCEL_ID))
+ return -EINVAL;
+
+ rtnl_lock();
+ mutex_lock(&knod_lock);
+
+ dev = __dev_get_by_index(genl_info_net(info),
+ nla_get_u32(
+ info->attrs[KNOD_A_DEV_NIC_IFINDEX]));
+ if (!dev) {
+ NL_SET_ERR_MSG(info->extack, "no such netdevice");
+ err = -ENODEV;
+ goto unlock;
+ }
+
+ knetdev = knod_netdev_lookup(dev);
+ if (!knetdev) {
+ NL_SET_ERR_MSG(info->extack, "netdevice not registered with knod");
+ err = -ENODEV;
+ goto unlock;
+ }
+
+ if (netif_running(dev)) {
+ NL_SET_ERR_MSG(info->extack, "bring the netdevice down first");
+ err = -EBUSY;
+ goto unlock;
+ }
+
+ accel = knod_accel_lookup(
+ nla_get_u32(info->attrs[KNOD_A_DEV_ACCEL_ID]));
+ if (!accel) {
+ NL_SET_ERR_MSG(info->extack, "no such accelerator");
+ err = -ENODEV;
+ goto unlock;
+ }
+
+ err = knod_dev_attach(knetdev, accel);
+ if (!err)
+ knod_nl_notify_dev(knetdev->knodev, KNOD_CMD_DEV_ADD_NTF);
+
+unlock:
+ mutex_unlock(&knod_lock);
+ rtnl_unlock();
+ return err;
+}
+
+int knod_nl_detach_doit(struct sk_buff *skb, struct genl_info *info)
+{
+ struct knod_dev *knodev;
+ struct net_device *dev;
+ int err;
+
+ if (GENL_REQ_ATTR_CHECK(info, KNOD_A_DEV_NIC_IFINDEX))
+ return -EINVAL;
+
+ rtnl_lock();
+
+ dev = __dev_get_by_index(genl_info_net(info),
+ nla_get_u32(
+ info->attrs[KNOD_A_DEV_NIC_IFINDEX]));
+ knodev = dev ? knod_dev_lookup(dev) : NULL;
+ if (!knodev) {
+ NL_SET_ERR_MSG(info->extack, "netdevice not attached");
+ err = -ENODEV;
+ goto unlock;
+ }
+
+ if (netif_running(dev)) {
+ NL_SET_ERR_MSG(info->extack, "bring the netdevice down first");
+ err = -EBUSY;
+ goto unlock;
+ }
+
+ knod_nl_notify_dev(knodev, KNOD_CMD_DEV_DEL_NTF);
+ err = knod_dev_detach(knodev);
+
+unlock:
+ rtnl_unlock();
+ return err;
+}
+
+int knod_nl_dev_get_doit(struct sk_buff *skb, struct genl_info *info)
+{
+ struct knod_dev *knodev;
+ struct net_device *dev;
+ struct sk_buff *rsp;
+ int err;
+
+ if (GENL_REQ_ATTR_CHECK(info, KNOD_A_DEV_NIC_IFINDEX))
+ return -EINVAL;
+
+ rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
+ if (!rsp)
+ return -ENOMEM;
+
+ rtnl_lock();
+ dev = __dev_get_by_index(genl_info_net(info),
+ nla_get_u32(
+ info->attrs[KNOD_A_DEV_NIC_IFINDEX]));
+ knodev = dev ? knod_dev_lookup(dev) : NULL;
+ if (!knodev) {
+ rtnl_unlock();
+ err = -ENODEV;
+ goto err_free;
+ }
+ err = knod_nl_dev_fill(knodev, rsp, info);
+ rtnl_unlock();
+ if (err)
+ goto err_free;
+
+ return genlmsg_reply(rsp, info);
+
+err_free:
+ nlmsg_free(rsp);
+ return err;
+}
+
+int knod_nl_dev_get_dumpit(struct sk_buff *rsp, struct netlink_callback *cb)
+{
+ struct net *net = sock_net(rsp->sk);
+ struct knod_dev *knodev;
+ int idx = 0, s_idx = cb->args[0];
+
+ rtnl_lock();
+ list_for_each_entry(knodev, &knod_dev_list, list) {
+ if (idx < s_idx)
+ goto cont;
+ if (dev_net(knodev->netdev) != net)
+ goto cont;
+ if (knod_nl_dev_fill(knodev, rsp, genl_info_dump(cb)) < 0)
+ break;
+cont:
+ idx++;
+ }
+ rtnl_unlock();
+
+ cb->args[0] = idx;
+ return rsp->len;
+}
--
2.43.0
^ permalink raw reply related
* [RFC PATCH net-next 05/13] bpf: offload: allow PERCPU_ARRAY maps for offloaded programs
From: Taehee Yoo @ 2026-07-19 17:58 UTC (permalink / raw)
To: Alex Deucher, Alexei Starovoitov, amd-gfx, Andrew Lunn,
Andrii Nakryiko, Bill Wendling, bpf, Christian König,
Daniel Borkmann, David Airlie, David S. Miller, Donald Hunter,
dri-devel, Eduard Zingerman, Emil Tsalapatis, Eric Dumazet,
Felix Kuehling, Hoyeon Lee, Ilias Apalodimas, Jakub Kicinski,
Jesper Dangaard Brouer, Jiri Olsa, John Fastabend, Justin Stitt,
Kees Cook, Kumar Kartikeya Dwivedi, Leon Romanovsky,
linaro-mm-sig, linux-hardening, linux-kernel, linux-kselftest,
linux-media, linux-rdma, llvm, Mark Bloch, Martin KaFai Lau,
Michael Chan, Nathan Chancellor, netdev, Nick Desaulniers,
Paolo Abeni, Pavan Chebbi, Saeed Mahameed, Shuah Khan,
Simona Vetter, Simon Horman, Song Liu, Stanislav Fomichev,
Sumit Semwal, Taehee Yoo, Tariq Toukan, Yonghong Song
In-Reply-To: <20260719175857.4071636-1-ap420073@gmail.com>
The knod BPF offload keeps hot per-CPU statistics in a PERCPU_ARRAY
map that is mirrored into accelerator memory and aggregated back on the
host. Permit this map type on the offload path.
Signed-off-by: Taehee Yoo <ap420073@gmail.com>
(cherry picked from commit c447c9012940d48c1ec20a8ebac8d779e7d5a6fa)
---
kernel/bpf/offload.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/kernel/bpf/offload.c b/kernel/bpf/offload.c
index 0d6f5569588c..36e747a12cbe 100644
--- a/kernel/bpf/offload.c
+++ b/kernel/bpf/offload.c
@@ -510,7 +510,8 @@ struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr)
if (!capable(CAP_SYS_ADMIN))
return ERR_PTR(-EPERM);
if (attr->map_type != BPF_MAP_TYPE_ARRAY &&
- attr->map_type != BPF_MAP_TYPE_HASH)
+ attr->map_type != BPF_MAP_TYPE_HASH &&
+ attr->map_type != BPF_MAP_TYPE_PERCPU_ARRAY)
return ERR_PTR(-EINVAL);
offmap = bpf_map_area_alloc(sizeof(*offmap), NUMA_NO_NODE);
--
2.43.0
^ permalink raw reply related
* [RFC PATCH net-next 06/13] drm/amdkfd: prepare kfd core for the knod provider
From: Taehee Yoo @ 2026-07-19 17:58 UTC (permalink / raw)
To: Alex Deucher, Alexei Starovoitov, amd-gfx, Andrew Lunn,
Andrii Nakryiko, Bill Wendling, bpf, Christian König,
Daniel Borkmann, David Airlie, David S. Miller, Donald Hunter,
dri-devel, Eduard Zingerman, Emil Tsalapatis, Eric Dumazet,
Felix Kuehling, Hoyeon Lee, Ilias Apalodimas, Jakub Kicinski,
Jesper Dangaard Brouer, Jiri Olsa, John Fastabend, Justin Stitt,
Kees Cook, Kumar Kartikeya Dwivedi, Leon Romanovsky,
linaro-mm-sig, linux-hardening, linux-kernel, linux-kselftest,
linux-media, linux-rdma, llvm, Mark Bloch, Martin KaFai Lau,
Michael Chan, Nathan Chancellor, netdev, Nick Desaulniers,
Paolo Abeni, Pavan Chebbi, Saeed Mahameed, Shuah Khan,
Simona Vetter, Simon Horman, Song Liu, Stanislav Fomichev,
Sumit Semwal, Taehee Yoo, Tariq Toukan, Yonghong Song
In-Reply-To: <20260719175857.4071636-1-ap420073@gmail.com>
Expose the kfd internals the knod provider needs to build and run GPU
queues on behalf of the kernel: kernel-side event waiting, GPUVM
allocation for VRAM/GTT, doorbell access, and the HSA queue/packet
layout (kfd_hsa.h). No functional change for existing user-mode
queue users; the provider itself is added in a following patch.
Signed-off-by: Taehee Yoo <ap420073@gmail.com>
(cherry picked from commit 7d2c7cbfbb2a4a80052b5793841f6a229271973d)
---
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 13 +-
.../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 65 +++
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 3 +
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c | 117 ++---
drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c | 20 +-
drivers/gpu/drm/amd/amdkfd/kfd_events.c | 112 ++++-
drivers/gpu/drm/amd/amdkfd/kfd_events.h | 4 +
drivers/gpu/drm/amd/amdkfd/kfd_hsa.h | 451 ++++++++++++++++++
drivers/gpu/drm/amd/amdkfd/kfd_module.c | 1 +
drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 34 ++
drivers/gpu/drm/amd/amdkfd/kfd_process.c | 26 +-
11 files changed, 778 insertions(+), 68 deletions(-)
create mode 100644 drivers/gpu/drm/amd/amdkfd/kfd_hsa.h
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index e443a7277299..cf906b32eacb 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -328,6 +328,8 @@ int amdgpu_amdkfd_gpuvm_sync_memory(
struct amdgpu_device *adev, struct kgd_mem *mem, bool intr);
int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_mem *mem,
void **kptr, uint64_t *size);
+int amdgpu_amdkfd_gpuvm_map_vram_bo_to_kernel(struct kgd_mem *mem,
+ void **kptr, uint64_t *size);
void amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(struct kgd_mem *mem);
int amdgpu_amdkfd_map_gtt_bo_to_gart(struct amdgpu_bo *bo, struct amdgpu_bo **bo_gart);
@@ -446,7 +448,6 @@ bool kgd2kfd_vmfault_fast_path(struct amdgpu_device *adev, struct amdgpu_iv_entr
bool retry_fault);
void kgd2kfd_lock_kfd(void);
void kgd2kfd_teardown_processes(struct amdgpu_device *adev);
-
#else
static inline int kgd2kfd_init(void)
{
@@ -576,5 +577,15 @@ static inline void kgd2kfd_teardown_processes(struct amdgpu_device *adev)
{
}
+#endif
+
+#if defined(CONFIG_HSA_AMD_KNOD)
+int knod_init(struct amdgpu_device *adev);
+void knod_fini(struct amdgpu_device *adev);
+void knod_exit(void);
+#else
+static inline int knod_init(struct amdgpu_device *adev) { return 0; }
+static inline void knod_fini(struct amdgpu_device *adev) { }
+static inline void knod_exit(void) { }
#endif
#endif /* AMDGPU_AMDKFD_H_INCLUDED */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 35fe2c974699..79fd0ce7fc24 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -2336,6 +2336,71 @@ int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_mem *mem,
return ret;
}
+/** amdgpu_amdkfd_gpuvm_map_vram_bo_to_kernel() - Map a VRAM BO for kernel CPU access
+ *
+ * @mem: Buffer object to be mapped for CPU access
+ * @kptr[out]: pointer in kernel CPU address space
+ * @size[out]: size of the buffer
+ *
+ * Pins the BO and maps it for kernel CPU access. The eviction fence is removed
+ * from the BO, since pinned BOs cannot be evicted. The bo must remain on the
+ * validate_list, so the GPU mapping can be restored after a page table was
+ * evicted.
+ *
+ * Return: 0 on success, error code on failure
+ */
+int amdgpu_amdkfd_gpuvm_map_vram_bo_to_kernel(struct kgd_mem *mem,
+ void **kptr, uint64_t *size)
+{
+ int ret;
+ struct amdgpu_bo *bo = mem->bo;
+
+ if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {
+ pr_err("userptr can't be mapped to kernel\n");
+ return -EINVAL;
+ }
+
+ mutex_lock(&mem->process_info->lock);
+
+ ret = amdgpu_bo_reserve(bo, true);
+ if (ret) {
+ pr_err("Failed to reserve bo. ret %d\n", ret);
+ goto bo_reserve_failed;
+ }
+
+ ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_VRAM);
+ if (ret) {
+ pr_err("Failed to pin bo. ret %d\n", ret);
+ goto pin_failed;
+ }
+
+ ret = amdgpu_bo_kmap(bo, kptr);
+ if (ret) {
+ pr_err("Failed to map bo to kernel. ret %d\n", ret);
+ goto kmap_failed;
+ }
+
+ amdgpu_amdkfd_remove_eviction_fence(
+ bo, mem->process_info->eviction_fence);
+
+ if (size)
+ *size = amdgpu_bo_size(bo);
+
+ amdgpu_bo_unreserve(bo);
+
+ mutex_unlock(&mem->process_info->lock);
+ return 0;
+
+kmap_failed:
+ amdgpu_bo_unpin(bo);
+pin_failed:
+ amdgpu_bo_unreserve(bo);
+bo_reserve_failed:
+ mutex_unlock(&mem->process_info->lock);
+
+ return ret;
+}
+
/** amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel() - Unmap a GTT BO for kernel CPU access
*
* @mem: Buffer object to be unmapped for CPU access
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index 4c0c77eafbd1..aa974929d22a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -2480,6 +2480,8 @@ static int amdgpu_pci_probe(struct pci_dev *pdev,
drm_client_setup(adev_to_drm(adev), format);
}
+ knod_init(adev);
+
ret = amdgpu_debugfs_init(adev);
if (ret)
DRM_ERROR("Creating debugfs files failed (%d).\n", ret);
@@ -2540,6 +2542,7 @@ amdgpu_pci_remove(struct pci_dev *pdev)
struct drm_device *dev = pci_get_drvdata(pdev);
struct amdgpu_device *adev = drm_to_adev(dev);
+ knod_fini(adev);
amdgpu_ras_eeprom_check_and_recover(adev);
amdgpu_xcp_dev_unplug(adev);
amdgpu_gmc_prepare_nps_mode_change(adev);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index c7edebd2fd8a..7879a6ffaa76 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -335,32 +335,27 @@ static int set_queue_properties_from_user(struct queue_properties *q_properties,
return 0;
}
-static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
- void *data)
+/*
+ * Create a queue from a kernel-filled queue_properties. @q_properties is
+ * owned by the caller and must already be populated (the ioctl handler does
+ * this from user args via set_queue_properties_from_user(); in-kernel callers
+ * fill it directly). On success *@queue_id_out and *@doorbell_offset_out are
+ * set.
+ */
+int kfd_create_queue(struct kfd_process *p, struct queue_properties *q_properties,
+ u32 gpu_id, u32 *queue_id_out, u64 *doorbell_offset_out)
{
- struct kfd_ioctl_create_queue_args *args = data;
struct kfd_node *dev;
int err = 0;
unsigned int queue_id;
struct kfd_process_device *pdd;
- struct queue_properties q_properties;
uint32_t doorbell_offset_in_process = 0;
- memset(&q_properties, 0, sizeof(struct queue_properties));
-
- pr_debug("Creating queue ioctl\n");
-
- err = set_queue_properties_from_user(&q_properties, args);
- if (err)
- return err;
-
- pr_debug("Looking for gpu id 0x%x\n", args->gpu_id);
-
mutex_lock(&p->mutex);
- pdd = kfd_process_device_data_by_id(p, args->gpu_id);
+ pdd = kfd_process_device_data_by_id(p, gpu_id);
if (!pdd) {
- pr_debug("Could not find gpu id 0x%x\n", args->gpu_id);
+ pr_debug("Could not find gpu id 0x%x\n", gpu_id);
err = -EINVAL;
goto err_pdd;
}
@@ -372,14 +367,14 @@ static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
goto err_bind_process;
}
- if (q_properties.type == KFD_QUEUE_TYPE_SDMA_BY_ENG_ID) {
+ if (q_properties->type == KFD_QUEUE_TYPE_SDMA_BY_ENG_ID) {
int max_sdma_eng_id = kfd_get_num_sdma_engines(dev) +
kfd_get_num_xgmi_sdma_engines(dev) - 1;
- if (q_properties.sdma_engine_id > max_sdma_eng_id) {
+ if (q_properties->sdma_engine_id > max_sdma_eng_id) {
err = -EINVAL;
pr_err("sdma_engine_id %i exceeds maximum id of %i\n",
- q_properties.sdma_engine_id, max_sdma_eng_id);
+ q_properties->sdma_engine_id, max_sdma_eng_id);
goto err_sdma_engine_id;
}
}
@@ -392,7 +387,7 @@ static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
}
}
- err = kfd_queue_acquire_buffers(pdd, &q_properties);
+ err = kfd_queue_acquire_buffers(pdd, q_properties);
if (err) {
pr_debug("failed to acquire user queue buffers\n");
goto err_acquire_queue_buf;
@@ -402,42 +397,32 @@ static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
p->lead_thread->pid,
dev->id);
- err = pqm_create_queue(&p->pqm, dev, &q_properties, &queue_id,
+ err = pqm_create_queue(&p->pqm, dev, q_properties, &queue_id,
NULL, NULL, NULL, &doorbell_offset_in_process);
if (err != 0)
goto err_create_queue;
- args->queue_id = queue_id;
-
+ *queue_id_out = queue_id;
/* Return gpu_id as doorbell offset for mmap usage */
- args->doorbell_offset = KFD_MMAP_TYPE_DOORBELL;
- args->doorbell_offset |= KFD_MMAP_GPU_ID(args->gpu_id);
+ *doorbell_offset_out = KFD_MMAP_TYPE_DOORBELL;
+ *doorbell_offset_out |= KFD_MMAP_GPU_ID(gpu_id);
if (KFD_IS_SOC15(dev))
/* On SOC15 ASICs, include the doorbell offset within the
* process doorbell frame, which is 2 pages.
*/
- args->doorbell_offset |= doorbell_offset_in_process;
+ *doorbell_offset_out |= doorbell_offset_in_process;
mutex_unlock(&p->mutex);
- pr_debug("Queue id %d was created successfully\n", args->queue_id);
-
- pr_debug("Ring buffer address == 0x%016llX\n",
- args->ring_base_address);
-
- pr_debug("Read ptr address == 0x%016llX\n",
- args->read_pointer_address);
-
- pr_debug("Write ptr address == 0x%016llX\n",
- args->write_pointer_address);
+ pr_debug("Queue id %d was created successfully\n", queue_id);
kfd_dbg_ev_raise(KFD_EC_MASK(EC_QUEUE_NEW), p, dev, queue_id, false, NULL, 0);
return 0;
err_create_queue:
- kfd_queue_unref_bo_vas(pdd, &q_properties);
- kfd_queue_release_buffers(pdd, &q_properties);
+ kfd_queue_unref_bo_vas(pdd, q_properties);
+ kfd_queue_release_buffers(pdd, q_properties);
err_acquire_queue_buf:
err_sdma_engine_id:
err_bind_process:
@@ -446,6 +431,23 @@ static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
return err;
}
+static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
+ void *data)
+{
+ struct kfd_ioctl_create_queue_args *args = data;
+ struct queue_properties q_properties;
+ int err;
+
+ memset(&q_properties, 0, sizeof(q_properties));
+
+ err = set_queue_properties_from_user(&q_properties, args);
+ if (err)
+ return err;
+
+ return kfd_create_queue(p, &q_properties, args->gpu_id,
+ &args->queue_id, &args->doorbell_offset);
+}
+
static int kfd_ioctl_destroy_queue(struct file *filp, struct kfd_process *p,
void *data)
{
@@ -643,7 +645,7 @@ static int kfd_ioctl_set_memory_policy(struct file *filep,
return err;
}
-static int kfd_ioctl_set_trap_handler(struct file *filep,
+int kfd_ioctl_set_trap_handler(struct file *filep,
struct kfd_process *p, void *data)
{
struct kfd_ioctl_set_trap_handler_args *args = data;
@@ -726,7 +728,7 @@ static int kfd_ioctl_get_clock_counters(struct file *filep,
static int kfd_ioctl_get_process_apertures(struct file *filp,
- struct kfd_process *p, void *data)
+ struct kfd_process *p, void *data)
{
struct kfd_ioctl_get_process_apertures_args *args = data;
struct kfd_process_device_apertures *pAperture;
@@ -859,7 +861,7 @@ static int kfd_ioctl_get_process_apertures_new(struct file *filp,
}
static int kfd_ioctl_create_event(struct file *filp, struct kfd_process *p,
- void *data)
+ void *data)
{
struct kfd_ioctl_create_event_args *args = data;
int err;
@@ -894,24 +896,24 @@ static int kfd_ioctl_destroy_event(struct file *filp, struct kfd_process *p,
return kfd_event_destroy(p, args->event_id);
}
-static int kfd_ioctl_set_event(struct file *filp, struct kfd_process *p,
- void *data)
+int kfd_ioctl_set_event(struct file *filp, struct kfd_process *p,
+ void *data)
{
struct kfd_ioctl_set_event_args *args = data;
return kfd_set_event(p, args->event_id);
}
-static int kfd_ioctl_reset_event(struct file *filp, struct kfd_process *p,
- void *data)
+int kfd_ioctl_reset_event(struct file *filp, struct kfd_process *p,
+ void *data)
{
struct kfd_ioctl_reset_event_args *args = data;
return kfd_reset_event(p, args->event_id);
}
-static int kfd_ioctl_wait_events(struct file *filp, struct kfd_process *p,
- void *data)
+int kfd_ioctl_wait_events(struct file *filp, struct kfd_process *p,
+ void *data)
{
struct kfd_ioctl_wait_events_args *args = data;
@@ -920,8 +922,9 @@ static int kfd_ioctl_wait_events(struct file *filp, struct kfd_process *p,
(args->wait_for_all != 0),
&args->timeout, &args->wait_result);
}
-static int kfd_ioctl_set_scratch_backing_va(struct file *filep,
- struct kfd_process *p, void *data)
+
+int kfd_ioctl_set_scratch_backing_va(struct file *filep,
+ struct kfd_process *p, void *data)
{
struct kfd_ioctl_set_scratch_backing_va_args *args = data;
struct kfd_process_device *pdd;
@@ -1003,8 +1006,8 @@ static int kfd_ioctl_get_tile_config(struct file *filep,
return 0;
}
-static int kfd_ioctl_acquire_vm(struct file *filep, struct kfd_process *p,
- void *data)
+int kfd_ioctl_acquire_vm(struct file *filep, struct kfd_process *p,
+ void *data)
{
struct kfd_ioctl_acquire_vm_args *args = data;
struct kfd_process_device *pdd;
@@ -1078,8 +1081,8 @@ static int kfd_ioctl_get_available_memory(struct file *filep,
return 0;
}
-static int kfd_ioctl_alloc_memory_of_gpu(struct file *filep,
- struct kfd_process *p, void *data)
+int kfd_ioctl_alloc_memory_of_gpu(struct file *filep,
+ struct kfd_process *p, void *data)
{
struct kfd_ioctl_alloc_memory_of_gpu_args *args = data;
struct kfd_process_device *pdd;
@@ -1279,8 +1282,8 @@ static int kfd_ioctl_free_memory_of_gpu(struct file *filep,
return ret;
}
-static int kfd_ioctl_map_memory_to_gpu(struct file *filep,
- struct kfd_process *p, void *data)
+int kfd_ioctl_map_memory_to_gpu(struct file *filep,
+ struct kfd_process *p, void *data)
{
struct kfd_ioctl_map_memory_to_gpu_args *args = data;
struct kfd_process_device *pdd, *peer_pdd;
@@ -1624,8 +1627,8 @@ static int kfd_ioctl_import_dmabuf(struct file *filep,
return r;
}
-static int kfd_ioctl_export_dmabuf(struct file *filep,
- struct kfd_process *p, void *data)
+int kfd_ioctl_export_dmabuf(struct file *filep,
+ struct kfd_process *p, void *data)
{
struct kfd_ioctl_export_dmabuf_args *args = data;
struct kfd_process_device *pdd;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c b/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c
index fdcf7f2d1b5b..78b7e956f590 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c
@@ -145,10 +145,28 @@ int kfd_doorbell_mmap(struct kfd_node *dev, struct kfd_process *process,
vma->vm_page_prot);
}
+void __iomem *kfd_kernel_doorbell_mmap(struct kfd_node *dev,
+ struct kfd_process *process)
+{
+ phys_addr_t address;
+ struct kfd_process_device *pdd;
+
+ pdd = kfd_get_process_device_data(dev, process);
+ if (!pdd)
+ return NULL;
+
+ /* Calculate physical address of doorbell */
+ address = kfd_get_process_doorbells(pdd);
+ if (!address)
+ return NULL;
+
+ return ioremap(address, kfd_doorbell_process_slice(dev->kfd));
+}
+
/* get kernel iomem pointer for a doorbell */
void __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd,
- unsigned int *doorbell_off)
+ unsigned int *doorbell_off)
{
u32 inx;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_events.c
index 81900b49d9d5..91ef4c61a8bc 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_events.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.c
@@ -326,7 +326,7 @@ static bool event_can_be_cpu_signaled(const struct kfd_event *ev)
return ev->type == KFD_EVENT_TYPE_SIGNAL;
}
-static int kfd_event_page_set(struct kfd_process *p, void *kernel_address,
+int kfd_event_page_set(struct kfd_process *p, void *kernel_address,
uint64_t size, uint64_t user_handle)
{
struct kfd_signal_page *page;
@@ -1068,6 +1068,116 @@ int kfd_wait_on_events(struct kfd_process *p,
return ret;
}
+int kfd_wait_on_events_kernel(struct kfd_process *p,
+ uint32_t num_events, void __user *data,
+ bool all, uint32_t *user_timeout_ms,
+ uint32_t *wait_result)
+{
+ struct kfd_event_data *events = (struct kfd_event_data *) data;
+ uint32_t i;
+ int ret = 0;
+
+ struct kfd_event_waiter *event_waiters = NULL;
+ long timeout = user_timeout_to_jiffies(*user_timeout_ms);
+
+ event_waiters = alloc_event_waiters(num_events);
+ if (!event_waiters) {
+ ret = -ENOMEM;
+ goto out;
+ }
+
+ /* Use p->event_mutex here to protect against concurrent creation and
+ * destruction of events while we initialize event_waiters.
+ */
+ mutex_lock(&p->event_mutex);
+
+ for (i = 0; i < num_events; i++) {
+ struct kfd_event_data event_data;
+
+ memcpy(&event_data, &events[i], sizeof(struct kfd_event_data));
+ ret = init_event_waiter(p, &event_waiters[i], &event_data);
+ if (ret)
+ goto out_unlock;
+ }
+
+ /* Check condition once. */
+ *wait_result = test_event_condition(all, num_events, event_waiters);
+ if (*wait_result == KFD_IOC_WAIT_RESULT_COMPLETE) {
+ ret = copy_signaled_event_data(num_events,
+ event_waiters, events);
+ goto out_unlock;
+ } else if (WARN_ON(*wait_result == KFD_IOC_WAIT_RESULT_FAIL)) {
+ /* This should not happen. Events shouldn't be
+ * destroyed while we're holding the event_mutex
+ */
+ goto out_unlock;
+ }
+
+ mutex_unlock(&p->event_mutex);
+
+ while (true) {
+ if (fatal_signal_pending(current)) {
+ ret = -EINTR;
+ break;
+ }
+
+ if (signal_pending(current)) {
+ ret = -ERESTARTSYS;
+ if (*user_timeout_ms != KFD_EVENT_TIMEOUT_IMMEDIATE &&
+ *user_timeout_ms != KFD_EVENT_TIMEOUT_INFINITE)
+ *user_timeout_ms = jiffies_to_msecs(
+ max(0l, timeout-1));
+ break;
+ }
+
+ /* Set task state to interruptible sleep before
+ * checking wake-up conditions. A concurrent wake-up
+ * will put the task back into runnable state. In that
+ * case schedule_timeout will not put the task to
+ * sleep and we'll get a chance to re-check the
+ * updated conditions almost immediately. Otherwise,
+ * this race condition would lead to a soft hang or a
+ * very long sleep.
+ */
+ set_current_state(TASK_INTERRUPTIBLE);
+
+ *wait_result = test_event_condition(all, num_events,
+ event_waiters);
+ if (*wait_result != KFD_IOC_WAIT_RESULT_TIMEOUT)
+ break;
+
+ if (timeout <= 0)
+ break;
+
+ timeout = schedule_timeout(timeout);
+ }
+ __set_current_state(TASK_RUNNING);
+
+ mutex_lock(&p->event_mutex);
+ /* copy_signaled_event_data may sleep. So this has to happen
+ * after the task state is set back to RUNNING.
+ *
+ * The event may also have been destroyed after signaling. So
+ * copy_signaled_event_data also must confirm that the event
+ * still exists. Therefore this must be under the p->event_mutex
+ * which is also held when events are destroyed.
+ */
+ if (!ret && *wait_result == KFD_IOC_WAIT_RESULT_COMPLETE)
+ ret = copy_signaled_event_data(num_events,
+ event_waiters, events);
+
+out_unlock:
+ free_waiters(num_events, event_waiters, ret == -ERESTARTSYS);
+ mutex_unlock(&p->event_mutex);
+out:
+ if (ret)
+ *wait_result = KFD_IOC_WAIT_RESULT_FAIL;
+ else if (*wait_result == KFD_IOC_WAIT_RESULT_FAIL)
+ ret = -EIO;
+
+ return ret;
+}
+
int kfd_event_mmap(struct kfd_process *p, struct vm_area_struct *vma)
{
unsigned long pfn;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.h b/drivers/gpu/drm/amd/amdkfd/kfd_events.h
index 1dc21c13833b..aa6ead122084 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_events.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.h
@@ -87,5 +87,9 @@ struct kfd_event {
extern void kfd_signal_event_interrupt(u32 pasid, uint32_t partial_id,
uint32_t valid_id_bits,
bool signal_mailbox_updated);
+int kfd_wait_on_events_kernel(struct kfd_process *p,
+ uint32_t num_events, void __user *data,
+ bool all, uint32_t *user_timeout_ms,
+ uint32_t *wait_result);
#endif
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_hsa.h b/drivers/gpu/drm/amd/amdkfd/kfd_hsa.h
new file mode 100644
index 000000000000..794688c615f0
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_hsa.h
@@ -0,0 +1,451 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+#ifndef KFD_HSA_CODE_H_
+#define KFD_HSA_CODE_H_
+
+struct amd_queue_properties {
+ u32 enable_trap_handler:1,
+ is_ptr64:1,
+ enable_trap_handler_debug_sgprs:1,
+ enable_profiling:1,
+ use_scratch_once:1,
+ reserved1:27;
+};
+
+enum hsa_packet_type {
+ /*
+ * Vendor-specific packet.
+ */
+ HSA_PACKET_TYPE_VENDOR_SPECIFIC = 0,
+ /*
+ * The packet has been processed in the past, but has not been reassigned to
+ * the packet processor. A packet processor must not process a packet of this
+ * type. All queues support this packet type.
+ */
+ HSA_PACKET_TYPE_INVALID = 1,
+ /*
+ * Packet used by agents for dispatching jobs to kernel agents. Not all
+ * queues support packets of this type (see ::hsa_queue_feature_t).
+ */
+ HSA_PACKET_TYPE_KERNEL_DISPATCH = 2,
+ /*
+ * Packet used by agents to delay processing of subsequent packets, and to
+ * express complex dependencies between multiple packets. All queues support
+ * this packet type.
+ */
+ HSA_PACKET_TYPE_BARRIER_AND = 3,
+ /*
+ * Packet used by agents for dispatching jobs to agents. Not all
+ * queues support packets of this type (see ::hsa_queue_feature_t).
+ */
+ HSA_PACKET_TYPE_AGENT_DISPATCH = 4,
+ /*
+ * Packet used by agents to delay processing of subsequent packets, and to
+ * express complex dependencies between multiple packets. All queues support
+ * this packet type.
+ */
+ HSA_PACKET_TYPE_BARRIER_OR = 5
+};
+
+enum hsa_packet_header {
+ /*
+ * Packet type. The value of this sub-field must be one of
+ * ::hsa_packet_type_t. If the type is ::HSA_PACKET_TYPE_VENDOR_SPECIFIC, the
+ * packet layout is vendor-specific.
+ */
+ HSA_PACKET_HEADER_TYPE = 0,
+ /*
+ * Barrier bit. If the barrier bit is set, the processing of the current
+ * packet only launches when all preceding packets (within the same queue) are
+ * complete.
+ */
+ HSA_PACKET_HEADER_BARRIER = 8,
+ /*
+ * Acquire fence scope. The value of this sub-field determines the scope and
+ * type of the memory fence operation applied before the packet enters the
+ * active phase. An acquire fence ensures that any subsequent global segment
+ * or image loads by any unit of execution that belongs to a dispatch that has
+ * not yet entered the active phase on any queue of the same kernel agent,
+ * sees any data previously released at the scopes specified by the acquire
+ * fence. The value of this sub-field must be one of ::hsa_fence_scope_t.
+ */
+ HSA_PACKET_HEADER_ACQUIRE_FENCE_SCOPE = 9,
+ /*
+ * Release fence scope, The value of this sub-field determines the scope and
+ * type of the memory fence operation applied after kernel completion but
+ * before the packet is completed. A release fence makes any global segment or
+ * image data that was stored by any unit of execution that belonged to a
+ * dispatch that has completed the active phase on any queue of the same
+ * kernel agent visible in all the scopes specified by the release fence. The
+ * value of this sub-field must be one of ::hsa_fence_scope_t.
+ */
+ HSA_PACKET_HEADER_RELEASE_FENCE_SCOPE = 11
+};
+
+struct code_properties {
+ /* 4-sgprs */
+ u16 enable_sgpr_private_segment_buffer:1,
+ /* 2-sgprs */
+ enable_sgpr_dispatch_ptr:1,
+ /* 2-sgprs */
+ enable_sgpr_queue_ptr:1,
+ /* 2-sgprs */
+ enable_sgpr_kernarg_segment_ptr:1,
+ /* 2-sgprs */
+ enable_sgpr_dispatch_id:1,
+ /* 2-sgprs */
+ enable_sgpr_flat_scratch_init:1,
+ /* 2-sgprs */
+ enable_sgpr_private_segment_size:1,
+ reserved0:3,
+ enable_wavefront_size32:1, // gfx10+
+ uses_dynamic_stack:1,
+ reserved1:4;
+};
+
+struct compute_pgm_rsrc1 {
+ u32 granulated_workitem_vgpr_count:6,
+ granulated_wavefront_sgpr_count:4,
+ priority:2,
+ float_round_mode_32:2,
+ float_round_mode_16_64:2,
+ float_denorm_mode_32:2,
+ float_denorm_mode_16_64:2,
+ priv:1,
+ enable_dx10_clamp:1,
+ debug_mode:1,
+ enable_ieee_mode:1,
+ bulky:1,
+ cdbg_user:1,
+ fp16_ovfl:1, /* gfx9+ */
+ reserved0:2,
+ wgp_mode:1, /* gfx10+ */
+ mem_ordered:1, /* gfx10+ */
+ fwd_progress:1; /* gfx10+ */
+};
+
+struct compute_pgm_rsrc2 {
+ /* 1-sgprs
+ * enable_sgpr_private_segment_wavefront_offset
+ */
+ u32 enable_private_segment:1,
+ /* More or equal to enable_sgpr_*.
+ * Exceed 16 will be ignored.
+ */
+ user_sgpr_count:5,
+ enable_trap_handler:1,
+ /* 1-sgpr */
+ enable_sgpr_workgroup_id_x:1,
+ /* 1-sgpr */
+ enable_sgpr_workgroup_id_y:1,
+ /* 1-sgpr */
+ enable_sgpr_workgroup_id_z:1,
+ /* 1-sgpr */
+ enable_sgpr_workgroup_info:1,
+ enable_vgpr_workitem_id:2,
+ enable_exception_address_watch:1,
+ enable_exception_memory:1,
+ granulated_lds_size:9,
+ enable_exception_ieee_754_fp_invalid_operation:1,
+ enable_exception_fp_denormal_source:1,
+ enable_exception_ieee_754_fp_division_by_zero:1,
+ enable_exception_ieee_754_fp_overflow:1,
+ enable_exception_ieee_754_fp_underflow:1,
+ enable_exception_ieee_754_fp_inexact:1,
+ enable_exception_int_divide_by_zero:1,
+ reserved0:1;
+};
+
+struct compute_pgm_rsrc3 {
+ u32 accum_offset:6,
+ reserved0:10,
+ tg_split:1,
+ reserved1:15;
+};
+
+struct kernel_descriptor {
+ u32 group_segment_fixed_size;
+ u32 private_segment_fixed_size;
+ u32 kernarg_size;
+ u8 reserved0[4];
+ s64 kernel_code_entry_byte_offset;
+ u8 reserved1[20];
+ struct compute_pgm_rsrc3 compute_pgm_rsrc3; /* GFX10+ and GFX90A+ */
+ struct compute_pgm_rsrc1 compute_pgm_rsrc1;
+ struct compute_pgm_rsrc2 compute_pgm_rsrc2;
+ struct code_properties code_properties;
+ u8 reserved2[6];
+};
+
+enum hsa_queue_type {
+ HSA_QUEUE_TYPE_MULTI = 0,
+ HSA_QUEUE_TYPE_SINGLE = 1
+};
+
+enum hsa_queue_feature {
+ HSA_QUEUE_FEATURE_KERNEL_DISPATCH = 1,
+ HSA_QUEUE_FEATURE_AGENT_DISPATCH = 2
+};
+
+struct hsa_kernel_dispatch_packet {
+ u16 header;
+ u16 setup;
+ u16 workgroup_size_x;
+ u16 workgroup_size_y;
+ u16 workgroup_size_z;
+ u16 reserved0;
+ u32 grid_size_x;
+ u32 grid_size_y;
+ u32 grid_size_z;
+ u32 private_segment_size;
+ u32 group_segment_size;
+ u64 kernel_object;
+ void *kernarg_address;
+ u64 reserved2;
+ u64 completion_signal;
+};
+
+enum amd_signal_kind {
+ AMD_SIGNAL_KIND_INVALID = 0,
+ AMD_SIGNAL_KIND_USER = 1,
+ AMD_SIGNAL_KIND_DOORBELL = -1,
+ AMD_SIGNAL_KIND_LEGACY_DOORBELL = -2
+};
+
+/* An AMD Signal object must always be 64 byte aligned to ensure it cannot
+ * span a page boundary. This is required by CP microcode which optimizes
+ * access to the structure by only doing a single SUA (System Uniform Address)
+ * translation when accessing signal fields. This optimization is used in GFX8.
+ */
+struct amd_signal {
+ u64 kind;
+ union {
+ volatile s64 value;
+ volatile u32 *legacy_hardware_doorbell_ptr;
+ volatile u64 *hardware_doorbell_ptr;
+ };
+ /* For AMD_SIGNAL_KIND_USER: mailbox address for event notification
+ * in Signal operations.
+ */
+ u64 event_mailbox_ptr;
+ /* For AMD_SIGNAL_KIND_USER: event id for event notification in Signal
+ * operations.
+ */
+ u32 event_id;
+ u32 reserved1;
+ /* Start of the AQL packet timestamp, when profiled. */
+ u64 start_ts;
+ /* End of the AQL packet timestamp, when profiled. */
+ u64 end_ts;
+ union {
+ /* For AMD_SIGNAL_KIND_*DOORBELL: the address of the associated
+ * amd_queue, otherwise reserved and must be 0.
+ */
+ void *queue_ptr;
+ u64 reserved2;
+ };
+ u32 reserved3[2];
+} __aligned(64);
+
+struct hsa_queue {
+ u32 type;
+
+ u32 features;
+
+ void *base_address;
+ /*
+ * Signal object used by the application to indicate the ID of a packet that
+ * is ready to be processed. The HSA runtime manages the doorbell signal. If
+ * the application tries to replace or destroy this signal, the behavior is
+ * undefined.
+ *
+ * If @a type is ::HSA_QUEUE_TYPE_SINGLE, the doorbell signal value must be
+ * updated in a monotonically increasing fashion. If @a type is
+ * ::HSA_QUEUE_TYPE_MULTI, the doorbell signal value can be updated with any
+ * value.
+ */
+ u64 doorbell_signal;
+
+ /*
+ * Maximum number of packets the queue can hold. Must be a power of 2.
+ */
+ u32 size;
+ /* Reserved. Must be 0. */
+ u32 reserved1;
+ /*
+ * Queue identifier, which is unique over the lifetime of the application.
+ */
+ u64 id;
+
+};
+
+enum hsa_fence_scope {
+ HSA_FENCE_SCOPE_NONE = 0,
+ HSA_FENCE_SCOPE_AGENT = 1,
+ HSA_FENCE_SCOPE_SYSTEM = 2
+};
+
+struct amd_queue {
+ struct hsa_queue hsa_queue;
+ u32 reserved1[4];
+ volatile u64 write_dispatch_id;
+ u32 group_segment_aperture_base_hi;
+ u32 private_segment_aperture_base_hi;
+ u32 max_cu_id;
+ u32 max_wave_id;
+ volatile u64 max_legacy_doorbell_dispatch_id_plus_1;
+ volatile u32 legacy_doorbell_lock;
+ u32 reserved2[9];
+ volatile u64 read_dispatch_id;
+ u32 read_dispatch_id_field_base_byte_offset;
+ u32 compute_tmpring_size;
+ u32 scratch_resource_descriptor[4];
+ u64 scratch_backing_memory_location;
+ u64 scratch_backing_memory_byte_size;
+ u32 scratch_wave64_lane_byte_size;
+ struct amd_queue_properties queue_properties;
+ u32 reserved3[2];
+ u64 queue_inactive_signal;
+ u32 reserved4[14];
+} __aligned(64);
+
+struct hsa_sync_var {
+ union {
+ /* pointer to user mode data */
+ void *user_data;
+ /* 64bit compatibility of value */
+ u64 user_data_ptr_value;
+ };
+ u64 SyncVarSize;
+};
+
+enum hsa_event_type {
+ /* user-mode generated GPU signal */
+ HSA_EVENTTYPE_SIGNAL = 0,
+ /* HSA node change (attach/detach) */
+ HSA_EVENTTYPE_NODECHANGE = 1,
+ /* HSA device state change( start/stop ) */
+ HSA_EVENTTYPE_DEVICESTATECHANGE = 2,
+ /* GPU shader exception event */
+ HSA_EVENTTYPE_HW_EXCEPTION = 3,
+ /* GPU SYSCALL with parameter info */
+ HSA_EVENTTYPE_SYSTEM_EVENT = 4,
+ /* GPU signal for debugging */
+ HSA_EVENTTYPE_DEBUG_EVENT = 5,
+ /* GPU signal for profiling */
+ HSA_EVENTTYPE_PROFILE_EVENT = 6,
+ /* GPU signal queue idle state (EOP pm4) */
+ HSA_EVENTTYPE_QUEUE_EVENT = 7,
+ /* GPU signal for signaling memory access faults and memory subsystem issues */
+ HSA_EVENTTYPE_MEMORY = 8,
+ /* ... */
+ HSA_EVENTTYPE_MAXID,
+ HSA_EVENTTYPE_TYPE_SIZE = 0xffffffff
+};
+
+enum hsa_eventtype_nodechange_flags {
+ HSA_EVENTTYPE_NODECHANGE_ADD = 0,
+ HSA_EVENTTYPE_NODECHANGE_REMOVE = 1,
+ HSA_EVENTTYPE_NODECHANGE_SIZE = 0xffffffff
+};
+
+struct hsa_node_change {
+ /* HSA node added/removed on the platform */
+ enum hsa_eventtype_nodechange_flags flags;
+};
+
+enum hsa_device {
+ HSA_DEVICE_CPU = 0,
+ HSA_DEVICE_GPU = 1,
+ MAX_HSA_DEVICE = 2
+};
+
+enum hsa_eventtype_devicestatechange_flags {
+ /* device started (and available) */
+ HSA_EVENTTYPE_DEVICESTATUSCHANGE_START = 0,
+ /* device stopped (i.e. unavailable) */
+ HSA_EVENTTYPE_DEVICESTATUSCHANGE_STOP = 1,
+ HSA_EVENTTYPE_DEVICESTATUSCHANGE_SIZE = 0xffffffff
+};
+
+struct hsa_device_state_change {
+ /* F-NUMA node that contains the device */
+ u32 node_id;
+ /* device type: GPU or CPU */
+ enum hsa_device device;
+ /* event flags */
+ enum hsa_eventtype_devicestatechange_flags flags;
+};
+
+struct hsa_access_attribute_failure {
+ /* Page not present or supervisor privilege */
+ unsigned int not_present:1;
+ /* Write access to a read-only page */
+ unsigned int readonly:1;
+ /* Execute access to a page marked NX */
+ unsigned int no_execute:1;
+ /* Host access only */
+ unsigned int gpu_access:1;
+ /* RAS ECC failure (notification of DRAM ECC - non-recoverable -
+ * error, if supported by HW)
+ */
+ unsigned int ecc:1;
+ /* Can't determine the exact fault address */
+ unsigned int imprecise:1;
+ /* Indicates RAS errors or other errors causing the access to GPU to fail
+ * 0 = no RAS error,
+ * 1 = ECC_SRAM,
+ * 2 = Link_SYNFLOOD (poison),
+ * 3 = GPU hang (not attributable to a specific cause), other values reserved
+ */
+ unsigned int error_type:3;
+ /* must be 0 */
+ unsigned int Reserved:23;
+};
+
+enum hsa_eventid_memory_flags {
+ /* access fault, recoverable after page adjustment */
+ HSA_EVENTID_MEMORY_RECOVERABLE = 0,
+ /* memory access requires process context destruction, unrecoverable */
+ HSA_EVENTID_MEMORY_FATAL_PROCESS = 1,
+ /* memory access requires all GPU VA context destruction, unrecoverable */
+ HSA_EVENTID_MEMORY_FATAL_VM = 2,
+};
+
+struct hsa_memory_access_fault {
+ /* H-NUMA node that contains the device where the memory access occurred */
+ u32 node_id;
+ /* virtual address this occurred on */
+ u64 virtual_address;
+ /* failure attribute */
+ struct hsa_access_attribute_failure failure;
+ /* event flags */
+ enum hsa_eventid_memory_flags flags;
+};
+
+struct hsa_event_data {
+ enum hsa_event_type event_type;
+
+ union {
+ /* return data associated with HSA_EVENTTYPE_SIGNAL and other events */
+ struct hsa_sync_var sync_var;
+ /* data associated with HSA_EVENTTYPE_NODE_CHANGE */
+ struct hsa_node_change node_change_state;
+ /* data associated with HSA_EVENTTYPE_DEVICE_STATE_CHANGE */
+ struct hsa_device_state_change device_state;
+ /* data associated with HSA_EVENTTYPE_MEMORY */
+ struct hsa_memory_access_fault memory_access_fault;
+ };
+
+ // the following data entries are internal to the KFD & thunk itself.
+
+ u64 hw_data1; // internal thunk store for Event data (OsEventHandle)
+ u64 hw_data2; // internal thunk store for Event data (HWAddress)
+ u32 hw_data3; // internal thunk store for Event data (HWData)
+};
+
+struct hsa_event {
+ u32 event_id;
+ struct hsa_event_data event_data;
+};
+
+#endif /* KFD_HSA_CODE_H_ */
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_module.c b/drivers/gpu/drm/amd/amdkfd/kfd_module.c
index 33aa23450b3f..2a2405db5b9c 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_module.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_module.c
@@ -77,6 +77,7 @@ static int kfd_init(void)
static void kfd_exit(void)
{
+ knod_exit();
kfd_cleanup_processes();
kfd_process_destroy_wq();
kfd_debugfs_fini();
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index acd0e41e744c..4f5d4b0bfa89 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -1069,6 +1069,7 @@ bool kfd_dev_is_large_bar(struct kfd_node *dev);
struct kfd_process *create_process(const struct task_struct *thread, bool primary);
int kfd_process_create_wq(void);
void kfd_process_destroy_wq(void);
+void kfd_process_flush_wq(void);
void kfd_cleanup_processes(void);
struct kfd_process *kfd_create_process(struct task_struct *thread);
int kfd_create_process_sysfs(struct kfd_process *process);
@@ -1536,6 +1537,9 @@ void kfd_signal_hw_exception_event(u32 pasid);
int kfd_set_event(struct kfd_process *p, uint32_t event_id);
int kfd_reset_event(struct kfd_process *p, uint32_t event_id);
int kfd_kmap_event_page(struct kfd_process *p, uint64_t event_page_offset);
+int kfd_event_page_set(struct kfd_process *p, void *kernel_address,
+ uint64_t size, uint64_t user_handle);
+
int kfd_event_create(struct file *devkfd, struct kfd_process *p,
uint32_t event_type, bool auto_reset, uint32_t node_id,
@@ -1641,3 +1645,33 @@ static inline void kfd_debugfs_remove_process(struct kfd_process *p) {}
#endif
#endif
+
+int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd,
+ uint64_t gpu_va, uint32_t size,
+ uint32_t flags, struct kgd_mem **mem, void **kptr);
+void kfd_process_free_gpuvm(struct kgd_mem *mem,
+ struct kfd_process_device *pdd, void **kptr);
+int kfd_create_queue(struct kfd_process *p, struct queue_properties *q_properties,
+ u32 gpu_id, u32 *queue_id_out, u64 *doorbell_offset_out);
+int kfd_ioctl_set_event(struct file *filp, struct kfd_process *p,
+ void *data);
+int kfd_ioctl_reset_event(struct file *filp, struct kfd_process *p,
+ void *data);
+int kfd_ioctl_wait_events(struct file *filp, struct kfd_process *p, void *data);
+
+int kfd_ioctl_set_trap_handler(struct file *filep,
+ struct kfd_process *p, void *data);
+int kfd_ioctl_set_scratch_backing_va(struct file *filep,
+ struct kfd_process *p, void *data);
+
+
+int kfd_ioctl_acquire_vm(struct file *filep, struct kfd_process *p, void *data);
+int kfd_ioctl_map_memory_to_gpu(struct file *filep, struct kfd_process *p,
+ void *data);
+int kfd_ioctl_alloc_memory_of_gpu(struct file *filep, struct kfd_process *p,
+ void *data);
+int kfd_ioctl_export_dmabuf(struct file *filep,
+ struct kfd_process *p, void *data);
+
+void __iomem *kfd_kernel_doorbell_mmap(struct kfd_node *dev,
+ struct kfd_process *process);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
index ca71fa726e32..3c9268241800 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
@@ -715,8 +715,14 @@ void kfd_process_destroy_wq(void)
}
}
-static void kfd_process_free_gpuvm(struct kgd_mem *mem,
- struct kfd_process_device *pdd, void **kptr)
+void kfd_process_flush_wq(void)
+{
+ if (kfd_process_wq)
+ flush_workqueue(kfd_process_wq);
+}
+
+void kfd_process_free_gpuvm(struct kgd_mem *mem,
+ struct kfd_process_device *pdd, void **kptr)
{
struct kfd_node *dev = pdd->dev;
@@ -736,9 +742,9 @@ static void kfd_process_free_gpuvm(struct kgd_mem *mem,
* to avoid concurrency. Because of that exclusiveness, we do
* not need to take p->mutex.
*/
-static int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd,
- uint64_t gpu_va, uint32_t size,
- uint32_t flags, struct kgd_mem **mem, void **kptr)
+int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd,
+ uint64_t gpu_va, uint32_t size,
+ uint32_t flags, struct kgd_mem **mem, void **kptr)
{
struct kfd_node *kdev = pdd->dev;
int err;
@@ -761,10 +767,14 @@ static int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd,
}
if (kptr) {
- err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(
- (struct kgd_mem *)*mem, kptr, NULL);
+ if (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM)
+ err = amdgpu_amdkfd_gpuvm_map_vram_bo_to_kernel(
+ (struct kgd_mem *)*mem, kptr, NULL);
+ else
+ err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(
+ (struct kgd_mem *)*mem, kptr, NULL);
if (err) {
- pr_debug("Map GTT BO to kernel failed\n");
+ pr_debug("Map BO to kernel failed\n");
goto sync_memory_failed;
}
}
--
2.43.0
^ permalink raw reply related
* [RFC PATCH net-next 07/13] drm/amdkfd: add knod provider core
From: Taehee Yoo @ 2026-07-19 17:58 UTC (permalink / raw)
To: Alex Deucher, Alexei Starovoitov, amd-gfx, Andrew Lunn,
Andrii Nakryiko, Bill Wendling, bpf, Christian König,
Daniel Borkmann, David Airlie, David S. Miller, Donald Hunter,
dri-devel, Eduard Zingerman, Emil Tsalapatis, Eric Dumazet,
Felix Kuehling, Hoyeon Lee, Ilias Apalodimas, Jakub Kicinski,
Jesper Dangaard Brouer, Jiri Olsa, John Fastabend, Justin Stitt,
Kees Cook, Kumar Kartikeya Dwivedi, Leon Romanovsky,
linaro-mm-sig, linux-hardening, linux-kernel, linux-kselftest,
linux-media, linux-rdma, llvm, Mark Bloch, Martin KaFai Lau,
Michael Chan, Nathan Chancellor, netdev, Nick Desaulniers,
Paolo Abeni, Pavan Chebbi, Saeed Mahameed, Shuah Khan,
Simona Vetter, Simon Horman, Song Liu, Stanislav Fomichev,
Sumit Semwal, Taehee Yoo, Tariq Toukan, Yonghong Song
In-Reply-To: <20260719175857.4071636-1-ap420073@gmail.com>
Add the knod accelerator provider, built into amdgpu. It allocates and
drives GPU AQL/SDMA queues, manages GPU memory for the RX data path,
and registers a knod accelerator that binds to a NIC via the knod core.
Feature workers (BPF, IPsec) plug in on top through the accel ops.
Signed-off-by: Taehee Yoo <ap420073@gmail.com>
(cherry picked from commit acaff825a0e87188906fd59f8a377c04d0b0ed2e)
---
drivers/gpu/drm/amd/amdgpu/Makefile | 1 +
drivers/gpu/drm/amd/amdkfd/Kconfig | 11 +
drivers/gpu/drm/amd/amdkfd/Makefile | 4 +
drivers/gpu/drm/amd/amdkfd/kfd_knod.c | 2202 ++++++++++++++++++++
drivers/gpu/drm/amd/amdkfd/knod/kfd_knod.h | 270 +++
5 files changed, 2488 insertions(+)
create mode 100644 drivers/gpu/drm/amd/amdkfd/kfd_knod.c
create mode 100644 drivers/gpu/drm/amd/amdkfd/knod/kfd_knod.h
diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
index ba80542ead9d..7a9826df9f37 100644
--- a/drivers/gpu/drm/amd/amdgpu/Makefile
+++ b/drivers/gpu/drm/amd/amdgpu/Makefile
@@ -42,6 +42,7 @@ ccflags-y := -I$(FULL_AMD_PATH)/include/asic_reg \
-I$(FULL_AMD_DISPLAY_PATH)/dc \
-I$(FULL_AMD_DISPLAY_PATH)/amdgpu_dm \
-I$(FULL_AMD_PATH)/amdkfd \
+ -I$(FULL_AMD_PATH)/amdkfd/knod \
-I$(FULL_AMD_PATH)/ras/ras_mgr
# Locally disable W=1 warnings enabled in drm subsystem Makefile
diff --git a/drivers/gpu/drm/amd/amdkfd/Kconfig b/drivers/gpu/drm/amd/amdkfd/Kconfig
index a5d7467c2f34..d93f1af749ff 100644
--- a/drivers/gpu/drm/amd/amdkfd/Kconfig
+++ b/drivers/gpu/drm/amd/amdkfd/Kconfig
@@ -38,3 +38,14 @@ config HSA_AMD_P2P
GPUs with large memory BARs that expose the entire VRAM in PCIe bus
address space within the physical address limits of the GPUs.
+config HSA_AMD_KNOD
+ bool "KNOD GPU network offload core"
+ depends on HSA_AMD
+ default y
+ help
+ Core framework for KNOD, GPU-accelerated zero-copy network packet
+ processing on AMD GPUs. It is built into amdgpu and cannot be a
+ module because it relies on non-exported amdgpu/amdkfd internals.
+
+ Say N to drop the KNOD core along with the BPF and IPsec offloads
+ layered on top of it. If unsure, say Y.
diff --git a/drivers/gpu/drm/amd/amdkfd/Makefile b/drivers/gpu/drm/amd/amdkfd/Makefile
index 85fc67d521e5..1834faa54863 100644
--- a/drivers/gpu/drm/amd/amdkfd/Makefile
+++ b/drivers/gpu/drm/amd/amdkfd/Makefile
@@ -71,3 +71,7 @@ ifneq ($(CONFIG_HSA_AMD_SVM),)
AMDKFD_FILES += $(AMDKFD_PATH)/kfd_svm.o \
$(AMDKFD_PATH)/kfd_migrate.o
endif
+
+ifneq ($(CONFIG_HSA_AMD_KNOD),)
+AMDKFD_FILES += $(AMDKFD_PATH)/kfd_knod.o
+endif
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_knod.c b/drivers/gpu/drm/amd/amdkfd/kfd_knod.c
new file mode 100644
index 000000000000..03fff054dcb9
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_knod.c
@@ -0,0 +1,2202 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright (c) 2021 Taehee Yoo <ap420073@gmail.com>
+ * Copyright (c) 2021 Hoyeon Lee <hoyeon.rhee@gmail.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/types.h>
+#include <linux/mutex.h>
+#include <linux/slab.h>
+#include <linux/sched.h>
+#include <linux/workqueue.h>
+#include <linux/file.h>
+#include <linux/dma-buf.h>
+#include <crypto/algapi.h>
+#include <crypto/internal/simd.h>
+#include "kfd_priv.h"
+#include "kfd_hsa.h"
+#include "kfd_knod.h"
+#include "kfd_topology.h"
+#include "kfd_device_queue_manager.h"
+#include "kfd_events.h"
+#include <crypto/skcipher.h>
+#include <crypto/internal/skcipher.h>
+#include <linux/pid.h>
+#include <linux/debugfs.h>
+#include <linux/sched/signal.h>
+#include "../amdgpu/amdgpu_amdkfd.h"
+#include "../amdgpu/amdgpu_gfx.h"
+#include "../amdgpu/./navi10_sdma_pkt_open.h"
+#include "amdgpu_dpm.h"
+#include "kgd_pp_interface.h"
+#include <linux/kthread.h>
+#include <linux/delay.h>
+#include <drm/ttm/ttm_tt.h>
+#include <linux/seq_file.h>
+#include "knod_bpf.h"
+#include <net/page_pool/helpers.h>
+#include <linux/netdevice.h>
+
+struct umh_data {
+ pid_t pid;
+};
+
+LIST_HEAD(ctx_list);
+
+/*
+ * AQL/SDMA queue pair count requested by the highest-demand accel
+ * consumer (currently knod_ipsec's parallel-dispatcher machinery).
+ * Each accel module sets this via knod_request_queue_cnt() during its
+ * module_init BEFORE the NOD attach happens, so knod_attach() creates
+ * a context with enough kaql[]/sdma[] pairs for the worst-case
+ * consumer.
+ *
+ * The value is a high-water mark across all accel types - whichever
+ * consumer asks for the most queue pairs wins; a consumer that needs
+ * fewer just leaves the extra pairs unused (a minor, harmless GPU
+ * resource waste).
+ *
+ * Default is 1 when nothing has called the setter - mirrors the
+ * historical single-queue behaviour.
+ */
+static int knod_requested_queue_cnt = 1;
+
+static struct knod_accel *accels[KNOD_MAX_AQL];
+static int nr_accels;
+static struct knod_accel_ops accel_ops;
+
+/*
+ * Thin wrappers around the KFD core entry points. knod drives KFD from
+ * kernel context using kernel-internal types, so it calls these directly
+ * instead of going through the kfd_ioctl_* handlers (which marshal user
+ * data and stay static to KFD).
+ */
+static int knod_create_queue(struct kfd_process *p,
+ struct queue_properties *qp, u32 gpu_id,
+ u32 *queue_id, u64 *doorbell_offset)
+{
+ return kfd_create_queue(p, qp, gpu_id, queue_id, doorbell_offset);
+}
+
+static int knod_destroy_queue(struct kfd_process *p, u32 queue_id)
+{
+ int ret;
+
+ mutex_lock(&p->mutex);
+ ret = pqm_destroy_queue(&p->pqm, queue_id);
+ mutex_unlock(&p->mutex);
+ return ret;
+}
+
+static int knod_create_event(struct kfd_process *p, u32 event_type,
+ bool auto_reset, u32 node_id,
+ struct knod_event *event)
+{
+ u32 event_trigger_data;
+ u64 event_page_offset = 0;
+
+ return kfd_event_create(NULL, p, event_type, auto_reset, node_id,
+ &event->id, &event_trigger_data,
+ &event_page_offset, &event->slot);
+}
+
+static int knod_destroy_event(struct kfd_process *p, u32 event_id)
+{
+ return kfd_event_destroy(p, event_id);
+}
+
+struct knod_mem *__knod_alloc_mem(struct knod *knod, size_t size,
+ int flags)
+{
+ struct kfd_process_device *pdd = knod->process->pdds[0];
+ struct kfd_node *kdev = pdd->dev;
+ struct knod_mem *mem;
+ int err;
+
+ mem = kzalloc_obj(struct knod_mem, GFP_KERNEL);
+ if (!mem)
+ return ERR_PTR(-ENOMEM);
+
+ size = ALIGN(size, PAGE_SIZE);
+ mem->flags = flags;
+ mem->size = size;
+ mem->gaddr = gen_pool_alloc(knod->pool, size);
+ if (!mem->gaddr) {
+ kfree(mem);
+ return ERR_PTR(-ENOMEM);
+ }
+
+ err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(kdev->adev, mem->gaddr,
+ size, pdd->drm_priv,
+ &mem->mem, NULL, flags,
+ false);
+ if (err) {
+ knod_err(" failed to alloc mem\n");
+ gen_pool_free(knod->pool, mem->gaddr, mem->size);
+ kfree(mem);
+ return ERR_PTR(-ENOMEM);
+ }
+
+ err = amdgpu_amdkfd_gpuvm_sync_memory(kdev->adev, mem->mem, true);
+ if (err) {
+ pr_debug("Sync memory failed, wait interrupted by user signal\n");
+ amdgpu_amdkfd_gpuvm_free_memory_of_gpu(kdev->adev, mem->mem,
+ pdd->drm_priv, NULL);
+ gen_pool_free(knod->pool, mem->gaddr, mem->size);
+ kfree(mem);
+ return ERR_PTR(-ENOMEM);
+ }
+
+ list_add_tail(&mem->list, &knod->active_list);
+
+ return mem;
+}
+
+int __knod_export_dma_buf(struct knod *knod, struct knod_mem *mem)
+{
+ struct dma_buf *dmabuf;
+ int err;
+
+ err = amdgpu_amdkfd_gpuvm_export_dmabuf(mem->mem, &dmabuf);
+ if (err) {
+ pr_debug("export dmabuf failed\n");
+ return -ENOMEM;
+ }
+
+ dma_buf_put(dmabuf);
+
+ return 0;
+}
+
+struct knod_mem *knod_alloc_mem(struct knod *knod, size_t size, int flags)
+{
+ struct knod_mem *mem;
+ int err;
+
+ mem = kzalloc_obj(struct knod_mem, GFP_KERNEL);
+ if (!mem)
+ return ERR_PTR(-ENOMEM);
+
+ size = ALIGN(size, PAGE_SIZE);
+ mem->flags = flags;
+ mem->size = size;
+ mem->gaddr = gen_pool_alloc(knod->pool, size);
+ if (!mem->gaddr) {
+ kfree(mem);
+ return ERR_PTR(-ENOMEM);
+ }
+
+ err = kfd_process_alloc_gpuvm(knod->process->pdds[0],
+ mem->gaddr,
+ mem->size,
+ flags,
+ &mem->mem,
+ &mem->kaddr);
+
+ if (err) {
+ knod_err(" err = %d\n", err);
+ gen_pool_free(knod->pool, mem->gaddr, mem->size);
+ kfree(mem);
+ return ERR_PTR(-ENOMEM);
+ }
+
+ list_add_tail(&mem->list, &knod->active_list);
+
+ return mem;
+}
+EXPORT_SYMBOL(knod_alloc_mem);
+
+int __knod_map_kaddr(struct knod *knod, struct knod_mem *mem)
+{
+ int err = 0;
+
+ if (mem->flags & KFD_IOC_ALLOC_MEM_FLAGS_GTT) {
+ err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(mem->mem,
+ &mem->kaddr,
+ NULL);
+ if (err) {
+ pr_debug("Map GTT BO to kernel failed\n");
+ err = -ENOMEM;
+ }
+ } else if (mem->flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
+ err = amdgpu_amdkfd_gpuvm_map_vram_bo_to_kernel(mem->mem,
+ &mem->kaddr,
+ NULL);
+ if (err) {
+ pr_debug("Map VRAM BO to kernel failed\n");
+ err = -ENOMEM;
+ }
+ }
+
+ return err;
+}
+
+int __knod_map_mem(struct knod *knod, struct knod_mem *mem)
+{
+ struct kfd_process_device *pdd = knod->process->pdds[0];
+ struct kfd_node *kdev = pdd->dev;
+ int err;
+
+ err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(kdev->adev, mem->mem,
+ pdd->drm_priv);
+ if (err) {
+ knod_err(" failed to map gpu\n");
+ return 1;
+ }
+
+ err = amdgpu_amdkfd_gpuvm_sync_memory(kdev->adev, mem->mem, true);
+ if (err) {
+ pr_debug("Sync memory failed, wait interrupted by user signal\n");
+ return 1;
+ }
+
+ return 0;
+}
+EXPORT_SYMBOL(__knod_map_mem);
+
+void knod_free_mem(struct knod *knod, struct knod_mem *mem)
+{
+ if (mem) {
+ list_del_init(&mem->list);
+ kfd_process_free_gpuvm(mem->mem, knod->process->pdds[0],
+ &mem->kaddr);
+ gen_pool_free(knod->pool, mem->gaddr, mem->size);
+ kfree(mem);
+ }
+}
+EXPORT_SYMBOL(knod_free_mem);
+
+void knod_sdma_copy(struct knod *knod, u64 dst_offset,
+ u64 src_offset, int idx, int size)
+{
+ struct knod_sdma *sdma = &knod->sdma[idx];
+ u32 ring_mask = (sdma->sdma->size / 4) - 1;
+ u64 *wptr = (u64 *)sdma->queue->kaddr + 1;
+ u32 *ptr = sdma->sdma->kaddr;
+
+ ptr[sdma->idx++ & ring_mask] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |
+ SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR) |
+ SDMA_PKT_COPY_LINEAR_HEADER_TMZ((0));
+ ptr[sdma->idx++ & ring_mask] = size - 1;
+ ptr[sdma->idx++ & ring_mask] = 0; /* src/dst endian swap */
+ ptr[sdma->idx++ & ring_mask] = lower_32_bits(src_offset);
+ ptr[sdma->idx++ & ring_mask] = upper_32_bits(src_offset);
+ ptr[sdma->idx++ & ring_mask] = lower_32_bits(dst_offset);
+ ptr[sdma->idx++ & ring_mask] = upper_32_bits(dst_offset);
+
+ *wptr += 7 * 4;
+}
+
+void knod_sdma_fence(struct knod *knod, u64 fence_addr, u32 fence_val,
+ int idx)
+{
+ struct knod_sdma *sdma = &knod->sdma[idx];
+ u32 ring_mask = (sdma->sdma->size / 4) - 1;
+ u64 *wptr = (u64 *)sdma->queue->kaddr + 1;
+ u32 *ptr = sdma->sdma->kaddr;
+ u32 hdr = SDMA_PKT_HEADER_OP(SDMA_OP_FENCE);
+
+ if (knod->isa_version >= 10)
+ hdr |= SDMA_PKT_FENCE_HEADER_MTYPE(3);
+
+ ptr[sdma->idx++ & ring_mask] = hdr;
+ ptr[sdma->idx++ & ring_mask] = lower_32_bits(fence_addr);
+ ptr[sdma->idx++ & ring_mask] = upper_32_bits(fence_addr);
+ ptr[sdma->idx++ & ring_mask] = fence_val;
+
+ *wptr += 4 * 4;
+}
+
+void knod_sdma_trap(struct knod *knod, int idx)
+{
+ struct knod_sdma *sdma = &knod->sdma[idx];
+ u32 ring_mask = (sdma->sdma->size / 4) - 1;
+ u64 *wptr = (u64 *)sdma->queue->kaddr + 1;
+ u32 *ptr = sdma->sdma->kaddr;
+ u32 ctx = knod->sdma_event[idx].id & 0x0fffffff;
+ u64 slot_addr = knod->mailbox->gaddr +
+ knod->sdma_event[idx].slot * 8;
+ u32 hdr;
+
+ /* Write 0 to the signal page slot so lookup_signaled_event
+ * sees it as non-UNSIGNALED (0xFFFFFFFFFFFFFFFF).
+ */
+ hdr = SDMA_PKT_HEADER_OP(SDMA_OP_FENCE);
+ if (knod->isa_version >= 10)
+ hdr |= SDMA_PKT_FENCE_HEADER_MTYPE(3);
+
+ ptr[sdma->idx++ & ring_mask] = hdr;
+ ptr[sdma->idx++ & ring_mask] = lower_32_bits(slot_addr);
+ ptr[sdma->idx++ & ring_mask] = upper_32_bits(slot_addr);
+ ptr[sdma->idx++ & ring_mask] = 0;
+
+ ptr[sdma->idx++ & ring_mask] = SDMA_PKT_HEADER_OP(SDMA_OP_TRAP);
+ ptr[sdma->idx++ & ring_mask] = ctx;
+
+ *wptr += 6 * 4;
+}
+
+void knod_sdma_doorbell(struct knod *knod, int idx)
+{
+ struct knod_sdma *sdma = &knod->sdma[idx];
+ u64 *wptr = (u64 *)sdma->queue->kaddr + 1;
+
+ /* ensure the SDMA ring writes are visible before ringing doorbell */
+ wmb();
+ writeq(*wptr, sdma->doorbell);
+}
+
+u32 knod_sdma_submit(struct knod *knod, int idx,
+ const struct knod_sdma_copy_desc *copies, int n)
+{
+ struct knod_sdma *sdma = &knod->sdma[idx];
+ u32 capacity = sdma->sdma->size / 4;
+ u32 completed, inflight;
+ int i;
+
+ /*
+ * knod_sdma_copy has no overflow guard (sdma->idx is the dword write
+ * cursor that just wraps), so drop the whole batch once the ring is
+ * near full. @completed is the last fenced cursor (knod_sdma_kick
+ * writes sdma->idx into the signal), so @inflight is the unprocessed
+ * span; reserve room for these @n copies (7 dwords each) plus a fence.
+ * Signed compare so a stale signal ahead of the cursor (e.g. at
+ * startup) reads as "negative" inflight rather than a false full.
+ */
+ completed = (u32)READ_ONCE(((struct amd_signal *)
+ sdma->queue_signal->kaddr)->value);
+ inflight = (u32)sdma->idx - completed;
+ if ((s32)(inflight + n * 7) >= (s32)(capacity - 64))
+ return 0;
+
+ for (i = 0; i < n; i++)
+ knod_sdma_copy(knod, copies[i].dst, copies[i].src, idx,
+ copies[i].len);
+
+ return (u32)sdma->idx;
+}
+EXPORT_SYMBOL(knod_sdma_submit);
+
+void knod_sdma_kick(struct knod *knod, int idx)
+{
+ struct knod_sdma *sdma = &knod->sdma[idx];
+ u64 fence_addr;
+
+ fence_addr = sdma->queue_signal->gaddr +
+ offsetof(struct amd_signal, value);
+ knod_sdma_fence(knod, fence_addr, (u32)sdma->idx, idx);
+ knod_sdma_doorbell(knod, idx);
+}
+EXPORT_SYMBOL(knod_sdma_kick);
+
+int knod_gart_map(struct amdgpu_device *adev, u64 npages,
+ dma_addr_t *addr, u64 *gart_addr, u64 flags)
+{
+ struct amdgpu_ring *ring =
+ to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]);
+ struct amdgpu_job *job;
+ unsigned int num_dw, num_bytes;
+ struct dma_fence *fence;
+ u64 src_addr, dst_addr;
+ u64 pte_flags;
+ void *cpu_addr;
+ int r;
+
+ /* use gart window 0 */
+ *gart_addr = adev->gmc.gart_start;
+
+ num_dw = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8);
+ num_bytes = npages * 8;
+
+ r = amdgpu_job_alloc_with_ib(adev, &adev->mman.default_entity.base,
+ AMDGPU_FENCE_OWNER_UNDEFINED,
+ num_dw * 4 + num_bytes,
+ AMDGPU_IB_POOL_DELAYED,
+ &job,
+ AMDGPU_KERNEL_JOB_ID_TTM_MAP_BUFFER);
+ if (r)
+ return r;
+
+ src_addr = num_dw * 4;
+ src_addr += job->ibs[0].gpu_addr;
+
+ dst_addr = amdgpu_bo_gpu_offset(adev->gart.bo);
+ amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr,
+ dst_addr, num_bytes, 0);
+
+ amdgpu_ring_pad_ib(ring, &job->ibs[0]);
+ WARN_ON(job->ibs[0].length_dw > num_dw);
+
+ pte_flags = AMDGPU_PTE_VALID | AMDGPU_PTE_READABLE;
+ pte_flags |= AMDGPU_PTE_SYSTEM | AMDGPU_PTE_SNOOPED;
+ if (!(flags & KFD_IOCTL_SVM_FLAG_GPU_RO))
+ pte_flags |= AMDGPU_PTE_WRITEABLE;
+ pte_flags |= adev->gart.gart_pte_flags;
+
+ cpu_addr = &job->ibs[0].ptr[num_dw];
+
+ amdgpu_gart_map(adev, 0, npages, addr, pte_flags, cpu_addr);
+ fence = amdgpu_job_submit(job);
+ dma_fence_put(fence);
+
+ return r;
+}
+
+static void knod_init_aql_queue(struct knod *knod, int qid, int idx)
+{
+ struct amd_queue *amd_queue =
+ (struct amd_queue *)knod->kaql[idx].queue->kaddr;
+ union knod_aql_rsrc1 rsrc1;
+ u32 scratch_gaddr;
+ int i;
+
+ for (i = 0; i < knod->nr_aql_ring; i++)
+ knod_setup_invalidate(knod, i, idx);
+
+ amd_queue->hsa_queue.type = HSA_QUEUE_TYPE_MULTI;
+ amd_queue->hsa_queue.features = HSA_QUEUE_FEATURE_KERNEL_DISPATCH;
+ amd_queue->hsa_queue.base_address = (void *)knod->kaql[idx].aql->gaddr;
+ amd_queue->hsa_queue.doorbell_signal =
+ knod->kaql[idx].queue_signal->gaddr;
+ amd_queue->hsa_queue.size = knod->nr_aql_ring;
+ amd_queue->hsa_queue.reserved1 = 0;
+ amd_queue->hsa_queue.id = qid;
+
+ amd_queue->write_dispatch_id = 0; /* id is index */
+ amd_queue->group_segment_aperture_base_hi = 0;
+ amd_queue->private_segment_aperture_base_hi = 0;
+ amd_queue->max_cu_id = -1;
+ amd_queue->max_wave_id = -1;
+ amd_queue->max_legacy_doorbell_dispatch_id_plus_1 = 0;
+ amd_queue->legacy_doorbell_lock = 0;
+ amd_queue->read_dispatch_id = 0; /* id is index */
+ amd_queue->read_dispatch_id_field_base_byte_offset = 0x80;
+ /* scratch resource descriptor - use allocated scratch BO address */
+ scratch_gaddr = (u32)(knod->kaql[idx].scratch->gaddr & 0xffffffff);
+ amd_queue->scratch_resource_descriptor[0] = scratch_gaddr;
+ scratch_gaddr = (u32)((knod->kaql[idx].scratch->gaddr >> 32) & 0xffff);
+ rsrc1.base_address_hi = scratch_gaddr;
+ rsrc1.stride = 0;
+ rsrc1.cache_swizzle = 0;
+ rsrc1.swizzle_enable = 1;
+ memcpy(&amd_queue->scratch_resource_descriptor[1], &rsrc1, sizeof(u32));
+ amd_queue->scratch_resource_descriptor[2] =
+ knod->kaql[idx].scratch->size;
+ amd_queue->scratch_resource_descriptor[3] = 0x00ffffff;
+ amd_queue->scratch_backing_memory_location =
+ knod->kaql[idx].scratch->gaddr;
+ amd_queue->scratch_backing_memory_byte_size =
+ knod->kaql[idx].scratch->size;
+ amd_queue->scratch_wave64_lane_byte_size = 64;
+
+ amd_queue->queue_properties.is_ptr64 = 1;
+ amd_queue->queue_properties.enable_trap_handler_debug_sgprs = 0;
+}
+
+static void knod_init_sdma_queue(struct knod *knod, int qid, int idx)
+{
+}
+
+static void knod_init_queue(struct knod *knod, int qid, int idx, int type)
+{
+ if (type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL)
+ knod_init_aql_queue(knod, qid, idx);
+ else if (type == KFD_IOC_QUEUE_TYPE_SDMA)
+ knod_init_sdma_queue(knod, qid, idx);
+}
+
+static void stop_umh(pid_t umh_pid)
+{
+ struct pid *p = find_get_pid(umh_pid);
+
+ if (!p)
+ return;
+
+ kill_pid(p, SIGKILL, 1);
+ put_pid(p);
+}
+
+static int umh_init(struct subprocess_info *info, struct cred *new)
+{
+ struct umh_data *d = info->data;
+
+ d->pid = current->pid;
+
+ return 0;
+}
+
+static int launch_and_get_pid(void)
+{
+ static const char * const argv[] = { "/bin/sleep", "2147483647", NULL };
+ static const char * const envp[] = { "HOME=/", "PATH=/sbin:/bin", NULL };
+ struct umh_data data = { .pid = -1 };
+ struct subprocess_info *info;
+ int ret;
+
+ info = call_usermodehelper_setup(argv[0], (char **)argv,
+ (char **)envp, GFP_KERNEL,
+ umh_init, NULL, &data);
+ if (!info) {
+ pr_err("UMH setup failed\n");
+ return 0;
+ }
+
+ ret = call_usermodehelper_exec(info, UMH_WAIT_EXEC);
+ if (ret < 0) {
+ pr_err("UMH exec failed: %d\n", ret);
+ return 0;
+ }
+
+ return data.pid;
+}
+
+static int knod_set_isa(struct knod *knod)
+{
+ enum amd_asic_type asic_type;
+
+ asic_type = knod->process->pdds[0]->dev->adev->asic_type;
+ if (knod->isa_version != 9 && knod->isa_version != 10)
+ knod->isa_version = 0;
+ if (knod->isa_version == 0) {
+ if (asic_type <= CHIP_VEGAM) {
+ pr_err("Not supported chip");
+ return -EOPNOTSUPP;
+ } else if (asic_type == CHIP_VEGA10 ||
+ asic_type == CHIP_VEGA12 ||
+ asic_type == CHIP_VEGA20 ||
+ asic_type == CHIP_RAVEN ||
+ asic_type == CHIP_RENOIR) {
+ pr_debug("GCN5 is detected");
+ knod->isa_version = 9;
+ } else if (asic_type == CHIP_NAVI10 ||
+ asic_type == CHIP_NAVI12 ||
+ asic_type == CHIP_NAVI14 ||
+ asic_type == CHIP_CYAN_SKILLFISH) {
+ pr_err("RDNA1 is not supported yet");
+ return -EOPNOTSUPP;
+ } else if (asic_type == CHIP_SIENNA_CICHLID ||
+ asic_type == CHIP_NAVY_FLOUNDER ||
+ asic_type == CHIP_DIMGREY_CAVEFISH ||
+ asic_type == CHIP_BEIGE_GOBY ||
+ asic_type == CHIP_YELLOW_CARP) {
+ pr_debug("RDNA2 is detected");
+ knod->isa_version = 10;
+ } else if (asic_type == CHIP_ARCTURUS ||
+ asic_type == CHIP_ALDEBARAN) {
+ pr_err("CDNA is not supported yet");
+ return -EOPNOTSUPP;
+ } else if (asic_type == CHIP_IP_DISCOVERY) {
+ pr_err("Can't detect chip version, firmware update may be needed");
+ return -EOPNOTSUPP;
+ } else {
+ pr_err("Not supported chip");
+ return -EOPNOTSUPP;
+ }
+ }
+
+ knod->igpu = false;
+ if (asic_type == CHIP_RENOIR || asic_type == CHIP_RAVEN ||
+ asic_type == CHIP_CYAN_SKILLFISH || asic_type == CHIP_BEIGE_GOBY ||
+ asic_type == CHIP_YELLOW_CARP) {
+ pr_debug("iGPU is detected");
+ knod->igpu = true;
+ }
+
+ return 0;
+}
+
+static void knod_destroy_one_queue(struct knod *knod, int idx)
+{
+ /* Destroy queue and event BEFORE freeing BOs.
+ * Queue holds references to BO VAs; freeing BOs first causes
+ * NULL deref in kfd_queue_unref_bo_vas when UMH exits.
+ * Use the created flag (not queue_id value) since KFD IDR can
+ * assign queue_id=0 to the first queue.
+ */
+ if (knod->aql_queue_created[idx]) {
+ int ret;
+
+ ret = knod_destroy_queue(knod->process,
+ knod->aql_queue_id[idx]);
+ if (ret)
+ pr_err("knod: destroy_queue failed: %d (queue_id=%u)\n",
+ ret, knod->aql_queue_id[idx]);
+ knod->aql_queue_created[idx] = false;
+ knod->aql_queue_id[idx] = 0;
+ }
+ if (knod->aql_event[idx].id) {
+ knod_destroy_event(knod->process, knod->aql_event[idx].id);
+ knod->aql_event[idx].id = 0;
+ }
+
+ knod_free_mem(knod, knod->kaql[idx].aql);
+ knod_free_mem(knod, knod->kaql[idx].queue);
+ knod_free_mem(knod, knod->kaql[idx].eop);
+ knod_free_mem(knod, knod->kaql[idx].ctx);
+ knod_free_mem(knod, knod->kaql[idx].queue_signal);
+ knod_free_mem(knod, knod->kaql[idx].amd_queue);
+ knod_free_mem(knod, knod->kaql[idx].scratch);
+}
+
+static int knod_alloc_one_queue(struct knod *knod, int idx,
+ struct kfd_topology_device *topo_dev,
+ struct kfd_process_device *pdd, void *ptr)
+{
+ int buf_flags = KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE |
+ KFD_IOC_ALLOC_MEM_FLAGS_COHERENT |
+ KFD_IOC_ALLOC_MEM_FLAGS_VRAM;
+ int flags = KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE |
+ KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE |
+ KFD_IOC_ALLOC_MEM_FLAGS_COHERENT |
+ KFD_IOC_ALLOC_MEM_FLAGS_GTT;
+ struct amd_signal *queue_signal;
+ struct amd_queue *amd_queue;
+ struct queue_properties qp;
+ u32 total_cwsr_size;
+ int err;
+
+ knod->kaql[idx].aql = knod_alloc_mem(knod,
+ (NR_AQL_RING *
+ sizeof(struct hsa_kernel_dispatch_packet) * 2),
+ KFD_IOC_ALLOC_MEM_FLAGS_GTT |
+ KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE |
+ KFD_IOC_ALLOC_MEM_FLAGS_COHERENT |
+ KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED |
+ KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM |
+ KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE);
+ if (IS_ERR(knod->kaql[idx].aql)) {
+ err = PTR_ERR(knod->kaql[idx].aql);
+ knod->kaql[idx].aql = NULL;
+ goto err_mem;
+ }
+
+ knod->kaql[idx].queue = knod_alloc_mem(knod, PAGE_SIZE, flags);
+ if (IS_ERR(knod->kaql[idx].queue)) {
+ err = PTR_ERR(knod->kaql[idx].queue);
+ knod->kaql[idx].queue = NULL;
+ goto err_mem;
+ }
+
+ knod->kaql[idx].eop = knod_alloc_mem(knod, PAGE_SIZE, flags);
+ if (IS_ERR(knod->kaql[idx].eop)) {
+ err = PTR_ERR(knod->kaql[idx].eop);
+ knod->kaql[idx].eop = NULL;
+ goto err_mem;
+ }
+
+ total_cwsr_size = (topo_dev->node_props.cwsr_size +
+ topo_dev->node_props.debug_memory_size)
+ * NUM_XCC(pdd->dev->xcc_mask);
+ total_cwsr_size = ALIGN(total_cwsr_size, PAGE_SIZE);
+
+ knod->kaql[idx].ctx = knod_alloc_mem(knod, total_cwsr_size, buf_flags);
+ if (IS_ERR(knod->kaql[idx].ctx)) {
+ err = PTR_ERR(knod->kaql[idx].ctx);
+ knod->kaql[idx].ctx = NULL;
+ goto err_mem;
+ }
+
+ knod->kaql[idx].queue_signal = knod_alloc_mem(knod,
+ PAGE_SIZE << 5,
+ KFD_IOC_ALLOC_MEM_FLAGS_GTT |
+ KFD_IOC_ALLOC_MEM_FLAGS_COHERENT |
+ KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE |
+ KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE);
+ if (IS_ERR(knod->kaql[idx].queue_signal)) {
+ err = PTR_ERR(knod->kaql[idx].queue_signal);
+ knod->kaql[idx].queue_signal = NULL;
+ goto err_mem;
+ }
+
+ knod->kaql[idx].amd_queue = knod_alloc_mem(knod,
+ PAGE_SIZE << 5,
+ KFD_IOC_ALLOC_MEM_FLAGS_GTT |
+ KFD_IOC_ALLOC_MEM_FLAGS_COHERENT);
+ if (IS_ERR(knod->kaql[idx].amd_queue)) {
+ err = PTR_ERR(knod->kaql[idx].amd_queue);
+ knod->kaql[idx].amd_queue = NULL;
+ goto err_mem;
+ }
+
+ knod->kaql[idx].scratch = knod_alloc_mem(knod,
+ PAGE_SIZE << 5,
+ KFD_IOC_ALLOC_MEM_FLAGS_VRAM |
+ KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE |
+ KFD_IOC_ALLOC_MEM_FLAGS_COHERENT);
+ if (IS_ERR(knod->kaql[idx].scratch)) {
+ err = PTR_ERR(knod->kaql[idx].scratch);
+ knod->kaql[idx].scratch = NULL;
+ goto err_mem;
+ }
+
+ knod->kaql[idx].idx = 0;
+
+ /*
+ * KNOD dispatchers poll queue_signal->value directly. Do not attach a
+ * KFD signal event to every AQL completion; at high packet rates the
+ * unused event notifications overflow the KFD interrupt/event path and
+ * add latency without contributing to completion detection.
+ */
+ knod->aql_event[idx].id = 0;
+ knod->aql_event[idx].slot = 0;
+ knod->signal_eid = 0;
+
+ /* Zero ALL queue-related GPU memory BEFORE creating the HW queue.
+ * kfd_ioctl_create_queue loads HQD immediately - if the memory
+ * contains stale data from a previous session (write/read pointers,
+ * AQL dispatch packets, doorbell values), the GPU starts processing
+ * garbage packets and goes to 100%.
+ */
+ memset(knod->kaql[idx].queue->kaddr, 0, knod->kaql[idx].queue->size);
+ /* AQL ring is allocated 2x with AQL_QUEUE_MEM flag; the second
+ * half is a GPU-side mirror for wrap-around. kaddr only covers
+ * the first half.
+ */
+ memset(knod->kaql[idx].aql->kaddr, 0, knod->kaql[idx].aql->size / 2);
+ memset(knod->kaql[idx].eop->kaddr, 0, knod->kaql[idx].eop->size);
+ memset(knod->kaql[idx].amd_queue->kaddr, 0,
+ knod->kaql[idx].amd_queue->size);
+ knod_init_queue(knod, idx, idx, KFD_IOC_QUEUE_TYPE_COMPUTE_AQL);
+
+ memset(&qp, 0, sizeof(qp));
+ qp.type = KFD_QUEUE_TYPE_COMPUTE;
+ qp.format = KFD_QUEUE_FORMAT_AQL;
+ qp.queue_percent = 100;
+ qp.priority = 15;
+ qp.ctl_stack_size = topo_dev->node_props.ctl_stack_size;
+ qp.ctx_save_restore_area_size = topo_dev->node_props.cwsr_size;
+ qp.ctx_save_restore_area_address = (u64)knod->kaql[idx].ctx->gaddr;
+ qp.queue_address = (u64)knod->kaql[idx].aql->gaddr;
+ qp.queue_size = knod->kaql[idx].aql->size / 2;
+ qp.write_ptr =
+ (void __user *)((u64)knod->kaql[idx].queue->gaddr + 0x38);
+ qp.read_ptr = (void __user *)((u64)knod->kaql[idx].queue->gaddr + 0x80);
+ qp.eop_ring_buffer_address = (u64)knod->kaql[idx].eop->gaddr;
+ qp.eop_ring_buffer_size = knod->kaql[idx].eop->size;
+
+ err = knod_create_event(knod->process, KFD_IOC_EVENT_SIGNAL, true, 1,
+ &knod->aql_event[idx]);
+ if (err) {
+ knod_err(" failed to create AQL event[%d] err=%d\n", idx, err);
+ goto err_mem;
+ }
+
+ queue_signal = (struct amd_signal *)knod->kaql[idx].queue_signal->kaddr;
+ queue_signal->kind = AMD_SIGNAL_KIND_USER;
+ if (queue_signal->kind == AMD_SIGNAL_KIND_DOORBELL) {
+ queue_signal->value = 0;
+ queue_signal->event_id = 0;
+ queue_signal->queue_ptr = (void *)knod->kaql[idx].queue->gaddr;
+ queue_signal->hardware_doorbell_ptr = ptr;
+ queue_signal->event_mailbox_ptr = 0;
+ } else if (queue_signal->kind == AMD_SIGNAL_KIND_USER) {
+ queue_signal->event_id = knod->aql_event[idx].id;
+ queue_signal->queue_ptr = 0;
+ queue_signal->event_mailbox_ptr = knod->mailbox->gaddr +
+ knod->aql_event[idx].slot * 8;
+ /* value is in union with hardware_doorbell_ptr - set last */
+ queue_signal->value = 0xffffffffffffffff;
+ }
+ knod_dbg(" pre-create-queue signal=%lld kaddr=%p gaddr=0x%llx\n",
+ READ_ONCE(queue_signal->value), queue_signal,
+ knod->kaql[idx].queue_signal->gaddr);
+
+ err = knod_create_queue(knod->process, &qp, pdd->user_gpu_id,
+ &knod->aql_queue_id[idx],
+ &knod->aql_doorbell_offset[idx]);
+ if (err) {
+ knod_err(" failed to create queue, err = %d\n", err);
+ goto err_mem;
+ }
+ knod->aql_queue_created[idx] = true;
+
+ knod_dbg(" post-create-queue signal=%lld queue_id=%d\n",
+ READ_ONCE(queue_signal->value), knod->aql_queue_id[idx]);
+
+ /* Update queue id with the actual KFD-assigned value */
+ amd_queue = (struct amd_queue *)knod->kaql[idx].queue->kaddr;
+ amd_queue->hsa_queue.id = knod->aql_queue_id[idx];
+ return 0;
+
+err_mem:
+ if (knod->aql_event[idx].id) {
+ knod_destroy_event(knod->process, knod->aql_event[idx].id);
+ knod->aql_event[idx].id = 0;
+ }
+ knod_free_mem(knod, knod->kaql[idx].aql);
+ knod_free_mem(knod, knod->kaql[idx].queue);
+ knod_free_mem(knod, knod->kaql[idx].eop);
+ knod_free_mem(knod, knod->kaql[idx].ctx);
+ knod_free_mem(knod, knod->kaql[idx].queue_signal);
+ knod_free_mem(knod, knod->kaql[idx].amd_queue);
+ knod_free_mem(knod, knod->kaql[idx].scratch);
+ memset(&knod->kaql[idx], 0, sizeof(knod->kaql[idx]));
+ return err;
+}
+
+/*
+ * Write a minimal no-op kernel into the kernel BO so that any dispatch
+ * before a real shader (BPF/IPsec/MACsec/WG) is loaded executes a
+ * harmless s_endpgm instead of faulting on uninitialised VRAM.
+ *
+ * Layout:
+ * [0..63] kernel_descriptor (kernel_code_entry_byte_offset = 256)
+ * [256..259] s_endpgm (0xBF810000)
+ * [260..1023] s_code_end padding (GFX10 SQC prefetch safety)
+ */
+#define KNOD_DEFAULT_KD_ENTRY_OFFSET 256
+#define KNOD_S_ENDPGM 0xBF810000u
+#define KNOD_S_CODE_END 0xBF9F0000u
+
+static void knod_init_default_kernel(struct knod *knod)
+{
+ struct kernel_descriptor *kd = knod->kernels[0]->kaddr;
+ u32 *code = (u32 *)((u8 *)knod->kernels[0]->kaddr +
+ KNOD_DEFAULT_KD_ENTRY_OFFSET);
+ int i;
+
+ memset(kd, 0, sizeof(*kd));
+ kd->kernel_code_entry_byte_offset = KNOD_DEFAULT_KD_ENTRY_OFFSET;
+ kd->compute_pgm_rsrc1.granulated_workitem_vgpr_count = 0;
+ kd->compute_pgm_rsrc1.granulated_wavefront_sgpr_count = 0;
+ kd->compute_pgm_rsrc1.float_denorm_mode_32 = 3;
+ kd->compute_pgm_rsrc1.float_denorm_mode_16_64 = 3;
+ kd->compute_pgm_rsrc1.enable_dx10_clamp = 1;
+ kd->compute_pgm_rsrc1.enable_ieee_mode = 1;
+ if (knod->isa_version >= 10)
+ kd->compute_pgm_rsrc1.mem_ordered = 1;
+ kd->compute_pgm_rsrc2.enable_sgpr_workgroup_id_x = 1;
+
+ code[0] = KNOD_S_ENDPGM;
+ for (i = 1; i < (1024 - KNOD_DEFAULT_KD_ENTRY_OFFSET) / 4; i++)
+ code[i] = KNOD_S_CODE_END;
+}
+
+#define KNOD_DEFAULT_BATCH 64
+
+static struct knod_accel_xdp_ops *registered_xdp_ops;
+
+/*
+ * feature=none has no per-feature ops: the default worker stamps XDP_PASS and
+ * the NIC act handler delivers via knod_d2h_copy / knod_d2h_drain.
+ */
+static struct knod_accel_xdp_ops default_xdp_ops = {
+};
+
+static int knod_default_worker(void *arg)
+{
+ struct knod *knod = arg;
+ struct spsc_bd *bds[KNOD_DEFAULT_BATCH];
+ struct knod_dev *knodev;
+ unsigned int cnt;
+ int qi, i;
+
+ while (!kthread_should_stop()) {
+ knodev = READ_ONCE(knod->accel->knodev);
+ if (!knodev || !knodev->started) {
+ usleep_range(1000, 2000);
+ continue;
+ }
+
+ for (qi = 0; qi < knod->channels; qi++) {
+ struct knod_work_priv *wpriv = &knodev->wpriv[qi];
+
+ if (!wpriv->napi)
+ continue;
+
+ if (spsc_peek(&wpriv->spsc_bds, (void **)bds,
+ KNOD_DEFAULT_BATCH, &cnt))
+ continue;
+
+ /*
+ * feature=none has no program, so every packet passes.
+ * Stamp the verdict before advancing the consumer
+ * cursor: spsc_acquire() publishes the window with a
+ * release barrier the act handler pairs with, so the
+ * verdict has to be written first or the act handler
+ * races a POISON read. The NIC act handler does the
+ * device->host delivery via knod_d2h_copy.
+ */
+ for (i = 0; i < cnt; i++)
+ WRITE_ONCE(bds[i]->act, XDP_PASS);
+
+ spsc_acquire(&wpriv->spsc_bds, NULL, cnt, NULL);
+ knod_napi_kick(wpriv);
+ }
+
+ usleep_range(100, 200);
+ }
+ return 0;
+}
+
+static int knod_start_default_worker(struct knod *knod)
+{
+ struct task_struct *p;
+
+ knod->worker_fn = knod_default_worker;
+ knod->flush_fn = NULL;
+ knod->worker_ctx = knod;
+ p = kthread_run(knod_default_worker, knod, "knod_dflt_%d",
+ knod->accel->id);
+ if (IS_ERR(p))
+ return PTR_ERR(p);
+
+ get_task_struct(p);
+ knod->worker = p;
+ return 0;
+}
+
+static void knod_stop_worker(struct knod *knod)
+{
+ if (!knod->worker)
+ return;
+
+ if (knod->flush_fn)
+ knod->flush_fn(knod->worker_ctx);
+
+ kthread_stop(knod->worker);
+ put_task_struct(knod->worker);
+ knod->worker = NULL;
+ knod->worker_fn = NULL;
+ knod->flush_fn = NULL;
+ knod->worker_ctx = NULL;
+}
+
+int knod_register_worker(struct knod *knod, knod_worker_fn_t fn,
+ knod_flush_fn_t flush, void *ctx)
+{
+ struct task_struct *p;
+
+ knod_stop_worker(knod);
+
+ knod->worker_fn = fn;
+ knod->flush_fn = flush;
+ knod->worker_ctx = ctx;
+ p = kthread_run(fn, ctx, "knod_%d", knod->accel->id);
+ if (IS_ERR(p)) {
+ knod->worker_fn = NULL;
+ knod->flush_fn = NULL;
+ knod->worker_ctx = NULL;
+ return PTR_ERR(p);
+ }
+
+ get_task_struct(p);
+ knod->worker = p;
+ return 0;
+}
+
+void knod_unregister_worker(struct knod *knod)
+{
+ knod_stop_worker(knod);
+ knod_start_default_worker(knod);
+}
+
+int knod_wait_on_events(struct kfd_process *p, u32 num_events,
+ void __user *data, bool all, u32 *user_timeout_ms,
+ u32 *wait_result)
+{
+ return kfd_wait_on_events_kernel(p, num_events, data, all,
+ user_timeout_ms, wait_result);
+}
+EXPORT_SYMBOL(knod_wait_on_events);
+
+static int knod_alloc_ctx_init(struct knod *knod, int id, void **doorbell,
+ struct kfd_topology_device **out_topo_dev,
+ struct kfd_process_device **out_pdd)
+{
+ struct kfd_topology_device *topo_dev;
+ struct kfd_process_device *pdd;
+ struct file *drm_file;
+ size_t mem_size;
+ char path[64];
+ int err;
+
+ sprintf(path, "/dev/dri/renderD%d", id);
+
+ drm_file = filp_open(path, O_RDWR, 0);
+ if (IS_ERR(drm_file))
+ return PTR_ERR(drm_file);
+
+ knod->drm_file = drm_file;
+
+ knod->process = kfd_create_process(knod->umh_task);
+ if (IS_ERR(knod->process)) {
+ err = PTR_ERR(knod->process);
+ goto err_filp_close;
+ }
+ kref_get(&knod->process->ref);
+
+ err = knod_set_isa(knod);
+ if (err < 0)
+ goto err_unref_process;
+
+ pdd = knod->process->pdds[0];
+
+ topo_dev = kfd_topology_device_by_id(pdd->dev->id);
+ if (!topo_dev) {
+ pr_err("knod: can't find topo_dev for dev id %u\n",
+ pdd->dev->id);
+ err = -ENODEV;
+ goto err_unref_process;
+ }
+
+ /* Acquire VM directly without installing an fd into any process's
+ * fdtable. get_file() provides the ref that pdd->drm_file will own.
+ * kfd_process_destroy_pdds will fput it during process cleanup.
+ */
+ get_file(drm_file);
+ mutex_lock(&knod->process->mutex);
+ err = kfd_process_device_init_vm(pdd, drm_file);
+ mutex_unlock(&knod->process->mutex);
+ if (err) {
+ fput(drm_file);
+ goto err_unref_process;
+ }
+
+ *doorbell = kfd_kernel_doorbell_mmap(pdd->dev, knod->process);
+ if (!*doorbell) {
+ err = -ENOMEM;
+ goto err_unref_process;
+ }
+
+ knod->pool = gen_pool_create(PAGE_SHIFT,
+ dev_to_node(pdd->dev->adev->dev));
+ if (!knod->pool) {
+ err = -ENOMEM;
+ goto err_release_doorbell;
+ }
+
+ knod->dev = pdd->dev;
+ knod->reserved_addr = pdd->gpuvm_base << 2;
+ knod->limit_addr = pdd->gpuvm_limit;
+ mem_size = knod->limit_addr - knod->reserved_addr;
+
+ err = gen_pool_add(knod->pool, knod->reserved_addr, mem_size,
+ dev_to_node(pdd->dev->adev->dev));
+ if (err)
+ goto err_gen_pool_destroy;
+
+ knod->kernels[0] = knod_alloc_mem(knod, PAGE_SIZE << 10,
+ KFD_IOC_ALLOC_MEM_FLAGS_VRAM |
+ KFD_IOC_ALLOC_MEM_FLAGS_COHERENT |
+ KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE |
+ KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE);
+ if (IS_ERR(knod->kernels[0])) {
+ err = PTR_ERR(knod->kernels[0]);
+ knod->kernels[0] = NULL;
+ goto err_gen_pool_destroy;
+ }
+ knod_init_default_kernel(knod);
+
+ /*
+ * Second dispatch slot for the BPF ping-pong swap, allocated right
+ * after knod->kernels[0] so both kernel BOs sit in the same low VA
+ * region.
+ * Allocating it later (at feature activate, past the RX buffers) put it
+ * at a high VA, and switching the dispatch kernel_object to that BO
+ * mid-stream wedged the compute queue.
+ */
+ knod->kernels[1] = knod_alloc_mem(knod, PAGE_SIZE << 10,
+ KFD_IOC_ALLOC_MEM_FLAGS_VRAM |
+ KFD_IOC_ALLOC_MEM_FLAGS_COHERENT |
+ KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE |
+ KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE);
+ if (IS_ERR(knod->kernels[1])) {
+ err = PTR_ERR(knod->kernels[1]);
+ knod->kernels[1] = NULL;
+ goto err_free_kernel;
+ }
+
+ knod->mailbox = knod_alloc_mem(knod, PAGE_SIZE << 5,
+ KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE |
+ KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE |
+ KFD_IOC_ALLOC_MEM_FLAGS_COHERENT |
+ KFD_IOC_ALLOC_MEM_FLAGS_VRAM);
+ if (IS_ERR(knod->mailbox)) {
+ err = PTR_ERR(knod->mailbox);
+ knod->mailbox = NULL;
+ goto err_free_kernel;
+ }
+
+ *out_topo_dev = topo_dev;
+ *out_pdd = pdd;
+ return 0;
+
+err_free_kernel:
+ if (knod->kernels[1])
+ knod_free_mem(knod, knod->kernels[1]);
+ knod->kernels[1] = NULL;
+ knod_free_mem(knod, knod->kernels[0]);
+ knod->kernels[0] = NULL;
+err_gen_pool_destroy:
+ gen_pool_destroy(knod->pool);
+ knod->pool = NULL;
+err_release_doorbell:
+ iounmap(*doorbell);
+ *doorbell = NULL;
+err_unref_process:
+ kfd_unref_process(knod->process);
+err_filp_close:
+ fput(knod->drm_file);
+ return err;
+}
+
+struct knod *knod_alloc_ctx(struct knod_dev *knodev, int queue_cnt, int id,
+ int channels)
+{
+ int buf_flags = KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE |
+ KFD_IOC_ALLOC_MEM_FLAGS_COHERENT |
+ KFD_IOC_ALLOC_MEM_FLAGS_VRAM;
+ struct kfd_topology_device *topo_dev;
+ struct kfd_process_device *pdd;
+ struct process_queue_node *pqn;
+ struct knod_work_priv *wpriv;
+ struct task_struct *task;
+ unsigned int sdma_cnt;
+ void *ptr = NULL;
+ struct knod *knod;
+ struct knod_mem *buf;
+ struct pid *spid;
+ struct queue *q;
+ int err, size;
+ pid_t pid;
+ int idx;
+
+ pid = launch_and_get_pid();
+ if (!pid) {
+ knod_err(" failed to create UMH\n");
+ return ERR_PTR(-EINVAL);
+ }
+
+ spid = find_get_pid(pid);
+ if (!spid) {
+ knod_err(" failed to find pid\n");
+ return ERR_PTR(-ESRCH);
+ }
+
+ task = get_pid_task(spid, PIDTYPE_PID);
+ put_pid(spid);
+ if (!task) {
+ knod_err(" no task found\n");
+ return ERR_PTR(-ESRCH);
+ }
+
+ knod = kzalloc(sizeof(struct knod), GFP_KERNEL);
+ if (!knod) {
+ put_task_struct(task);
+ return ERR_PTR(-ENOMEM);
+ }
+
+ knod->queue_cnt = queue_cnt;
+ knod->sdma_cnt = queue_cnt;
+ knod->channels = channels;
+ knod->umh_pid = pid;
+ knod->umh_task = task;
+ knod->nr_aql_ring = NR_AQL_RING;
+ INIT_LIST_HEAD(&knod->list);
+ INIT_LIST_HEAD(&knod->active_list);
+
+ err = knod_alloc_ctx_init(knod, id, &ptr, &topo_dev, &pdd);
+ if (err)
+ goto err_free_knod;
+
+ sdma_cnt = topo_dev->node_props.num_sdma_engines *
+ topo_dev->node_props.num_sdma_queues_per_engine;
+ if (!sdma_cnt)
+ sdma_cnt = queue_cnt;
+ knod->sdma_cnt = min_t(int, queue_cnt, sdma_cnt);
+ if (!knod->sdma_cnt)
+ knod->sdma_cnt = 1;
+ pr_info("knod: AQL queues=%d SDMA queues=%d channels=%d\n",
+ queue_cnt, knod->sdma_cnt, channels);
+
+ knod->doorbell_base = ptr;
+
+ knod->buf = kmalloc_array(channels, sizeof(struct knod_mem *),
+ GFP_KERNEL | __GFP_ZERO);
+ if (!knod->buf) {
+ err = -ENOMEM;
+ goto err_free_mailbox;
+ }
+
+ if (knod->igpu)
+ size = PAGE_SIZE << MAX_PAGE_ORDER;
+ else
+ size = PAGE_SIZE << 14;
+
+ for (idx = 0; idx < channels; idx++) {
+ wpriv = &knodev->wpriv[idx];
+ buf = __knod_alloc_mem(knod, size, buf_flags);
+ if (IS_ERR(buf)) {
+ err = PTR_ERR(buf);
+ goto err_free_bufs;
+ }
+ if (__knod_export_dma_buf(knod, buf)) {
+ knod_free_mem(knod, buf);
+ err = -ENOMEM;
+ goto err_free_bufs;
+ }
+ if (__knod_map_kaddr(knod, buf)) {
+ knod_free_mem(knod, buf);
+ err = -ENOMEM;
+ goto err_free_bufs;
+ }
+ if (__knod_map_mem(knod, buf)) {
+ knod_free_mem(knod, buf);
+ err = -ENOMEM;
+ goto err_free_bufs;
+ }
+ wpriv->dmabuf = buf->mem->dmabuf;
+ wpriv->index = idx;
+ knod->buf[idx] = buf;
+ }
+
+ /*
+ * GPU->host delivery buffer + per-queue page_pools are owned by the
+ * NOD framework (knod_pass_attach), allocated via accel_ops->alloc_mem.
+ */
+
+ memset(knod->mailbox->kaddr, 0, knod->mailbox->size);
+
+ err = kfd_event_page_set(knod->process,
+ knod->mailbox->kaddr,
+ KFD_SIGNAL_EVENT_LIMIT * 8,
+ knod->mailbox->gaddr);
+ if (err < 0)
+ goto err_free_bufs;
+
+ for (idx = 0; idx < queue_cnt; idx++) {
+ err = knod_alloc_one_queue(knod, idx, topo_dev, pdd, ptr);
+ if (err)
+ goto err_free_queues;
+ }
+
+ /* Allocate only the SDMA queues the device can actually provide. BPF
+ * XDP_TX does not use SDMA for the verdict path, while PASS/d2h maps RX
+ * queues onto the available SDMA queues modulo sdma_cnt.
+ */
+ for (idx = 0; idx < knod->sdma_cnt; idx++) {
+ struct amd_signal *sdma_signal;
+ struct queue_properties qp;
+ long *sdma_lptr;
+ int sdma_flags = KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE |
+ KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE |
+ KFD_IOC_ALLOC_MEM_FLAGS_COHERENT |
+ KFD_IOC_ALLOC_MEM_FLAGS_GTT;
+
+ knod->sdma[idx].sdma = knod_alloc_mem(knod, PAGE_SIZE << 4,
+ KFD_IOC_ALLOC_MEM_FLAGS_GTT |
+ KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE |
+ KFD_IOC_ALLOC_MEM_FLAGS_COHERENT |
+ KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED |
+ KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE);
+ if (IS_ERR(knod->sdma[idx].sdma)) {
+ err = PTR_ERR(knod->sdma[idx].sdma);
+ knod->sdma[idx].sdma = NULL;
+ goto err_free_sdma;
+ }
+ knod->sdma[idx].queue = knod_alloc_mem(knod, PAGE_SIZE,
+ sdma_flags);
+ if (IS_ERR(knod->sdma[idx].queue)) {
+ err = PTR_ERR(knod->sdma[idx].queue);
+ knod->sdma[idx].queue = NULL;
+ goto err_free_sdma;
+ }
+ knod->sdma[idx].queue_signal = knod_alloc_mem(knod,
+ PAGE_SIZE << 5,
+ KFD_IOC_ALLOC_MEM_FLAGS_GTT |
+ KFD_IOC_ALLOC_MEM_FLAGS_COHERENT |
+ KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE |
+ KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE);
+ if (IS_ERR(knod->sdma[idx].queue_signal)) {
+ err = PTR_ERR(knod->sdma[idx].queue_signal);
+ knod->sdma[idx].queue_signal = NULL;
+ goto err_free_sdma;
+ }
+ knod->sdma[idx].idx = 0;
+
+ err = knod_create_event(knod->process, KFD_IOC_EVENT_SIGNAL,
+ true, 1, &knod->sdma_event[idx]);
+ if (err) {
+ knod_err(" failed to create SDMA event[%d] err=%d\n",
+ idx, err);
+ goto err_free_sdma;
+ }
+
+ memset(&qp, 0, sizeof(qp));
+ qp.type = KFD_QUEUE_TYPE_SDMA;
+ qp.format = KFD_QUEUE_FORMAT_PM4;
+ qp.queue_percent = 100;
+ qp.priority = 15;
+ qp.queue_address = (u64)knod->sdma[idx].sdma->gaddr;
+ qp.queue_size = knod->sdma[idx].sdma->size;
+ qp.write_ptr = (void __user *)
+ ((u64)knod->sdma[idx].queue->gaddr + 0x08);
+ qp.read_ptr = (void __user *)
+ ((u64)knod->sdma[idx].queue->gaddr + 0x10);
+
+ sdma_signal = (struct amd_signal *)
+ knod->sdma[idx].queue_signal->kaddr;
+ sdma_signal->kind = AMD_SIGNAL_KIND_USER;
+ sdma_signal->event_id = knod->sdma_event[idx].id;
+ sdma_signal->queue_ptr = 0;
+ sdma_signal->event_mailbox_ptr = knod->mailbox->gaddr +
+ (knod->sdma_event[idx].slot * 8);
+ /* SDMA fence writes u32 to low 32 bits of value.
+ * Init to 0 (not -1 like AQL) so comparison works.
+ * value is in union with hardware_doorbell_ptr - set last.
+ */
+ sdma_signal->value = 0;
+ sdma_lptr = knod->mailbox->kaddr + (idx * 8);
+ sdma_lptr[knod->sdma_event[idx].slot] = 0;
+
+ err = knod_create_queue(knod->process, &qp, pdd->user_gpu_id,
+ &knod->sdma_queue_id[idx],
+ &knod->sdma_doorbell_offset[idx]);
+ if (err) {
+ knod_err(" failed to create SDMA queue[%d] err=%d\n",
+ idx, err);
+ goto err_free_sdma;
+ }
+ knod->sdma_queue_created[idx] = true;
+
+ knod_init_queue(knod, knod->sdma_queue_id[idx], idx,
+ KFD_IOC_QUEUE_TYPE_SDMA);
+ knod->sdma[idx].doorbell =
+ (u64 *)((u8 *)ptr +
+ (u32)knod->sdma_doorbell_offset[idx]);
+ knod_dbg(" SDMA queue[%d] created: id=%d doorbell_offset=0x%x doorbell=%p\n",
+ idx, knod->sdma_queue_id[idx],
+ (u32)knod->sdma_doorbell_offset[idx],
+ knod->sdma[idx].doorbell);
+ }
+
+ list_for_each_entry(pqn, &knod->process->pqm.queues,
+ process_queue_list) {
+ if (!pqn->q)
+ continue;
+ q = pqn->q;
+ for (idx = 0; idx < queue_cnt; idx++) {
+ if (q->properties.queue_id == idx) {
+ knod->kaql[idx].doorbell =
+ (u64 *)((u8 *)ptr +
+ (u32)knod->aql_doorbell_offset[idx]);
+ q->properties.doorbell_ptr =
+ knod->kaql[idx].doorbell;
+ knod_dbg(" doorbell: idx=%d qid=%d door_off=0x%x doorbell=%p\n",
+ idx, q->properties.queue_id,
+ (u32)knod->aql_doorbell_offset[idx],
+ knod->kaql[idx].doorbell);
+ }
+ }
+ }
+
+ list_add_tail(&knod->list, &ctx_list);
+
+ /* Create shared debugfs directory under dri/<N>/knod/ */
+ {
+ struct drm_minor *minor = adev_to_drm(pdd->dev->adev)->render;
+
+ if (minor && minor->debugfs_root) {
+ struct dentry *dir;
+
+ dir = debugfs_lookup("knod", minor->debugfs_root);
+ if (!dir) {
+ dir = debugfs_create_dir("knod",
+ minor->debugfs_root);
+ if (IS_ERR(dir))
+ dir = NULL;
+ }
+ knod->debug_dir = dir;
+ }
+ }
+
+ return knod;
+
+err_free_sdma:
+ {
+ int j;
+
+ for (j = idx; j >= 0; j--) {
+ if (knod->sdma_queue_created[j]) {
+ knod_destroy_queue(knod->process,
+ knod->sdma_queue_id[j]);
+ knod->sdma_queue_created[j] = false;
+ }
+ if (knod->sdma_event[j].id) {
+ knod_destroy_event(knod->process,
+ knod->sdma_event[j].id);
+ knod->sdma_event[j].id = 0;
+ }
+ knod_free_mem(knod, knod->sdma[j].sdma);
+ knod_free_mem(knod, knod->sdma[j].queue);
+ knod_free_mem(knod, knod->sdma[j].queue_signal);
+ }
+ }
+ idx = queue_cnt;
+err_free_queues:
+ while (idx-- > 0)
+ knod_destroy_one_queue(knod, idx);
+err_free_bufs:
+ for (idx = 0; idx < channels; idx++)
+ knod_free_mem(knod, knod->buf[idx]);
+ kfree(knod->buf);
+err_free_mailbox:
+ knod_free_mem(knod, knod->mailbox);
+ knod_free_mem(knod, knod->kernels[1]);
+ knod_free_mem(knod, knod->kernels[0]);
+ gen_pool_destroy(knod->pool);
+ iounmap(ptr);
+ kfd_unref_process(knod->process);
+ fput(knod->drm_file);
+err_free_knod:
+ put_task_struct(knod->umh_task);
+ stop_umh(knod->umh_pid);
+ kfree(knod);
+ return ERR_PTR(err);
+}
+EXPORT_SYMBOL(knod_alloc_ctx);
+
+void knod_release_ctx(struct knod *knod)
+{
+ int idx;
+
+ list_del(&knod->list);
+
+ debugfs_remove_recursive(knod->debug_dir);
+
+ /* Release SDMA queues - destroy queue/event BEFORE freeing BOs.
+ * Same ordering as knod_destroy_one_queue() for AQL: the queue
+ * holds references to BO VAs, so freeing BOs first causes the
+ * queue destroy to fail and leaves internal BOs in kfd_bo_list.
+ */
+ for (idx = 0; idx < knod->sdma_cnt; idx++) {
+ knod_destroy_queue(knod->process, knod->sdma_queue_id[idx]);
+ knod_destroy_event(knod->process, knod->sdma_event[idx].id);
+ knod_free_mem(knod, knod->sdma[idx].sdma);
+ knod_free_mem(knod, knod->sdma[idx].queue);
+ knod_free_mem(knod, knod->sdma[idx].queue_signal);
+ }
+
+ for (idx = 0; idx < knod->queue_cnt; idx++)
+ knod_destroy_one_queue(knod, idx);
+
+ for (idx = 0; idx < knod->channels; idx++)
+ knod_free_mem(knod, knod->buf[idx]);
+ kfree(knod->buf);
+
+ knod_free_mem(knod, knod->mailbox);
+ knod_free_mem(knod, knod->kernels[1]);
+ knod_free_mem(knod, knod->kernels[0]);
+ gen_pool_destroy(knod->pool);
+ iounmap(knod->doorbell_base);
+
+ /* Tear down the KFD process. It holds 3 refs:
+ * ref 1: "open" ref from kfd_create_process (normally dropped
+ * by kfd_release when /dev/kfd is closed - KNOD never
+ * opens /dev/kfd so we must drop this ourselves)
+ * ref 2: KNOD's explicit kref_get in knod_alloc_ctx
+ * ref 3: mmu_notifier alloc_notifier ref (dropped by
+ * free_notifier callback via SRCU after mmu_notifier_put)
+ *
+ * kfd_process_notifier_release_internal removes from hash,
+ * destroys queues, and calls mmu_notifier_put which schedules
+ * the SRCU callback to drop ref 3. We drop refs 1 and 2 here.
+ * After SRCU fires, ref reaches 0 -> kfd_process_wq_release
+ * runs (sysfs removal, BO/PDD/doorbell/event cleanup, kfree).
+ *
+ * kfd_process_destroy_pdds fput's pdd->drm_file (the get_file
+ * ref from init_vm), bringing the DRM file ref from 2->1.
+ * The file/VM stays alive until we fput the filp_open ref below.
+ */
+ kfd_process_notifier_release_internal(knod->process);
+ kfd_unref_process(knod->process);
+ kfd_unref_process(knod->process);
+ mmu_notifier_synchronize();
+ kfd_process_flush_wq();
+
+ /* Drop the filp_open ref (file ref 1->0). kfd_process_destroy_pdds
+ * already fput'd the get_file ref during wq_release above.
+ */
+ fput(knod->drm_file);
+ flush_delayed_fput();
+
+ stop_umh(knod->umh_pid);
+ put_task_struct(knod->umh_task);
+ kfree(knod);
+}
+EXPORT_SYMBOL(knod_release_ctx);
+
+/* ---- feature control (knod genetlink) ---- */
+
+static void *knod_feature_ops(enum knod_feature feat)
+{
+ switch (feat) {
+ case KNOD_FEATURE_BPF:
+ return registered_xdp_ops;
+ case KNOD_FEATURE_IPSEC:
+ return accel_ops.ipsec_ops;
+ default:
+ return NULL;
+ }
+}
+
+/*
+ * Feature lifecycle across three independent axes:
+ * init/exit attach/detach permanent per-attach state
+ * activate/deactivate feature select feature GPU resources
+ * start/stop interface up/down worker + GPU in-flight drain
+ *
+ * The worker only does useful work while (interface up AND a feature is
+ * selected), so a feature switch is bracketed stop -> change -> start: the
+ * worker is stopped and its GPU dispatch drained before deactivate() frees
+ * the resources it used, then restarted afterwards. start/stop never touch
+ * the framework/NIC-owned RX SPSC ring (mlx5 co-owns it via rq->knodev), so
+ * they are safe to run while the interface is up and traffic is flowing.
+ */
+static void knod_feature_stop(struct knod *knod)
+{
+ struct knod_dev *knodev = knod->accel->knodev;
+
+ knod_stop_worker(knod);
+
+ switch (knod->active_feature) {
+ case KNOD_FEATURE_BPF:
+ if (registered_xdp_ops && registered_xdp_ops->stop)
+ registered_xdp_ops->stop(knodev);
+ break;
+ case KNOD_FEATURE_IPSEC:
+ if (accel_ops.ipsec_ops && accel_ops.ipsec_ops->stop)
+ accel_ops.ipsec_ops->stop(knodev);
+ break;
+ default:
+ break;
+ }
+}
+
+static void knod_feature_start(struct knod *knod)
+{
+ struct knod_dev *knodev = knod->accel->knodev;
+
+ switch (knod->active_feature) {
+ case KNOD_FEATURE_BPF:
+ if (registered_xdp_ops && registered_xdp_ops->start)
+ registered_xdp_ops->start(knodev);
+ break;
+ case KNOD_FEATURE_IPSEC:
+ if (accel_ops.ipsec_ops && accel_ops.ipsec_ops->start)
+ accel_ops.ipsec_ops->start(knodev);
+ break;
+ default:
+ knod_start_default_worker(knod);
+ break;
+ }
+}
+
+static void knod_feature_deactivate(struct knod *knod)
+{
+ struct knod_dev *knodev = knod->accel->knodev;
+
+ switch (knod->active_feature) {
+ case KNOD_FEATURE_BPF:
+ /*
+ * Phase 1: unregister the offload dev - this force-frees any
+ * user XDP progs/maps still bound. The map-free ndo routes
+ * back through accel_ops.xdp_ops->xdp_install, so xdp_ops must
+ * still point at the BPF ops (and priv must be alive) here.
+ */
+ if (registered_xdp_ops &&
+ registered_xdp_ops->xdp_offload_uninit)
+ registered_xdp_ops->xdp_offload_uninit(knodev);
+ /*
+ * Phase 2: repoint RX delivery at the default drain_pass and
+ * wait out the in-flight NAPI readers.
+ */
+ WRITE_ONCE(accel_ops.xdp_ops, &default_xdp_ops);
+ synchronize_net();
+ knod_stop_worker(knod);
+ if (registered_xdp_ops && registered_xdp_ops->deactivate)
+ registered_xdp_ops->deactivate(knodev);
+ break;
+ case KNOD_FEATURE_IPSEC:
+ /*
+ * knod_ipsec_detach() clears the netdev xfrm flags and runs
+ * ->deactivate(), which NULLs ipsec_priv and does its own
+ * synchronize_net() before freeing.
+ */
+ knod_ipsec_detach(knodev);
+ break;
+ default:
+ break;
+ }
+}
+
+static int knod_feature_activate(struct knod *knod)
+{
+ struct knod_dev *knodev = knod->accel->knodev;
+ int err;
+
+ switch (knod->active_feature) {
+ case KNOD_FEATURE_BPF:
+ if (!registered_xdp_ops)
+ return -ENODEV;
+ /* Phase A: GPU compute buffers. */
+ if (registered_xdp_ops->activate) {
+ err = registered_xdp_ops->activate(knodev);
+ if (err)
+ return err;
+ }
+ /*
+ * Publish xdp_ops, then phase B: register the offload dev so
+ * user XDP progs/maps can bind (their install/free route
+ * through accel_ops.xdp_ops->xdp_install).
+ */
+ WRITE_ONCE(accel_ops.xdp_ops, registered_xdp_ops);
+ if (registered_xdp_ops->xdp_offload_init) {
+ err = registered_xdp_ops->xdp_offload_init(knodev);
+ if (err) {
+ WRITE_ONCE(accel_ops.xdp_ops, &default_xdp_ops);
+ synchronize_net();
+ knod_stop_worker(knod);
+ if (registered_xdp_ops->deactivate)
+ registered_xdp_ops->deactivate(knodev);
+ return err;
+ }
+ }
+ return 0;
+ case KNOD_FEATURE_IPSEC:
+ if (!accel_ops.ipsec_ops)
+ return -ENODEV;
+ /* knod_ipsec_attach() runs ->activate() + sets netdev flags. */
+ return knod_ipsec_attach(knodev);
+ case KNOD_FEATURE_NONE:
+ return 0;
+ default:
+ return -EINVAL;
+ }
+}
+
+static int knod_accel_feature_get(struct knod_accel *accel,
+ u32 *ena, u32 *cap)
+{
+ struct knod *knod = READ_ONCE(accel->priv);
+ u32 mask = 0;
+ int i;
+
+ /* A registered-but-not-attached accel has no context yet. */
+ *ena = knod ? knod->active_feature : KNOD_FEATURE_NONE;
+ for (i = 0; i < KNOD_FEATURE_MAX; i++)
+ if (i == KNOD_FEATURE_NONE || knod_feature_ops(i))
+ mask |= BIT(i);
+ *cap = mask;
+ return 0;
+}
+
+static int knod_accel_feature_set(struct knod_accel *accel, u32 feature,
+ struct netlink_ext_ack *extack)
+{
+ struct knod *knod = READ_ONCE(accel->priv);
+ struct knod_dev *knodev;
+ bool started;
+ int err = 0;
+
+ if (feature >= KNOD_FEATURE_MAX) {
+ NL_SET_ERR_MSG(extack, "unknown feature");
+ return -EINVAL;
+ }
+ if (feature != KNOD_FEATURE_NONE && !knod_feature_ops(feature)) {
+ NL_SET_ERR_MSG(extack, "feature not available on this accelerator");
+ return -ENOENT;
+ }
+ knodev = knod ? READ_ONCE(knod->accel->knodev) : NULL;
+ if (!knod || !knodev) {
+ NL_SET_ERR_MSG(extack, "accelerator not attached to a NIC");
+ return -ENODEV;
+ }
+ if (feature == knod->active_feature)
+ return 0;
+
+ /*
+ * Refuse to leave BPF while a user XDP prog or offloaded map is still
+ * bound - tearing the offload down underneath them is unsafe (the GPU
+ * dispatch keeps running against freed state). The user must detach
+ * first, e.g. "ip link set dev <if> xdp off".
+ */
+ if (knod->active_feature == KNOD_FEATURE_BPF && registered_xdp_ops &&
+ registered_xdp_ops->busy && registered_xdp_ops->busy(knodev)) {
+ NL_SET_ERR_MSG(extack, "detach the XDP program/maps first");
+ return -EBUSY;
+ }
+ if (knod->active_feature == KNOD_FEATURE_IPSEC && accel_ops.ipsec_ops &&
+ accel_ops.ipsec_ops->busy && accel_ops.ipsec_ops->busy(knodev)) {
+ NL_SET_ERR_MSG(extack, "remove the offloaded xfrm SAs first");
+ return -EBUSY;
+ }
+
+ /*
+ * stop -> change -> start. The worker only runs while the interface
+ * is up; bracket the resource change with worker stop/start in that
+ * case. stop() drains the GPU in-flight so deactivate() frees safely.
+ */
+ started = READ_ONCE(knodev->started);
+ if (started)
+ knod_feature_stop(knod);
+ knod_feature_deactivate(knod);
+ knod->active_feature = KNOD_FEATURE_NONE;
+
+ if (feature != KNOD_FEATURE_NONE) {
+ knod->active_feature = feature;
+ err = knod_feature_activate(knod);
+ if (err) {
+ knod->active_feature = KNOD_FEATURE_NONE;
+ NL_SET_ERR_MSG(extack, "failed to activate feature");
+ }
+ }
+ if (started)
+ knod_feature_start(knod);
+ return err;
+}
+
+static int knod_attach(struct knod_dev *knodev)
+{
+ struct knod_accel *accel = knodev->accel;
+ int render_idx = accel->id / KNOD_MAX_AQL;
+ struct net_device *netdev = knodev->netdev;
+ struct amdgpu_device *adev;
+ struct knod *knod;
+
+ {
+ int q_cnt = clamp(READ_ONCE(knod_requested_queue_cnt), 1,
+ KNOD_MAX_QUEUE_CNT);
+
+ knod = knod_alloc_ctx(knodev, q_cnt, render_idx,
+ min(netdev->num_rx_queues,
+ KNOD_SPSC_MAX));
+ }
+ if (IS_ERR(knod)) {
+ pr_err("knod: Failed to allocate context\n");
+ return -EINVAL;
+ }
+
+ accel->priv = knod;
+ knod->accel = accel;
+
+ /*
+ * Keep GFX engine out of GFXOFF while a KNOD accel is attached.
+ * On RDNA2 (tested RX6600 / gfx1032) the very first AQL dispatch
+ * after boot hangs with signal=-1 when GFXOFF is active - SMU exit
+ * from GFXOFF races with the doorbell ring and the completion
+ * signal is never decremented. Forcing GFXOFF off for the attached
+ * lifetime avoids the race entirely.
+ *
+ * Pin MCLK soft-min to HW max. KNOD's RX path is bandwidth-bound
+ * on VRAM (NIC->GPU p2pdma delivers frames directly to VRAM, then
+ * the shader streams them back out) but the individual dispatches
+ * are too short for SMU's activity monitor to react - MCLK sticks
+ * at the lowest DPM (~96 MHz on RX6600) in AUTO mode and caps
+ * throughput far below what the compute path can sustain. Switching
+ * to the COMPUTE power profile did not help on RDNA2: the COMPUTE
+ * DpmActivityMonitor coefficients tune GFX upclock aggressively
+ * but leave memory activity detection conservative.
+ *
+ * Setting soft_min via SetSoftMinByFreq is independent of
+ * pp_power_profile_mode and of power_dpm_force_performance_level,
+ * so AUTO governance stays in effect for SCLK/voltage - we get the
+ * same 30W full-throughput state the user reaches via "force
+ * performance = manual + echo 3 > pp_dpm_mclk", without the 75W
+ * voltage pin that PROFILE_PEAK imposes. Passing 0xFFFF MHz lets
+ * SMU firmware clamp to the actual hardware max.
+ */
+ adev = knod->process->pdds[0]->dev->adev;
+ amdgpu_gfx_off_ctrl_immediate(adev, false);
+ amdgpu_dpm_set_soft_freq_range(adev, PP_MCLK, 0xFFFF, 0xFFFF);
+
+ /*
+ * Attach settles in KNOD_FEATURE_NONE with no worker running. The
+ * worker is started by the NIC driver bringing the interface up
+ * (knod_dev_start -> ->dev_start), and each feature's GPU resources
+ * are allocated when the feature is selected (->activate).
+ */
+ knod->active_feature = KNOD_FEATURE_NONE;
+
+ /*
+ * Permanent per-attach feature state (e.g. the BPF bpf_offload_dev,
+ * which must outlive feature switches so user XDP progs/maps survive).
+ */
+ if (registered_xdp_ops && registered_xdp_ops->init)
+ registered_xdp_ops->init(knodev);
+ return 0;
+}
+
+static void knod_pre_detach(struct knod_dev *knodev)
+{
+ struct knod *knod = knodev->accel->priv;
+
+ /*
+ * Detach requires the interface down, so the worker is already
+ * stopped; stop again defensively, free the active feature's
+ * resources, then tear down the permanent per-attach state.
+ */
+ knod_feature_stop(knod);
+ knod_feature_deactivate(knod);
+ knod->active_feature = KNOD_FEATURE_NONE;
+
+ if (registered_xdp_ops && registered_xdp_ops->exit)
+ registered_xdp_ops->exit(knodev);
+}
+
+static void knod_dev_start_worker(struct knod_dev *knodev)
+{
+ struct knod *knod = knodev->accel->priv;
+
+ /* Interface up: start the current feature's worker. */
+ if (knod)
+ knod_feature_start(knod);
+}
+
+static void knod_dev_stop_worker(struct knod_dev *knodev)
+{
+ struct knod *knod = knodev->accel->priv;
+
+ /* Interface down: stop the worker + drain the GPU in-flight. */
+ if (knod)
+ knod_feature_stop(knod);
+}
+
+static void knod_detach(struct knod_dev *knodev)
+{
+ struct knod_accel *accel = knodev->accel;
+ struct knod *knod = accel->priv;
+ struct amdgpu_device *adev = knod->process->pdds[0]->dev->adev;
+
+ knod_stop_worker(knod);
+ knod_release_ctx(knod);
+ WRITE_ONCE(accel->priv, NULL);
+
+ /*
+ * Restore default MCLK range. min=1 triggers SetSoftMinByFreq (the
+ * API skips the call when min==0) and SMU clamps to HW min; max
+ * 0xFFFF clamps to HW max. Together this matches the pre-attach
+ * "no soft constraint" state so DPM can idle MCLK back down.
+ */
+ amdgpu_dpm_set_soft_freq_range(adev, PP_MCLK, 1, 0xFFFF);
+ amdgpu_gfx_off_ctrl(adev, true);
+}
+
+static void *knod_accel_alloc_mem(struct knod_dev *knodev, size_t size,
+ u64 *gaddr, struct page ***pages, void **priv)
+{
+ struct knod_accel *accel = knodev->accel;
+ struct knod *knod = accel->priv;
+ struct knod_mem *mem;
+ struct ttm_tt *tt;
+ u32 flags;
+
+ /* SPSC rings are hot producer/consumer control data. Keep them plain
+ * GTT; only host-read delivery buffers need coherent CPU visibility.
+ */
+ flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT | KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE;
+ if (pages)
+ flags |= KFD_IOC_ALLOC_MEM_FLAGS_COHERENT;
+
+ mem = knod_alloc_mem(knod, size, flags);
+ if (IS_ERR(mem))
+ return NULL;
+
+ if (pages) {
+ tt = mem->mem->bo ? mem->mem->bo->tbo.ttm : NULL;
+ if (!tt || !tt->pages) {
+ knod_free_mem(knod, mem);
+ return NULL;
+ }
+ *pages = tt->pages;
+ }
+
+ *gaddr = mem->gaddr;
+ *priv = mem;
+ return mem->kaddr;
+}
+
+static void knod_accel_free_mem(struct knod_dev *knodev, void *priv)
+{
+ struct knod_accel *accel = knodev->accel;
+ struct knod *knod = accel->priv;
+ struct knod_mem *mem = priv;
+
+ knod_free_mem(knod, mem);
+}
+
+/*
+ * Device->host copy primitives for the common knod_d2h_copy/knod_d2h_drain
+ * path. Thin wrappers over the SDMA engine (sdma[0]); the framework owns the
+ * pending ring, fence counter and dst pool.
+ */
+static u32 knod_accel_d2h_submit(struct knod_dev *knodev, u64 dst, int queue,
+ u32 page_idx, u16 off, u32 len)
+{
+ struct knod *knod = knodev->accel->priv;
+ struct knod_sdma_copy_desc c;
+
+ /* The netmem page_pool dma_addr is the NIC address; SDMA needs
+ * the GPU VM address, so derive the source from buf[queue].
+ */
+ c.src = knod->buf[queue]->gaddr + ((u64)page_idx << PAGE_SHIFT) + off;
+ c.dst = dst;
+ c.len = len;
+ return knod_sdma_submit(knod, 0, &c, 1);
+}
+
+static void knod_accel_d2h_kick(struct knod_dev *knodev)
+{
+ knod_sdma_kick(knodev->accel->priv, 0);
+}
+
+static u32 knod_accel_d2h_fence(struct knod_dev *knodev, int sdma_idx)
+{
+ struct knod *knod = READ_ONCE(knodev->accel->priv);
+ struct knod_mem *signal;
+
+ if (!knod || sdma_idx < 0 || sdma_idx >= READ_ONCE(knod->sdma_cnt))
+ return 0;
+ signal = READ_ONCE(knod->sdma[sdma_idx].queue_signal);
+ if (!signal || !signal->kaddr)
+ return 0;
+
+ return (u32)READ_ONCE(((struct amd_signal *)signal->kaddr)->value);
+}
+
+static int knod_accel_mp_map(struct knod_dev *knodev)
+{
+ struct knod *knod = knodev->accel->priv;
+ int i;
+
+ if (!knod)
+ return -ENODEV;
+
+ for (i = 0; i < knod->channels; i++)
+ if (__knod_map_mem(knod, knod->buf[i]))
+ knod_err(" mp_map failed ch %d\n", i);
+
+ return 0;
+}
+
+static struct knod_accel_ops accel_ops = {
+ .attach = knod_attach,
+ .pre_detach = knod_pre_detach,
+ .detach = knod_detach,
+ .dev_start = knod_dev_start_worker,
+ .dev_stop = knod_dev_stop_worker,
+ .alloc_mem = knod_accel_alloc_mem,
+ .free_mem = knod_accel_free_mem,
+ .mp_map = knod_accel_mp_map,
+ .d2h_submit = knod_accel_d2h_submit,
+ .d2h_kick = knod_accel_d2h_kick,
+ .d2h_fence = knod_accel_d2h_fence,
+ .xdp_ops = &default_xdp_ops,
+ .feature_get = knod_accel_feature_get,
+ .feature_set = knod_accel_feature_set,
+};
+
+/*
+ * Tear a feature down on every accel still using it before its ops pointer
+ * is cleared (module unload). Mirrors the feature_set transition to NONE.
+ */
+static void knod_feature_force_none(enum knod_feature feat)
+{
+ int i;
+
+ for (i = 0; i < nr_accels; i++) {
+ struct knod_dev *knodev;
+ struct knod *knod;
+ bool started;
+
+ if (!accels[i])
+ continue;
+ knod = READ_ONCE(accels[i]->priv);
+ knodev = knod ? READ_ONCE(accels[i]->knodev) : NULL;
+ if (!knod || !knodev || knod->active_feature != feat)
+ continue;
+ started = READ_ONCE(knodev->started);
+ if (started)
+ knod_feature_stop(knod);
+ knod_feature_deactivate(knod);
+ knod->active_feature = KNOD_FEATURE_NONE;
+ if (started)
+ knod_feature_start(knod);
+ }
+}
+
+void knod_accel_xdp_register(struct knod_accel_xdp_ops *xdp_ops)
+{
+ int i;
+
+ /*
+ * Module load advertises the feature and sets up the permanent
+ * per-attach state (bpf_offload_dev) on already-attached accels;
+ * GPU compute resources wait for ->activate() on feature select.
+ */
+ WRITE_ONCE(registered_xdp_ops, xdp_ops);
+ for (i = 0; i < nr_accels; i++) {
+ struct knod_dev *knodev;
+
+ if (!accels[i])
+ continue;
+ knodev = READ_ONCE(accels[i]->knodev);
+ if (knodev && xdp_ops->init)
+ xdp_ops->init(knodev);
+ }
+}
+EXPORT_SYMBOL(knod_accel_xdp_register);
+
+void knod_accel_xdp_unregister(void)
+{
+ int i;
+
+ /* Force any active BPF feature off, then drop the permanent state. */
+ knod_feature_force_none(KNOD_FEATURE_BPF);
+ for (i = 0; i < nr_accels; i++) {
+ struct knod_dev *knodev;
+
+ if (!accels[i])
+ continue;
+ knodev = READ_ONCE(accels[i]->knodev);
+ if (knodev && registered_xdp_ops && registered_xdp_ops->exit)
+ registered_xdp_ops->exit(knodev);
+ }
+ WRITE_ONCE(registered_xdp_ops, NULL);
+}
+EXPORT_SYMBOL(knod_accel_xdp_unregister);
+
+void knod_accel_ipsec_register(struct knod_accel_ipsec_ops *ipsec_ops)
+{
+ /* Advertise only; resources are allocated by ->activate() on select. */
+ WRITE_ONCE(accel_ops.ipsec_ops, ipsec_ops);
+}
+EXPORT_SYMBOL(knod_accel_ipsec_register);
+
+void knod_accel_ipsec_unregister(void)
+{
+ knod_feature_force_none(KNOD_FEATURE_IPSEC);
+ WRITE_ONCE(accel_ops.ipsec_ops, NULL);
+}
+EXPORT_SYMBOL(knod_accel_ipsec_unregister);
+
+/*
+ * Accel modules call this from their module_init before NOD attach to
+ * request the minimum AQL/SDMA queue pair count their dispatcher
+ * infrastructure needs. The value is a high-water mark: multiple
+ * callers raise it but never lower it, so whichever module needs the
+ * most queues wins.
+ *
+ * Must be called before knod_attach() runs - i.e. before any
+ * `echo X,0 > /sys/kernel/debug/knod_dev/attach` - otherwise
+ * the already-created knod context keeps its old queue_cnt and the
+ * caller must unbind/rebind to pick up the new value.
+ *
+ * Clamped to [1, KNOD_MAX_QUEUE_CNT]. Values outside that range are ignored
+ * (the internal cap can grow later without ABI break).
+ */
+void knod_request_queue_cnt(int n)
+{
+ int cur;
+
+ if (n < 1 || n > KNOD_MAX_QUEUE_CNT)
+ return;
+
+ do {
+ cur = READ_ONCE(knod_requested_queue_cnt);
+ if (n <= cur)
+ return;
+ } while (cmpxchg(&knod_requested_queue_cnt, cur, n) != cur);
+}
+EXPORT_SYMBOL(knod_request_queue_cnt);
+
+int knod_init(struct amdgpu_device *adev)
+{
+ int base_id = adev_to_drm(adev)->render->index * KNOD_MAX_AQL;
+ struct knod_accel *accel;
+ int i, n;
+
+ n = KNOD_MAX_AQL;
+
+ for (i = 0; i < n; i++) {
+ accel = kzalloc_obj(struct knod_accel, GFP_KERNEL);
+ if (!accel) {
+ pr_err("knod: Failed to allocate accel node %d\n", i);
+ goto err;
+ }
+
+ INIT_LIST_HEAD(&accel->list);
+ accel->type = KNOD_TYPE_GPU;
+ accel->accel_ops = &accel_ops;
+ accel->owner = THIS_MODULE;
+ accel->id = base_id + i;
+ snprintf(accel->name, sizeof(accel->name), "amdgpu-%d", i);
+ knod_accel_register(accel);
+ accels[i] = accel;
+ }
+ nr_accels = n;
+
+ return 0;
+
+err:
+ while (i-- > 0) {
+ knod_accel_unregister(accels[i]);
+ kfree(accels[i]);
+ accels[i] = NULL;
+ }
+ return 0;
+}
+
+void knod_fini(struct amdgpu_device *adev)
+{
+ int base_id = adev_to_drm(adev)->render->index * KNOD_MAX_AQL;
+ int i;
+
+ for (i = 0; i < nr_accels; i++) {
+ if (!accels[i] ||
+ accels[i]->id / KNOD_MAX_AQL != base_id / KNOD_MAX_AQL)
+ continue;
+ knod_accel_unregister(accels[i]);
+ kfree(accels[i]);
+ accels[i] = NULL;
+ }
+}
+
+void knod_exit(void)
+{
+ struct knod *knod, *tmp;
+ int i;
+
+ for (i = 0; i < nr_accels; i++) {
+ if (!accels[i])
+ continue;
+ knod_accel_unregister(accels[i]);
+ kfree(accels[i]);
+ accels[i] = NULL;
+ }
+ nr_accels = 0;
+
+ list_for_each_entry_safe(knod, tmp, &ctx_list, list) {
+ knod_release_ctx(knod);
+ }
+}
diff --git a/drivers/gpu/drm/amd/amdkfd/knod/kfd_knod.h b/drivers/gpu/drm/amd/amdkfd/knod/kfd_knod.h
new file mode 100644
index 000000000000..a1eafb13a410
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdkfd/knod/kfd_knod.h
@@ -0,0 +1,270 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/* Copyright (c) 2021 Taehee Yoo <ap420073@gmail.com>
+ * Copyright (c) 2021 Hoyeon Lee <hoyeon.rhee@gmail.com>
+ */
+
+#ifndef KFD_KNOD_H_
+#define KFD_KNOD_H_
+#include <linux/kfd_ioctl.h>
+#include <net/knod.h>
+#include <linux/genalloc.h>
+#include "kfd_hsa.h"
+#include <linux/completion.h>
+
+/*
+ * knod_dbg() is a pr_debug(), so it is off by default and toggled with
+ * dynamic debug; knod_err() always fires.
+ */
+#define knod_dbg(fmt, ...) \
+ pr_debug("knod %s:%d" fmt, __func__, __LINE__, ##__VA_ARGS__)
+#define knod_err(fmt, ...) \
+ pr_err("knod %s:%d" fmt, __func__, __LINE__, ##__VA_ARGS__)
+
+struct page_pool;
+
+struct knod_mem {
+ struct list_head list;
+ struct kgd_mem *mem;
+ void *kaddr;
+ u32 flags;
+ u32 size;
+ u32 order;
+ u64 gaddr;
+};
+
+union knod_aql_rsrc1 {
+ struct {
+#if defined(__LITTLE_ENDIAN)
+ unsigned int base_address_hi : 16;
+ unsigned int stride : 14;
+ unsigned int cache_swizzle : 1;
+ unsigned int swizzle_enable : 1;
+#elif defined(__LITTLE_ENDIAN)
+ unsigned int swizzle_enable : 1;
+ unsigned int cache_swizzle : 1;
+ unsigned int stride : 14;
+ unsigned int base_address_hi : 16;
+#endif
+ };
+};
+
+struct knod_aql {
+ struct knod_mem *aql;
+ struct knod_mem *ctx;
+ struct knod_mem *queue;
+ struct knod_mem *scratch;
+ struct knod_mem *eop;
+ struct knod_mem *queue_signal;
+ struct knod_mem *tba;
+ struct knod_mem *tma;
+ struct knod_mem *amd_queue;
+ u64 *doorbell;
+ int idx;
+};
+
+struct knod_sdma {
+ struct knod_mem *sdma;
+ struct knod_mem *queue;
+ struct knod_mem *queue_signal;
+ u64 *doorbell;
+ int idx;
+};
+
+/* KFD event handle as knod tracks it: signal event id + its slot index. */
+struct knod_event {
+ u32 id;
+ u32 slot;
+};
+
+typedef int (*knod_worker_fn_t)(void *ctx);
+typedef void (*knod_flush_fn_t)(void *ctx);
+
+enum knod_feature {
+ KNOD_FEATURE_NONE = 0,
+ KNOD_FEATURE_BPF,
+ KNOD_FEATURE_IPSEC,
+ KNOD_FEATURE_MAX,
+};
+
+/*
+ * One accel per GPU: NIC:GPU is fixed 1:1 so a single NIC owns the whole
+ * device. The accel id is just the DRM render index (stride 1). Pipeline
+ * depth within the one accel is provided by KNOD_MAX_QUEUE_CNT HW queues,
+ * unrelated to this stride.
+ */
+#define KNOD_MAX_AQL 1
+
+/* Internal AQL/SDMA queue pairs per attached knod context. BPF can keep these
+ * queues in flight independently while the public accel-id ABI remains stable.
+ */
+#define KNOD_MAX_QUEUE_CNT 32
+
+#define NR_AQL_RING 16384
+#define AQL_STRUCT_SIZE 128
+struct knod {
+ struct list_head list;
+ struct list_head active_list;
+
+ struct gen_pool *pool;
+
+ struct hsa_kernel_dispatch_packet *dp;
+ pid_t umh_pid;
+ struct task_struct *umh_task;
+ u32 nr_aql_ring;
+ /* AQLs */
+ int queue_cnt;
+ int sdma_cnt;
+ int igpu;
+ int isa_version;
+ /* NAPIs */
+ int channels;
+ struct kfd_process *process;
+ struct mm_struct *mm;
+ struct kfd_node *dev;
+ struct file *drm_file;
+ void __iomem *doorbell_base;
+ u64 reserved_addr;
+ u64 limit_addr;
+ struct mutex lock;
+ struct hsa_event *event;
+ struct knod_mem *kernels[2]; /* dispatch slots: [0] default/pass, [1] BPF alt */
+ struct knod_mem *mailbox;
+ /* packet data path buf */
+ struct knod_mem **buf;
+
+ u32 signal_eid;
+ u32 completion_eid;
+ struct knod_aql kaql[NR_CPUS];
+ struct knod_sdma sdma[NR_CPUS];
+ struct knod_event aql_event[NR_CPUS];
+ struct knod_event sdma_event[NR_CPUS];
+ u32 aql_queue_id[NR_CPUS];
+ u32 sdma_queue_id[NR_CPUS];
+ u64 aql_doorbell_offset[NR_CPUS];
+ u64 sdma_doorbell_offset[NR_CPUS];
+ bool aql_queue_created[NR_CPUS];
+ bool sdma_queue_created[NR_CPUS];
+ struct kfd_event_data *event_data;
+ struct knod_accel *accel;
+ struct dentry *debug_dir;
+ /* Worker callback - one active worker at a time */
+ enum knod_feature active_feature;
+ knod_worker_fn_t worker_fn;
+ knod_flush_fn_t flush_fn;
+ void *worker_ctx;
+ struct task_struct *worker;
+};
+
+struct knod_dispatch_params {
+ u16 workgroup_size_x;
+ u32 grid_size_x;
+ u32 grid_size_y;
+ u32 private_segment_size;
+ u32 group_segment_size;
+ u64 kernel_object;
+ u64 kernarg_address;
+};
+
+static inline void
+knod_setup_invalidate(struct knod *knod, int idx, int q_idx)
+{
+ struct hsa_kernel_dispatch_packet *dp = knod->kaql[q_idx].aql->kaddr;
+
+ dp += idx;
+ dp->header = HSA_PACKET_TYPE_INVALID << HSA_PACKET_HEADER_TYPE;
+}
+
+static inline void
+knod_setup_dispatch(struct knod *knod, int idx,
+ const struct knod_dispatch_params *p, int q_idx)
+{
+ struct hsa_kernel_dispatch_packet *dp = knod->kaql[q_idx].aql->kaddr;
+
+ dp += idx;
+ dp->setup = 2;
+ dp->workgroup_size_x = p->workgroup_size_x;
+ dp->workgroup_size_y = 1;
+ dp->workgroup_size_z = 1;
+ dp->grid_size_x = p->grid_size_x;
+ dp->grid_size_y = p->grid_size_y;
+ dp->grid_size_z = 1;
+ dp->private_segment_size = p->private_segment_size;
+ dp->group_segment_size = p->group_segment_size;
+ dp->kernel_object = p->kernel_object;
+ dp->kernarg_address = (void *)p->kernarg_address;
+ dp->completion_signal = knod->kaql[q_idx].queue_signal->gaddr;
+ /* publish the packet body before the valid header (WRITE_ONCE below) */
+ wmb();
+ WRITE_ONCE(dp->header,
+ (HSA_PACKET_TYPE_KERNEL_DISPATCH <<
+ HSA_PACKET_HEADER_TYPE) |
+ (HSA_FENCE_SCOPE_SYSTEM <<
+ HSA_PACKET_HEADER_ACQUIRE_FENCE_SCOPE) |
+ (HSA_FENCE_SCOPE_SYSTEM <<
+ HSA_PACKET_HEADER_RELEASE_FENCE_SCOPE));
+}
+
+static inline void
+knod_setup_header(struct knod *knod,
+ const struct knod_dispatch_params *p, int q_idx)
+{
+ struct amd_queue *amd_queue = (struct amd_queue *)knod->kaql[q_idx].amd_queue->kaddr;
+ int curr_idx = knod->kaql[q_idx].idx;
+ int next_idx = curr_idx + 1;
+ u64 *ptr = knod->kaql[q_idx].doorbell;
+
+ knod_setup_invalidate(knod, next_idx % knod->nr_aql_ring, q_idx);
+ knod_setup_dispatch(knod, curr_idx % knod->nr_aql_ring, p, q_idx);
+ WRITE_ONCE(amd_queue->write_dispatch_id, curr_idx);
+ writeq(curr_idx, ptr);
+ knod->kaql[q_idx].idx = next_idx;
+}
+
+#define KNOD_NR_AQL_DEFAULT 1
+struct knod *knod_alloc_ctx(struct knod_dev *knodev, int queue_cnt, int id,
+ int channels);
+void knod_release_ctx(struct knod *knod);
+void knod_accel_xdp_register(struct knod_accel_xdp_ops *xdp_ops);
+void knod_accel_xdp_unregister(void);
+void knod_accel_ipsec_register(struct knod_accel_ipsec_ops *ipsec_ops);
+void knod_accel_ipsec_unregister(void);
+void knod_request_queue_cnt(int n);
+struct knod_mem *knod_alloc_mem(struct knod *knod, size_t size, int flags);
+struct knod_mem *__knod_alloc_mem(struct knod *knod, size_t size, int flags);
+int __knod_map_mem(struct knod *knod, struct knod_mem *mem);
+int __knod_export_dma_buf(struct knod *knod, struct knod_mem *mem);
+int __knod_map_kaddr(struct knod *knod, struct knod_mem *mem);
+void knod_free_mem(struct knod *pknod, struct knod_mem *mem);
+void knod_sdma_copy(struct knod *knod, u64 dst_gart_addr, u64 src_gart_addr,
+ int idx, int size);
+void knod_sdma_fence(struct knod *knod, u64 fence_addr, u32 fence_val,
+ int idx);
+void knod_sdma_trap(struct knod *knod, int idx);
+void knod_sdma_doorbell(struct knod *knod, int idx);
+
+/* One linear GPU->host SDMA copy (GPU VM addresses). */
+struct knod_sdma_copy_desc {
+ u64 dst;
+ u64 src;
+ u32 len;
+};
+
+/*
+ * Emit @n copies on sdma[@idx] as one batch with ring backpressure.
+ * Returns the post-batch ring position to fence/await, or 0 if the ring
+ * is too full (caller drops the whole batch). Pair with knod_sdma_kick().
+ */
+u32 knod_sdma_submit(struct knod *knod, int idx,
+ const struct knod_sdma_copy_desc *copies, int n);
+void knod_sdma_kick(struct knod *knod, int idx);
+
+int knod_gart_map(struct amdgpu_device *adev, u64 npages,
+ dma_addr_t *addr, u64 *gart_addr, u64 flags);
+int knod_register_worker(struct knod *knod, knod_worker_fn_t fn,
+ knod_flush_fn_t flush, void *ctx);
+void knod_unregister_worker(struct knod *knod);
+int knod_wait_on_events(struct kfd_process *p, u32 num_events,
+ void __user *data, bool all, u32 *user_timeout_ms,
+ u32 *wait_result);
+
+#endif /* KFD_KNOD_H_ */
--
2.43.0
^ permalink raw reply related
* [RFC PATCH net-next 08/13] drm/amdkfd: add GPU instruction emitter and disassembler
From: Taehee Yoo @ 2026-07-19 17:58 UTC (permalink / raw)
To: Alex Deucher, Alexei Starovoitov, amd-gfx, Andrew Lunn,
Andrii Nakryiko, Bill Wendling, bpf, Christian König,
Daniel Borkmann, David Airlie, David S. Miller, Donald Hunter,
dri-devel, Eduard Zingerman, Emil Tsalapatis, Eric Dumazet,
Felix Kuehling, Hoyeon Lee, Ilias Apalodimas, Jakub Kicinski,
Jesper Dangaard Brouer, Jiri Olsa, John Fastabend, Justin Stitt,
Kees Cook, Kumar Kartikeya Dwivedi, Leon Romanovsky,
linaro-mm-sig, linux-hardening, linux-kernel, linux-kselftest,
linux-media, linux-rdma, llvm, Mark Bloch, Martin KaFai Lau,
Michael Chan, Nathan Chancellor, netdev, Nick Desaulniers,
Paolo Abeni, Pavan Chebbi, Saeed Mahameed, Shuah Khan,
Simona Vetter, Simon Horman, Song Liu, Stanislav Fomichev,
Sumit Semwal, Taehee Yoo, Tariq Toukan, Yonghong Song
In-Reply-To: <20260719175857.4071636-1-ap420073@gmail.com>
Add the AMD GCN (gfx9/gfx10) instruction encoder used to build the GPU
shaders that knod dispatches, plus a matching disassembler used for
debugging the generated code.
Signed-off-by: Taehee Yoo <ap420073@gmail.com>
(cherry picked from commit bbbe8531de11017f565a922b072d4aa5f99674fc)
---
drivers/gpu/drm/amd/amdkfd/knod/knod_amdgpu.h | 173 +
.../drm/amd/amdkfd/knod/knod_amdgpu_insn.h | 6362 +++++++++++++++++
.../gpu/drm/amd/amdkfd/knod/knod_gfx10_insn.h | 3978 +++++++++++
.../gpu/drm/amd/amdkfd/knod/knod_gfx9_insn.h | 4068 +++++++++++
4 files changed, 14581 insertions(+)
create mode 100644 drivers/gpu/drm/amd/amdkfd/knod/knod_amdgpu.h
create mode 100644 drivers/gpu/drm/amd/amdkfd/knod/knod_amdgpu_insn.h
create mode 100644 drivers/gpu/drm/amd/amdkfd/knod/knod_gfx10_insn.h
create mode 100644 drivers/gpu/drm/amd/amdkfd/knod/knod_gfx9_insn.h
diff --git a/drivers/gpu/drm/amd/amdkfd/knod/knod_amdgpu.h b/drivers/gpu/drm/amd/amdkfd/knod/knod_amdgpu.h
new file mode 100644
index 000000000000..77d1c6afdd0a
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdkfd/knod/knod_amdgpu.h
@@ -0,0 +1,173 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/* Copyright (c) 2021 Taehee Yoo <ap420073@gmail.com>
+ * Copyright (c) 2021 Hoyeon Lee <hoyeon.rhee@gmail.com>
+ */
+
+#ifndef KFD_AMDGPU_H_INCLUDED
+#define KFD_AMDGPU_H_INCLUDED
+
+#define KNOD_AMDGPU_REG_PAIR(x) ((x) / 2)
+
+enum amdgcn_param_type {
+ AMDGCN_PARAM_TYPE_SGPR,
+ AMDGCN_PARAM_TYPE_VCC_LO,
+ AMDGCN_PARAM_TYPE_VCC_HI,
+ AMDGCN_PARAM_TYPE_TTPM_BASE,
+ AMDGCN_PARAM_TYPE_M0,
+ AMDGCN_PARAM_TYPE_NULL,
+ AMDGCN_PARAM_TYPE_EXEC_LO,
+ AMDGCN_PARAM_TYPE_EXEC_HI,
+ AMDGCN_PARAM_TYPE_INTEGER_0,
+ AMDGCN_PARAM_TYPE_INTEGER_MINUS_1,
+ AMDGCN_PARAM_TYPE_SHARED_BASE,
+ AMDGCN_PARAM_TYPE_SHARED_LIMIT,
+ AMDGCN_PARAM_TYPE_PRIVATE_BASE,
+ AMDGCN_PARAM_TYPE_PRIVATE_LIMIT,
+ AMDGCN_PARAM_TYPE_POPS_EXITING_WAVE_ID,
+ AMDGCN_PARAM_TYPE_SDWA,
+ AMDGCN_PARAM_TYPE_DPP16,
+ AMDGCN_PARAM_TYPE_VCCZ,
+ AMDGCN_PARAM_TYPE_EXECZ,
+ AMDGCN_PARAM_TYPE_SCC,
+ AMDGCN_PARAM_TYPE_LITERAL_CONST,
+ AMDGCN_PARAM_TYPE_VGPR,
+ __AMDGCN_PARAM_TYPE_MAX,
+};
+
+struct amdgcn_param32 {
+ enum amdgcn_param_type type;
+ int v;
+};
+
+struct amdgcn_param64 {
+ struct amdgcn_param32 lo;
+ struct amdgcn_param32 hi;
+ u64 imm;
+};
+
+inline void knod_set(struct amdgcn_param32 *param,
+ enum amdgcn_param_type type, int v)
+{
+ param->type = type;
+ param->v = v;
+}
+
+inline void knod_vset32(struct amdgcn_param32 *param, int v)
+{
+ param->type = AMDGCN_PARAM_TYPE_VGPR;
+ param->v = v;
+}
+
+inline void knod_vset64(struct amdgcn_param64 *param, int v)
+{
+ param->lo.type = AMDGCN_PARAM_TYPE_VGPR;
+ param->lo.v = v;
+ param->hi.type = AMDGCN_PARAM_TYPE_VGPR;
+ param->hi.v = v + 1;
+}
+
+inline void knod_sset32(struct amdgcn_param32 *param, int v)
+{
+ param->type = AMDGCN_PARAM_TYPE_SGPR;
+ param->v = v;
+}
+
+inline void knod_sset64(struct amdgcn_param64 *param, int v)
+{
+ param->lo.type = AMDGCN_PARAM_TYPE_SGPR;
+ param->lo.v = v;
+ param->hi.type = AMDGCN_PARAM_TYPE_SGPR;
+ param->hi.v = v + 1;
+}
+
+inline void knod_iset32(struct amdgcn_param32 *param, int v)
+{
+ if (v > 64 || v < -16) {
+ param->type = AMDGCN_PARAM_TYPE_LITERAL_CONST;
+ param->v = v;
+ } else if (v >= 0) {
+ param->type = AMDGCN_PARAM_TYPE_INTEGER_0;
+ param->v = v;
+ } else {
+ param->type = AMDGCN_PARAM_TYPE_INTEGER_MINUS_1;
+ param->v = ~v;
+ }
+}
+
+inline void knod_iset64(struct amdgcn_param64 *param,
+ u64 v)
+{
+ int imm0 = v & ~0U;
+ int imm1 = v >> 32;
+
+ param->imm = v;
+
+ if (imm0 > 64 || imm0 < -16) {
+ param->lo.type = AMDGCN_PARAM_TYPE_LITERAL_CONST;
+ param->lo.v = imm0;
+ } else if (imm0 >= 0) {
+ param->lo.type = AMDGCN_PARAM_TYPE_INTEGER_0;
+ param->lo.v = imm0;
+ } else {
+ param->lo.type = AMDGCN_PARAM_TYPE_INTEGER_MINUS_1;
+ param->lo.v = ~imm0;
+ }
+
+ if (imm1 > 64 || imm1 < -16) {
+ param->hi.type = AMDGCN_PARAM_TYPE_LITERAL_CONST;
+ WARN_ON_ONCE(1);
+ param->hi.v = imm1;
+ } else if (imm1 >= 0) {
+ param->hi.type = AMDGCN_PARAM_TYPE_INTEGER_0;
+ param->hi.v = imm1;
+ } else {
+ param->hi.type = AMDGCN_PARAM_TYPE_INTEGER_MINUS_1;
+ param->hi.v = ~imm1;
+ }
+}
+
+inline void knod_lset32(struct amdgcn_param32 *param, int v)
+{
+ param->type = AMDGCN_PARAM_TYPE_LITERAL_CONST;
+ param->v = v;
+}
+
+inline void knod_vccset(struct amdgcn_param32 *param)
+{
+ param->type = AMDGCN_PARAM_TYPE_VCC_LO;
+ param->v = 0;
+}
+
+inline bool knod_param_is_literal(struct amdgcn_param32 param)
+{
+ return param.type == AMDGCN_PARAM_TYPE_LITERAL_CONST;
+}
+
+/* ======================================================================
+ * Param constructors - common to GFX9/GFX10 shader emitters.
+ *
+ * Usage: pass directly to emit_gfx{9,10}_* functions that take
+ * struct amdgcn_param32 operands.
+ * ======================================================================
+ */
+
+#define P_S(n) ((struct amdgcn_param32){ AMDGCN_PARAM_TYPE_SGPR, (n) })
+#define P_V(n) ((struct amdgcn_param32){ AMDGCN_PARAM_TYPE_VGPR, (n) })
+#define P_I(n) ((struct amdgcn_param32){ AMDGCN_PARAM_TYPE_INTEGER_0, (n) })
+#define P_L(v) ((struct amdgcn_param32){ AMDGCN_PARAM_TYPE_LITERAL_CONST, (v) })
+#define P_VCC ((struct amdgcn_param32){ AMDGCN_PARAM_TYPE_VCC_LO, 0 })
+#define P_EXEC ((struct amdgcn_param32){ AMDGCN_PARAM_TYPE_EXEC_LO, 0 })
+
+/* ======================================================================
+ * Branch patching - operates directly on u32 *buf
+ * ======================================================================
+ */
+
+static inline void patch_branch(u32 *buf, int patch_pos, int target_pos)
+{
+ int offset = target_pos - (patch_pos + 1);
+
+ buf[patch_pos] = (buf[patch_pos] & 0xFFFF0000u) | (offset & 0xFFFF);
+}
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdkfd/knod/knod_amdgpu_insn.h b/drivers/gpu/drm/amd/amdkfd/knod/knod_amdgpu_insn.h
new file mode 100644
index 000000000000..9703bf781ff5
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdkfd/knod/knod_amdgpu_insn.h
@@ -0,0 +1,6362 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/* Copyright (c) 2021 Taehee Yoo <ap420073@gmail.com>
+ * Copyright (c) 2021 Hoyeon Lee <hoyeon.rhee@gmail.com>
+ */
+
+#ifndef KFD_AMDGPU_INSN_H_INCLUDED
+#define KFD_AMDGPU_INSN_H_INCLUDED
+
+#include "knod_amdgpu.h"
+#include "knod_gfx10_insn.h"
+#include "knod_gfx9_insn.h"
+
+enum amdgcn_insn_type {
+ AMDGCN_INSN_TYPE_SOP2,
+ AMDGCN_INSN_TYPE_SOPK,
+ AMDGCN_INSN_TYPE_SOP1,
+ AMDGCN_INSN_TYPE_SOPC,
+ AMDGCN_INSN_TYPE_SOPP,
+ AMDGCN_INSN_TYPE_SMEM,
+ AMDGCN_INSN_TYPE_VOP2,
+ AMDGCN_INSN_TYPE_VOP1,
+ AMDGCN_INSN_TYPE_VOPC,
+ AMDGCN_INSN_TYPE_VOP3A,
+ AMDGCN_INSN_TYPE_VOP3B,
+ AMDGCN_INSN_TYPE_VOP3P,
+ AMDGCN_INSN_TYPE_SDWA,
+ AMDGCN_INSN_TYPE_SDWAB,
+ AMDGCN_INSN_TYPE_DPP16,
+ AMDGCN_INSN_TYPE_DPP8,
+ AMDGCN_INSN_TYPE_VINTRP,
+ AMDGCN_INSN_TYPE_DS,
+ AMDGCN_INSN_TYPE_MTBUF,
+ AMDGCN_INSN_TYPE_MUBUF,
+ AMDGCN_INSN_TYPE_MIMG,
+ AMDGCN_INSN_TYPE_FLAT,
+ AMDGCN_INSN_TYPE_EXP,
+ __AMDGCN_INSN_TYPE_MAX,
+};
+
+static const char opnames_gfx10[__AMDGCN_INSN_TYPE_MAX][900][30] = {
+ [AMDGCN_INSN_TYPE_VOP3A] = {
+ [GFX10_V_FMA_LEGACY_F32] = "v_fma_legacy_f32",
+ [GFX10_V_MAD_I32_I24] = "V_MAD_I32_I24",
+ [GFX10_V_MAD_U32_U24] = "v_mad_u32_u24",
+ [GFX10_V_CUBEID_F32] = "v_cubeid_f32",
+ [GFX10_V_CUBESC_F32] = "v_cubesc_f32",
+ [GFX10_V_CUBETC_F32] = "v_cubetc_f32",
+ [GFX10_V_CUBEMA_F32] = "v_cubema_f32",
+ [GFX10_V_BFE_U32] = "v_bfe_u32",
+ [GFX10_V_BFE_I32] = "v_bfe_i32",
+ [GFX10_V_BFI_B32] = "v_bfi_b32",
+ [GFX10_V_FMA_F32] = "v_fma_f32",
+ [GFX10_V_FMA_F64] = "v_fma_f64",
+ [GFX10_V_LERP_U8] = "v_lerp_u8",
+ [GFX10_V_ALIGNBIT_B32] = "v_alignbit_b32",
+ [GFX10_V_ALIGNBYTE_B32] = "v_alignbyte_b32",
+ [GFX10_V_MULLIT_F32] = "v_mullit_f32",
+ [GFX10_V_MIN3_F32] = "v_min3_f32",
+ [GFX10_V_MIN3_I32] = "v_min3_i32",
+ [GFX10_V_MIN3_U32] = "v_min3_u32",
+ [GFX10_V_MAX3_F32] = "v_max3_f32",
+ [GFX10_V_MAX3_I32] = "v_max3_i32",
+ [GFX10_V_MAX3_U32] = "v_max3_u32",
+ [GFX10_V_MED3_F32] = "v_med3_f32",
+ [GFX10_V_MED3_I32] = "v_med3_i32",
+ [GFX10_V_MED3_U32] = "v_med3_u32",
+ [GFX10_V_SAD_U8] = "v_sad_u8",
+ [GFX10_V_SAD_HI_U8] = "v_sad_hi_u8",
+ [GFX10_V_SAD_U16] = "v_sad_u16",
+ [GFX10_V_SAD_U32] = "v_sad_u32",
+ [GFX10_V_CVT_PK_U8_F32] = "v_cvt_pk_u8_f32",
+ [GFX10_V_DIV_FIXUP_F32] = "v_div_fixup_f32",
+ [GFX10_V_DIV_FIXUP_F64] = "v_div_fixup_f64",
+ [GFX10_V_ADD_F64] = "v_add_f64",
+ [GFX10_V_MUL_F64] = "v_mul_f64",
+ [GFX10_V_MIN_F64] = "v_min_f64",
+ [GFX10_V_MAX_F64] = "v_max_f64",
+ [GFX10_V_LDEXP_F64] = "v_ldexp_f64",
+ [GFX10_V_MUL_LO_U32] = "v_mul_lo_u32",
+ [GFX10_V_MUL_HI_U32] = "v_mul_hi_u32",
+ [GFX10_V_MUL_HI_I32] = "v_mul_hi_i32",
+ [GFX10_V_DIV_FMAS_F32] = "v_div_fmas_f32",
+ [GFX10_V_DIV_FMAS_F64] = "v_div_fmas_f64",
+ [GFX10_V_MSAD_U8] = "v_msad_u8",
+ [GFX10_V_QSAD_PK_U16_U8] = "v_qsad_pk_u16_u8",
+ [GFX10_V_MQSAD_PK_U16_U8] = "v_mqsad_pk_u16_u8",
+ [GFX10_V_TRIG_PREOP_F64] = "v_trig_preop_f64",
+ [GFX10_V_MQSAD_U32_U8] = "v_mqsad_u32_u8",
+ [GFX10_V_XOR3_B32] = "v_xor3_b32",
+ [GFX10_V_LSHLREV_B64] = "v_lshlrev_b64",
+ [GFX10_V_LSHRREV_B64] = "v_lshrrev_b64",
+ [GFX10_V_ASHRREV_I64] = "v_ashrrev_i64",
+ [GFX10_V_ADD_NC_U16] = "v_add_nc_u16",
+ [GFX10_V_SUB_NC_U16] = "v_sub_nc_u16",
+ [GFX10_V_MUL_LO_U16] = "v_mul_lo_u16",
+ [GFX10_V_LSHRREV_B16] = "v_lshrrev_b16",
+ [GFX10_V_ASHRREV_I16] = "v_ashrrev_i16",
+ [GFX10_V_MAX_U16] = "v_max_u16",
+ [GFX10_V_MAX_I16] = "v_max_i16",
+ [GFX10_V_MIN_U16] = "v_min_u16",
+ [GFX10_V_MIN_I16] = "v_min_i16",
+ [GFX10_V_ADD_NC_I16] = "v_add_nc_i16",
+ [GFX10_V_SUB_NC_I16] = "v_sub_nc_i16",
+ [GFX10_V_PACK_B32_F16] = "v_pack_b32_f16",
+ [GFX10_V_CVT_PKNORM_I16_F16] = "v_cvt_pknorm_i16_f16",
+ [GFX10_V_CVT_PKNORM_U16_F16] = "v_cvt_pknorm_u16_f16",
+ [GFX10_V_LSHLREV_B16] = "v_lshlrev_b16",
+ [GFX10_V_MAD_U16] = "v_mad_u16",
+ [GFX10_V_INTERP_P1LL_F16] = "v_interp_p1ll_f16",
+ [GFX10_V_INTERP_P1LV_F16] = "v_interp_p1lv_f16",
+ [GFX10_V_PERM_B32] = "v_perm_b32",
+ [GFX10_V_XAD_U32] = "v_xad_u32",
+ [GFX10_V_LSHL_ADD_U32] = "v_lshl_add_u32",
+ [GFX10_V_ADD_LSHL_U32] = "v_add_lshl_u32",
+ [GFX10_V_FMA_F16] = "v_fma_f16",
+ [GFX10_V_MIN3_F16] = "v_min3_f16",
+ [GFX10_V_MIN3_I16] = "v_min3_i16",
+ [GFX10_V_MIN3_U16] = "v_min3_u16",
+ [GFX10_V_MAX3_F16] = "v_max3_f16",
+ [GFX10_V_MAX3_I16] = "v_max3_i16",
+ [GFX10_V_MAX3_U16] = "v_max3_u16",
+ [GFX10_V_MED3_F16] = "v_med3_f16",
+ [GFX10_V_MED3_I16] = "v_med3_i16",
+ [GFX10_V_MED3_U16] = "v_med3_u16",
+ [GFX10_V_INTERP_P2_F16] = "v_interp_p2_f16",
+ [GFX10_V_MAD_I16] = "v_mad_i16",
+ [GFX10_V_DIV_FIXUP_F16] = "v_div_fixup_f16",
+ [GFX10_V_READLANE_B32] = "v_readlane_b32",
+ [GFX10_V_WRITELANE_B32] = "v_writelane_b32",
+ [GFX10_V_LDEXP_F32] = "v_ldexp_f32",
+ [GFX10_V_BFM_B32] = "v_bfm_b32",
+ [GFX10_V_BCNT_U32_B32] = "v_bcnt_u32_b32",
+ [GFX10_V_MBCNT_LO_U32_B32] = "v_mbcnt_lo_u32_b32",
+ [GFX10_V_MBCNT_HI_U32_B32] = "v_mbcnt_hi_u32_b32",
+ [GFX10_V_CVT_PKNORM_I16_F32] = "v_cvt_pknorm_i16_f32",
+ [GFX10_V_CVT_PKNORM_U16_F32] = "v_cvt_pknorm_u16_f32",
+ [GFX10_V_CVT_PK_U16_U32] = "v_cvt_pk_u16_u32",
+ [GFX10_V_CVT_PK_I16_I32] = "v_cvt_pk_i16_i32",
+ [GFX10_V_ADD3_U32] = "v_add3_u32",
+ [GFX10_V_LSHL_OR_B32] = "v_lshl_or_b32",
+ [GFX10_V_AND_OR_B32] = "v_and_or_b32",
+ [GFX10_V_OR3_B32] = "v_or3_b32",
+ [GFX10_V_MAD_U32_U16] = "v_mad_u32_u16",
+ [GFX10_V_MAD_I32_I16] = "v_mad_i32_i16",
+ [GFX10_V_SUB_NC_I32] = "v_sub_nc_i32",
+ [GFX10_V_PERMLANE16_B32] = "v_permlane16_b32",
+ [GFX10_V_PERMLANEX16_B32] = "v_permlanex16_b32",
+ [GFX10_V_ADD_NC_I32] = "v_add_nc_i32",
+ },
+ [AMDGCN_INSN_TYPE_SOP2] = {
+ [GFX10_S_ADD_U32] = "s_add_u32",
+ [GFX10_S_SUB_U32] = "s_sub_u32",
+ [GFX10_S_ADD_I32] = "s_add_i32",
+ [GFX10_S_SUB_I32] = "s_sub_i32",
+ [GFX10_S_ADDC_U32] = "s_addc_u32",
+ [GFX10_S_SUBB_U32] = "s_subb_u32",
+ [GFX10_S_MIN_I32] = "s_min_i32",
+ [GFX10_S_MIN_U32] = "s_min_u32",
+ [GFX10_S_MAX_I32] = "s_max_i32",
+ [GFX10_S_MAX_U32] = "s_max_u32",
+ [GFX10_S_CSELECT_B32] = "s_cselect_b32",
+ [GFX10_S_CELECT_B64] = "s_celect_b64",
+ [GFX10_S_AND_B32] = "s_and_b32",
+ [GFX10_S_AND_B64] = "s_and_b64",
+ [GFX10_S_OR_B32] = "s_or_b32",
+ [GFX10_S_OR_B64] = "s_or_b64",
+ [GFX10_S_XOR_B32] = "s_xor_b32",
+ [GFX10_S_XOR_B64] = "s_xor_b64",
+ [GFX10_S_ANDN2_B32] = "s_andn2_b32",
+ [GFX10_S_ANDN2_B64] = "s_andn2_b64",
+ [GFX10_S_ORN2_B32] = "s_orn2_b32",
+ [GFX10_S_ORN2_B64] = "s_orn2_b64",
+ [GFX10_S_NAND_B32] = "s_nand_b32",
+ [GFX10_S_NAND_B64] = "s_nand_b64",
+ [GFX10_S_NOR_B32] = "s_nor_b32",
+ [GFX10_S_NOR_B64] = "s_nor_b64",
+ [GFX10_S_XNOR_B32] = "s_xnor_b32",
+ [GFX10_S_XNOR_B64] = "s_xnor_b64",
+ [GFX10_S_LSHL_B32] = "s_lshl_b32",
+ [GFX10_S_LSHL_B64] = "s_lshl_b64",
+ [GFX10_S_LSHR_B32] = "s_lshr_b32",
+ [GFX10_S_LSHR_B64] = "s_lshr_b64",
+ [GFX10_S_ASHR_I32] = "s_ashr_i32",
+ [GFX10_S_ASHR_I64] = "s_ashr_i64",
+ [GFX10_S_BFM_B32] = "s_bfm_b32",
+ [GFX10_S_BFM_B64] = "s_bfm_b64",
+ [GFX10_S_MUL_I32] = "s_mul_i32",
+ [GFX10_S_MUL_I64] = "s_mul_i64",
+ [GFX10_S_BFE_U32] = "s_bfe_u32",
+ [GFX10_S_BFE_I32] = "s_bfe_i32",
+ [GFX10_S_BFE_U64] = "s_bfe_u64",
+ [GFX10_S_ABSDIFF_I32] = "s_absdiff_i32",
+ [GFX10_S_LSHL1_ADD_U32] = "s_lshl1_add_u32",
+ [GFX10_S_LSHL2_ADD_U32] = "s_lshl2_add_u32",
+ [GFX10_S_LSHL3_ADD_U32] = "s_lshl3_add_u32",
+ [GFX10_S_LSHL4_ADD_U32] = "s_lshl4_add_u32",
+ [GFX10_S_PACK_LL_B32_B16] = "s_pack_ll_b32_b16",
+ [GFX10_S_PACK_LH_B32_B16] = "s_pack_lh_b32_b16",
+ [GFX10_S_MUL_HI_U32] = "s_mul_hi_u32",
+ [GFX10_S_MUL_HI_I32] = "s_mul_hi_i32",
+ },
+ [AMDGCN_INSN_TYPE_SOPK] = {
+ [GFX10_S_MOVK_I32] = "s_movk_i32",
+ [GFX10_S_VERSION] = "s_version",
+ [GFX10_S_CMOVK_I32] = "s_cmovk_i32",
+ [GFX10_S_CMPK_EQ_I32] = "s_cmpk_eq_i32",
+ [GFX10_S_CMPK_LG_I32] = "s_cmpk_lg_i32",
+ [GFX10_S_CMPK_GT_I32] = "s_cmpk_gt_i32",
+ [GFX10_S_CMPK_GE_I32] = "s_cmpk_ge_i32",
+ [GFX10_S_CMPK_LT_I32] = "s_cmpk_lt_i32",
+ [GFX10_S_CMPK_LE_I32] = "s_cmpk_le_i32",
+ [GFX10_S_CMPK_EQ_U32] = "s_cmpk_eq_u32",
+ [GFX10_S_CMPK_LG_U32] = "s_cmpk_lg_u32",
+ [GFX10_S_CMPK_GT_U32] = "s_cmpk_gt_u32",
+ [GFX10_S_CMPK_GE_U32] = "s_cmpk_ge_u32",
+ [GFX10_S_CMPK_LT_U32] = "s_cmpk_lt_u32",
+ [GFX10_S_CMPK_LE_U32] = "s_cmpk_le_u32",
+ [GFX10_S_ADDK_I32] = "s_addk_i32",
+ [GFX10_S_MULK_I32] = "s_mulk_i32",
+ [GFX10_S_GETREG_B32] = "s_getreg_b32",
+ [GFX10_S_SETREG_B32] = "s_setreg_b32",
+ [GFX10_S_SETREG_IMM32_B32] = "s_setreg_imm32_b32",
+ [GFX10_S_CALL_B64] = "s_call_b64",
+ [GFX10_S_WAITCNT_VSCNT] = "s_waitcnt_vscnt",
+ [GFX10_S_WAITCNT_VMCNT] = "s_waitcnt_vmcnt",
+ [GFX10_S_WAITCNT_EXPCNT] = "s_waitcnt_expcnt",
+ [GFX10_S_WAITCNT_LGKMCNT] = "s_waitcnt_lgkmcnt",
+ [GFX10_S_SUBVECTOR_LOOP_BEGIN] = "s_subvector_loop_begin",
+ [GFX10_S_SUBVECTOR_LOOP_END] = "s_subvector_loop_end",
+ },
+ [AMDGCN_INSN_TYPE_SOP1] = {
+ [GFX10_S_MOV_B32] = "s_mov_b32",
+ [GFX10_S_MOV_B64] = "s_mov_b64",
+ [GFX10_S_CMOV_B32] = "s_cmov_b32",
+ [GFX10_S_CMOV_B64] = "s_cmov_b64",
+ [GFX10_S_NOT_B32] = "s_not_b32",
+ [GFX10_S_NOT_B64] = "s_not_b64",
+ [GFX10_S_WQM_B32] = "s_wqm_b32",
+ [GFX10_S_WQM_B64] = "s_wqm_b64",
+ [GFX10_S_BREV_B32] = "s_brev_b32",
+ [GFX10_S_BREV_B64] = "s_brev_b64",
+ [GFX10_S_BCNT0_I32_B32] = "s_bcnt0_i32_b32",
+ [GFX10_S_BCNT0_I32_B64] = "s_bcnt0_i32_b64",
+ [GFX10_S_BCNT1_I32_B32] = "s_bcnt1_i32_b32",
+ [GFX10_S_BCNT1_I32_B64] = "s_bcnt1_i32_b64",
+ [GFX10_S_FF0_I32_B32] = "s_ff0_i32_b32",
+ [GFX10_S_FF0_I32_B64] = "s_ff0_i32_b64",
+ [GFX10_S_FF1_I32_B32] = "s_ff1_i32_b32",
+ [GFX10_S_FF1_I32_B64] = "s_ff1_i32_b64",
+ [GFX10_S_FLBIT_I32_B32] = "s_flbit_i32_b32",
+ [GFX10_S_FLBIT_I32_B64] = "s_flbit_i32_b64",
+ [GFX10_S_FLBIT_I32] = "s_flbit_i32",
+ [GFX10_S_FLBIT_I32_I64] = "s_flbit_i32_i64",
+ [GFX10_S_SEXT_I32_I8] = "s_sext_i32_i8",
+ [GFX10_S_SEXT_I32_I16] = "s_sext_i32_i16",
+ [GFX10_S_BITSET0_B32] = "s_bitset0_b32",
+ [GFX10_S_BITSET0_B64] = "s_bitset0_b64",
+ [GFX10_S_BITSET1_B32] = "s_bitset1_b32",
+ [GFX10_S_BITSET1_B64] = "s_bitset1_b64",
+ [GFX10_S_GETPC_B64] = "s_getpc_b64",
+ [GFX10_S_SETPC_B64] = "s_setpc_b64",
+ [GFX10_S_SWAPPC_B64] = "s_swappc_b64",
+ [GFX10_S_RFE_B64] = "s_rfe_b64",
+ [GFX10_S_AND_SAVEEXEC_B64] = "s_and_saveexec_b64",
+ [GFX10_S_XNOR_SAVEEXEC_B64] = "s_xnor_saveexec_b64",
+ [GFX10_S_QUADMASK_B32] = "s_quadmask_b32",
+ [GFX10_S_QUADMASK_B64] = "s_quadmask_b64",
+ [GFX10_S_MOVRELS_B32] = "s_movrels_b32",
+ [GFX10_S_MOVRELS_B64] = "s_movrels_b64",
+ [GFX10_S_MOVRELD_B32] = "s_movreld_b32",
+ [GFX10_S_MOVRELD_B64] = "s_movreld_b64",
+ [GFX10_S_ABS_I32] = "s_abs_i32",
+ [GFX10_S_ANDN1_SAVEEXEC_B64] = "s_andn1_saveexec_b64",
+ [GFX10_S_ORN1_SAVEEXEC_B64] = "s_orn1_saveexec_b64",
+ [GFX10_S_ANDN1_WREXEC_B64] = "s_andn1_wrexec_b64",
+ [GFX10_S_ANDN2_WREXEC_B64] = "s_andn2_wrexec_b64",
+ [GFX10_S_BITREPLICATE_B64_B32] = "s_bitreplicate_b64_b32",
+ [GFX10_S_AND_SAVEEXEC_B32] = "s_and_saveexec_b32",
+ [GFX10_S_OR_SAVEEXEC_B32] = "s_or_saveexec_b32",
+ [GFX10_S_XOR_SAVEEXEC_B32] = "s_xor_saveexec_b32",
+ [GFX10_S_ANDN2_SAVEEXEC_B32] = "s_andn2_saveexec_b32",
+ [GFX10_S_ORN2_SAVEEXEC_B32] = "s_orn2_saveexec_b32",
+ [GFX10_S_NAND_SAVEEXEC_B32] = "s_nand_saveexec_b32",
+ [GFX10_S_NOR_SAVEEXEC_B32] = "s_nor_saveexec_b32",
+ [GFX10_S_XNOR_SAVEEXEC_B32] = "s_xnor_saveexec_b32",
+ [GFX10_S_ANDN1_SAVEEXEC_B32] = "s_andn1_saveexec_b32",
+ [GFX10_S_ORN1_SAVEEXEC_B32] = "s_orn1_saveexec_b32",
+ [GFX10_S_ANDN1_WREXEC_B32] = "s_andn1_wrexec_b32",
+ [GFX10_S_ANDN2_WREXEC_B32] = "s_andn2_wrexec_b32",
+ [GFX10_S_MOVRELSD_2_B32] = "s_movrelsd_2_b32",
+ },
+ [AMDGCN_INSN_TYPE_SOPC] = {
+ [GFX10_S_CMP_EQ_I32] = "s_cmp_eq_i32",
+ [GFX10_S_CMP_LG_I32] = "s_cmp_lg_i32",
+ [GFX10_S_CMP_GT_I32] = "s_cmp_gt_i32",
+ [GFX10_S_CMP_GE_I32] = "s_cmp_ge_i32",
+ [GFX10_S_CMP_LT_I32] = "s_cmp_lt_i32",
+ [GFX10_S_CMP_LE_I32] = "s_cmp_le_i32",
+ [GFX10_S_CMP_EQ_U32] = "s_cmp_eq_u32",
+ [GFX10_S_CMP_LG_U32] = "s_cmp_lg_u32",
+ [GFX10_S_CMP_GT_U32] = "s_cmp_gt_u32",
+ [GFX10_S_CMP_GE_U32] = "s_cmp_ge_u32",
+ [GFX10_S_CMP_LT_U32] = "s_cmp_lt_u32",
+ [GFX10_S_CMP_LE_U32] = "s_cmp_le_u32",
+ [GFX10_S_BITCMP0_B32] = "s_bitcmp0_b32",
+ [GFX10_S_BITCMP1_B32] = "s_bitcmp1_b32",
+ [GFX10_S_BITCMP0_B64] = "s_bitcmp0_b64",
+ [GFX10_S_BITCMP1_B64] = "s_bitcmp1_b64",
+ [GFX10_S_CMP_EQ_U64] = "s_cmp_eq_u64",
+ [GFX10_S_CMP_LG_U64] = "s_cmp_lg_u64",
+ },
+ [AMDGCN_INSN_TYPE_SOPP] = {
+ [GFX10_S_NOP] = "s_nop",
+ [GFX10_S_ENDPGM] = "s_endpgm",
+ [GFX10_S_BRANCH] = "s_branch",
+ [GFX10_S_WAKEUP] = "s_wakeup",
+ [GFX10_S_CBRANCH_SCC0] = "s_cbranch_scc0",
+ [GFX10_S_CBRANCH_SCC1] = "s_cbranch_scc1",
+ [GFX10_S_CBRANCH_VCCZ] = "s_cbranch_vccz",
+ [GFX10_S_CBRANCH_VCCNZ] = "s_cbranch_vccnz",
+ [GFX10_S_CBRANCH_EXECZ] = "s_cbranch_execz",
+ [GFX10_S_CBRANCH_EXECNZ] = "s_cbranch_execnz",
+ [GFX10_S_BARRIER] = "s_barrier",
+ [GFX10_S_SETKILL] = "s_setkill",
+ [GFX10_S_WAITCNT] = "s_waitcnt",
+ [GFX10_S_SETHALT] = "s_sethalt",
+ [GFX10_S_SLEEP] = "s_sleep",
+ [GFX10_S_SETPRIO] = "s_setprio",
+ [GFX10_S_SENDMSG] = "s_sendmsg",
+ [GFX10_S_SENDMSGHALT] = "s_sendmsghalt",
+ [GFX10_S_TRAP] = "s_trap",
+ [GFX10_S_ICACHE_INV] = "s_icache_inv",
+ [GFX10_S_INCPERFLEVEL] = "s_incperflevel",
+ [GFX10_S_DECPERFLEVEL] = "s_decperflevel",
+ [GFX10_S_TTRACEDATA] = "s_ttracedata",
+ [GFX10_S_CBRANCH_CDBGSYS] = "s_cbranch_cdbgsys",
+ [GFX10_S_CBRANCH_CDBGUSER] = "s_cbranch_cdbguser",
+ [GFX10_S_CBRANCH_CDBGSYS_OR_USER] = "s_cbranch_cdbgsys_or_user",
+ [GFX10_S_CBRANCH_CDBGSYS_AND_USER] = "s_cbranch_cdbgsys_and_user",
+ [GFX10_S_ENDPGM_SAVED] = "s_endpgm_saved",
+ [GFX10_S_ENDPGM_ORDERED_PS_DONE] = "s_endpgm_ordered_ps_done",
+ [GFX10_S_CODE_END] = "s_code_end",
+ [GFX10_S_INST_PREFETCH] = "s_inst_prefetch",
+ [GFX10_S_CLAUSE] = "s_clause",
+ [GFX10_S_WAITCNT_DEPCTR] = "s_waitcnt_depctr",
+ [GFX10_S_ROUND_MODE] = "s_round_mode",
+ [GFX10_S_DENORM_MODE] = "s_denorm_mode",
+ [GFX10_S_TTRACEDATA_IMM] = "s_ttracedata_imm",
+ },
+ [AMDGCN_INSN_TYPE_SMEM] = {
+ [GFX10_S_LOAD_DWORD] = "s_load_dword",
+ [GFX10_S_LOAD_DWORDX2] = "s_load_dwordx2",
+ [GFX10_S_LOAD_DWORDX4] = "s_load_dwordx4",
+ [GFX10_S_LOAD_DWORDX8] = "s_load_dwordx8",
+ [GFX10_S_LOAD_DWORDX16] = "s_load_dwordx16",
+ [GFX10_S_BUFFER_LOAD_DWORD] = "s_buffer_load_dword",
+ [GFX10_S_BUFFER_LOAD_DWORDX2] = "s_buffer_load_dwordx2",
+ [GFX10_S_BUFFER_LOAD_DWORDX4] = "s_buffer_load_dwordx4",
+ [GFX10_S_BUFFER_LOAD_DWORDX8] = "s_buffer_load_dwordx8",
+ [GFX10_S_BUFFER_LOAD_DWORDX16] = "s_buffer_load_dwordx16",
+ [GFX10_S_GL1_INV] = "s_gl1_inv",
+ [GFX10_S_DCACHE_INV] = "s_dcache_inv",
+ [GFX10_S_MEMTIME] = "s_memtime",
+ [GFX10_S_MEMREALTIME] = "s_memrealtime",
+ [GFX10_S_ATC_PROBE] = "s_atc_probe",
+ [GFX10_S_ATC_PROBE_BUFFER] = "s_atc_probe_buffer",
+ },
+ [AMDGCN_INSN_TYPE_VOP2] = {
+ [GFX10_V_CNDMASK_B32] = "v_cndmask_b32",
+ [GFX10_V_DOT2C_F32_F16] = "v_dot2c_f32_f16",
+ [GFX10_V_ADD_F32] = "v_add_f32",
+ [GFX10_V_SUB_F32] = "v_sub_f32",
+ [GFX10_V_SUBREV_F32] = "v_subrev_f32",
+ [GFX10_V_FMAC_LEGACY_F32] = "v_fmac_legacy_f32",
+ [GFX10_V_MUL_LEGACY_F32] = "v_mul_legacy_f32",
+ [GFX10_V_MUL_F32] = "v_mul_f32",
+ [GFX10_V_MUL_I32_I24] = "v_mul_i32_i24",
+ [GFX10_V_MUL_HI_I32_I24] = "v_mul_hi_i32_i24",
+ [GFX10_V_MUL_U32_U24] = "v_mul_u32_u24",
+ [GFX10_V_MUL_HI_U32_U24] = "v_mul_hi_u32_u24",
+ [GFX10_V_DOT4C_I32_I8] = "v_dot4c_i32_i8",
+ [GFX10_V_MIN_F32] = "v_min_f32",
+ [GFX10_V_MAX_F32] = "v_max_f32",
+ [GFX10_V_MIN_I32] = "v_min_i32",
+ [GFX10_V_MAX_I32] = "v_max_i32",
+ [GFX10_V_MIN_U32] = "v_min_u32",
+ [GFX10_V_MAX_U32] = "v_max_u32",
+ [GFX10_V_LSHRREV_B32] = "v_lshrrev_b32",
+ [GFX10_V_ASHRREV_I32] = "v_ashrrev_i32",
+ [GFX10_V_LSHLREV_B32] = "v_lshlrev_b32",
+ [GFX10_V_AND_B32] = "v_and_b32",
+ [GFX10_V_OR_B32] = "v_or_b32",
+ [GFX10_V_XOR_B32] = "v_xor_b32",
+ [GFX10_V_XNOR_B32] = "v_xnor_b32",
+ [GFX10_V_ADD_NC_U32] = "v_add_nc_u32",
+ [GFX10_V_SUB_NC_U32] = "v_sub_nc_u32",
+ [GFX10_V_SUBREV_NC_U32] = "v_subrev_nc_u32",
+ [GFX10_V_ADD_CO_CI_U32] = "v_add_co_ci_u32",
+ [GFX10_V_SUB_CO_CI_U32] = "v_sub_co_ci_u32",
+ [GFX10_V_SUBREV_CO_CI_U32] = "v_subrev_co_ci_u32",
+ [GFX10_V_FMAC_F32] = "v_fmac_f32",
+ [GFX10_V_FMAMK_F32] = "v_fmamk_f32",
+ [GFX10_V_FMAAK_F32] = "v_fmaak_f32",
+ [GFX10_V_CVT_PKRTZ_F16_F32] = "v_cvt_pkrtz_f16_f32",
+ [GFX10_V_ADD_F16] = "v_add_f16",
+ [GFX10_V_SUB_F16] = "v_sub_f16",
+ [GFX10_V_SUBREV_F16] = "v_subrev_f16",
+ [GFX10_V_MUL_F16] = "v_mul_f16",
+ [GFX10_V_FMAC_F16] = "v_fmac_f16",
+ [GFX10_V_FMAMK_F16] = "v_fmamk_f16",
+ [GFX10_V_FMAAK_F16] = "v_fmaak_f16",
+ [GFX10_V_MAX_F16] = "v_max_f16",
+ [GFX10_V_MIN_F16] = "v_min_f16",
+ [GFX10_V_LDEXP_F16] = "v_ldexp_f16",
+ [GFX10_V_PK_FMAC_F16] = "v_pk_fmac_f16",
+ },
+ [AMDGCN_INSN_TYPE_VOP1] = {
+ [GFX10_V_NOP] = "v_nop",
+ [GFX10_V_MOV_B32] = "v_mov_b32",
+ [GFX10_V_READFIRSTLANE_B32] = "v_readfirstlane_b32",
+ [GFX10_V_CVT_I32_F64] = "v_cvt_i32_f64",
+ [GFX10_V_CVT_F64_I32] = "v_cvt_f64_i32",
+ [GFX10_V_CVT_F32_I32] = "v_cvt_f32_i32",
+ [GFX10_V_CVT_F32_U32] = "v_cvt_f32_u32",
+ [GFX10_V_CVT_U32_F32] = "v_cvt_u32_f32",
+ [GFX10_V_CVT_I32_F32] = "v_cvt_i32_f32",
+ [GFX10_V_CVT_F16_F32] = "v_cvt_f16_f32",
+ [GFX10_V_CVT_F32_F16] = "v_cvt_f32_f16",
+ [GFX10_V_CVT_RPI_I32_F32] = "v_cvt_rpi_i32_f32",
+ [GFX10_V_CVT_FLR_I32_F32] = "v_cvt_flr_i32_f32",
+ [GFX10_V_CVT_OFF_F32_I4] = "v_cvt_off_f32_i4",
+ [GFX10_V_CVT_F32_F64] = "v_cvt_f32_f64",
+ [GFX10_V_CVT_F64_F32] = "v_cvt_f64_f32",
+ [GFX10_V_CVT_F32_UBYTE0] = "v_cvt_f32_ubyte0",
+ [GFX10_V_CVT_F32_UBYTE1] = "v_cvt_f32_ubyte1",
+ [GFX10_V_CVT_F32_UBYTE2] = "v_cvt_f32_ubyte2",
+ [GFX10_V_CVT_F32_UBYTE3] = "v_cvt_f32_ubyte3",
+ [GFX10_V_CVT_U32_F64] = "v_cvt_u32_f64",
+ [GFX10_V_CVT_F64_U32] = "v_cvt_f64_u32",
+ [GFX10_V_TRUNC_F64] = "v_trunc_f64",
+ [GFX10_V_CEIL_F64] = "v_ceil_f64",
+ [GFX10_V_RNDNE_F64] = "v_rndne_f64",
+ [GFX10_V_FLOOR_F64] = "v_floor_f64",
+ [GFX10_V_PIPEFLUSH] = "v_pipeflush",
+ [GFX10_V_FRACT_F32] = "v_fract_f32",
+ [GFX10_V_TRUNC_F32] = "v_trunc_f32",
+ [GFX10_V_CEIL_F32] = "v_ceil_f32",
+ [GFX10_V_RNDNE_F32] = "v_rndne_f32",
+ [GFX10_V_FLOOR_F32] = "v_floor_f32",
+ [GFX10_V_EXP_F32] = "v_exp_f32",
+ [GFX10_V_LOG_F32] = "v_log_f32",
+ [GFX10_V_RCP_F32] = "v_rcp_f32",
+ [GFX10_V_RCP_IFLAG_F32] = "v_rcp_iflag_f32",
+ [GFX10_V_RSQ_F32] = "v_rsq_f32",
+ [GFX10_V_RCP_F64] = "v_rcp_f64",
+ [GFX10_V_RSQ_F64] = "v_rsq_f64",
+ [GFX10_V_SQRT_F32] = "v_sqrt_f32",
+ [GFX10_V_SQRT_F64] = "v_sqrt_f64",
+ [GFX10_V_SIN_F32] = "v_sin_f32",
+ [GFX10_V_COS_F32] = "v_cos_f32",
+ [GFX10_V_NOT_B32] = "v_not_b32",
+ [GFX10_V_BFREV_B32] = "v_bfrev_b32",
+ [GFX10_V_FFBH_U32] = "v_ffbh_u32",
+ [GFX10_V_FFBL_B32] = "v_ffbl_b32",
+ [GFX10_V_FFBH_I32] = "v_ffbh_i32",
+ [GFX10_V_FREXP_EXP_I32_F64] = "v_frexp_exp_i32_f64",
+ [GFX10_V_FREXP_MANT_F64] = "v_frexp_mant_f64",
+ [GFX10_V_FRACT_F64] = "v_fract_f64",
+ [GFX10_V_FREXP_EXP_I32_F32] = "v_frexp_exp_i32_f32",
+ [GFX10_V_FREXP_MANT_F32] = "v_frexp_mant_f32",
+ [GFX10_V_CLREXCP] = "v_clrexcp",
+ [GFX10_V_MOVRELD_B32] = "v_movreld_b32",
+ [GFX10_V_MOVRELS_B32] = "v_movrels_b32",
+ [GFX10_V_MOVRELSD_B32] = "v_movrelsd_b32",
+ [GFX10_V_MOVRELSD_2_B32] = "v_movrelsd_2_b32",
+ [GFX10_V_CVT_F16_U16] = "v_cvt_f16_u16",
+ [GFX10_V_CVT_F16_I16] = "v_cvt_f16_i16",
+ [GFX10_V_CVT_U16_F16] = "v_cvt_u16_f16",
+ [GFX10_V_CVT_I16_F16] = "v_cvt_i16_f16",
+ [GFX10_V_RCP_F16] = "v_rcp_f16",
+ [GFX10_V_SQRT_F16] = "v_sqrt_f16",
+ [GFX10_V_RSQ_F16] = "v_rsq_f16",
+ [GFX10_V_LOG_F16] = "v_log_f16",
+ [GFX10_V_EXP_F16] = "v_exp_f16",
+ [GFX10_V_FREXP_MANT_F16] = "v_frexp_mant_f16",
+ [GFX10_V_FREXP_EXP_I16_F16] = "v_frexp_exp_i16_f16",
+ [GFX10_V_FLOOR_F16] = "v_floor_f16",
+ [GFX10_V_CEIL_F16] = "v_ceil_f16",
+ [GFX10_V_TRUNC_F16] = "v_trunc_f16",
+ [GFX10_V_RNDNE_F16] = "v_rndne_f16",
+ [GFX10_V_FRACT_F16] = "v_fract_f16",
+ [GFX10_V_SIN_F16] = "v_sin_f16",
+ [GFX10_V_COS_F16] = "v_cos_f16",
+ [GFX10_V_SAT_PK_U8_I16] = "v_sat_pk_u8_i16",
+ [GFX10_V_CVT_NORM_I16_F16] = "v_cvt_norm_i16_f16",
+ [GFX10_V_CVT_NORM_U16_F16] = "v_cvt_norm_u16_f16",
+ [GFX10_V_SWAP_B32] = "v_swap_b32",
+ [GFX10_V_SWAPREL_B32] = "v_swaprel_b32",
+ },
+ [AMDGCN_INSN_TYPE_VOPC] = {
+ [GFX10_V_CMP_F_F32] = "v_cmp_f_f32",
+ [GFX10_V_CMP_LT_F32] = "v_cmp_lt_f32",
+ [GFX10_V_CMP_EQ_F32] = "v_cmp_eq_f32",
+ [GFX10_V_CMP_LE_F32] = "v_cmp_le_f32",
+ [GFX10_V_CMP_GT_F32] = "v_cmp_gt_f32",
+ [GFX10_V_CMP_LG_F32] = "v_cmp_lg_f32",
+ [GFX10_V_CMP_GE_F32] = "v_cmp_ge_f32",
+ [GFX10_V_CMP_O_F32] = "v_cmp_o_f32",
+ [GFX10_V_CMP_U_F32] = "v_cmp_u_f32",
+ [GFX10_V_CMP_NGE_F32] = "v_cmp_nge_f32",
+ [GFX10_V_CMP_NLG_F32] = "v_cmp_nlg_f32",
+ [GFX10_V_CMP_NGT_F32] = "v_cmp_ngt_f32",
+ [GFX10_V_CMP_NLE_F32] = "v_cmp_nle_f32",
+ [GFX10_V_CMP_NEQ_F32] = "v_cmp_neq_f32",
+ [GFX10_V_CMP_NLT_F32] = "v_cmp_nlt_f32",
+ [GFX10_V_CMP_TRU_F32] = "v_cmp_tru_f32",
+ [GFX10_V_CMPX_F_F32] = "v_cmpx_f_f32",
+ [GFX10_V_CMPX_LT_F32] = "v_cmpx_lt_f32",
+ [GFX10_V_CMPX_EQ_F32] = "v_cmpx_eq_f32",
+ [GFX10_V_CMPX_LE_F32] = "v_cmpx_le_f32",
+ [GFX10_V_CMPX_GT_F32] = "v_cmpx_gt_f32",
+ [GFX10_V_CMPX_LG_F32] = "v_cmpx_lg_f32",
+ [GFX10_V_CMPX_GE_F32] = "v_cmpx_ge_f32",
+ [GFX10_V_CMPX_O_F32] = "v_cmpx_o_f32",
+ [GFX10_V_CMPX_U_F32] = "v_cmpx_u_f32",
+ [GFX10_V_CMPX_NGE_F32] = "v_cmpx_nge_f32",
+ [GFX10_V_CMPX_NLG_F32] = "v_cmpx_nlg_f32",
+ [GFX10_V_CMPX_NGT_F32] = "v_cmpx_ngt_f32",
+ [GFX10_V_CMPX_NLE_F32] = "v_cmpx_nle_f32",
+ [GFX10_V_CMPX_NEQ_F32] = "v_cmpx_neq_f32",
+ [GFX10_V_CMPX_NLT_F32] = "v_cmpx_nlt_f32",
+ [GFX10_V_CMPX_TRU_F32] = "v_cmpx_tru_f32",
+ [GFX10_V_CMP_F_F64] = "v_cmp_f_f64",
+ [GFX10_V_CMP_LT_F64] = "v_cmp_lt_f64",
+ [GFX10_V_CMP_EQ_F64] = "v_cmp_eq_f64",
+ [GFX10_V_CMP_LE_F64] = "v_cmp_le_f64",
+ [GFX10_V_CMP_GT_F64] = "v_cmp_gt_f64",
+ [GFX10_V_CMP_LG_F64] = "v_cmp_lg_f64",
+ [GFX10_V_CMP_GE_F64] = "v_cmp_ge_f64",
+ [GFX10_V_CMP_O_F64] = "v_cmp_o_f64",
+ [GFX10_V_CMP_U_F64] = "v_cmp_u_f64",
+ [GFX10_V_CMP_NGE_F64] = "v_cmp_nge_f64",
+ [GFX10_V_CMP_NLG_F64] = "v_cmp_nlg_f64",
+ [GFX10_V_CMP_NGT_F64] = "v_cmp_ngt_f64",
+ [GFX10_V_CMP_NLE_F64] = "v_cmp_nle_f64",
+ [GFX10_V_CMP_NEQ_F64] = "v_cmp_neq_f64",
+ [GFX10_V_CMP_NLT_F64] = "v_cmp_nlt_f64",
+ [GFX10_V_CMP_TRU_F64] = "v_cmp_tru_f64",
+ [GFX10_V_CMPX_F_F64] = "v_cmpx_f_f64",
+ [GFX10_V_CMPX_LT_F64] = "v_cmpx_lt_f64",
+ [GFX10_V_CMPX_EQ_F64] = "v_cmpx_eq_f64",
+ [GFX10_V_CMPX_LE_F64] = "v_cmpx_le_f64",
+ [GFX10_V_CMPX_GT_F64] = "v_cmpx_gt_f64",
+ [GFX10_V_CMPX_LG_F64] = "v_cmpx_lg_f64",
+ [GFX10_V_CMPX_GE_F64] = "v_cmpx_ge_f64",
+ [GFX10_V_CMPX_O_F64] = "v_cmpx_o_f64",
+ [GFX10_V_CMPX_U_F64] = "v_cmpx_u_f64",
+ [GFX10_V_CMPX_NGE_F64] = "v_cmpx_nge_f64",
+ [GFX10_V_CMPX_NLG_F64] = "v_cmpx_nlg_f64",
+ [GFX10_V_CMPX_NGT_F64] = "v_cmpx_ngt_f64",
+ [GFX10_V_CMPX_NLE_F64] = "v_cmpx_nle_f64",
+ [GFX10_V_CMPX_NEQ_F64] = "v_cmpx_neq_f64",
+ [GFX10_V_CMPX_NLT_F64] = "v_cmpx_nlt_f64",
+ [GFX10_V_CMPX_TRU_F64] = "v_cmpx_tru_f64",
+ [GFX10_V_CMP_F_I32] = "v_cmp_f_i32",
+ [GFX10_V_CMP_LT_I32] = "v_cmp_lt_i32",
+ [GFX10_V_CMP_EQ_I32] = "v_cmp_eq_i32",
+ [GFX10_V_CMP_LE_I32] = "v_cmp_le_i32",
+ [GFX10_V_CMP_GT_I32] = "v_cmp_gt_i32",
+ [GFX10_V_CMP_NE_I32] = "v_cmp_ne_i32",
+ [GFX10_V_CMP_GE_I32] = "v_cmp_ge_i32",
+ [GFX10_V_CMP_T_I32] = "v_cmp_t_i32",
+ [GFX10_V_CMP_CLASS_F32] = "v_cmp_class_f32",
+ [GFX10_V_CMP_LT_I16] = "v_cmp_lt_i16",
+ [GFX10_V_CMP_EQ_I16] = "v_cmp_eq_i16",
+ [GFX10_V_CMP_LE_I16] = "v_cmp_le_i16",
+ [GFX10_V_CMP_GT_I16] = "v_cmp_gt_i16",
+ [GFX10_V_CMP_NE_I16] = "v_cmp_ne_i16",
+ [GFX10_V_CMP_GE_I16] = "v_cmp_ge_i16",
+ [GFX10_V_CMP_CLASS_F16] = "v_cmp_class_f16",
+ [GFX10_V_CMPX_F_I32] = "v_cmpx_f_i32",
+ [GFX10_V_CMPX_LT_I32] = "v_cmpx_lt_i32",
+ [GFX10_V_CMPX_EQ_I32] = "v_cmpx_eq_i32",
+ [GFX10_V_CMPX_LE_I32] = "v_cmpx_le_i32",
+ [GFX10_V_CMPX_GT_I32] = "v_cmpx_gt_i32",
+ [GFX10_V_CMPX_NE_I32] = "v_cmpx_ne_i32",
+ [GFX10_V_CMPX_GE_I32] = "v_cmpx_ge_i32",
+ [GFX10_V_CMPX_T_I32] = "v_cmpx_t_i32",
+ [GFX10_V_CMPX_CLASS_F32] = "v_cmpx_class_f32",
+ [GFX10_V_CMPX_LT_I16] = "v_cmpx_lt_i16",
+ [GFX10_V_CMPX_EQ_I16] = "v_cmpx_eq_i16",
+ [GFX10_V_CMPX_LE_I16] = "v_cmpx_le_i16",
+ [GFX10_V_CMPX_GT_I16] = "v_cmpx_gt_i16",
+ [GFX10_V_CMPX_NE_I16] = "v_cmpx_ne_i16",
+ [GFX10_V_CMPX_GE_I16] = "v_cmpx_ge_i16",
+ [GFX10_V_CMPX_CLASS_F16] = "v_cmpx_class_f16",
+ [GFX10_V_CMP_F_I64] = "v_cmp_f_i64",
+ [GFX10_V_CMP_LT_I64] = "v_cmp_lt_i64",
+ [GFX10_V_CMP_EQ_I64] = "v_cmp_eq_i64",
+ [GFX10_V_CMP_LE_I64] = "v_cmp_le_i64",
+ [GFX10_V_CMP_GT_I64] = "v_cmp_gt_i64",
+ [GFX10_V_CMP_NE_I64] = "v_cmp_ne_i64",
+ [GFX10_V_CMP_GE_I64] = "v_cmp_ge_i64",
+ [GFX10_V_CMP_T_I64] = "v_cmp_t_i64",
+ [GFX10_V_CMP_CLASS_F64] = "v_cmp_class_f64",
+ [GFX10_V_CMP_LT_U16] = "v_cmp_lt_u16",
+ [GFX10_V_CMP_EQ_U16] = "v_cmp_eq_u16",
+ [GFX10_V_CMP_LE_U16] = "v_cmp_le_u16",
+ [GFX10_V_CMP_GT_U16] = "v_cmp_gt_u16",
+ [GFX10_V_CMP_NE_U16] = "v_cmp_ne_u16",
+ [GFX10_V_CMP_GE_U16] = "v_cmp_ge_u16",
+ [GFX10_V_CMPX_F_I64] = "v_cmpx_f_i64",
+ [GFX10_V_CMPX_LT_I64] = "v_cmpx_lt_i64",
+ [GFX10_V_CMPX_EQ_I64] = "v_cmpx_eq_i64",
+ [GFX10_V_CMPX_LE_I64] = "v_cmpx_le_i64",
+ [GFX10_V_CMPX_GT_I64] = "v_cmpx_gt_i64",
+ [GFX10_V_CMPX_NE_I64] = "v_cmpx_ne_i64",
+ [GFX10_V_CMPX_GE_I64] = "v_cmpx_ge_i64",
+ [GFX10_V_CMPX_T_I64] = "v_cmpx_t_i64",
+ [GFX10_V_CMPX_CLASS_F64] = "v_cmpx_class_f64",
+ [GFX10_V_CMPX_LT_U16] = "v_cmpx_lt_u16",
+ [GFX10_V_CMPX_EQ_U16] = "v_cmpx_eq_u16",
+ [GFX10_V_CMPX_LE_U16] = "v_cmpx_le_u16",
+ [GFX10_V_CMPX_GT_U16] = "v_cmpx_gt_u16",
+ [GFX10_V_CMPX_NE_U16] = "v_cmpx_ne_u16",
+ [GFX10_V_CMPX_GE_U16] = "v_cmpx_ge_u16",
+ [GFX10_V_CMP_F_U32] = "v_cmp_f_u32",
+ [GFX10_V_CMP_LT_U32] = "v_cmp_lt_u32",
+ [GFX10_V_CMP_EQ_U32] = "v_cmp_eq_u32",
+ [GFX10_V_CMP_LE_U32] = "v_cmp_le_u32",
+ [GFX10_V_CMP_GT_U32] = "v_cmp_gt_u32",
+ [GFX10_V_CMP_NE_U32] = "v_cmp_ne_u32",
+ [GFX10_V_CMP_GE_U32] = "v_cmp_ge_u32",
+ [GFX10_V_CMP_T_U32] = "v_cmp_t_u32",
+ [GFX10_V_CMP_F_F16] = "v_cmp_f_f16",
+ [GFX10_V_CMP_LT_F16] = "v_cmp_lt_f16",
+ [GFX10_V_CMP_EQ_F16] = "v_cmp_eq_f16",
+ [GFX10_V_CMP_LE_F16] = "v_cmp_le_f16",
+ [GFX10_V_CMP_GT_F16] = "v_cmp_gt_f16",
+ [GFX10_V_CMP_LG_F16] = "v_cmp_lg_f16",
+ [GFX10_V_CMP_GE_F16] = "v_cmp_ge_f16",
+ [GFX10_V_CMP_O_F16] = "v_cmp_o_f16",
+ [GFX10_V_CMPX_F_U32] = "v_cmpx_f_u32",
+ [GFX10_V_CMPX_LT_U32] = "v_cmpx_lt_u32",
+ [GFX10_V_CMPX_EQ_U32] = "v_cmpx_eq_u32",
+ [GFX10_V_CMPX_LE_U32] = "v_cmpx_le_u32",
+ [GFX10_V_CMPX_GT_U32] = "v_cmpx_gt_u32",
+ [GFX10_V_CMPX_NE_U32] = "v_cmpx_ne_u32",
+ [GFX10_V_CMPX_GE_U32] = "v_cmpx_ge_u32",
+ [GFX10_V_CMPX_T_U32] = "v_cmpx_t_u32",
+ [GFX10_V_CMPX_F_F16] = "v_cmpx_f_f16",
+ [GFX10_V_CMPX_LT_F16] = "v_cmpx_lt_f16",
+ [GFX10_V_CMPX_EQ_F16] = "v_cmpx_eq_f16",
+ [GFX10_V_CMPX_LE_F16] = "v_cmpx_le_f16",
+ [GFX10_V_CMPX_GT_F16] = "v_cmpx_gt_f16",
+ [GFX10_V_CMPX_LG_F16] = "v_cmpx_lg_f16",
+ [GFX10_V_CMPX_GE_F16] = "v_cmpx_ge_f16",
+ [GFX10_V_CMPX_O_F16] = "v_cmpx_o_f16",
+ [GFX10_V_CMP_F_U64] = "v_cmp_f_u64",
+ [GFX10_V_CMP_LT_U64] = "v_cmp_lt_u64",
+ [GFX10_V_CMP_EQ_U64] = "v_cmp_eq_u64",
+ [GFX10_V_CMP_LE_U64] = "v_cmp_le_u64",
+ [GFX10_V_CMP_GT_U64] = "v_cmp_gt_u64",
+ [GFX10_V_CMP_NE_U64] = "v_cmp_ne_u64",
+ [GFX10_V_CMP_GE_U64] = "v_cmp_ge_u64",
+ [GFX10_V_CMP_T_U64] = "v_cmp_t_u64",
+ [GFX10_V_CMP_U_F16] = "v_cmp_u_f16",
+ [GFX10_V_CMP_NGE_F16] = "v_cmp_nge_f16",
+ [GFX10_V_CMP_NLG_F16] = "v_cmp_nlg_f16",
+ [GFX10_V_CMP_NGT_F16] = "v_cmp_ngt_f16",
+ [GFX10_V_CMP_NLE_F16] = "v_cmp_nle_f16",
+ [GFX10_V_CMP_NEQ_F16] = "v_cmp_neq_f16",
+ [GFX10_V_CMP_NLT_F16] = "v_cmp_nlt_f16",
+ [GFX10_V_CMP_TRU_F16] = "v_cmp_tru_f16",
+ [GFX10_V_CMPX_F_U64] = "v_cmpx_f_u64",
+ [GFX10_V_CMPX_LT_U64] = "v_cmpx_lt_u64",
+ [GFX10_V_CMPX_EQ_U64] = "v_cmpx_eq_u64",
+ [GFX10_V_CMPX_LE_U64] = "v_cmpx_le_u64",
+ [GFX10_V_CMPX_GT_U64] = "v_cmpx_gt_u64",
+ [GFX10_V_CMPX_NE_U64] = "v_cmpx_ne_u64",
+ [GFX10_V_CMPX_GE_U64] = "v_cmpx_ge_u64",
+ [GFX10_V_CMPX_T_U64] = "v_cmpx_t_u64",
+ [GFX10_V_CMPX_U_F16] = "v_cmpx_u_f16",
+ [GFX10_V_CMPX_NGE_F16] = "v_cmpx_nge_f16",
+ [GFX10_V_CMPX_NLG_F16] = "v_cmpx_nlg_f16",
+ [GFX10_V_CMPX_NGT_F16] = "v_cmpx_ngt_f16",
+ [GFX10_V_CMPX_NLE_F16] = "v_cmpx_nle_f16",
+ [GFX10_V_CMPX_NEQ_F16] = "v_cmpx_neq_f16",
+ [GFX10_V_CMPX_NLT_F16] = "v_cmpx_nlt_f16",
+ [GFX10_V_CMPX_TRU_F16] = "v_cmpx_tru_f16",
+ },
+ [AMDGCN_INSN_TYPE_VOP3B] = {
+ [GFX10_V_DIV_SCALE_F32] = "v_div_scale_f32",
+ [GFX10_V_DIV_SCALE_F64] = "v_div_scale_f64",
+ [GFX10_V_MAD_U64_U32] = "v_mad_u64_u32",
+ [GFX10_V_MAD_I64_I32] = "v_mad_i64_i32",
+ [GFX10_V_ADD_CO_U32] = "v_add_co_u32",
+ [GFX10_V_SUB_CO_U32] = "v_sub_co_u32",
+ [GFX10_V_SUBREV_CO_U32] = "v_subrev_co_u32",
+ },
+ [AMDGCN_INSN_TYPE_VOP3P] = {
+ [GFX10_V_PK_MAD_I16] = "v_pk_mad_i16",
+ [GFX10_V_PK_MUL_LO_U16] = "v_pk_mul_lo_u16",
+ [GFX10_V_PK_ADD_I16] = "v_pk_add_i16",
+ [GFX10_V_PK_SUB_I16] = "v_pk_sub_i16",
+ [GFX10_V_PK_LSHLREV_B16] = "v_pk_lshlrev_b16",
+ [GFX10_V_PK_LSHRREV_B16] = "v_pk_lshrrev_b16",
+ [GFX10_V_PK_ASHRREV_I16] = "v_pk_ashrrev_i16",
+ [GFX10_V_PK_MAX_I16] = "v_pk_max_i16",
+ [GFX10_V_PK_MIN_I16] = "v_pk_min_i16",
+ [GFX10_V_PK_MAD_U16] = "v_pk_mad_u16",
+ [GFX10_V_PK_ADD_U16] = "v_pk_add_u16",
+ [GFX10_V_PK_SUB_U16] = "v_pk_sub_u16",
+ [GFX10_V_PK_MAX_U16] = "v_pk_max_u16",
+ [GFX10_V_PK_MIN_U16] = "v_pk_min_u16",
+ [GFX10_V_PK_FMA_F16] = "v_pk_fma_f16",
+ [GFX10_V_PK_ADD_F16] = "v_pk_add_f16",
+ [GFX10_V_PK_MUL_F16] = "v_pk_mul_f16",
+ [GFX10_V_PK_MIN_F16] = "v_pk_min_f16",
+ [GFX10_V_PK_MAX_F16] = "v_pk_max_f16",
+ [GFX10_V_DOT2_F32_F16] = "v_dot2_f32_f16",
+ [GFX10_V_DOT2_I32_I16] = "v_dot2_i32_i16",
+ [GFX10_V_DOT2_U32_U16] = "v_dot2_u32_u16",
+ [GFX10_V_DOT4_I32_I8] = "v_dot4_i32_i8",
+ [GFX10_V_DOT4_U32_U8] = "v_dot4_u32_u8",
+ [GFX10_V_DOT8_I32_I4] = "v_dot8_i32_i4",
+ [GFX10_V_DOT8_U32_U4] = "v_dot8_u32_u4",
+ [GFX10_V_FMA_MIX_F32] = "v_fma_mix_f32",
+ [GFX10_V_FMA_MIXLO_F16] = "v_fma_mixlo_f16",
+ [GFX10_V_FMA_MIXHI_F16] = "v_fma_mixhi_f16",
+ },
+ [AMDGCN_INSN_TYPE_MUBUF] = {
+ [GFX10_BUFFER_LOAD_FORMAT_X] = "buffer_load_format_x",
+ [GFX10_BUFFER_LOAD_FORMAT_XY] = "buffer_load_format_xy",
+ [GFX10_BUFFER_LOAD_FORMAT_XYZ] = "buffer_load_format_xyz",
+ [GFX10_BUFFER_LOAD_FORMAT_XYZW] = "buffer_load_format_xyzw",
+ [GFX10_BUFFER_STORE_FORMAT_X] = "buffer_store_format_x",
+ [GFX10_BUFFER_STORE_FORMAT_XY] = "buffer_store_format_xy",
+ [GFX10_BUFFER_STORE_FORMAT_XYZ] = "buffer_store_format_xyz",
+ [GFX10_BUFFER_STORE_FORMAT_XYZW] = "buffer_store_format_xyzw",
+ [GFX10_BUFFER_LOAD_UBYTE] = "buffer_load_ubyte",
+ [GFX10_BUFFER_LOAD_SBYTE] = "buffer_load_sbyte",
+ [GFX10_BUFFER_LOAD_USHORT] = "buffer_load_ushort",
+ [GFX10_BUFFER_LOAD_SSHORT] = "buffer_load_sshort",
+ [GFX10_BUFFER_LOAD_DWORD] = "buffer_load_dword",
+ [GFX10_BUFFER_LOAD_DWORDX2] = "buffer_load_dwordx2",
+ [GFX10_BUFFER_LOAD_DWORDX4] = "buffer_load_dwordx4",
+ [GFX10_BUFFER_LOAD_DWORDX3] = "buffer_load_dwordx3",
+ [GFX10_BUFFER_STORE_BYTE] = "buffer_store_byte",
+ [GFX10_BUFFER_STORE_BYTE_D16_HI] = "buffer_store_byte_d16_hi",
+ [GFX10_BUFFER_STORE_SHORT] = "buffer_store_short",
+ [GFX10_BUFFER_STORE_SHORT_D16_HI] = "buffer_store_short_d16_hi",
+ [GFX10_BUFFER_STORE_DWORD] = "buffer_store_dword",
+ [GFX10_BUFFER_STORE_DWORDX2] = "buffer_store_dwordx2",
+ [GFX10_BUFFER_STORE_DWORDX4] = "buffer_store_dwordx4",
+ [GFX10_BUFFER_STORE_DWORDX3] = "buffer_store_dwordx3",
+ [GFX10_BUFFER_LOAD_UBYTE_D16] = "buffer_load_ubyte_d16",
+ [GFX10_BUFFER_LOAD_UBYTE_D16_HI] = "buffer_load_ubyte_d16_hi",
+ [GFX10_BUFFER_LOAD_SBYTE_D16] = "buffer_load_sbyte_d16",
+ [GFX10_BUFFER_LOAD_SBYTE_D16_HI] = "buffer_load_sbyte_d16_hi",
+ [GFX10_BUFFER_LOAD_SHORT_D16] = "buffer_load_short_d16",
+ [GFX10_BUFFER_LOAD_SHORT_D16_HI] = "buffer_load_short_d16_hi",
+ [GFX10_BUFFER_LOAD_FORMAT_D16_HI_X] = "buffer_load_format_d16_hi_x",
+ [GFX10_BUFFER_STORE_FORMAT_D16_HI_X] = "buffer_store_format_d16_hi_x",
+ [GFX10_BUFFER_ATOMIC_SWAP] = "buffer_atomic_swap",
+ [GFX10_BUFFER_ATOMIC_CMPSWAP] = "buffer_atomic_cmpswap",
+ [GFX10_BUFFER_ATOMIC_ADD] = "buffer_atomic_add",
+ [GFX10_BUFFER_ATOMIC_SUB] = "buffer_atomic_sub",
+ [GFX10_BUFFER_ATOMIC_CSUB] = "buffer_atomic_csub",
+ [GFX10_BUFFER_ATOMIC_SMIN] = "buffer_atomic_smin",
+ [GFX10_BUFFER_ATOMIC_UMIN] = "buffer_atomic_umin",
+ [GFX10_BUFFER_ATOMIC_SMAX] = "buffer_atomic_smax",
+ [GFX10_BUFFER_ATOMIC_UMAX] = "buffer_atomic_umax",
+ [GFX10_BUFFER_ATOMIC_AND] = "buffer_atomic_and",
+ [GFX10_BUFFER_ATOMIC_OR] = "buffer_atomic_or",
+ [GFX10_BUFFER_ATOMIC_XOR] = "buffer_atomic_xor",
+ [GFX10_BUFFER_ATOMIC_INC] = "buffer_atomic_inc",
+ [GFX10_BUFFER_ATOMIC_DEC] = "buffer_atomic_dec",
+ [GFX10_BUFFER_ATOMIC_FCMPSWAP] = "buffer_atomic_fcmpswap",
+ [GFX10_BUFFER_ATOMIC_FMIN] = "buffer_atomic_fmin",
+ [GFX10_BUFFER_ATOMIC_FMAX] = "buffer_atomic_fmax",
+ [GFX10_BUFFER_ATOMIC_SWAP_X2] = "buffer_atomic_swap_x2",
+ [GFX10_BUFFER_ATOMIC_CMPSWAP_X2] = "buffer_atomic_cmpswap_x2",
+ [GFX10_BUFFER_ATOMIC_ADD_X2] = "buffer_atomic_add_x2",
+ [GFX10_BUFFER_ATOMIC_SUB_X2] = "buffer_atomic_sub_x2",
+ [GFX10_BUFFER_ATOMIC_SMIN_X2] = "buffer_atomic_smin_x2",
+ [GFX10_BUFFER_ATOMIC_UMIN_X2] = "buffer_atomic_umin_x2",
+ [GFX10_BUFFER_ATOMIC_SMAX_X2] = "buffer_atomic_smax_x2",
+ [GFX10_BUFFER_ATOMIC_UMAX_X2] = "buffer_atomic_umax_x2",
+ [GFX10_BUFFER_ATOMIC_AND_X2] = "buffer_atomic_and_x2",
+ [GFX10_BUFFER_ATOMIC_OR_X2] = "buffer_atomic_or_x2",
+ [GFX10_BUFFER_ATOMIC_XOR_X2] = "buffer_atomic_xor_x2",
+ [GFX10_BUFFER_ATOMIC_INC_X2] = "buffer_atomic_inc_x2",
+ [GFX10_BUFFER_ATOMIC_DEC_X2] = "buffer_atomic_dec_x2",
+ [GFX10_BUFFER_ATOMIC_FCMPSWAP_X2] = "buffer_atomic_fcmpswap_x2",
+ [GFX10_BUFFER_ATOMIC_FMIN_X2] = "buffer_atomic_fmin_x2",
+ [GFX10_BUFFER_ATOMIC_FMAX_X2] = "buffer_atomic_fmax_x2",
+ [GFX10_BUFFER_GL0_INV] = "buffer_gl0_inv",
+ [GFX10_BUFFER_GL1_INV] = "buffer_gl1_inv",
+ [GFX10_BUFFER_LOAD_FORMAT_D16_X] = "buffer_load_format_d16_x",
+ [GFX10_BUFFER_LOAD_FORMAT_D16_XY] = "buffer_load_format_d16_xy",
+ [GFX10_BUFFER_LOAD_FORMAT_D16_XYZ] = "buffer_load_format_d16_xyz",
+ [GFX10_BUFFER_LOAD_FORMAT_D16_XYZW] = "buffer_load_format_d16_xyzw",
+ [GFX10_BUFFER_STORE_FORMAT_D16_X] = "buffer_store_format_d16_x",
+ [GFX10_BUFFER_STORE_FORMAT_D16_XY] = "buffer_store_format_d16_xy",
+ [GFX10_BUFFER_STORE_FORMAT_D16_XYZ] = "buffer_store_format_d16_xyz",
+ [GFX10_BUFFER_STORE_FORMAT_D16_XYZW] = "buffer_store_format_d16_xyzw",
+ },
+ [AMDGCN_INSN_TYPE_FLAT] = {
+ [GFX10_GLOBAL_LOAD_UBYTE] = "global_load_ubyte",
+ [GFX10_GLOBAL_LOAD_SBYTE] = "global_load_sbyte",
+ [GFX10_GLOBAL_LOAD_USHORT] = "global_load_ushort",
+ [GFX10_GLOBAL_LOAD_SSHORT] = "global_load_sshort",
+ [GFX10_GLOBAL_LOAD_DWORD] = "global_load_dword",
+ [GFX10_GLOBAL_LOAD_DWORDX2] = "global_load_dwordx2",
+ [GFX10_GLOBAL_LOAD_DWORDX4] = "global_load_dwordx4",
+ [GFX10_GLOBAL_LOAD_DWORDX3] = "global_load_dwordx3",
+ [GFX10_GLOBAL_LOAD_DWORD_ADDTID] = "global_load_dword_addtid",
+ [GFX10_GLOBAL_STORE_DWORD_ADDTID] = "global_store_dword_addtid",
+ [GFX10_GLOBAL_STORE_BYTE] = "global_store_byte",
+ [GFX10_GLOBAL_STORE_BYTE_D16_HI] = "global_store_byte_d16_hi",
+ [GFX10_GLOBAL_STORE_SHORT] = "global_store_short",
+ [GFX10_GLOBAL_STORE_SHORT_D16_HI] = "global_store_short_d16_hi",
+ [GFX10_GLOBAL_STORE_DWORD] = "global_store_dword",
+ [GFX10_GLOBAL_STORE_DWORDX2] = "global_store_dwordx2",
+ [GFX10_GLOBAL_STORE_DWORDX4] = "global_store_dwordx4",
+ [GFX10_GLOBAL_STORE_DWORDX3] = "global_store_dwordx3",
+ [GFX10_GLOBAL_LOAD_UBYTE_D16] = "global_load_ubyte_d16",
+ [GFX10_GLOBAL_LOAD_UBYTE_D16_HI] = "global_load_ubyte_d16_hi",
+ [GFX10_GLOBAL_LOAD_SBYTE_D16] = "global_load_sbyte_d16",
+ [GFX10_GLOBAL_LOAD_SBYTE_D16_HI] = "global_load_sbyte_d16_hi",
+ [GFX10_GLOBAL_LOAD_SHORT_D16] = "global_load_short_d16",
+ [GFX10_GLOBAL_LOAD_SHORT_D16_HI] = "global_load_short_d16_hi",
+ [GFX10_GLOBAL_ATOMIC_SWAP] = "global_atomic_swap",
+ [GFX10_GLOBAL_ATOMIC_CMPSWAP] = "global_atomic_cmpswap",
+ [GFX10_GLOBAL_ATOMIC_ADD] = "global_atomic_add",
+ [GFX10_GLOBAL_ATOMIC_SUB] = "global_atomic_sub",
+ [GFX10_GLOBAL_ATOMIC_CSUB] = "global_atomic_csub",
+ [GFX10_GLOBAL_ATOMIC_SMIN] = "global_atomic_smin",
+ [GFX10_GLOBAL_ATOMIC_UMIN] = "global_atomic_umin",
+ [GFX10_GLOBAL_ATOMIC_SMAX] = "global_atomic_smax",
+ [GFX10_GLOBAL_ATOMIC_UMAX] = "global_atomic_umax",
+ [GFX10_GLOBAL_ATOMIC_AND] = "global_atomic_and",
+ [GFX10_GLOBAL_ATOMIC_OR] = "global_atomic_or",
+ [GFX10_GLOBAL_ATOMIC_XOR] = "global_atomic_xor",
+ [GFX10_GLOBAL_ATOMIC_INC] = "global_atomic_inc",
+ [GFX10_GLOBAL_ATOMIC_DEC] = "global_atomic_dec",
+ [GFX10_GLOBAL_ATOMIC_FCMPSWAP] = "global_atomic_fcmpswap",
+ [GFX10_GLOBAL_ATOMIC_FMIN] = "global_atomic_fmin",
+ [GFX10_GLOBAL_ATOMIC_FMAX] = "global_atomic_fmax",
+ [GFX10_GLOBAL_ATOMIC_SWAP_X2] = "global_atomic_swap_x2",
+ [GFX10_GLOBAL_ATOMIC_CMPSWAP_X2] = "global_atomic_cmpswap_x2",
+ [GFX10_GLOBAL_ATOMIC_ADD_X2] = "global_atomic_add_x2",
+ [GFX10_GLOBAL_ATOMIC_SUB_X2] = "global_atomic_sub_x2",
+ [GFX10_GLOBAL_ATOMIC_SMIN_X2] = "global_atomic_smin_x2",
+ [GFX10_GLOBAL_ATOMIC_UMIN_X2] = "global_atomic_umin_x2",
+ [GFX10_GLOBAL_ATOMIC_SMAX_X2] = "global_atomic_smax_x2",
+ [GFX10_GLOBAL_ATOMIC_UMAX_X2] = "global_atomic_umax_x2",
+ [GFX10_GLOBAL_ATOMIC_AND_X2] = "global_atomic_and_x2",
+ [GFX10_GLOBAL_ATOMIC_OR_X2] = "global_atomic_or_x2",
+ [GFX10_GLOBAL_ATOMIC_XOR_X2] = "global_atomic_xor_x2",
+ [GFX10_GLOBAL_ATOMIC_INC_X2] = "global_atomic_inc_x2",
+ [GFX10_GLOBAL_ATOMIC_DEC_X2] = "global_atomic_dec_x2",
+ [GFX10_GLOBAL_ATOMIC_FCMPSWAP_X2] = "global_atomic_fcmpswap_x2",
+ [GFX10_GLOBAL_ATOMIC_FMIN_X2] = "global_atomic_fmin_x2",
+ [GFX10_GLOBAL_ATOMIC_FMAX_X2] = "global_atomic_fmax_x2",
+ },
+};
+
+static const char opnames_gfx9[__AMDGCN_INSN_TYPE_MAX][900][30] = {
+ [AMDGCN_INSN_TYPE_SOP2] = {
+ [GFX9_S_ADD_U32] = "s_add_u32",
+ [GFX9_S_SUB_U32] = "s_sub_u32",
+ [GFX9_S_ADD_I32] = "s_add_i32",
+ [GFX9_S_SUB_I32] = "s_sub_i32",
+ [GFX9_S_ADDC_U32] = "s_addc_u32",
+ [GFX9_S_SUBB_U32] = "s_subb_u32",
+ [GFX9_S_MIN_I32] = "s_min_i32",
+ [GFX9_S_MIN_U32] = "s_min_u32",
+ [GFX9_S_MAX_I32] = "s_max_i32",
+ [GFX9_S_MAX_U32] = "s_max_u32",
+ [GFX9_S_CSELECT_B32] = "s_cselect_b32",
+ [GFX9_S_CSELECT_B64] = "s_cselect_b64",
+ [GFX9_S_AND_B32] = "s_and_b32",
+ [GFX9_S_AND_B64] = "s_and_b64",
+ [GFX9_S_OR_B32] = "s_or_b32",
+ [GFX9_S_OR_B64] = "s_or_b64",
+ [GFX9_S_XOR_B32] = "s_xor_b32",
+ [GFX9_S_XOR_B64] = "s_xor_b64",
+ [GFX9_S_ANDN2_B32] = "s_andn2_b32",
+ [GFX9_S_ANDN2_B64] = "s_andn2_b64",
+ [GFX9_S_ORN2_B32] = "s_orn2_b32",
+ [GFX9_S_ORN2_B64] = "s_orn2_b64",
+ [GFX9_S_NAND_B32] = "s_nand_b32",
+ [GFX9_S_NAND_B64] = "s_nand_b64",
+ [GFX9_S_NOR_B32] = "s_nor_b32",
+ [GFX9_S_NOR_B64] = "s_nor_b64",
+ [GFX9_S_XNOR_B32] = "s_xnor_b32",
+ [GFX9_S_XNOR_B64] = "s_xnor_b64",
+ [GFX9_S_LSHL_B32] = "s_lshl_b32",
+ [GFX9_S_LSHL_B64] = "s_lshl_b64",
+ [GFX9_S_LSHR_B32] = "s_lshr_b32",
+ [GFX9_S_LSHR_B64] = "s_lshr_b64",
+ [GFX9_S_ASHR_I32] = "s_ashr_i32",
+ [GFX9_S_ASHR_I64] = "s_ashr_i64",
+ [GFX9_S_BFM_B32] = "s_bfm_b32",
+ [GFX9_S_BFM_B64] = "s_bfm_b64",
+ [GFX9_S_MUL_I32] = "s_mul_i32",
+ [GFX9_S_BFE_U32] = "s_bfe_u32",
+ [GFX9_S_BFE_I32] = "s_bfe_i32",
+ [GFX9_S_BFE_U64] = "s_bfe_u64",
+ [GFX9_S_BFE_I64] = "s_bfe_i64",
+ [GFX9_S_CBRANCH_G_FORK] = "s_cbranch_g_fork",
+ [GFX9_S_ABSDIFF_I32] = "s_absdiff_i32",
+ [GFX9_S_RFE_RESTORE_B64] = "s_rfe_restore_b64",
+ [GFX9_S_MUL_HI_U32] = "s_mul_hi_u32",
+ [GFX9_S_MUL_HI_I32] = "s_mul_hi_i32",
+ [GFX9_S_LSHL1_ADD_U32] = "s_lshl1_add_u32",
+ [GFX9_S_LSHL2_ADD_U32] = "s_lshl2_add_u32",
+ [GFX9_S_LSHL3_ADD_U32] = "s_lshl3_add_u32",
+ [GFX9_S_LSHL4_ADD_U32] = "s_lshl4_add_u32",
+ [GFX9_S_PACK_LL_B32_B16] = "s_pack_ll_b32_b16",
+ [GFX9_S_PACK_LH_B32_B16] = "s_pack_lh_b32_b16",
+ [GFX9_S_PACK_HH_B32_B16] = "s_pack_hh_b32_b16",
+ },
+ [AMDGCN_INSN_TYPE_SOPK] = {
+ [GFX9_S_MOVK_I32] = "s_movk_i32",
+ [GFX9_S_CMOVK_I32] = "s_cmovk_i32",
+ [GFX9_S_CMPK_EQ_I32] = "s_cmpk_eq_i32",
+ [GFX9_S_CMPK_LG_I32] = "s_cmpk_lg_i32",
+ [GFX9_S_CMPK_GT_I32] = "s_cmpk_gt_i32",
+ [GFX9_S_CMPK_GE_I32] = "s_cmpk_ge_i32",
+ [GFX9_S_CMPK_LT_I32] = "s_cmpk_lt_i32",
+ [GFX9_S_CMPK_LE_I32] = "s_cmpk_le_i32",
+ [GFX9_S_CMPK_EQ_U32] = "s_cmpk_eq_u32",
+ [GFX9_S_CMPK_LG_U32] = "s_cmpk_lg_u32",
+ [GFX9_S_CMPK_GT_U32] = "s_cmpk_gt_u32",
+ [GFX9_S_CMPK_GE_U32] = "s_cmpk_ge_u32",
+ [GFX9_S_CMPK_LT_U32] = "s_cmpk_lt_u32",
+ [GFX9_S_CMPK_LE_U32] = "s_cmpk_le_u32",
+ [GFX9_S_ADDK_I32] = "s_addk_i32",
+ [GFX9_S_MULK_I32] = "s_mulk_i32",
+ [GFX9_S_CBRANCH_I_FORK] = "s_cbranch_i_fork",
+ [GFX9_S_GETREG_B32] = "s_getreg_b32",
+ [GFX9_S_SETREG_B32] = "s_setreg_b32",
+ [GFX9_S_SETREG_IMM32_B32] = "s_setreg_imm32_b32",
+ [GFX9_S_CALL_B64] = "s_call_b64",
+ },
+ [AMDGCN_INSN_TYPE_SOP1] = {
+ [GFX9_S_MOV_B32] = "s_mov_b32",
+ [GFX9_S_MOV_B64] = "s_mov_b64",
+ [GFX9_S_CMOV_B32] = "s_cmov_b32",
+ [GFX9_S_CMOV_B64] = "s_cmov_b64",
+ [GFX9_S_NOT_B32] = "s_not_b32",
+ [GFX9_S_NOT_B64] = "s_not_b64",
+ [GFX9_S_WQM_B32] = "s_wqm_b32",
+ [GFX9_S_WQM_B64] = "s_wqm_b64",
+ [GFX9_S_BREV_B32] = "s_brev_b32",
+ [GFX9_S_BREV_B64] = "s_brev_b64",
+ [GFX9_S_BCNT0_I32_B32] = "s_bcnt0_i32_b32",
+ [GFX9_S_BCNT0_I32_B64] = "s_bcnt0_i32_b64",
+ [GFX9_S_BCNT1_I32_B32] = "s_bcnt1_i32_b32",
+ [GFX9_S_BCNT1_I32_B64] = "s_bcnt1_i32_b64",
+ [GFX9_S_FF0_I32_B32] = "s_ff0_i32_b32",
+ [GFX9_S_FF0_I32_B64] = "s_ff0_i32_b64",
+ [GFX9_S_FF1_I32_B32] = "s_ff1_i32_b32",
+ [GFX9_S_FF1_I32_B64] = "s_ff1_i32_b64",
+ [GFX9_S_FLBIT_I32_B32] = "s_flbit_i32_b32",
+ [GFX9_S_FLBIT_I32_B64] = "s_flbit_i32_b64",
+ [GFX9_S_FLBIT_I32] = "s_flbit_i32",
+ [GFX9_S_FLBIT_I32_I64] = "s_flbit_i32_i64",
+ [GFX9_S_SEXT_I32_I8] = "s_sext_i32_i8",
+ [GFX9_S_SEXT_I32_I16] = "s_sext_i32_i16",
+ [GFX9_S_BITSET0_B32] = "s_bitset0_b32",
+ [GFX9_S_BITSET0_B64] = "s_bitset0_b64",
+ [GFX9_S_BITSET1_B32] = "s_bitset1_b32",
+ [GFX9_S_BITSET1_B64] = "s_bitset1_b64",
+ [GFX9_S_GETPC_B64] = "s_getpc_b64",
+ [GFX9_S_SETPC_B64] = "s_setpc_b64",
+ [GFX9_S_SWAPPC_B64] = "s_swappc_b64",
+ [GFX9_S_RFE_B64] = "s_rfe_b64",
+ [GFX9_S_AND_SAVEEXEC_B64] = "s_and_saveexec_b64",
+ [GFX9_S_XNOR_SAVEEXEC_B64] = "s_xnor_saveexec_b64",
+ [GFX9_S_QUADMASK_B32] = "s_quadmask_b32",
+ [GFX9_S_QUADMASK_B64] = "s_quadmask_b64",
+ [GFX9_S_MOVRELS_B32] = "s_movrels_b32",
+ [GFX9_S_MOVRELS_B64] = "s_movrels_b64",
+ [GFX9_S_MOVRELD_B32] = "s_movreld_b32",
+ [GFX9_S_MOVRELD_B64] = "s_movreld_b64",
+ [GFX9_S_ABS_I32] = "s_abs_i32",
+ [GFX9_S_ANDN1_SAVEEXEC_B64] = "s_andn1_saveexec_b64",
+ [GFX9_S_ORN1_SAVEEXEC_B64] = "s_orn1_saveexec_b64",
+ [GFX9_S_ANDN1_WREXEC_B64] = "s_andn1_wrexec_b64",
+ [GFX9_S_ANDN2_WREXEC_B64] = "s_andn2_wrexec_b64",
+ [GFX9_S_BITREPLICATE_B64_B32] = "s_bitreplicate_b64_b32",
+ /* SOP1 opcodes missing from the table (Vega ISA 12.3). */
+ [GFX9_S_OR_SAVEEXEC_B64] = "s_or_saveexec_b64",
+ [GFX9_S_XOR_SAVEEXEC_B64] = "s_xor_saveexec_b64",
+ [GFX9_S_ANDN2_SAVEEXEC_B64] = "s_andn2_saveexec_b64",
+ [GFX9_S_ORN2_SAVEEXEC_B64] = "s_orn2_saveexec_b64",
+ [GFX9_S_NAND_SAVEEXEC_B64] = "s_nand_saveexec_b64",
+ [GFX9_S_NOR_SAVEEXEC_B64] = "s_nor_saveexec_b64",
+ [GFX9_S_SET_GPR_IDX_IDX] = "s_set_gpr_idx_idx",
+ },
+ [AMDGCN_INSN_TYPE_SOPC] = {
+ [GFX9_S_CMP_EQ_I32] = "s_cmp_eq_i32",
+ [GFX9_S_CMP_LG_I32] = "s_cmp_lg_i32",
+ [GFX9_S_CMP_GT_I32] = "s_cmp_gt_i32",
+ [GFX9_S_CMP_GE_I32] = "s_cmp_ge_i32",
+ [GFX9_S_CMP_LT_I32] = "s_cmp_lt_i32",
+ [GFX9_S_CMP_LE_I32] = "s_cmp_le_i32",
+ [GFX9_S_CMP_EQ_U32] = "s_cmp_eq_u32",
+ [GFX9_S_CMP_LG_U32] = "s_cmp_lg_u32",
+ [GFX9_S_CMP_GT_U32] = "s_cmp_gt_u32",
+ [GFX9_S_CMP_GE_U32] = "s_cmp_ge_u32",
+ [GFX9_S_CMP_LT_U32] = "s_cmp_lt_u32",
+ [GFX9_S_CMP_LE_U32] = "s_cmp_le_u32",
+ [GFX9_S_BITCMP0_B32] = "s_bitcmp0_b32",
+ [GFX9_S_BITCMP1_B32] = "s_bitcmp1_b32",
+ [GFX9_S_BITCMP0_B64] = "s_bitcmp0_b64",
+ [GFX9_S_BITCMP1_B64] = "s_bitcmp1_b64",
+ [GFX9_S_SETVSKIP] = "s_setvskip",
+ [GFX9_S_SET_GPR_IDX_ON] = "s_set_gpr_idx_on",
+ [GFX9_S_CMP_EQ_U64] = "s_cmp_eq_u64",
+ [GFX9_S_CMP_LG_U64] = "s_cmp_lg_u64",
+ },
+ [AMDGCN_INSN_TYPE_SOPP] = {
+ [GFX9_S_NOP] = "s_nop",
+ [GFX9_S_ENDPGM] = "s_endpgm",
+ [GFX9_S_BRANCH] = "s_branch",
+ [GFX9_S_WAKEUP] = "s_wakeup",
+ [GFX9_S_CBRANCH_SCC0] = "s_cbranch_scc0",
+ [GFX9_S_CBRANCH_SCC1] = "s_cbranch_scc1",
+ [GFX9_S_CBRANCH_VCCZ] = "s_cbranch_vccz",
+ [GFX9_S_CBRANCH_VCCNZ] = "s_cbranch_vccnz",
+ [GFX9_S_CBRANCH_EXECZ] = "s_cbranch_execz",
+ [GFX9_S_CBRANCH_EXECNZ] = "s_cbranch_execnz",
+ [GFX9_S_BARRIER] = "s_barrier",
+ [GFX9_S_SETKILL] = "s_setkill",
+ [GFX9_S_WAITCNT] = "s_waitcnt",
+ [GFX9_S_SETHALT] = "s_sethalt",
+ [GFX9_S_SLEEP] = "s_sleep",
+ [GFX9_S_SETPRIO] = "s_setprio",
+ [GFX9_S_SENDMSG] = "s_sendmsg",
+ [GFX9_S_SENDMSGHALT] = "s_sendmsghalt",
+ [GFX9_S_TRAP] = "s_trap",
+ [GFX9_S_ICACHE_INV] = "s_icache_inv",
+ [GFX9_S_INCPERFLEVEL] = "s_incperflevel",
+ [GFX9_S_DECPERFLEVEL] = "s_decperflevel",
+ [GFX9_S_TTRACEDATA] = "s_ttracedata",
+ [GFX9_S_CBRANCH_CDBGSYS] = "s_cbranch_cdbgsys",
+ [GFX9_S_CBRANCH_CDBGUSER] = "s_cbranch_cdbguser",
+ [GFX9_S_CBRANCH_CDBGSYS_OR_USER] = "s_cbranch_cdbgsys_or_user",
+ [GFX9_S_CBRANCH_CDBGSYS_AND_USER] = "s_cbranch_cdbgsys_and_user",
+ [GFX9_S_ENDPGM_SAVED] = "s_endpgm_saved",
+ [GFX9_S_SET_GPR_IDX_OFF] = "s_set_gpr_idx_off",
+ [GFX9_S_SET_GPR_IDX_MODE] = "s_set_gpr_idx_mode",
+ [GFX9_S_ENDPGM_ORDERED_PS_DONE] = "s_endpgm_ordered_ps_done",
+ },
+ [AMDGCN_INSN_TYPE_SMEM] = {
+ [GFX9_S_LOAD_DWORD] = "s_load_dword",
+ [GFX9_S_LOAD_DWORDX2] = "s_load_dwordx2",
+ [GFX9_S_LOAD_DWORDX4] = "s_load_dwordx4",
+ [GFX9_S_LOAD_DWORDX8] = "s_load_dwordx8",
+ [GFX9_S_LOAD_DWORDX16] = "s_load_dwordx16",
+ [GFX9_S_SCRATCH_LOAD_DWORD] = "s_scratch_load_dword",
+ [GFX9_S_SCRATCH_LOAD_DWORDX2] = "s_scratch_load_dwordx2",
+ [GFX9_S_SCRATCH_LOAD_DWORDX4] = "s_scratch_load_dwordx4",
+ [GFX9_S_BUFFER_LOAD_DWORD] = "s_buffer_load_dword",
+ [GFX9_S_BUFFER_LOAD_DWORDX2] = "s_buffer_load_dwordx2",
+ [GFX9_S_BUFFER_LOAD_DWORDX4] = "s_buffer_load_dwordx4",
+ [GFX9_S_BUFFER_LOAD_DWORDX8] = "s_buffer_load_dwordx8",
+ [GFX9_S_BUFFER_LOAD_DWORDX16] = "s_buffer_load_dwordx16",
+ [GFX9_S_STORE_DWORD] = "s_store_dword",
+ [GFX9_S_STORE_DWORDX2] = "s_store_dwordx2",
+ [GFX9_S_STORE_DWORDX4] = "s_store_dwordx4",
+ [GFX9_S_SCRATCH_STORE_DWORD] = "s_scratch_store_dword",
+ [GFX9_S_SCRATCH_STORE_DWORDX2] = "s_scratch_store_dwordx2",
+ [GFX9_S_SCRATCH_STORE_DWORDX4] = "s_scratch_store_dwordx4",
+ [GFX9_S_BUFFER_STORE_DWORD] = "s_buffer_store_dword",
+ [GFX9_S_BUFFER_STORE_DWORDX2] = "s_buffer_store_dwordx2",
+ [GFX9_S_BUFFER_STORE_DWORDX4] = "s_buffer_store_dwordx4",
+ [GFX9_S_DCACHE_INV] = "s_dcache_inv",
+ [GFX9_S_DCACHE_WB] = "s_dcache_wb",
+ [GFX9_S_DCACHE_INV_VOL] = "s_dcache_inv_vol",
+ [GFX9_S_DCACHE_WB_VOL] = "s_dcache_wb_vol",
+ [GFX9_S_MEMTIME] = "s_memtime",
+ [GFX9_S_MEMREALTIME] = "s_memrealtime",
+ [GFX9_S_ATC_PROBE] = "s_atc_probe",
+ [GFX9_S_ATC_PROBE_BUFFER] = "s_atc_probe_buffer",
+ [GFX9_S_DCACHE_DISCARD] = "s_dcache_discard",
+ [GFX9_S_DCACHE_DISCARD_X2] = "s_dcache_discard_x2",
+ [GFX9_S_BUFFER_ATOMIC_SWAP] = "s_buffer_atomic_swap",
+ [GFX9_S_BUFFER_ATOMIC_CMPSWAP] = "s_buffer_atomic_cmpswap",
+ [GFX9_S_BUFFER_ATOMIC_ADD] = "s_buffer_atomic_add",
+ [GFX9_S_BUFFER_ATOMIC_SUB] = "s_buffer_atomic_sub",
+ [GFX9_S_BUFFER_ATOMIC_SMIN] = "s_buffer_atomic_smin",
+ [GFX9_S_BUFFER_ATOMIC_UMIN] = "s_buffer_atomic_umin",
+ [GFX9_S_BUFFER_ATOMIC_SMAX] = "s_buffer_atomic_smax",
+ [GFX9_S_BUFFER_ATOMIC_UMAX] = "s_buffer_atomic_umax",
+ [GFX9_S_BUFFER_ATOMIC_AND] = "s_buffer_atomic_and",
+ [GFX9_S_BUFFER_ATOMIC_OR] = "s_buffer_atomic_or",
+ [GFX9_S_BUFFER_ATOMIC_XOR] = "s_buffer_atomic_xor",
+ [GFX9_S_BUFFER_ATOMIC_INC] = "s_buffer_atomic_inc",
+ [GFX9_S_BUFFER_ATOMIC_DEC] = "s_buffer_atomic_dec",
+ [GFX9_S_BUFFER_ATOMIC_SWAP_X2] = "s_buffer_atomic_swap_x2",
+ [GFX9_S_BUFFER_ATOMIC_CMPSWAP_X2] = "s_buffer_atomic_cmpswap_x2",
+ [GFX9_S_BUFFER_ATOMIC_ADD_X2] = "s_buffer_atomic_add_x2",
+ [GFX9_S_BUFFER_ATOMIC_SUB_X2] = "s_buffer_atomic_sub_x2",
+ [GFX9_S_BUFFER_ATOMIC_SMIN_X2] = "s_buffer_atomic_smin_x2",
+ [GFX9_S_BUFFER_ATOMIC_UMIN_X2] = "s_buffer_atomic_umin_x2",
+ [GFX9_S_BUFFER_ATOMIC_SMAX_X2] = "s_buffer_atomic_smax_x2",
+ [GFX9_S_BUFFER_ATOMIC_UMAX_X2] = "s_buffer_atomic_umax_x2",
+ [GFX9_S_BUFFER_ATOMIC_AND_X2] = "s_buffer_atomic_and_x2",
+ [GFX9_S_BUFFER_ATOMIC_OR_X2] = "s_buffer_atomic_or_x2",
+ [GFX9_S_BUFFER_ATOMIC_XOR_X2] = "s_buffer_atomic_xor_x2",
+ [GFX9_S_BUFFER_ATOMIC_INC_X2] = "s_buffer_atomic_inc_x2",
+ [GFX9_S_BUFFER_ATOMIC_DEC_X2] = "s_buffer_atomic_dec_x2",
+ [GFX9_S_ATOMIC_SWAP] = "s_atomic_swap",
+ [GFX9_S_ATOMIC_CMPSWAP] = "s_atomic_cmpswap",
+ [GFX9_S_ATOMIC_ADD] = "s_atomic_add",
+ [GFX9_S_ATOMIC_SUB] = "s_atomic_sub",
+ [GFX9_S_ATOMIC_SMIN] = "s_atomic_smin",
+ [GFX9_S_ATOMIC_UMIN] = "s_atomic_umin",
+ [GFX9_S_ATOMIC_SMAX] = "s_atomic_smax",
+ [GFX9_S_ATOMIC_UMAX] = "s_atomic_umax",
+ [GFX9_S_ATOMIC_AND] = "s_atomic_and",
+ [GFX9_S_ATOMIC_OR] = "s_atomic_or",
+ [GFX9_S_ATOMIC_XOR] = "s_atomic_xor",
+ [GFX9_S_ATOMIC_INC] = "s_atomic_inc",
+ [GFX9_S_ATOMIC_DEC] = "s_atomic_dec",
+ [GFX9_S_ATOMIC_SWAP_X2] = "s_atomic_swap_x2",
+ [GFX9_S_ATOMIC_CMPSWAP_X2] = "s_atomic_cmpswap_x2",
+ [GFX9_S_ATOMIC_ADD_X2] = "s_atomic_add_x2",
+ [GFX9_S_ATOMIC_SUB_X2] = "s_atomic_sub_x2",
+ [GFX9_S_ATOMIC_SMIN_X2] = "s_atomic_smin_x2",
+ [GFX9_S_ATOMIC_UMIN_X2] = "s_atomic_umin_x2",
+ [GFX9_S_ATOMIC_SMAX_X2] = "s_atomic_smax_x2",
+ [GFX9_S_ATOMIC_UMAX_X2] = "s_atomic_umax_x2",
+ [GFX9_S_ATOMIC_AND_X2] = "s_atomic_and_x2",
+ [GFX9_S_ATOMIC_OR_X2] = "s_atomic_or_x2",
+ [GFX9_S_ATOMIC_XOR_X2] = "s_atomic_xor_x2",
+ [GFX9_S_ATOMIC_INC_X2] = "s_atomic_inc_x2",
+ [GFX9_S_ATOMIC_DEC_X2] = "s_atomic_dec_x2",
+ },
+ [AMDGCN_INSN_TYPE_VOP2] = {
+ [GFX9_V_CNDMASK_B32] = "v_cndmask_b32",
+ [GFX9_V_ADD_F32] = "v_add_f32",
+ [GFX9_V_SUB_F32] = "v_sub_f32",
+ [GFX9_V_SUBREV_F32] = "v_subrev_f32",
+ [GFX9_V_MUL_LEGACY_F32] = "v_mul_legacy_f32",
+ [GFX9_V_MUL_F32] = "v_mul_f32",
+ [GFX9_V_MUL_I32_I24] = "v_mul_i32_i24",
+ [GFX9_V_MUL_HI_I32_I24] = "v_mul_hi_i32_i24",
+ [GFX9_V_MUL_U32_U24] = "v_mul_u32_u24",
+ [GFX9_V_MUL_HI_U32_U24] = "v_mul_hi_u32_u24",
+ [GFX9_V_MIN_F32] = "v_min_f32",
+ [GFX9_V_MAX_F32] = "v_max_f32",
+ [GFX9_V_MIN_I32] = "v_min_i32",
+ [GFX9_V_MAX_I32] = "v_max_i32",
+ [GFX9_V_MIN_U32] = "v_min_u32",
+ [GFX9_V_MAX_U32] = "v_max_u32",
+ [GFX9_V_LSHRREV_B32] = "v_lshrrev_b32",
+ [GFX9_V_ASHRREV_I32] = "v_ashrrev_i32",
+ [GFX9_V_LSHLREV_B32] = "v_lshlrev_b32",
+ [GFX9_V_AND_B32] = "v_and_b32",
+ [GFX9_V_OR_B32] = "v_or_b32",
+ [GFX9_V_XOR_B32] = "v_xor_b32",
+ [GFX9_V_MAC_F32] = "v_mac_f32",
+ [GFX9_V_MADMK_F32] = "v_madmk_f32",
+ [GFX9_V_MADAK_F32] = "v_madak_f32",
+ [GFX9_V_ADD_CO_U32] = "v_add_co_u32",
+ [GFX9_V_SUB_CO_U32] = "v_sub_co_u32",
+ [GFX9_V_SUBREV_CO_U32] = "v_subrev_co_u32",
+ [GFX9_V_ADDC_CO_U32] = "v_addc_co_u32",
+ [GFX9_V_SUBB_CO_U32] = "v_subb_co_u32",
+ [GFX9_V_SUBBREV_CO_U32] = "v_subbrev_co_u32",
+ [GFX9_V_ADD_F16] = "v_add_f16",
+ [GFX9_V_SUB_F16] = "v_sub_f16",
+ [GFX9_V_SUBREV_F16] = "v_subrev_f16",
+ [GFX9_V_MUL_F16] = "v_mul_f16",
+ [GFX9_V_MAC_F16] = "v_mac_f16",
+ [GFX9_V_MADMK_F16] = "v_madmk_f16",
+ [GFX9_V_MADAK_F16] = "v_madak_f16",
+ [GFX9_V_ADD_U16] = "v_add_u16",
+ [GFX9_V_SUB_U16] = "v_sub_u16",
+ [GFX9_V_SUBREV_U16] = "v_subrev_u16",
+ [GFX9_V_MUL_LO_U16] = "v_mul_lo_u16",
+ [GFX9_V_LSHLREV_B16] = "v_lshlrev_b16",
+ [GFX9_V_LSHRREV_B16] = "v_lshrrev_b16",
+ [GFX9_V_ASHRREV_I16] = "v_ashrrev_i16",
+ [GFX9_V_MAX_F16] = "v_max_f16",
+ [GFX9_V_MIN_F16] = "v_min_f16",
+ [GFX9_V_MAX_U16] = "v_max_u16",
+ [GFX9_V_MAX_I16] = "v_max_i16",
+ [GFX9_V_MIN_U16] = "v_min_u16",
+ [GFX9_V_MIN_I16] = "v_min_i16",
+ [GFX9_V_LDEXP_F16] = "v_ldexp_f16",
+ [GFX9_V_ADD_U32] = "v_add_u32",
+ [GFX9_V_SUB_U32] = "v_sub_u32",
+ [GFX9_V_SUBREV_U32] = "v_subrev_u32",
+ },
+ [AMDGCN_INSN_TYPE_VOP1] = {
+ [GFX9_V_NOP] = "v_nop",
+ [GFX9_V_MOV_B32] = "v_mov_b32",
+ [GFX9_V_READFIRSTLANE_B32] = "v_readfirstlane_b32",
+ [GFX9_V_CVT_I32_F64] = "v_cvt_i32_f64",
+ [GFX9_V_CVT_F64_I32] = "v_cvt_f64_i32",
+ [GFX9_V_CVT_F32_I32] = "v_cvt_f32_i32",
+ [GFX9_V_CVT_F32_U32] = "v_cvt_f32_u32",
+ [GFX9_V_CVT_U32_F32] = "v_cvt_u32_f32",
+ [GFX9_V_CVT_I32_F32] = "v_cvt_i32_f32",
+ [GFX9_V_CVT_F16_F32] = "v_cvt_f16_f32",
+ [GFX9_V_CVT_F32_F16] = "v_cvt_f32_f16",
+ [GFX9_V_CVT_RPI_I32_F32] = "v_cvt_rpi_i32_f32",
+ [GFX9_V_CVT_FLR_I32_F32] = "v_cvt_flr_i32_f32",
+ [GFX9_V_CVT_OFF_F32_I4] = "v_cvt_off_f32_i4",
+ [GFX9_V_CVT_F32_F64] = "v_cvt_f32_f64",
+ [GFX9_V_CVT_F64_F32] = "v_cvt_f64_f32",
+ [GFX9_V_CVT_F32_UBYTE0] = "v_cvt_f32_ubyte0",
+ [GFX9_V_CVT_F32_UBYTE1] = "v_cvt_f32_ubyte1",
+ [GFX9_V_CVT_F32_UBYTE2] = "v_cvt_f32_ubyte2",
+ [GFX9_V_CVT_F32_UBYTE3] = "v_cvt_f32_ubyte3",
+ [GFX9_V_CVT_U32_F64] = "v_cvt_u32_f64",
+ [GFX9_V_CVT_F64_U32] = "v_cvt_f64_u32",
+ [GFX9_V_TRUNC_F64] = "v_trunc_f64",
+ [GFX9_V_CEIL_F64] = "v_ceil_f64",
+ [GFX9_V_RNDNE_F64] = "v_rndne_f64",
+ [GFX9_V_FLOOR_F64] = "v_floor_f64",
+ [GFX9_V_FRACT_F32] = "v_fract_f32",
+ [GFX9_V_TRUNC_F32] = "v_trunc_f32",
+ [GFX9_V_CEIL_F32] = "v_ceil_f32",
+ [GFX9_V_RNDNE_F32] = "v_rndne_f32",
+ [GFX9_V_FLOOR_F32] = "v_floor_f32",
+ [GFX9_V_EXP_F32] = "v_exp_f32",
+ [GFX9_V_LOG_F32] = "v_log_f32",
+ [GFX9_V_RCP_F32] = "v_rcp_f32",
+ [GFX9_V_RCP_IFLAG_F32] = "v_rcp_iflag_f32",
+ [GFX9_V_RSQ_F32] = "v_rsq_f32",
+ [GFX9_V_RCP_F64] = "v_rcp_f64",
+ [GFX9_V_RSQ_F64] = "v_rsq_f64",
+ [GFX9_V_SQRT_F32] = "v_sqrt_f32",
+ [GFX9_V_SQRT_F64] = "v_sqrt_f64",
+ [GFX9_V_SIN_F32] = "v_sin_f32",
+ [GFX9_V_COS_F32] = "v_cos_f32",
+ [GFX9_V_NOT_B32] = "v_not_b32",
+ [GFX9_V_BFREV_B32] = "v_bfrev_b32",
+ [GFX9_V_FFBH_U32] = "v_ffbh_u32",
+ [GFX9_V_FFBL_B32] = "v_ffbl_b32",
+ [GFX9_V_FFBH_I32] = "v_ffbh_i32",
+ [GFX9_V_FREXP_EXP_I32_F64] = "v_frexp_exp_i32_f64",
+ [GFX9_V_FREXP_MANT_F64] = "v_frexp_mant_f64",
+ [GFX9_V_FRACT_F64] = "v_fract_f64",
+ [GFX9_V_FREXP_EXP_I32_F32] = "v_frexp_exp_i32_f32",
+ [GFX9_V_FREXP_MANT_F32] = "v_frexp_mant_f32",
+ [GFX9_V_CLREXCP] = "v_clrexcp",
+ [GFX9_V_SCREEN_PARTITION_4SE_B32] = "v_screen_partition_4se_b32",
+ [GFX9_V_CVT_F16_U16] = "v_cvt_f16_u16",
+ [GFX9_V_CVT_F16_I16] = "v_cvt_f16_i16",
+ [GFX9_V_CVT_U16_F16] = "v_cvt_u16_f16",
+ [GFX9_V_CVT_I16_F16] = "v_cvt_i16_f16",
+ [GFX9_V_RCP_F16] = "v_rcp_f16",
+ [GFX9_V_SQRT_F16] = "v_sqrt_f16",
+ [GFX9_V_RSQ_F16] = "v_rsq_f16",
+ [GFX9_V_LOG_F16] = "v_log_f16",
+ [GFX9_V_EXP_F16] = "v_exp_f16",
+ [GFX9_V_FREXP_MANT_F16] = "v_frexp_mant_f16",
+ [GFX9_V_FREXP_EXP_I16_F16] = "v_frexp_exp_i16_f16",
+ [GFX9_V_FLOOR_F16] = "v_floor_f16",
+ [GFX9_V_CEIL_F16] = "v_ceil_f16",
+ [GFX9_V_TRUNC_F16] = "v_trunc_f16",
+ [GFX9_V_RNDNE_F16] = "v_rndne_f16",
+ [GFX9_V_FRACT_F16] = "v_fract_f16",
+ [GFX9_V_SIN_F16] = "v_sin_f16",
+ [GFX9_V_COS_F16] = "v_cos_f16",
+ [GFX9_V_EXP_LEGACY_F32] = "v_exp_legacy_f32",
+ [GFX9_V_LOG_LEGACY_F32] = "v_log_legacy_f32",
+ [GFX9_V_CVT_NORM_I16_F16] = "v_cvt_norm_i16_f16",
+ [GFX9_V_CVT_NORM_U16_F16] = "v_cvt_norm_u16_f16",
+ [GFX9_V_SAT_PK_U8_I16] = "v_sat_pk_u8_i16",
+ [GFX9_V_SWAP_B32] = "v_swap_b32",
+ },
+ [AMDGCN_INSN_TYPE_VOPC] = {
+ [GFX9_V_CMP_CLASS_F32] = "v_cmp_class_f32",
+ [GFX9_V_CMPX_CLASS_F32] = "v_cmpx_class_f32",
+ [GFX9_V_CMP_CLASS_F64] = "v_cmp_class_f64",
+ [GFX9_V_CMPX_CLASS_F64] = "v_cmpx_class_f64",
+ [GFX9_V_CMP_CLASS_F16] = "v_cmp_class_f16",
+ [GFX9_V_CMPX_CLASS_F16] = "v_cmpx_class_f16",
+ [GFX9_V_CMP_F_F16] = "v_cmp_f_f16",
+ [GFX9_V_CMP_LT_F16] = "v_cmp_lt_f16",
+ [GFX9_V_CMP_EQ_F16] = "v_cmp_eq_f16",
+ [GFX9_V_CMP_LE_F16] = "v_cmp_le_f16",
+ [GFX9_V_CMP_GT_F16] = "v_cmp_gt_f16",
+ [GFX9_V_CMP_LG_F16] = "v_cmp_lg_f16",
+ [GFX9_V_CMP_GE_F16] = "v_cmp_ge_f16",
+ [GFX9_V_CMP_O_F16] = "v_cmp_o_f16",
+ [GFX9_V_CMP_U_F16] = "v_cmp_u_f16",
+ [GFX9_V_CMP_NGE_F16] = "v_cmp_nge_f16",
+ [GFX9_V_CMP_NLG_F16] = "v_cmp_nlg_f16",
+ [GFX9_V_CMP_NGT_F16] = "v_cmp_ngt_f16",
+ [GFX9_V_CMP_NLE_F16] = "v_cmp_nle_f16",
+ [GFX9_V_CMP_NEQ_F16] = "v_cmp_neq_f16",
+ [GFX9_V_CMP_NLT_F16] = "v_cmp_nlt_f16",
+ [GFX9_V_CMP_TRU_F16] = "v_cmp_tru_f16",
+ [GFX9_V_CMPX_F_F16] = "v_cmpx_f_f16",
+ [GFX9_V_CMPX_LT_F16] = "v_cmpx_lt_f16",
+ [GFX9_V_CMPX_EQ_F16] = "v_cmpx_eq_f16",
+ [GFX9_V_CMPX_LE_F16] = "v_cmpx_le_f16",
+ [GFX9_V_CMPX_GT_F16] = "v_cmpx_gt_f16",
+ [GFX9_V_CMPX_LG_F16] = "v_cmpx_lg_f16",
+ [GFX9_V_CMPX_GE_F16] = "v_cmpx_ge_f16",
+ [GFX9_V_CMPX_O_F16] = "v_cmpx_o_f16",
+ [GFX9_V_CMPX_U_F16] = "v_cmpx_u_f16",
+ [GFX9_V_CMPX_NGE_F16] = "v_cmpx_nge_f16",
+ [GFX9_V_CMPX_NLG_F16] = "v_cmpx_nlg_f16",
+ [GFX9_V_CMPX_NGT_F16] = "v_cmpx_ngt_f16",
+ [GFX9_V_CMPX_NLE_F16] = "v_cmpx_nle_f16",
+ [GFX9_V_CMPX_NEQ_F16] = "v_cmpx_neq_f16",
+ [GFX9_V_CMPX_NLT_F16] = "v_cmpx_nlt_f16",
+ [GFX9_V_CMPX_TRU_F16] = "v_cmpx_tru_f16",
+ [GFX9_V_CMP_F_F32] = "v_cmp_f_f32",
+ [GFX9_V_CMP_LT_F32] = "v_cmp_lt_f32",
+ [GFX9_V_CMP_EQ_F32] = "v_cmp_eq_f32",
+ [GFX9_V_CMP_LE_F32] = "v_cmp_le_f32",
+ [GFX9_V_CMP_GT_F32] = "v_cmp_gt_f32",
+ [GFX9_V_CMP_LG_F32] = "v_cmp_lg_f32",
+ [GFX9_V_CMP_GE_F32] = "v_cmp_ge_f32",
+ [GFX9_V_CMP_O_F32] = "v_cmp_o_f32",
+ [GFX9_V_CMP_U_F32] = "v_cmp_u_f32",
+ [GFX9_V_CMP_NGE_F32] = "v_cmp_nge_f32",
+ [GFX9_V_CMP_NLG_F32] = "v_cmp_nlg_f32",
+ [GFX9_V_CMP_NGT_F32] = "v_cmp_ngt_f32",
+ [GFX9_V_CMP_NLE_F32] = "v_cmp_nle_f32",
+ [GFX9_V_CMP_NEQ_F32] = "v_cmp_neq_f32",
+ [GFX9_V_CMP_NLT_F32] = "v_cmp_nlt_f32",
+ [GFX9_V_CMP_TRU_F32] = "v_cmp_tru_f32",
+ [GFX9_V_CMPX_F_F32] = "v_cmpx_f_f32",
+ [GFX9_V_CMPX_LT_F32] = "v_cmpx_lt_f32",
+ [GFX9_V_CMPX_EQ_F32] = "v_cmpx_eq_f32",
+ [GFX9_V_CMPX_LE_F32] = "v_cmpx_le_f32",
+ [GFX9_V_CMPX_GT_F32] = "v_cmpx_gt_f32",
+ [GFX9_V_CMPX_LG_F32] = "v_cmpx_lg_f32",
+ [GFX9_V_CMPX_GE_F32] = "v_cmpx_ge_f32",
+ [GFX9_V_CMPX_O_F32] = "v_cmpx_o_f32",
+ [GFX9_V_CMPX_U_F32] = "v_cmpx_u_f32",
+ [GFX9_V_CMPX_NGE_F32] = "v_cmpx_nge_f32",
+ [GFX9_V_CMPX_NLG_F32] = "v_cmpx_nlg_f32",
+ [GFX9_V_CMPX_NGT_F32] = "v_cmpx_ngt_f32",
+ [GFX9_V_CMPX_NLE_F32] = "v_cmpx_nle_f32",
+ [GFX9_V_CMPX_NEQ_F32] = "v_cmpx_neq_f32",
+ [GFX9_V_CMPX_NLT_F32] = "v_cmpx_nlt_f32",
+ [GFX9_V_CMPX_TRU_F32] = "v_cmpx_tru_f32",
+ [GFX9_V_CMP_F_F64] = "v_cmp_f_f64",
+ [GFX9_V_CMP_LT_F64] = "v_cmp_lt_f64",
+ [GFX9_V_CMP_EQ_F64] = "v_cmp_eq_f64",
+ [GFX9_V_CMP_LE_F64] = "v_cmp_le_f64",
+ [GFX9_V_CMP_GT_F64] = "v_cmp_gt_f64",
+ [GFX9_V_CMP_LG_F64] = "v_cmp_lg_f64",
+ [GFX9_V_CMP_GE_F64] = "v_cmp_ge_f64",
+ [GFX9_V_CMP_O_F64] = "v_cmp_o_f64",
+ [GFX9_V_CMP_U_F64] = "v_cmp_u_f64",
+ [GFX9_V_CMP_NGE_F64] = "v_cmp_nge_f64",
+ [GFX9_V_CMP_NLG_F64] = "v_cmp_nlg_f64",
+ [GFX9_V_CMP_NGT_F64] = "v_cmp_ngt_f64",
+ [GFX9_V_CMP_NLE_F64] = "v_cmp_nle_f64",
+ [GFX9_V_CMP_NEQ_F64] = "v_cmp_neq_f64",
+ [GFX9_V_CMP_NLT_F64] = "v_cmp_nlt_f64",
+ [GFX9_V_CMP_TRU_F64] = "v_cmp_tru_f64",
+ [GFX9_V_CMPX_F_F64] = "v_cmpx_f_f64",
+ [GFX9_V_CMPX_LT_F64] = "v_cmpx_lt_f64",
+ [GFX9_V_CMPX_EQ_F64] = "v_cmpx_eq_f64",
+ [GFX9_V_CMPX_LE_F64] = "v_cmpx_le_f64",
+ [GFX9_V_CMPX_GT_F64] = "v_cmpx_gt_f64",
+ [GFX9_V_CMPX_LG_F64] = "v_cmpx_lg_f64",
+ [GFX9_V_CMPX_GE_F64] = "v_cmpx_ge_f64",
+ [GFX9_V_CMPX_O_F64] = "v_cmpx_o_f64",
+ [GFX9_V_CMPX_U_F64] = "v_cmpx_u_f64",
+ [GFX9_V_CMPX_NGE_F64] = "v_cmpx_nge_f64",
+ [GFX9_V_CMPX_NLG_F64] = "v_cmpx_nlg_f64",
+ [GFX9_V_CMPX_NGT_F64] = "v_cmpx_ngt_f64",
+ [GFX9_V_CMPX_NLE_F64] = "v_cmpx_nle_f64",
+ [GFX9_V_CMPX_NEQ_F64] = "v_cmpx_neq_f64",
+ [GFX9_V_CMPX_NLT_F64] = "v_cmpx_nlt_f64",
+ [GFX9_V_CMPX_TRU_F64] = "v_cmpx_tru_f64",
+ [GFX9_V_CMP_F_I16] = "v_cmp_f_i16",
+ [GFX9_V_CMP_LT_I16] = "v_cmp_lt_i16",
+ [GFX9_V_CMP_EQ_I16] = "v_cmp_eq_i16",
+ [GFX9_V_CMP_LE_I16] = "v_cmp_le_i16",
+ [GFX9_V_CMP_GT_I16] = "v_cmp_gt_i16",
+ [GFX9_V_CMP_NE_I16] = "v_cmp_ne_i16",
+ [GFX9_V_CMP_GE_I16] = "v_cmp_ge_i16",
+ [GFX9_V_CMP_T_I16] = "v_cmp_t_i16",
+ [GFX9_V_CMP_F_U16] = "v_cmp_f_u16",
+ [GFX9_V_CMP_LT_U16] = "v_cmp_lt_u16",
+ [GFX9_V_CMP_EQ_U16] = "v_cmp_eq_u16",
+ [GFX9_V_CMP_LE_U16] = "v_cmp_le_u16",
+ [GFX9_V_CMP_GT_U16] = "v_cmp_gt_u16",
+ [GFX9_V_CMP_NE_U16] = "v_cmp_ne_u16",
+ [GFX9_V_CMP_GE_U16] = "v_cmp_ge_u16",
+ [GFX9_V_CMP_T_U16] = "v_cmp_t_u16",
+ [GFX9_V_CMPX_F_I16] = "v_cmpx_f_i16",
+ [GFX9_V_CMPX_LT_I16] = "v_cmpx_lt_i16",
+ [GFX9_V_CMPX_EQ_I16] = "v_cmpx_eq_i16",
+ [GFX9_V_CMPX_LE_I16] = "v_cmpx_le_i16",
+ [GFX9_V_CMPX_GT_I16] = "v_cmpx_gt_i16",
+ [GFX9_V_CMPX_NE_I16] = "v_cmpx_ne_i16",
+ [GFX9_V_CMPX_GE_I16] = "v_cmpx_ge_i16",
+ [GFX9_V_CMPX_T_I16] = "v_cmpx_t_i16",
+ [GFX9_V_CMPX_F_U16] = "v_cmpx_f_u16",
+ [GFX9_V_CMPX_LT_U16] = "v_cmpx_lt_u16",
+ [GFX9_V_CMPX_EQ_U16] = "v_cmpx_eq_u16",
+ [GFX9_V_CMPX_LE_U16] = "v_cmpx_le_u16",
+ [GFX9_V_CMPX_GT_U16] = "v_cmpx_gt_u16",
+ [GFX9_V_CMPX_NE_U16] = "v_cmpx_ne_u16",
+ [GFX9_V_CMPX_GE_U16] = "v_cmpx_ge_u16",
+ [GFX9_V_CMPX_T_U16] = "v_cmpx_t_u16",
+ [GFX9_V_CMP_F_I32] = "v_cmp_f_i32",
+ [GFX9_V_CMP_LT_I32] = "v_cmp_lt_i32",
+ [GFX9_V_CMP_EQ_I32] = "v_cmp_eq_i32",
+ [GFX9_V_CMP_LE_I32] = "v_cmp_le_i32",
+ [GFX9_V_CMP_GT_I32] = "v_cmp_gt_i32",
+ [GFX9_V_CMP_NE_I32] = "v_cmp_ne_i32",
+ [GFX9_V_CMP_GE_I32] = "v_cmp_ge_i32",
+ [GFX9_V_CMP_T_I32] = "v_cmp_t_i32",
+ [GFX9_V_CMP_F_U32] = "v_cmp_f_u32",
+ [GFX9_V_CMP_LT_U32] = "v_cmp_lt_u32",
+ [GFX9_V_CMP_EQ_U32] = "v_cmp_eq_u32",
+ [GFX9_V_CMP_LE_U32] = "v_cmp_le_u32",
+ [GFX9_V_CMP_GT_U32] = "v_cmp_gt_u32",
+ [GFX9_V_CMP_NE_U32] = "v_cmp_ne_u32",
+ [GFX9_V_CMP_GE_U32] = "v_cmp_ge_u32",
+ [GFX9_V_CMP_T_U32] = "v_cmp_t_u32",
+ [GFX9_V_CMPX_F_I32] = "v_cmpx_f_i32",
+ [GFX9_V_CMPX_LT_I32] = "v_cmpx_lt_i32",
+ [GFX9_V_CMPX_EQ_I32] = "v_cmpx_eq_i32",
+ [GFX9_V_CMPX_LE_I32] = "v_cmpx_le_i32",
+ [GFX9_V_CMPX_GT_I32] = "v_cmpx_gt_i32",
+ [GFX9_V_CMPX_NE_I32] = "v_cmpx_ne_i32",
+ [GFX9_V_CMPX_GE_I32] = "v_cmpx_ge_i32",
+ [GFX9_V_CMPX_T_I32] = "v_cmpx_t_i32",
+ [GFX9_V_CMPX_F_U32] = "v_cmpx_f_u32",
+ [GFX9_V_CMPX_LT_U32] = "v_cmpx_lt_u32",
+ [GFX9_V_CMPX_EQ_U32] = "v_cmpx_eq_u32",
+ [GFX9_V_CMPX_LE_U32] = "v_cmpx_le_u32",
+ [GFX9_V_CMPX_GT_U32] = "v_cmpx_gt_u32",
+ [GFX9_V_CMPX_NE_U32] = "v_cmpx_ne_u32",
+ [GFX9_V_CMPX_GE_U32] = "v_cmpx_ge_u32",
+ [GFX9_V_CMPX_T_U32] = "v_cmpx_t_u32",
+ [GFX9_V_CMP_F_I64] = "v_cmp_f_i64",
+ [GFX9_V_CMP_LT_I64] = "v_cmp_lt_i64",
+ [GFX9_V_CMP_EQ_I64] = "v_cmp_eq_i64",
+ [GFX9_V_CMP_LE_I64] = "v_cmp_le_i64",
+ [GFX9_V_CMP_GT_I64] = "v_cmp_gt_i64",
+ [GFX9_V_CMP_NE_I64] = "v_cmp_ne_i64",
+ [GFX9_V_CMP_GE_I64] = "v_cmp_ge_i64",
+ [GFX9_V_CMP_T_I64] = "v_cmp_t_i64",
+ [GFX9_V_CMP_F_U64] = "v_cmp_f_u64",
+ [GFX9_V_CMP_LT_U64] = "v_cmp_lt_u64",
+ [GFX9_V_CMP_EQ_U64] = "v_cmp_eq_u64",
+ [GFX9_V_CMP_LE_U64] = "v_cmp_le_u64",
+ [GFX9_V_CMP_GT_U64] = "v_cmp_gt_u64",
+ [GFX9_V_CMP_NE_U64] = "v_cmp_ne_u64",
+ [GFX9_V_CMP_GE_U64] = "v_cmp_ge_u64",
+ [GFX9_V_CMP_T_U64] = "v_cmp_t_u64",
+ [GFX9_V_CMPX_F_I64] = "v_cmpx_f_i64",
+ [GFX9_V_CMPX_LT_I64] = "v_cmpx_lt_i64",
+ [GFX9_V_CMPX_EQ_I64] = "v_cmpx_eq_i64",
+ [GFX9_V_CMPX_LE_I64] = "v_cmpx_le_i64",
+ [GFX9_V_CMPX_GT_I64] = "v_cmpx_gt_i64",
+ [GFX9_V_CMPX_NE_I64] = "v_cmpx_ne_i64",
+ [GFX9_V_CMPX_GE_I64] = "v_cmpx_ge_i64",
+ [GFX9_V_CMPX_T_I64] = "v_cmpx_t_i64",
+ [GFX9_V_CMPX_F_U64] = "v_cmpx_f_u64",
+ [GFX9_V_CMPX_LT_U64] = "v_cmpx_lt_u64",
+ [GFX9_V_CMPX_EQ_U64] = "v_cmpx_eq_u64",
+ [GFX9_V_CMPX_LE_U64] = "v_cmpx_le_u64",
+ [GFX9_V_CMPX_GT_U64] = "v_cmpx_gt_u64",
+ [GFX9_V_CMPX_NE_U64] = "v_cmpx_ne_u64",
+ [GFX9_V_CMPX_GE_U64] = "v_cmpx_ge_u64",
+ [GFX9_V_CMPX_T_U64] = "v_cmpx_t_u64",
+ },
+ [AMDGCN_INSN_TYPE_VOP3A] = {
+ [GFX9_V_MAD_LEGACY_F32] = "v_mad_legacy_f32",
+ [GFX9_V_MAD_F32] = "v_mad_f32",
+ [GFX9_V_MAD_I32_I24] = "v_mad_i32_i24",
+ [GFX9_V_MAD_U32_U24] = "v_mad_u32_u24",
+ [GFX9_V_CUBEID_F32] = "v_cubeid_f32",
+ [GFX9_V_CUBESC_F32] = "v_cubesc_f32",
+ [GFX9_V_CUBETC_F32] = "v_cubetc_f32",
+ [GFX9_V_CUBEMA_F32] = "v_cubema_f32",
+ [GFX9_V_BFE_U32] = "v_bfe_u32",
+ [GFX9_V_BFE_I32] = "v_bfe_i32",
+ [GFX9_V_BFI_B32] = "v_bfi_b32",
+ [GFX9_V_FMA_F32] = "v_fma_f32",
+ [GFX9_V_FMA_F64] = "v_fma_f64",
+ [GFX9_V_LERP_U8] = "v_lerp_u8",
+ [GFX9_V_ALIGNBIT_B32] = "v_alignbit_b32",
+ [GFX9_V_ALIGNBYTE_B32] = "v_alignbyte_b32",
+ [GFX9_V_MIN3_F32] = "v_min3_f32",
+ [GFX9_V_MIN3_I32] = "v_min3_i32",
+ [GFX9_V_MIN3_U32] = "v_min3_u32",
+ [GFX9_V_MAX3_F32] = "v_max3_f32",
+ [GFX9_V_MAX3_I32] = "v_max3_i32",
+ [GFX9_V_MAX3_U32] = "v_max3_u32",
+ [GFX9_V_MED3_F32] = "v_med3_f32",
+ [GFX9_V_MED3_I32] = "v_med3_i32",
+ [GFX9_V_MED3_U32] = "v_med3_u32",
+ [GFX9_V_SAD_U8] = "v_sad_u8",
+ [GFX9_V_SAD_HI_U8] = "v_sad_hi_u8",
+ [GFX9_V_SAD_U16] = "v_sad_u16",
+ [GFX9_V_SAD_U32] = "v_sad_u32",
+ [GFX9_V_CVT_PK_U8_F32] = "v_cvt_pk_u8_f32",
+ [GFX9_V_DIV_FIXUP_F32] = "v_div_fixup_f32",
+ [GFX9_V_DIV_FIXUP_F64] = "v_div_fixup_f64",
+ [GFX9_V_DIV_FMAS_F32] = "v_div_fmas_f32",
+ [GFX9_V_DIV_FMAS_F64] = "v_div_fmas_f64",
+ [GFX9_V_MSAD_U8] = "v_msad_u8",
+ [GFX9_V_QSAD_PK_U16_U8] = "v_qsad_pk_u16_u8",
+ [GFX9_V_MQSAD_PK_U16_U8] = "v_mqsad_pk_u16_u8",
+ [GFX9_V_MQSAD_U32_U8] = "v_mqsad_u32_u8",
+ [GFX9_V_MAD_LEGACY_F16] = "v_mad_legacy_f16",
+ [GFX9_V_MAD_LEGACY_U16] = "v_mad_legacy_u16",
+ [GFX9_V_MAD_LEGACY_I16] = "v_mad_legacy_i16",
+ [GFX9_V_PERM_B32] = "v_perm_b32",
+ [GFX9_V_FMA_LEGACY_F16] = "v_fma_legacy_f16",
+ [GFX9_V_DIV_FIXUP_LEGACY_F16] = "v_div_fixup_legacy_f16",
+ [GFX9_V_CVT_PKACCUM_U8_F32] = "v_cvt_pkaccum_u8_f32",
+ [GFX9_V_MAD_U32_U16] = "v_mad_u32_u16",
+ [GFX9_V_MAD_I32_I16] = "v_mad_i32_i16",
+ [GFX9_V_XAD_U32] = "v_xad_u32",
+ [GFX9_V_MIN3_F16] = "v_min3_f16",
+ [GFX9_V_MIN3_I16] = "v_min3_i16",
+ [GFX9_V_MIN3_U16] = "v_min3_u16",
+ [GFX9_V_MAX3_F16] = "v_max3_f16",
+ [GFX9_V_MAX3_I16] = "v_max3_i16",
+ [GFX9_V_MAX3_U16] = "v_max3_u16",
+ [GFX9_V_MED3_F16] = "v_med3_f16",
+ [GFX9_V_MED3_I16] = "v_med3_i16",
+ [GFX9_V_MED3_U16] = "v_med3_u16",
+ [GFX9_V_LSHL_ADD_U32] = "v_lshl_add_u32",
+ [GFX9_V_ADD_LSHL_U32] = "v_add_lshl_u32",
+ [GFX9_V_ADD3_U32] = "v_add3_u32",
+ [GFX9_V_LSHL_OR_B32] = "v_lshl_or_b32",
+ [GFX9_V_AND_OR_B32] = "v_and_or_b32",
+ [GFX9_V_OR3_B32] = "v_or3_b32",
+ [GFX9_V_MAD_F16] = "v_mad_f16",
+ [GFX9_V_MAD_U16] = "v_mad_u16",
+ [GFX9_V_MAD_I16] = "v_mad_i16",
+ [GFX9_V_FMA_F16] = "v_fma_f16",
+ [GFX9_V_DIV_FIXUP_F16] = "v_div_fixup_f16",
+ [GFX9_V_INTERP_P1LL_F16] = "v_interp_p1ll_f16",
+ [GFX9_V_INTERP_P1LV_F16] = "v_interp_p1lv_f16",
+ [GFX9_V_INTERP_P2_LEGACY_F16] = "v_interp_p2_legacy_f16",
+ [GFX9_V_INTERP_P2_F16] = "v_interp_p2_f16",
+ [GFX9_V_ADD_F64] = "v_add_f64",
+ [GFX9_V_MUL_F64] = "v_mul_f64",
+ [GFX9_V_MIN_F64] = "v_min_f64",
+ [GFX9_V_MAX_F64] = "v_max_f64",
+ [GFX9_V_LDEXP_F64] = "v_ldexp_f64",
+ [GFX9_V_MUL_LO_U32] = "v_mul_lo_u32",
+ [GFX9_V_MUL_HI_U32] = "v_mul_hi_u32",
+ [GFX9_V_MUL_HI_I32] = "v_mul_hi_i32",
+ [GFX9_V_LDEXP_F32] = "v_ldexp_f32",
+ [GFX9_V_READLANE_B32] = "v_readlane_b32",
+ [GFX9_V_WRITELANE_B32] = "v_writelane_b32",
+ [GFX9_V_BCNT_U32_B32] = "v_bcnt_u32_b32",
+ [GFX9_V_MBCNT_LO_U32_B32] = "v_mbcnt_lo_u32_b32",
+ [GFX9_V_MBCNT_HI_U32_B32] = "v_mbcnt_hi_u32_b32",
+ [GFX9_V_LSHLREV_B64] = "v_lshlrev_b64",
+ [GFX9_V_LSHRREV_B64] = "v_lshrrev_b64",
+ [GFX9_V_ASHRREV_I64] = "v_ashrrev_i64",
+ [GFX9_V_TRIG_PREOP_F64] = "v_trig_preop_f64",
+ [GFX9_V_BFM_B32] = "v_bfm_b32",
+ [GFX9_V_CVT_PKNORM_I16_F32] = "v_cvt_pknorm_i16_f32",
+ [GFX9_V_CVT_PKNORM_U16_F32] = "v_cvt_pknorm_u16_f32",
+ [GFX9_V_CVT_PKRTZ_F16_F32] = "v_cvt_pkrtz_f16_f32",
+ [GFX9_V_CVT_PK_U16_U32] = "v_cvt_pk_u16_u32",
+ [GFX9_V_CVT_PK_I16_I32] = "v_cvt_pk_i16_i32",
+ [GFX9_V_CVT_PKNORM_I16_F16] = "v_cvt_pknorm_i16_f16",
+ [GFX9_V_CVT_PKNORM_U16_F16] = "v_cvt_pknorm_u16_f16",
+ [GFX9_V_ADD_I32] = "v_add_i32",
+ [GFX9_V_SUB_I32] = "v_sub_i32",
+ [GFX9_V_ADD_I16] = "v_add_i16",
+ [GFX9_V_SUB_I16] = "v_sub_i16",
+ [GFX9_V_PACK_B32_F16] = "v_pack_b32_f16",
+ },
+ [AMDGCN_INSN_TYPE_VOP3B] = {
+ [GFX9_V_DIV_SCALE_F64] = "v_div_scale_f64",
+ [GFX9_V_MAD_U64_U32] = "v_mad_u64_u32",
+ [GFX9_V_MAD_I64_I32] = "v_mad_i64_i32",
+ },
+ [AMDGCN_INSN_TYPE_VOP3P] = {
+ [GFX9_V_PK_MAD_I16] = "v_pk_mad_i16",
+ [GFX9_V_PK_MUL_LO_U16] = "v_pk_mul_lo_u16",
+ [GFX9_V_PK_ADD_I16] = "v_pk_add_i16",
+ [GFX9_V_PK_SUB_I16] = "v_pk_sub_i16",
+ [GFX9_V_PK_LSHLREV_B16] = "v_pk_lshlrev_b16",
+ [GFX9_V_PK_LSHRREV_B16] = "v_pk_lshrrev_b16",
+ [GFX9_V_PK_ASHRREV_I16] = "v_pk_ashrrev_i16",
+ [GFX9_V_PK_MAX_I16] = "v_pk_max_i16",
+ [GFX9_V_PK_MIN_I16] = "v_pk_min_i16",
+ [GFX9_V_PK_MAD_U16] = "v_pk_mad_u16",
+ [GFX9_V_PK_ADD_U16] = "v_pk_add_u16",
+ [GFX9_V_PK_SUB_U16] = "v_pk_sub_u16",
+ [GFX9_V_PK_MAX_U16] = "v_pk_max_u16",
+ [GFX9_V_PK_MIN_U16] = "v_pk_min_u16",
+ [GFX9_V_PK_FMA_F16] = "v_pk_fma_f16",
+ [GFX9_V_PK_ADD_F16] = "v_pk_add_f16",
+ [GFX9_V_PK_MUL_F16] = "v_pk_mul_f16",
+ [GFX9_V_PK_MIN_F16] = "v_pk_min_f16",
+ [GFX9_V_PK_MAX_F16] = "v_pk_max_f16",
+ [GFX9_V_MAD_MIX_F32] = "v_mad_mix_f32",
+ [GFX9_V_MAD_MIXLO_F16] = "v_mad_mixlo_f16",
+ [GFX9_V_MAD_MIXHI_F16] = "v_mad_mixhi_f16",
+ },
+ [AMDGCN_INSN_TYPE_MUBUF] = {
+ [GFX9_BUFFER_LOAD_FORMAT_X] = "buffer_load_format_x",
+ [GFX9_BUFFER_LOAD_FORMAT_XY] = "buffer_load_format_xy",
+ [GFX9_BUFFER_LOAD_FORMAT_XYZ] = "buffer_load_format_xyz",
+ [GFX9_BUFFER_LOAD_FORMAT_XYZW] = "buffer_load_format_xyzw",
+ [GFX9_BUFFER_STORE_FORMAT_X] = "buffer_store_format_x",
+ [GFX9_BUFFER_STORE_FORMAT_XY] = "buffer_store_format_xy",
+ [GFX9_BUFFER_STORE_FORMAT_XYZ] = "buffer_store_format_xyz",
+ [GFX9_BUFFER_STORE_FORMAT_XYZW] = "buffer_store_format_xyzw",
+ [GFX9_BUFFER_LOAD_FORMAT_D16_X] = "buffer_load_format_d16_x",
+ [GFX9_BUFFER_LOAD_FORMAT_D16_XY] = "buffer_load_format_d16_xy",
+ [GFX9_BUFFER_LOAD_FORMAT_D16_XYZ] = "buffer_load_format_d16_xyz",
+ [GFX9_BUFFER_LOAD_FORMAT_D16_XYZW] = "buffer_load_format_d16_xyzw",
+ [GFX9_BUFFER_STORE_FORMAT_D16_X] = "buffer_store_format_d16_x",
+ [GFX9_BUFFER_STORE_FORMAT_D16_XY] = "buffer_store_format_d16_xy",
+ [GFX9_BUFFER_STORE_FORMAT_D16_XYZ] = "buffer_store_format_d16_xyz",
+ [GFX9_BUFFER_STORE_FORMAT_D16_XYZW] = "buffer_store_format_d16_xyzw",
+ [GFX9_BUFFER_LOAD_UBYTE] = "buffer_load_ubyte",
+ [GFX9_BUFFER_LOAD_SBYTE] = "buffer_load_sbyte",
+ [GFX9_BUFFER_LOAD_USHORT] = "buffer_load_ushort",
+ [GFX9_BUFFER_LOAD_SSHORT] = "buffer_load_sshort",
+ [GFX9_BUFFER_LOAD_DWORD] = "buffer_load_dword",
+ [GFX9_BUFFER_LOAD_DWORDX2] = "buffer_load_dwordx2",
+ [GFX9_BUFFER_LOAD_DWORDX3] = "buffer_load_dwordx3",
+ [GFX9_BUFFER_LOAD_DWORDX4] = "buffer_load_dwordx4",
+ [GFX9_BUFFER_STORE_BYTE] = "buffer_store_byte",
+ [GFX9_BUFFER_STORE_BYTE_D16_HI] = "buffer_store_byte_d16_hi",
+ [GFX9_BUFFER_STORE_SHORT] = "buffer_store_short",
+ [GFX9_BUFFER_STORE_SHORT_D16_HI] = "buffer_store_short_d16_hi",
+ [GFX9_BUFFER_STORE_DWORD] = "buffer_store_dword",
+ [GFX9_BUFFER_STORE_DWORDX2] = "buffer_store_dwordx2",
+ [GFX9_BUFFER_STORE_DWORDX3] = "buffer_store_dwordx3",
+ [GFX9_BUFFER_STORE_DWORDX4] = "buffer_store_dwordx4",
+ [GFX9_BUFFER_LOAD_UBYTE_D16] = "buffer_load_ubyte_d16",
+ [GFX9_BUFFER_LOAD_UBYTE_D16_HI] = "buffer_load_ubyte_d16_hi",
+ [GFX9_BUFFER_LOAD_SBYTE_D16] = "buffer_load_sbyte_d16",
+ [GFX9_BUFFER_LOAD_SBYTE_D16_HI] = "buffer_load_sbyte_d16_hi",
+ [GFX9_BUFFER_LOAD_SHORT_D16] = "buffer_load_short_d16",
+ [GFX9_BUFFER_LOAD_SHORT_D16_HI] = "buffer_load_short_d16_hi",
+ [GFX9_BUFFER_LOAD_FORMAT_D16_HI_X] = "buffer_load_format_d16_hi_x",
+ [GFX9_BUFFER_STORE_FORMAT_D16_HI_X] = "buffer_store_format_d16_hi_x",
+ [GFX9_BUFFER_STORE_LDS_DWORD] = "buffer_store_lds_dword",
+ [GFX9_BUFFER_WBINVL1] = "buffer_wbinvl1",
+ [GFX9_BUFFER_WBINVL1_VOL] = "buffer_wbinvl1_vol",
+ [GFX9_BUFFER_ATOMIC_SWAP] = "buffer_atomic_swap",
+ [GFX9_BUFFER_ATOMIC_CMPSWAP] = "buffer_atomic_cmpswap",
+ [GFX9_BUFFER_ATOMIC_ADD] = "buffer_atomic_add",
+ [GFX9_BUFFER_ATOMIC_SUB] = "buffer_atomic_sub",
+ [GFX9_BUFFER_ATOMIC_SMIN] = "buffer_atomic_smin",
+ [GFX9_BUFFER_ATOMIC_UMIN] = "buffer_atomic_umin",
+ [GFX9_BUFFER_ATOMIC_SMAX] = "buffer_atomic_smax",
+ [GFX9_BUFFER_ATOMIC_UMAX] = "buffer_atomic_umax",
+ [GFX9_BUFFER_ATOMIC_AND] = "buffer_atomic_and",
+ [GFX9_BUFFER_ATOMIC_OR] = "buffer_atomic_or",
+ [GFX9_BUFFER_ATOMIC_XOR] = "buffer_atomic_xor",
+ [GFX9_BUFFER_ATOMIC_INC] = "buffer_atomic_inc",
+ [GFX9_BUFFER_ATOMIC_DEC] = "buffer_atomic_dec",
+ [GFX9_BUFFER_ATOMIC_SWAP_X2] = "buffer_atomic_swap_x2",
+ [GFX9_BUFFER_ATOMIC_CMPSWAP_X2] = "buffer_atomic_cmpswap_x2",
+ [GFX9_BUFFER_ATOMIC_ADD_X2] = "buffer_atomic_add_x2",
+ [GFX9_BUFFER_ATOMIC_SUB_X2] = "buffer_atomic_sub_x2",
+ [GFX9_BUFFER_ATOMIC_SMIN_X2] = "buffer_atomic_smin_x2",
+ [GFX9_BUFFER_ATOMIC_UMIN_X2] = "buffer_atomic_umin_x2",
+ [GFX9_BUFFER_ATOMIC_SMAX_X2] = "buffer_atomic_smax_x2",
+ [GFX9_BUFFER_ATOMIC_UMAX_X2] = "buffer_atomic_umax_x2",
+ [GFX9_BUFFER_ATOMIC_AND_X2] = "buffer_atomic_and_x2",
+ [GFX9_BUFFER_ATOMIC_OR_X2] = "buffer_atomic_or_x2",
+ [GFX9_BUFFER_ATOMIC_XOR_X2] = "buffer_atomic_xor_x2",
+ [GFX9_BUFFER_ATOMIC_INC_X2] = "buffer_atomic_inc_x2",
+ [GFX9_BUFFER_ATOMIC_DEC_X2] = "buffer_atomic_dec_x2",
+ },
+ [AMDGCN_INSN_TYPE_FLAT] = {
+ [GFX9_GLOBAL_LOAD_UBYTE] = "global_load_ubyte",
+ [GFX9_GLOBAL_LOAD_SBYTE] = "global_load_sbyte",
+ [GFX9_GLOBAL_LOAD_USHORT] = "global_load_ushort",
+ [GFX9_GLOBAL_LOAD_SSHORT] = "global_load_sshort",
+ [GFX9_GLOBAL_LOAD_DWORD] = "global_load_dword",
+ [GFX9_GLOBAL_LOAD_DWORDX2] = "global_load_dwordx2",
+ [GFX9_GLOBAL_LOAD_DWORDX3] = "global_load_dwordx3",
+ [GFX9_GLOBAL_LOAD_DWORDX4] = "global_load_dwordx4",
+ [GFX9_GLOBAL_STORE_BYTE] = "global_store_byte",
+ [GFX9_GLOBAL_STORE_BYTE_D16_HI] = "global_store_byte_d16_hi",
+ [GFX9_GLOBAL_STORE_SHORT] = "global_store_short",
+ [GFX9_GLOBAL_STORE_SHORT_D16_HI] = "global_store_short_d16_hi",
+ [GFX9_GLOBAL_STORE_DWORD] = "global_store_dword",
+ [GFX9_GLOBAL_STORE_DWORDX2] = "global_store_dwordx2",
+ [GFX9_GLOBAL_STORE_DWORDX3] = "global_store_dwordx3",
+ [GFX9_GLOBAL_STORE_DWORDX4] = "global_store_dwordx4",
+ [GFX9_GLOBAL_LOAD_UBYTE_D16] = "global_load_ubyte_d16",
+ [GFX9_GLOBAL_LOAD_UBYTE_D16_HI] = "global_load_ubyte_d16_hi",
+ [GFX9_GLOBAL_LOAD_SBYTE_D16] = "global_load_sbyte_d16",
+ [GFX9_GLOBAL_LOAD_SBYTE_D16_HI] = "global_load_sbyte_d16_hi",
+ [GFX9_GLOBAL_LOAD_SHORT_D16] = "global_load_short_d16",
+ [GFX9_GLOBAL_LOAD_SHORT_D16_HI] = "global_load_short_d16_hi",
+ [GFX9_GLOBAL_ATOMIC_SWAP] = "global_atomic_swap",
+ [GFX9_GLOBAL_ATOMIC_CMPSWAP] = "global_atomic_cmpswap",
+ [GFX9_GLOBAL_ATOMIC_ADD] = "global_atomic_add",
+ [GFX9_GLOBAL_ATOMIC_SUB] = "global_atomic_sub",
+ [GFX9_GLOBAL_ATOMIC_SMIN] = "global_atomic_smin",
+ [GFX9_GLOBAL_ATOMIC_UMIN] = "global_atomic_umin",
+ [GFX9_GLOBAL_ATOMIC_SMAX] = "global_atomic_smax",
+ [GFX9_GLOBAL_ATOMIC_UMAX] = "global_atomic_umax",
+ [GFX9_GLOBAL_ATOMIC_AND] = "global_atomic_and",
+ [GFX9_GLOBAL_ATOMIC_OR] = "global_atomic_or",
+ [GFX9_GLOBAL_ATOMIC_XOR] = "global_atomic_xor",
+ [GFX9_GLOBAL_ATOMIC_INC] = "global_atomic_inc",
+ [GFX9_GLOBAL_ATOMIC_DEC] = "global_atomic_dec",
+ [GFX9_GLOBAL_ATOMIC_SWAP_X2] = "global_atomic_swap_x2",
+ [GFX9_GLOBAL_ATOMIC_CMPSWAP_X2] = "global_atomic_cmpswap_x2",
+ [GFX9_GLOBAL_ATOMIC_ADD_X2] = "global_atomic_add_x2",
+ [GFX9_GLOBAL_ATOMIC_SUB_X2] = "global_atomic_sub_x2",
+ [GFX9_GLOBAL_ATOMIC_SMIN_X2] = "global_atomic_smin_x2",
+ [GFX9_GLOBAL_ATOMIC_UMIN_X2] = "global_atomic_umin_x2",
+ [GFX9_GLOBAL_ATOMIC_SMAX_X2] = "global_atomic_smax_x2",
+ [GFX9_GLOBAL_ATOMIC_UMAX_X2] = "global_atomic_umax_x2",
+ [GFX9_GLOBAL_ATOMIC_AND_X2] = "global_atomic_and_x2",
+ [GFX9_GLOBAL_ATOMIC_OR_X2] = "global_atomic_or_x2",
+ [GFX9_GLOBAL_ATOMIC_XOR_X2] = "global_atomic_xor_x2",
+ [GFX9_GLOBAL_ATOMIC_INC_X2] = "global_atomic_inc_x2",
+ [GFX9_GLOBAL_ATOMIC_DEC_X2] = "global_atomic_dec_x2",
+ },
+ [AMDGCN_INSN_TYPE_DS] = {
+ /* DS opcode names (Vega ISA 12.13, opcodes 0-255). The DS
+ * table was entirely unpopulated, so DS instructions
+ * disassembled with a blank mnemonic. Enum values were
+ * ISA-verified (one known value bug at DS_CONDXCHG32_RTN_B64
+ * is tracked separately).
+ */
+ [GFX9_DS_ADD_U32] = "ds_add_u32",
+ [GFX9_DS_SUB_U32] = "ds_sub_u32",
+ [GFX9_DS_RSUB_U32] = "ds_rsub_u32",
+ [GFX9_DS_INC_U32] = "ds_inc_u32",
+ [GFX9_DS_DEC_U32] = "ds_dec_u32",
+ [GFX9_DS_MIN_I32] = "ds_min_i32",
+ [GFX9_DS_MAX_I32] = "ds_max_i32",
+ [GFX9_DS_MIN_U32] = "ds_min_u32",
+ [GFX9_DS_MAX_U32] = "ds_max_u32",
+ [GFX9_DS_AND_B32] = "ds_and_b32",
+ [GFX9_DS_OR_B32] = "ds_or_b32",
+ [GFX9_DS_XOR_B32] = "ds_xor_b32",
+ [GFX9_DS_MSKOR_B32] = "ds_mskor_b32",
+ [GFX9_DS_WRITE_B32] = "ds_write_b32",
+ [GFX9_DS_WRITE2_B32] = "ds_write2_b32",
+ [GFX9_DS_WRITE2ST64_B32] = "ds_write2st64_b32",
+ [GFX9_DS_CMPST_B32] = "ds_cmpst_b32",
+ [GFX9_DS_CMPST_F32] = "ds_cmpst_f32",
+ [GFX9_DS_MIN_F32] = "ds_min_f32",
+ [GFX9_DS_MAX_F32] = "ds_max_f32",
+ [GFX9_DS_NOP] = "ds_nop",
+ [GFX9_DS_ADD_F32] = "ds_add_f32",
+ [GFX9_DS_WRITE_ADDTID_B32] = "ds_write_addtid_b32",
+ [GFX9_DS_WRITE_B8] = "ds_write_b8",
+ [GFX9_DS_WRITE_B16] = "ds_write_b16",
+ [GFX9_DS_ADD_RTN_U32] = "ds_add_rtn_u32",
+ [GFX9_DS_SUB_RTN_U32] = "ds_sub_rtn_u32",
+ [GFX9_DS_RSUB_RTN_U32] = "ds_rsub_rtn_u32",
+ [GFX9_DS_INC_RTN_U32] = "ds_inc_rtn_u32",
+ [GFX9_DS_DEC_RTN_U32] = "ds_dec_rtn_u32",
+ [GFX9_DS_MIN_RTN_I32] = "ds_min_rtn_i32",
+ [GFX9_DS_MAX_RTN_I32] = "ds_max_rtn_i32",
+ [GFX9_DS_MIN_RTN_U32] = "ds_min_rtn_u32",
+ [GFX9_DS_MAX_RTN_U32] = "ds_max_rtn_u32",
+ [GFX9_DS_AND_RTN_B32] = "ds_and_rtn_b32",
+ [GFX9_DS_OR_RTN_B32] = "ds_or_rtn_b32",
+ [GFX9_DS_XOR_RTN_B32] = "ds_xor_rtn_b32",
+ [GFX9_DS_MSKOR_RTN_B32] = "ds_mskor_rtn_b32",
+ [GFX9_DS_WRXCHG_RTN_B32] = "ds_wrxchg_rtn_b32",
+ [GFX9_DS_WRXCHG2_RTN_B32] = "ds_wrxchg2_rtn_b32",
+ [GFX9_DS_WRXCHG2ST64_RTN_B32] = "ds_wrxchg2st64_rtn_b32",
+ [GFX9_DS_CMPST_RTN_B32] = "ds_cmpst_rtn_b32",
+ [GFX9_DS_CMPST_RTN_F32] = "ds_cmpst_rtn_f32",
+ [GFX9_DS_MIN_RTN_F32] = "ds_min_rtn_f32",
+ [GFX9_DS_MAX_RTN_F32] = "ds_max_rtn_f32",
+ [GFX9_DS_WRAP_RTN_B32] = "ds_wrap_rtn_b32",
+ [GFX9_DS_ADD_RTN_F32] = "ds_add_rtn_f32",
+ [GFX9_DS_READ_B32] = "ds_read_b32",
+ [GFX9_DS_READ2_B32] = "ds_read2_b32",
+ [GFX9_DS_READ2ST64_B32] = "ds_read2st64_b32",
+ [GFX9_DS_READ_I8] = "ds_read_i8",
+ [GFX9_DS_READ_U8] = "ds_read_u8",
+ [GFX9_DS_READ_I16] = "ds_read_i16",
+ [GFX9_DS_READ_U16] = "ds_read_u16",
+ [GFX9_DS_SWIZZLE_B32] = "ds_swizzle_b32",
+ [GFX9_DS_PERMUTE_B32] = "ds_permute_b32",
+ [GFX9_DS_BPERMUTE_B32] = "ds_bpermute_b32",
+ [GFX9_DS_ADD_U64] = "ds_add_u64",
+ [GFX9_DS_SUB_U64] = "ds_sub_u64",
+ [GFX9_DS_RSUB_U64] = "ds_rsub_u64",
+ [GFX9_DS_INC_U64] = "ds_inc_u64",
+ [GFX9_DS_DEC_U64] = "ds_dec_u64",
+ [GFX9_DS_MIN_I64] = "ds_min_i64",
+ [GFX9_DS_MAX_I64] = "ds_max_i64",
+ [GFX9_DS_MIN_U64] = "ds_min_u64",
+ [GFX9_DS_MAX_U64] = "ds_max_u64",
+ [GFX9_DS_AND_B64] = "ds_and_b64",
+ [GFX9_DS_OR_B64] = "ds_or_b64",
+ [GFX9_DS_XOR_B64] = "ds_xor_b64",
+ [GFX9_DS_MSKOR_B64] = "ds_mskor_b64",
+ [GFX9_DS_WRITE_B64] = "ds_write_b64",
+ [GFX9_DS_WRITE2_B64] = "ds_write2_b64",
+ [GFX9_DS_WRITE2ST64_B64] = "ds_write2st64_b64",
+ [GFX9_DS_CMPST_B64] = "ds_cmpst_b64",
+ [GFX9_DS_CMPST_F64] = "ds_cmpst_f64",
+ [GFX9_DS_MIN_F64] = "ds_min_f64",
+ [GFX9_DS_MAX_F64] = "ds_max_f64",
+ [GFX9_DS_WRITE_B8_D16_HI] = "ds_write_b8_d16_hi",
+ [GFX9_DS_WRITE_B16_D16_HI] = "ds_write_b16_d16_hi",
+ [GFX9_DS_READ_U8_D16] = "ds_read_u8_d16",
+ [GFX9_DS_READ_U8_D16_HI] = "ds_read_u8_d16_hi",
+ [GFX9_DS_READ_I8_D16] = "ds_read_i8_d16",
+ [GFX9_DS_READ_I8_D16_HI] = "ds_read_i8_d16_hi",
+ [GFX9_DS_READ_U16_D16] = "ds_read_u16_d16",
+ [GFX9_DS_READ_U16_D16_HI] = "ds_read_u16_d16_hi",
+ [GFX9_DS_ADD_RTN_U64] = "ds_add_rtn_u64",
+ [GFX9_DS_SUB_RTN_U64] = "ds_sub_rtn_u64",
+ [GFX9_DS_RSUB_RTN_U64] = "ds_rsub_rtn_u64",
+ [GFX9_DS_INC_RTN_U64] = "ds_inc_rtn_u64",
+ [GFX9_DS_DEC_RTN_U64] = "ds_dec_rtn_u64",
+ [GFX9_DS_MIN_RTN_I64] = "ds_min_rtn_i64",
+ [GFX9_DS_MAX_RTN_I64] = "ds_max_rtn_i64",
+ [GFX9_DS_MIN_RTN_U64] = "ds_min_rtn_u64",
+ [GFX9_DS_MAX_RTN_U64] = "ds_max_rtn_u64",
+ [GFX9_DS_AND_RTN_B64] = "ds_and_rtn_b64",
+ [GFX9_DS_OR_RTN_B64] = "ds_or_rtn_b64",
+ [GFX9_DS_XOR_RTN_B64] = "ds_xor_rtn_b64",
+ [GFX9_DS_MSKOR_RTN_B64] = "ds_mskor_rtn_b64",
+ [GFX9_DS_WRXCHG_RTN_B64] = "ds_wrxchg_rtn_b64",
+ [GFX9_DS_WRXCHG2_RTN_B64] = "ds_wrxchg2_rtn_b64",
+ [GFX9_DS_WRXCHG2ST64_RTN_B64] = "ds_wrxchg2st64_rtn_b64",
+ [GFX9_DS_CMPST_RTN_B64] = "ds_cmpst_rtn_b64",
+ [GFX9_DS_CMPST_RTN_F64] = "ds_cmpst_rtn_f64",
+ [GFX9_DS_MIN_RTN_F64] = "ds_min_rtn_f64",
+ [GFX9_DS_MAX_RTN_F64] = "ds_max_rtn_f64",
+ [GFX9_DS_READ_B64] = "ds_read_b64",
+ [GFX9_DS_READ2_B64] = "ds_read2_b64",
+ [GFX9_DS_CONDXCHG32_RTN_B64] = "ds_condxchg32_rtn_b64",
+ [GFX9_DS_ADD_SRC2_U32] = "ds_add_src2_u32",
+ [GFX9_DS_SUB_SRC2_U32] = "ds_sub_src2_u32",
+ [GFX9_DS_RSUB_SRC2_U32] = "ds_rsub_src2_u32",
+ [GFX9_DS_INC_SRC2_U32] = "ds_inc_src2_u32",
+ [GFX9_DS_DEC_SRC2_U32] = "ds_dec_src2_u32",
+ [GFX9_DS_MIN_SRC2_I32] = "ds_min_src2_i32",
+ [GFX9_DS_MAX_SRC2_I32] = "ds_max_src2_i32",
+ [GFX9_DS_MIN_SRC2_U32] = "ds_min_src2_u32",
+ [GFX9_DS_MAX_SRC2_U32] = "ds_max_src2_u32",
+ [GFX9_DS_AND_SRC2_B32] = "ds_and_src2_b32",
+ [GFX9_DS_OR_SRC2_B32] = "ds_or_src2_b32",
+ [GFX9_DS_XOR_SRC2_B32] = "ds_xor_src2_b32",
+ [GFX9_DS_WRITE_SRC2_B32] = "ds_write_src2_b32",
+ [GFX9_DS_MIN_SRC2_F32] = "ds_min_src2_f32",
+ [GFX9_DS_MAX_SRC2_F32] = "ds_max_src2_f32",
+ [GFX9_DS_ADD_SRC2_F32] = "ds_add_src2_f32",
+ [GFX9_DS_GWS_SEMA_RELEASE_ALL] = "ds_gws_sema_release_all",
+ [GFX9_DS_ADD_SRC2_U64] = "ds_add_src2_u64",
+ [GFX9_DS_SUB_SRC2_U64] = "ds_sub_src2_u64",
+ [GFX9_DS_RSUB_SRC2_U64] = "ds_rsub_src2_u64",
+ [GFX9_DS_INC_SRC2_U64] = "ds_inc_src2_u64",
+ [GFX9_DS_DEC_SRC2_U64] = "ds_dec_src2_u64",
+ [GFX9_DS_MIN_SRC2_I64] = "ds_min_src2_i64",
+ [GFX9_DS_MAX_SRC2_I64] = "ds_max_src2_i64",
+ [GFX9_DS_MIN_SRC2_U64] = "ds_min_src2_u64",
+ [GFX9_DS_MAX_SRC2_U64] = "ds_max_src2_u64",
+ [GFX9_DS_AND_SRC2_B64] = "ds_and_src2_b64",
+ [GFX9_DS_OR_SRC2_B64] = "ds_or_src2_b64",
+ [GFX9_DS_XOR_SRC2_B64] = "ds_xor_src2_b64",
+ [GFX9_DS_WRITE_SRC2_B64] = "ds_write_src2_b64",
+ [GFX9_DS_MIN_SRC2_F64] = "ds_min_src2_f64",
+ [GFX9_DS_MAX_SRC2_F64] = "ds_max_src2_f64",
+ [GFX9_DS_WRITE_B96] = "ds_write_b96",
+ [GFX9_DS_WRITE_B128] = "ds_write_b128",
+ [GFX9_DS_READ_B96] = "ds_read_b96",
+ [GFX9_DS_READ_B128] = "ds_read_b128",
+ },
+};
+
+struct amdgcn_insn {
+ union {
+ union amdgcn_gfx10_insn gfx10;
+ union amdgcn_gfx9_insn gfx9;
+ };
+ u32 size;
+ u32 idx;
+ enum amdgcn_insn_type type;
+};
+
+static inline void emit_s_load_dwordx2(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src, int offset)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_s_load_dwordx2(&insn->gfx10, dst,
+ src,
+ offset);
+ insn->type = AMDGCN_INSN_TYPE_SMEM;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_s_load_dwordx2(&insn->gfx9, dst,
+ src,
+ offset);
+ insn->type = AMDGCN_INSN_TYPE_SMEM;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_s_load_dwordx2_soff(int version,
+ struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src,
+ int offset, u8 soffset)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_s_load_dwordx2(&insn->gfx10, dst,
+ src, offset);
+ insn->gfx10.smem.soffset = soffset;
+ insn->type = AMDGCN_INSN_TYPE_SMEM;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_s_load_dwordx2(&insn->gfx9, dst,
+ src, offset);
+ insn->gfx9.smem.soe = 1;
+ insn->gfx9.smem.soffset = soffset;
+ insn->type = AMDGCN_INSN_TYPE_SMEM;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_s_lshl_b32(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_s_lshl_b32(&insn->gfx10, dst,
+ src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_SOP2;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_s_lshl_b32(&insn->gfx9, dst,
+ src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_SOP2;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_bfe_i32(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1,
+ struct amdgcn_param32 src2)
+{
+ WARN_ON(knod_param_is_literal(src0) ||
+ knod_param_is_literal(src1) ||
+ knod_param_is_literal(src2));
+ if (version == 10) {
+ insn->size = emit_gfx10_v_bfe_i32(&insn->gfx10,
+ dst, src0, src1, src2);
+ insn->type = AMDGCN_INSN_TYPE_VOP3A;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_v_bfe_i32(&insn->gfx9,
+ dst, src0, src1, src2);
+ insn->type = AMDGCN_INSN_TYPE_VOP3A;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_bfe_u32(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1,
+ struct amdgcn_param32 src2)
+{
+ WARN_ON(knod_param_is_literal(src0) ||
+ knod_param_is_literal(src1) ||
+ knod_param_is_literal(src2));
+ if (version == 10) {
+ insn->size = emit_gfx10_v_bfe_u32(&insn->gfx10,
+ dst, src0, src1, src2);
+ insn->type = AMDGCN_INSN_TYPE_VOP3A;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_v_bfe_u32(&insn->gfx9,
+ dst, src0, src1, src2);
+ insn->type = AMDGCN_INSN_TYPE_VOP3A;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_bfi_b32(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1,
+ struct amdgcn_param32 src2)
+{
+ WARN_ON(knod_param_is_literal(src0) ||
+ knod_param_is_literal(src1) ||
+ knod_param_is_literal(src2));
+ if (version == 10) {
+ insn->size = emit_gfx10_v_bfi_b32(&insn->gfx10,
+ dst, src0, src1, src2);
+ insn->type = AMDGCN_INSN_TYPE_VOP3A;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_v_bfi_b32(&insn->gfx9,
+ dst, src0, src1, src2);
+ insn->type = AMDGCN_INSN_TYPE_VOP3A;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_lshl_add_u32(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1,
+ struct amdgcn_param32 src2)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_v_lshl_add_u32(&insn->gfx10,
+ dst, src0, src1, src2);
+ insn->type = AMDGCN_INSN_TYPE_VOP3A;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_v_lshl_add_u32(&insn->gfx9,
+ dst, src0, src1, src2);
+ insn->type = AMDGCN_INSN_TYPE_VOP3A;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_lshl_or_b32(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1,
+ struct amdgcn_param32 src2)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_v_lshl_or_b32(&insn->gfx10,
+ dst, src0, src1, src2);
+ insn->type = AMDGCN_INSN_TYPE_VOP3A;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_v_lshl_or_b32(&insn->gfx9,
+ dst, src0, src1, src2);
+ insn->type = AMDGCN_INSN_TYPE_VOP3A;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_alignbit_b32(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1,
+ struct amdgcn_param32 src2)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_v_alignbit_b32(&insn->gfx10,
+ dst, src0,
+ src1, src2);
+ insn->type = AMDGCN_INSN_TYPE_VOP3A;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_v_alignbit_b32(&insn->gfx9,
+ dst, src0,
+ src1, src2);
+ insn->type = AMDGCN_INSN_TYPE_VOP3A;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_perm_b32(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1,
+ struct amdgcn_param32 src2)
+{
+ WARN_ON(knod_param_is_literal(src0) ||
+ knod_param_is_literal(src1) ||
+ knod_param_is_literal(src2));
+ if (version == 10) {
+ insn->size = emit_gfx10_v_perm_b32(&insn->gfx10,
+ dst, src0, src1, src2);
+ insn->type = AMDGCN_INSN_TYPE_VOP3A;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_v_perm_b32(&insn->gfx9,
+ dst, src0, src1, src2);
+ insn->type = AMDGCN_INSN_TYPE_VOP3A;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_mad_u64_u32(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param32 dst2,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1,
+ struct amdgcn_param64 src2)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_v_mad_u64_u32(&insn->gfx10, dst,
+ dst2, src0,
+ src1, src2);
+ insn->type = AMDGCN_INSN_TYPE_VOP3B;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_v_mad_u64_u32(&insn->gfx9, dst,
+ dst2, src0,
+ src1, src2);
+ insn->type = AMDGCN_INSN_TYPE_VOP3B;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_s_mov_b32(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst, struct amdgcn_param32 src)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_s_mov_b32(&insn->gfx10, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_SOP1;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_s_mov_b32(&insn->gfx9, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_SOP1;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_mov_b32_e32(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_v_mov_b32_e32(&insn->gfx10, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_VOP1;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_v_mov_b32_e32(&insn->gfx9, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_VOP1;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_readfirstlane_b32(int version,
+ struct amdgcn_insn *insn,
+ u8 sdst, u8 vsrc)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_v_readfirstlane_b32(&insn->gfx10,
+ sdst, vsrc);
+ insn->type = AMDGCN_INSN_TYPE_VOP1;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_v_readfirstlane_b32(&insn->gfx9,
+ sdst, vsrc);
+ insn->type = AMDGCN_INSN_TYPE_VOP1;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_add_co_u32(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_v_add_co_u32(&insn->gfx10, dst,
+ src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP3B;
+ } else if (version == 9) {
+ WARN_ON_ONCE(src1.type != AMDGCN_PARAM_TYPE_VGPR);
+ insn->size = emit_gfx9_v_add_co_u32(&insn->gfx9, dst,
+ src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP2;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_add_co_ci_u32_e32(int version,
+ struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_v_add_co_ci_u32_e32(&insn->gfx10,
+ dst, src0,
+ src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP2;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_v_addc_co_u32(&insn->gfx9, dst,
+ src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP2;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+/* No carry in/out */
+static inline void emit_v_add_u32(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ if (version == 10) {
+ WARN_ON_ONCE(src1.type != AMDGCN_PARAM_TYPE_VGPR);
+ insn->size = emit_gfx10_v_add_nc_u32(&insn->gfx10, dst,
+ src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP2;
+ } else if (version == 9) {
+ WARN_ON_ONCE(src1.type != AMDGCN_PARAM_TYPE_VGPR);
+ insn->size = emit_gfx9_v_add_u32(&insn->gfx9, dst,
+ src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP2;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+/* No carry in/out */
+static inline void emit_v_sub_u32(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ if (version == 10) {
+ WARN_ON_ONCE(src1.type != AMDGCN_PARAM_TYPE_VGPR);
+ insn->size = emit_gfx10_v_sub_nc_u32(&insn->gfx10, dst,
+ src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP2;
+ } else if (version == 9) {
+ WARN_ON_ONCE(src1.type != AMDGCN_PARAM_TYPE_VGPR);
+ insn->size = emit_gfx9_v_sub_u32(&insn->gfx9, dst,
+ src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP2;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_xor_b32_e32(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ if (version == 10) {
+ WARN_ON_ONCE(src1.type != AMDGCN_PARAM_TYPE_VGPR);
+ insn->size = emit_gfx10_v_xor_b32_e32(&insn->gfx10, dst,
+ src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP2;
+ } else if (version == 9) {
+ WARN_ON_ONCE(src1.type != AMDGCN_PARAM_TYPE_VGPR);
+ insn->size = emit_gfx9_v_xor_b32_e32(&insn->gfx9, dst,
+ src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP2;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_or_b32_e32(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ if (version == 10) {
+ WARN_ON_ONCE(src1.type != AMDGCN_PARAM_TYPE_VGPR);
+ insn->size = emit_gfx10_v_or_b32_e32(&insn->gfx10, dst,
+ src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP2;
+ } else if (version == 9) {
+ WARN_ON_ONCE(src1.type != AMDGCN_PARAM_TYPE_VGPR);
+ insn->size = emit_gfx9_v_or_b32_e32(&insn->gfx9, dst,
+ src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP2;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_cndmask_b32_e32(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ if (version == 10) {
+ WARN_ON_ONCE(src1.type != AMDGCN_PARAM_TYPE_VGPR);
+ insn->size = emit_gfx10_v_cndmask_b32_e32(&insn->gfx10, dst,
+ src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP2;
+ } else if (version == 9) {
+ WARN_ON_ONCE(src1.type != AMDGCN_PARAM_TYPE_VGPR);
+ insn->size = emit_gfx9_v_cndmask_b32_e32(&insn->gfx9, dst,
+ src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP2;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_and_b32_e32(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ if (version == 10) {
+ WARN_ON_ONCE(src1.type != AMDGCN_PARAM_TYPE_VGPR);
+ insn->size = emit_gfx10_v_and_b32_e32(&insn->gfx10, dst,
+ src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP2;
+ } else if (version == 9) {
+ WARN_ON_ONCE(src1.type != AMDGCN_PARAM_TYPE_VGPR);
+ insn->size = emit_gfx9_v_and_b32_e32(&insn->gfx9, dst,
+ src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP2;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_sub_co_ci_u32_e32(int version,
+ struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ if (version == 10) {
+ WARN_ON_ONCE(src1.type != AMDGCN_PARAM_TYPE_VGPR);
+ insn->size = emit_gfx10_v_sub_co_ci_u32_e32(&insn->gfx10,
+ dst, src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP2;
+ } else if (version == 9) {
+ WARN_ON_ONCE(src1.type != AMDGCN_PARAM_TYPE_VGPR);
+ insn->size = emit_gfx9_v_subb_co_u32(&insn->gfx9, dst,
+ src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP2;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_subrev_co_ci_u32_e32(int version,
+ struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ if (version == 10) {
+ WARN_ON_ONCE(src1.type != AMDGCN_PARAM_TYPE_VGPR);
+ insn->size = emit_gfx10_v_subrev_co_ci_u32_e32(&insn->gfx10,
+ dst, src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP2;
+ } else if (version == 9) {
+ WARN_ON_ONCE(src1.type != AMDGCN_PARAM_TYPE_VGPR);
+ insn->size = emit_gfx9_v_subbrev_co_u32(&insn->gfx9, dst,
+ src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP2;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_sub_co_u32(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_v_sub_co_u32(&insn->gfx10, dst,
+ src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP3B;
+ } else if (version == 9) {
+ WARN_ON_ONCE(src1.type != AMDGCN_PARAM_TYPE_VGPR);
+ insn->size = emit_gfx9_v_sub_co_u32(&insn->gfx9, dst,
+ src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP2;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_subrev_co_u32(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_v_subrev_co_u32(&insn->gfx10, dst,
+ src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP3B;
+ } else if (version == 9) {
+ WARN_ON_ONCE(src1.type != AMDGCN_PARAM_TYPE_VGPR);
+ insn->size = emit_gfx9_v_subrev_co_u32(&insn->gfx9, dst,
+ src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP2;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_mul_lo_u32(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_v_mul_lo_u32(&insn->gfx10, dst,
+ src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP3A;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_v_mul_lo_u32(&insn->gfx9, dst,
+ src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP3A;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_mbcnt_lo_u32_b32(int version,
+ struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_v_mbcnt_lo_u32_b32(&insn->gfx10,
+ dst, src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP3A;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_v_mbcnt_lo_u32_b32(&insn->gfx9,
+ dst, src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP3A;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_mbcnt_hi_u32_b32(int version,
+ struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_v_mbcnt_hi_u32_b32(&insn->gfx10,
+ dst, src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP3A;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_v_mbcnt_hi_u32_b32(&insn->gfx9,
+ dst, src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP3A;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_mul_hi_u32(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_v_mul_hi_u32(&insn->gfx10, dst,
+ src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP3A;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_v_mul_hi_u32(&insn->gfx9, dst,
+ src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP3A;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_lshlrev_b64(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src0,
+ struct amdgcn_param64 src1)
+{
+ /* D.u64 = S1.u64 << S0.u[5:0]. */
+ if (version == 10) {
+ insn->size = emit_gfx10_v_lshlrev_b64(&insn->gfx10, dst,
+ src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP3A;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_v_lshlrev_b64(&insn->gfx9, dst,
+ src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP3A;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_lshrrev_b64(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src0,
+ struct amdgcn_param64 src1)
+{
+ WARN_ON(src1.lo.type == AMDGCN_PARAM_TYPE_LITERAL_CONST);
+ if (version == 10) {
+ insn->size = emit_gfx10_v_lshrrev_b64(&insn->gfx10, dst,
+ src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP3A;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_v_lshrrev_b64(&insn->gfx9, dst,
+ src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP3A;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_ashrrev_i64(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src0,
+ struct amdgcn_param64 src1)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_v_ashrrev_i64(&insn->gfx10, dst, src0,
+ src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP3A;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_v_ashrrev_i64(&insn->gfx9, dst, src0,
+ src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP3A;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_ashrrev_i32(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_v_ashrrev_i32(&insn->gfx10, dst, src0,
+ src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP3A;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_v_ashrrev_i32(&insn->gfx9, dst, src0,
+ src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP3A;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_lshlrev_b32(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ if (version == 10) {
+ WARN_ON_ONCE(src1.type != AMDGCN_PARAM_TYPE_VGPR);
+ insn->size = emit_gfx10_v_lshlrev_b32(&insn->gfx10, dst,
+ src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP2;
+ } else if (version == 9) {
+ WARN_ON_ONCE(src1.type != AMDGCN_PARAM_TYPE_VGPR);
+ insn->size = emit_gfx9_v_lshlrev_b32(&insn->gfx9, dst,
+ src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP2;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_lshrrev_b32(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ if (version == 10) {
+ WARN_ON_ONCE(src1.type != AMDGCN_PARAM_TYPE_VGPR);
+ insn->size = emit_gfx10_v_lshrrev_b32(&insn->gfx10, dst,
+ src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP2;
+ } else if (version == 9) {
+ WARN_ON_ONCE(src1.type != AMDGCN_PARAM_TYPE_VGPR);
+ insn->size = emit_gfx9_v_lshrrev_b32(&insn->gfx9, dst,
+ src0, src1);
+ insn->type = AMDGCN_INSN_TYPE_VOP2;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_cmp_eq_u64(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src)
+{
+ WARN_ON_ONCE(dst.type == AMDGCN_PARAM_TYPE_LITERAL_CONST);
+ WARN_ON(src.type != AMDGCN_PARAM_TYPE_VGPR);
+ if (version == 10) {
+ insn->size = emit_gfx10_v_cmp_eq_u64(&insn->gfx10, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_VOPC;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_v_cmp_eq_u64(&insn->gfx9, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_VOPC;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_cmp_eq_u32(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src)
+{
+ WARN_ON_ONCE(dst.type == AMDGCN_PARAM_TYPE_LITERAL_CONST);
+ WARN_ON(src.type != AMDGCN_PARAM_TYPE_VGPR);
+ if (version == 10) {
+ insn->size = emit_gfx10_v_cmp_eq_u32(&insn->gfx10, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_VOPC;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_v_cmp_eq_u32(&insn->gfx9, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_VOPC;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_cmp_gt_u64(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src)
+{
+ WARN_ON_ONCE(dst.lo.type == AMDGCN_PARAM_TYPE_LITERAL_CONST);
+ if (version == 10) {
+ insn->size = emit_gfx10_v_cmp_gt_u64(&insn->gfx10, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_VOPC;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_v_cmp_gt_u64(&insn->gfx9, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_VOPC;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_cmp_gt_i64(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src)
+{
+ WARN_ON_ONCE(dst.lo.type == AMDGCN_PARAM_TYPE_LITERAL_CONST);
+ if (version == 10) {
+ insn->size = emit_gfx10_v_cmp_gt_i64(&insn->gfx10, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_VOPC;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_v_cmp_gt_i64(&insn->gfx9, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_VOPC;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_cmp_ge_u32(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src)
+{
+ WARN_ON_ONCE(dst.type == AMDGCN_PARAM_TYPE_LITERAL_CONST);
+ if (version == 10) {
+ insn->size = emit_gfx10_v_cmp_ge_u32(&insn->gfx10, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_VOPC;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_v_cmp_ge_u32(&insn->gfx9, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_VOPC;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_cmp_gt_u32(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src)
+{
+ WARN_ON_ONCE(dst.type == AMDGCN_PARAM_TYPE_LITERAL_CONST);
+ WARN_ON(src.type != AMDGCN_PARAM_TYPE_VGPR);
+ if (version == 10) {
+ insn->size = emit_gfx10_v_cmp_gt_u32(&insn->gfx10, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_VOPC;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_v_cmp_gt_u32(&insn->gfx9, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_VOPC;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_cmp_lt_u32(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src)
+{
+ WARN_ON_ONCE(dst.type == AMDGCN_PARAM_TYPE_LITERAL_CONST);
+ WARN_ON(src.type != AMDGCN_PARAM_TYPE_VGPR);
+ if (version == 10) {
+ insn->size = emit_gfx10_v_cmp_lt_u32(&insn->gfx10, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_VOPC;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_v_cmp_lt_u32(&insn->gfx9, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_VOPC;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_cmp_le_u32(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src)
+{
+ WARN_ON_ONCE(dst.type == AMDGCN_PARAM_TYPE_LITERAL_CONST);
+ WARN_ON(src.type != AMDGCN_PARAM_TYPE_VGPR);
+ if (version == 10) {
+ insn->size = emit_gfx10_v_cmp_le_u32(&insn->gfx10, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_VOPC;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_v_cmp_le_u32(&insn->gfx9, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_VOPC;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_cmp_gt_i32(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src)
+{
+ WARN_ON_ONCE(dst.type == AMDGCN_PARAM_TYPE_LITERAL_CONST);
+ WARN_ON(src.type != AMDGCN_PARAM_TYPE_VGPR);
+ if (version == 10) {
+ insn->size = emit_gfx10_v_cmp_gt_i32(&insn->gfx10, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_VOPC;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_v_cmp_gt_i32(&insn->gfx9, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_VOPC;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_cmp_ge_i32(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src)
+{
+ WARN_ON_ONCE(dst.type == AMDGCN_PARAM_TYPE_LITERAL_CONST);
+ WARN_ON(src.type != AMDGCN_PARAM_TYPE_VGPR);
+ if (version == 10) {
+ insn->size = emit_gfx10_v_cmp_ge_i32(&insn->gfx10, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_VOPC;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_v_cmp_ge_i32(&insn->gfx9, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_VOPC;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_cmp_lt_i32(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src)
+{
+ WARN_ON_ONCE(dst.type == AMDGCN_PARAM_TYPE_LITERAL_CONST);
+ WARN_ON(src.type != AMDGCN_PARAM_TYPE_VGPR);
+ if (version == 10) {
+ insn->size = emit_gfx10_v_cmp_lt_i32(&insn->gfx10, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_VOPC;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_v_cmp_lt_i32(&insn->gfx9, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_VOPC;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_cmp_le_i32(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src)
+{
+ WARN_ON_ONCE(dst.type == AMDGCN_PARAM_TYPE_LITERAL_CONST);
+ WARN_ON(src.type != AMDGCN_PARAM_TYPE_VGPR);
+ if (version == 10) {
+ insn->size = emit_gfx10_v_cmp_le_i32(&insn->gfx10, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_VOPC;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_v_cmp_le_i32(&insn->gfx9, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_VOPC;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+/* EXEC &= (src0 < src1), per-lane mask update */
+static inline void emit_v_cmpx_lt_u32(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src)
+{
+ WARN_ON_ONCE(dst.type == AMDGCN_PARAM_TYPE_LITERAL_CONST);
+ if (version == 10) {
+ insn->size = emit_gfx10_v_cmpx_lt_u32(&insn->gfx10, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_VOPC;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_v_cmpx_lt_u32(&insn->gfx9, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_VOPC;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_cmp_ge_u64(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src)
+{
+ WARN_ON_ONCE(dst.lo.type == AMDGCN_PARAM_TYPE_LITERAL_CONST);
+ if (version == 10) {
+ insn->size = emit_gfx10_v_cmp_ge_u64(&insn->gfx10, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_VOPC;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_v_cmp_ge_u64(&insn->gfx9, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_VOPC;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_cmp_ge_i64(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src)
+{
+ WARN_ON_ONCE(dst.lo.type == AMDGCN_PARAM_TYPE_LITERAL_CONST);
+ if (version == 10) {
+ insn->size = emit_gfx10_v_cmp_ge_i64(&insn->gfx10, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_VOPC;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_v_cmp_ge_i64(&insn->gfx9, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_VOPC;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_cmp_lt_u64(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src)
+{
+ WARN_ON_ONCE(dst.lo.type == AMDGCN_PARAM_TYPE_LITERAL_CONST);
+ if (version == 10) {
+ insn->size = emit_gfx10_v_cmp_lt_u64(&insn->gfx10, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_VOPC;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_v_cmp_lt_u64(&insn->gfx9, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_VOPC;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_cmp_lt_i64(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src)
+{
+ WARN_ON_ONCE(dst.lo.type == AMDGCN_PARAM_TYPE_LITERAL_CONST);
+ if (version == 10) {
+ insn->size = emit_gfx10_v_cmp_lt_i64(&insn->gfx10, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_VOPC;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_v_cmp_lt_i64(&insn->gfx9, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_VOPC;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_cmp_le_u64(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src)
+{
+ WARN_ON_ONCE(dst.lo.type == AMDGCN_PARAM_TYPE_LITERAL_CONST);
+ if (version == 10) {
+ insn->size = emit_gfx10_v_cmp_le_u64(&insn->gfx10, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_VOPC;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_v_cmp_le_u64(&insn->gfx9, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_VOPC;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_v_cmp_le_i64(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src)
+{
+ WARN_ON_ONCE(dst.lo.type == AMDGCN_PARAM_TYPE_LITERAL_CONST);
+ if (version == 10) {
+ insn->size = emit_gfx10_v_cmp_le_i64(&insn->gfx10, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_VOPC;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_v_cmp_le_i64(&insn->gfx9, dst, src);
+ insn->type = AMDGCN_INSN_TYPE_VOPC;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_buffer_load_ubyte(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src,
+ short off)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_buffer_load_ubyte(&insn->gfx10,
+ dst, src, off);
+ insn->type = AMDGCN_INSN_TYPE_MUBUF;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_buffer_load_ubyte(&insn->gfx9,
+ dst, src, off);
+ insn->type = AMDGCN_INSN_TYPE_MUBUF;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_buffer_load_ushort(int version,
+ struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src,
+ short off)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_buffer_load_ushort(&insn->gfx10,
+ dst, src, off);
+ insn->type = AMDGCN_INSN_TYPE_MUBUF;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_buffer_load_ushort(&insn->gfx9,
+ dst, src, off);
+ insn->type = AMDGCN_INSN_TYPE_MUBUF;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_buffer_load_dword(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src,
+ short off)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_buffer_load_dword(&insn->gfx10,
+ dst, src, off);
+ insn->type = AMDGCN_INSN_TYPE_MUBUF;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_buffer_load_dword(&insn->gfx9,
+ dst, src, off);
+ insn->type = AMDGCN_INSN_TYPE_MUBUF;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_buffer_load_dwordx2(int version,
+ struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src,
+ short off)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_buffer_load_dwordx2(&insn->gfx10,
+ dst, src, off);
+ insn->type = AMDGCN_INSN_TYPE_MUBUF;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_buffer_load_dwordx2(&insn->gfx9,
+ dst, src, off);
+ insn->type = AMDGCN_INSN_TYPE_MUBUF;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_buffer_load_dwordx4(int version,
+ struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src,
+ short off)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_buffer_load_dwordx4(&insn->gfx10,
+ dst, src, off);
+ insn->type = AMDGCN_INSN_TYPE_MUBUF;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_buffer_load_dwordx4(&insn->gfx9,
+ dst, src, off);
+ insn->type = AMDGCN_INSN_TYPE_MUBUF;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_global_load_ubyte(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src,
+ short off)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_global_load_ubyte(&insn->gfx10,
+ dst, src, off);
+ insn->type = AMDGCN_INSN_TYPE_FLAT;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_global_load_ubyte(&insn->gfx9,
+ dst, src, off);
+ insn->type = AMDGCN_INSN_TYPE_FLAT;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_global_load_ushort(int version,
+ struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src,
+ short off)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_global_load_ushort(&insn->gfx10,
+ dst, src, off);
+ insn->type = AMDGCN_INSN_TYPE_FLAT;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_global_load_ushort(&insn->gfx9,
+ dst, src, off);
+ insn->type = AMDGCN_INSN_TYPE_FLAT;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_global_load_dword(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src,
+ short off)
+{
+ WARN_ON(dst.type != AMDGCN_PARAM_TYPE_VGPR);
+ WARN_ON(src.type != AMDGCN_PARAM_TYPE_VGPR);
+ if (version == 10) {
+ insn->size = emit_gfx10_global_load_dword(&insn->gfx10,
+ dst, src, off);
+ insn->type = AMDGCN_INSN_TYPE_FLAT;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_global_load_dword(&insn->gfx9,
+ dst, src, off);
+ insn->type = AMDGCN_INSN_TYPE_FLAT;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_global_load_dwordx2(int version,
+ struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src,
+ short off)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_global_load_dwordx2(&insn->gfx10,
+ dst, src, off);
+ insn->type = AMDGCN_INSN_TYPE_FLAT;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_global_load_dwordx2(&insn->gfx9,
+ dst, src, off);
+ insn->type = AMDGCN_INSN_TYPE_FLAT;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_global_load_dwordx4(int version,
+ struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src,
+ short off)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_global_load_dwordx4(&insn->gfx10,
+ dst, src, off);
+ insn->type = AMDGCN_INSN_TYPE_FLAT;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_global_load_dwordx4(&insn->gfx9,
+ dst, src, off);
+ insn->type = AMDGCN_INSN_TYPE_FLAT;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_global_store_byte(int version, struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src, int off)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_global_store_byte(&insn->gfx10,
+ src, dst, off);
+ insn->type = AMDGCN_INSN_TYPE_FLAT;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_global_store_byte(&insn->gfx9,
+ src, dst, off);
+ insn->type = AMDGCN_INSN_TYPE_FLAT;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+#define DEFINE_EMIT_GLOBAL_ATOMIC(name) \
+static inline void emit_global_atomic_##name(int version, \
+ struct amdgcn_insn *insn, \
+ struct amdgcn_param32 vdst, \
+ struct amdgcn_param32 addr, \
+ struct amdgcn_param32 data, \
+ int off, int glc) \
+{ \
+ if (version == 10) { \
+ insn->size = emit_gfx10_global_atomic_##name( \
+ &insn->gfx10, vdst, addr, data, off, glc); \
+ insn->type = AMDGCN_INSN_TYPE_FLAT; \
+ } else if (version == 9) { \
+ insn->size = emit_gfx9_global_atomic_##name( \
+ &insn->gfx9, vdst, addr, data, off, glc); \
+ insn->type = AMDGCN_INSN_TYPE_FLAT; \
+ } else { \
+ WARN_ON_ONCE(1); \
+ } \
+}
+
+DEFINE_EMIT_GLOBAL_ATOMIC(add)
+DEFINE_EMIT_GLOBAL_ATOMIC(and)
+DEFINE_EMIT_GLOBAL_ATOMIC(or)
+DEFINE_EMIT_GLOBAL_ATOMIC(xor)
+DEFINE_EMIT_GLOBAL_ATOMIC(swap)
+DEFINE_EMIT_GLOBAL_ATOMIC(cmpswap)
+DEFINE_EMIT_GLOBAL_ATOMIC(add_x2)
+DEFINE_EMIT_GLOBAL_ATOMIC(and_x2)
+DEFINE_EMIT_GLOBAL_ATOMIC(or_x2)
+DEFINE_EMIT_GLOBAL_ATOMIC(xor_x2)
+DEFINE_EMIT_GLOBAL_ATOMIC(swap_x2)
+DEFINE_EMIT_GLOBAL_ATOMIC(cmpswap_x2)
+
+static inline void emit_global_store_short(int version,
+ struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src, int off)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_global_store_short(&insn->gfx10,
+ src, dst, off);
+ insn->type = AMDGCN_INSN_TYPE_FLAT;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_global_store_short(&insn->gfx9,
+ src, dst, off);
+ insn->type = AMDGCN_INSN_TYPE_FLAT;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_global_store_dword(int version,
+ struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src, int off)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_global_store_dword(&insn->gfx10,
+ src, dst, off);
+ insn->type = AMDGCN_INSN_TYPE_FLAT;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_global_store_dword(&insn->gfx9,
+ src, dst, off);
+ insn->type = AMDGCN_INSN_TYPE_FLAT;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_global_store_dwordx2(int version,
+ struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src, int off)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_global_store_dwordx2(&insn->gfx10,
+ src, dst, off);
+ insn->type = AMDGCN_INSN_TYPE_FLAT;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_global_store_dwordx2(&insn->gfx9,
+ src, dst, off);
+ insn->type = AMDGCN_INSN_TYPE_FLAT;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_global_store_dwordx4(int version,
+ struct amdgcn_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src, int off)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_global_store_dwordx4(&insn->gfx10,
+ src, dst, off);
+ insn->type = AMDGCN_INSN_TYPE_FLAT;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_global_store_dwordx4(&insn->gfx9,
+ src, dst, off);
+ insn->type = AMDGCN_INSN_TYPE_FLAT;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_s_branch(int version, struct amdgcn_insn *insn,
+ short off)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_s_branch(&insn->gfx10, off);
+ insn->type = AMDGCN_INSN_TYPE_SOPP;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_s_branch(&insn->gfx9, off);
+ insn->type = AMDGCN_INSN_TYPE_SOPP;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_s_cbranch_vccz(int version, struct amdgcn_insn *insn,
+ short off)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_s_cbranch_vccz(&insn->gfx10, off);
+ insn->type = AMDGCN_INSN_TYPE_SOPP;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_s_cbranch_vccz(&insn->gfx9, off);
+ insn->type = AMDGCN_INSN_TYPE_SOPP;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_s_cbranch_vccnz(int version, struct amdgcn_insn *insn,
+ short off)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_s_cbranch_vccnz(&insn->gfx10, off);
+ insn->type = AMDGCN_INSN_TYPE_SOPP;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_s_cbranch_vccnz(&insn->gfx9, off);
+ insn->type = AMDGCN_INSN_TYPE_SOPP;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+/* Structurized CFG wrapper functions.
+ * Use raw SGPR indices; EXEC=126, VCC=106, integer_0=128.
+ */
+
+static inline void emit_s_and_saveexec_b64(int version,
+ struct amdgcn_insn *insn,
+ u8 sdst, u8 ssrc)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_s_and_saveexec_b64(&insn->gfx10,
+ sdst, ssrc);
+ insn->type = AMDGCN_INSN_TYPE_SOP1;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_s_and_saveexec_b64(&insn->gfx9,
+ sdst, ssrc);
+ insn->type = AMDGCN_INSN_TYPE_SOP1;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_s_bcnt1_i32_b64(int version, struct amdgcn_insn *insn,
+ u8 sdst, u8 ssrc)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_s_bcnt1_i32_b64(&insn->gfx10,
+ sdst, ssrc);
+ insn->type = AMDGCN_INSN_TYPE_SOP1;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_s_bcnt1_i32_b64(&insn->gfx9,
+ sdst, ssrc);
+ insn->type = AMDGCN_INSN_TYPE_SOP1;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_s_mov_b64(int version, struct amdgcn_insn *insn,
+ u8 sdst, u8 ssrc)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_s_mov_b64(&insn->gfx10, sdst, ssrc);
+ insn->type = AMDGCN_INSN_TYPE_SOP1;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_s_mov_b64(&insn->gfx9, sdst, ssrc);
+ insn->type = AMDGCN_INSN_TYPE_SOP1;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_s_and_b64(int version, struct amdgcn_insn *insn,
+ u8 sdst, u8 ssrc0, u8 ssrc1)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_s_and_b64(&insn->gfx10,
+ sdst, ssrc0, ssrc1);
+ insn->type = AMDGCN_INSN_TYPE_SOP2;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_s_and_b64(&insn->gfx9,
+ sdst, ssrc0, ssrc1);
+ insn->type = AMDGCN_INSN_TYPE_SOP2;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_s_or_b64(int version, struct amdgcn_insn *insn,
+ u8 sdst, u8 ssrc0, u8 ssrc1)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_s_or_b64(&insn->gfx10,
+ sdst, ssrc0, ssrc1);
+ insn->type = AMDGCN_INSN_TYPE_SOP2;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_s_or_b64(&insn->gfx9,
+ sdst, ssrc0, ssrc1);
+ insn->type = AMDGCN_INSN_TYPE_SOP2;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_s_andn2_b64(int version, struct amdgcn_insn *insn,
+ u8 sdst, u8 ssrc0, u8 ssrc1)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_s_andn2_b64(&insn->gfx10,
+ sdst, ssrc0, ssrc1);
+ insn->type = AMDGCN_INSN_TYPE_SOP2;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_s_andn2_b64(&insn->gfx9,
+ sdst, ssrc0, ssrc1);
+ insn->type = AMDGCN_INSN_TYPE_SOP2;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_s_cbranch_execz(int version, struct amdgcn_insn *insn,
+ short off)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_s_cbranch_execz(&insn->gfx10, off);
+ insn->type = AMDGCN_INSN_TYPE_SOPP;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_s_cbranch_execz(&insn->gfx9, off);
+ insn->type = AMDGCN_INSN_TYPE_SOPP;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_s_cbranch_execnz(int version, struct amdgcn_insn *insn,
+ short off)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_s_cbranch_execnz(&insn->gfx10, off);
+ insn->type = AMDGCN_INSN_TYPE_SOPP;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_s_cbranch_execnz(&insn->gfx9, off);
+ insn->type = AMDGCN_INSN_TYPE_SOPP;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_s_sub_u32(int version, struct amdgcn_insn *insn,
+ u8 sdst, u8 ssrc0, u8 ssrc1)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_s_sub_u32(&insn->gfx10,
+ sdst, ssrc0, ssrc1);
+ insn->type = AMDGCN_INSN_TYPE_SOP2;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_s_sub_u32(&insn->gfx9,
+ sdst, ssrc0, ssrc1);
+ insn->type = AMDGCN_INSN_TYPE_SOP2;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_s_cbranch_scc0(int version, struct amdgcn_insn *insn,
+ short off)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_s_cbranch_scc0(&insn->gfx10, off);
+ insn->type = AMDGCN_INSN_TYPE_SOPP;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_s_cbranch_scc0(&insn->gfx9, off);
+ insn->type = AMDGCN_INSN_TYPE_SOPP;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_branch_fixup(int version, struct amdgcn_insn *insn,
+ short off)
+{
+ if (version == 10)
+ insn->size = emit_gfx10_branch_fixup(&insn->gfx10, off);
+ else if (version == 9)
+ insn->size = emit_gfx9_branch_fixup(&insn->gfx9, off);
+ else
+ WARN_ON_ONCE(1);
+}
+
+static inline void emit_s_waitcnt_lgkmcnt(int version, struct amdgcn_insn *insn)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_s_waitcnt_lgkmcnt(&insn->gfx10);
+ insn->type = AMDGCN_INSN_TYPE_SOPP;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_s_waitcnt_lgkmcnt(&insn->gfx9);
+ insn->type = AMDGCN_INSN_TYPE_SOPP;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_s_waitcnt_vmcnt(int version, struct amdgcn_insn *insn)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_s_waitcnt_vmcnt(&insn->gfx10);
+ insn->type = AMDGCN_INSN_TYPE_SOPP;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_s_waitcnt_vmcnt(&insn->gfx9);
+ insn->type = AMDGCN_INSN_TYPE_SOPP;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_s_waitcnt_vmcnt_lgkmcnt(int version,
+ struct amdgcn_insn *insn)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_s_waitcnt_vmcnt_lgkmcnt(&insn->gfx10);
+ insn->type = AMDGCN_INSN_TYPE_SOPP;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_s_waitcnt_vmcnt_lgkmcnt(&insn->gfx9);
+ insn->type = AMDGCN_INSN_TYPE_SOPP;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_s_nop(int version, struct amdgcn_insn *insn)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_s_nop(&insn->gfx10);
+ insn->type = AMDGCN_INSN_TYPE_SOPP;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_s_nop(&insn->gfx9);
+ insn->type = AMDGCN_INSN_TYPE_SOPP;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_s_endpgm(int version, struct amdgcn_insn *insn)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_s_endpgm(&insn->gfx10);
+ insn->type = AMDGCN_INSN_TYPE_SOPP;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_s_endpgm(&insn->gfx9);
+ insn->type = AMDGCN_INSN_TYPE_SOPP;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_s_code_end(int version, struct amdgcn_insn *insn)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_s_code_end(&insn->gfx10);
+ insn->type = AMDGCN_INSN_TYPE_SOPP;
+ } else if (version == 9) {
+ WARN_ON_ONCE(1);
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void emit_s_icache_inv(int version, struct amdgcn_insn *insn)
+{
+ if (version == 10) {
+ insn->size = emit_gfx10_s_icache_inv(&insn->gfx10);
+ insn->type = AMDGCN_INSN_TYPE_SOPP;
+ } else if (version == 9) {
+ insn->size = emit_gfx9_s_icache_inv(&insn->gfx9);
+ insn->type = AMDGCN_INSN_TYPE_SOPP;
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void gfx10_debug_vop3a(union amdgcn_gfx10_insn *insn)
+{
+ char src0[20];
+ char src1[20];
+ char src2[20];
+
+ if (insn->vop3a.src0 < GFX10_VOP3A_SRC_VCC_LO)
+ sprintf(src0, "s%d", insn->vop3a.src0);
+ else if (insn->vop3a.src0 == GFX10_VOP3A_SRC_VCC_LO)
+ sprintf(src0, "%s", "vcc_lo");
+ else if (insn->vop3a.src0 == GFX10_VOP3A_SRC_VCC_HI)
+ sprintf(src0, "%s", "vcc_hi");
+ else if (insn->vop3a.src0 == GFX10_VOP3A_SRC_NULL)
+ sprintf(src0, "%s", "null");
+ else if (insn->vop3a.src0 == GFX10_VOP3A_SRC_EXEC_LO)
+ sprintf(src0, "%s", "exec_lo");
+ else if (insn->vop3a.src0 == GFX10_VOP3A_SRC_EXEC_HI)
+ sprintf(src0, "%s", "exec_hi");
+ else if (insn->vop3a.src0 < GFX10_VOP3A_SRC_INTEGER_MINUS_1)
+ sprintf(src0, "0x%x",
+ insn->vop3a.src0 - GFX10_VOP3A_SRC_INTEGER_0);
+ else if (insn->vop3a.src0 < GFX10_VOP3A_SRC_SHARED_BASE)
+ sprintf(src0, "-0x%x",
+ insn->vop3a.src0 - GFX10_VOP3A_SRC_INTEGER_MINUS_1);
+ else if (insn->vop3a.src0 == GFX10_VOP3A_SRC_VCCZ)
+ sprintf(src0, "%s", "vcc");
+ else if (insn->vop3a.src0 == GFX10_VOP3A_SRC_EXECZ)
+ sprintf(src0, "%s", "exec");
+ else if (insn->vop3a.src0 == GFX10_VOP3A_SRC_SCC)
+ sprintf(src0, "%s", "scc");
+ else if (insn->vop3a.src0 == GFX10_VOP3A_SRC_LITERAL_CONST)
+ sprintf(src0, "0x%x", insn->vop3a.literal);
+ else if (insn->vop3a.src0 >= GFX10_VOP3A_SRC_VGPR_BASE)
+ sprintf(src0, "v%d",
+ insn->vop3a.src0 - GFX10_VOP3A_SRC_VGPR_BASE);
+
+ if (insn->vop3a.src1 < GFX10_VOP3A_SRC_VCC_LO)
+ sprintf(src1, "s%d", insn->vop3a.src1);
+ else if (insn->vop3a.src1 == GFX10_VOP3A_SRC_VCC_LO)
+ sprintf(src1, "%s", "vcc_lo");
+ else if (insn->vop3a.src1 == GFX10_VOP3A_SRC_VCC_HI)
+ sprintf(src1, "%s", "vcc_hi");
+ else if (insn->vop3a.src1 == GFX10_VOP3A_SRC_NULL)
+ sprintf(src1, "%s", "null");
+ else if (insn->vop3a.src1 == GFX10_VOP3A_SRC_EXEC_LO)
+ sprintf(src1, "%s", "exec_lo");
+ else if (insn->vop3a.src1 == GFX10_VOP3A_SRC_EXEC_HI)
+ sprintf(src1, "%s", "exec_hi");
+ else if (insn->vop3a.src1 < GFX10_VOP3A_SRC_INTEGER_MINUS_1)
+ sprintf(src1, "0x%x",
+ insn->vop3a.src1 - GFX10_VOP3A_SRC_INTEGER_0);
+ else if (insn->vop3a.src1 < GFX10_VOP3A_SRC_SHARED_BASE)
+ sprintf(src1, "-0x%x",
+ insn->vop3a.src1 - GFX10_VOP3A_SRC_INTEGER_MINUS_1);
+ else if (insn->vop3a.src1 == GFX10_VOP3A_SRC_VCCZ)
+ sprintf(src1, "%s", "vcc");
+ else if (insn->vop3a.src1 == GFX10_VOP3A_SRC_EXECZ)
+ sprintf(src1, "%s", "exec");
+ else if (insn->vop3a.src1 == GFX10_VOP3A_SRC_SCC)
+ sprintf(src1, "%s", "scc");
+ else if (insn->vop3a.src1 == GFX10_VOP3A_SRC_LITERAL_CONST)
+ sprintf(src1, "0x%x", insn->vop3a.literal);
+ else if (insn->vop3a.src1 >= GFX10_VOP3A_SRC_VGPR_BASE)
+ sprintf(src1, "v%d",
+ insn->vop3a.src1 - GFX10_VOP3A_SRC_VGPR_BASE);
+
+ if (insn->vop3a.src2 < GFX10_VOP3A_SRC_VCC_LO)
+ sprintf(src2, "s%d", insn->vop3a.src2);
+ else if (insn->vop3a.src2 == GFX10_VOP3A_SRC_VCC_LO)
+ sprintf(src2, "%s", "vcc_lo");
+ else if (insn->vop3a.src2 == GFX10_VOP3A_SRC_VCC_HI)
+ sprintf(src2, "%s", "vcc_hi");
+ else if (insn->vop3a.src2 == GFX10_VOP3A_SRC_NULL)
+ sprintf(src2, "%s", "null");
+ else if (insn->vop3a.src2 == GFX10_VOP3A_SRC_EXEC_LO)
+ sprintf(src2, "%s", "exec_lo");
+ else if (insn->vop3a.src2 == GFX10_VOP3A_SRC_EXEC_HI)
+ sprintf(src2, "%s", "exec_hi");
+ else if (insn->vop3a.src2 < GFX10_VOP3A_SRC_INTEGER_MINUS_1)
+ sprintf(src2, "0x%x",
+ insn->vop3a.src2 - GFX10_VOP3A_SRC_INTEGER_0);
+ else if (insn->vop3a.src2 < GFX10_VOP3A_SRC_SHARED_BASE)
+ sprintf(src2, "-0x%x",
+ insn->vop3a.src2 - GFX10_VOP3A_SRC_INTEGER_MINUS_1);
+ else if (insn->vop3a.src2 == GFX10_VOP3A_SRC_VCCZ)
+ sprintf(src2, "%s", "vcc");
+ else if (insn->vop3a.src2 == GFX10_VOP3A_SRC_EXECZ)
+ sprintf(src2, "%s", "exec");
+ else if (insn->vop3a.src2 == GFX10_VOP3A_SRC_SCC)
+ sprintf(src2, "%s", "scc");
+ else if (insn->vop3a.src2 == GFX10_VOP3A_SRC_LITERAL_CONST)
+ sprintf(src2, "0x%x", insn->vop3a.literal);
+ else if (insn->vop3a.src2 >= GFX10_VOP3A_SRC_VGPR_BASE)
+ sprintf(src2, "v%d",
+ insn->vop3a.src2 - GFX10_VOP3A_SRC_VGPR_BASE);
+
+ pr_debug("knod_asm %s v%d, %s, %s, %s\n",
+ opnames_gfx10[AMDGCN_INSN_TYPE_VOP3A][insn->vop3a.op],
+ insn->vop3a.vdst, src0, src1, src2);
+}
+
+static inline void gfx10_debug_vop3b(union amdgcn_gfx10_insn *insn)
+{
+ char src0[20];
+ char src1[20];
+ char src2[20];
+ char sdst[20];
+
+ if (insn->vop3b.sdst < GFX10_VOP3A_SRC_VCC_LO)
+ sprintf(sdst, "s%d", insn->vop3b.sdst);
+ else if (insn->vop3b.sdst == GFX10_VOP3A_SRC_VCC_LO)
+ sprintf(sdst, "%s", "vcc_lo");
+ else if (insn->vop3b.sdst == GFX10_VOP3A_SRC_VCC_HI)
+ sprintf(sdst, "%s", "vcc_hi");
+ else if (insn->vop3b.sdst == GFX10_VOP3A_SRC_NULL)
+ sprintf(sdst, "%s", "null");
+ else if (insn->vop3b.sdst == GFX10_VOP3A_SRC_EXEC_LO)
+ sprintf(sdst, "%s", "exec_lo");
+ else if (insn->vop3b.sdst == GFX10_VOP3A_SRC_EXEC_HI)
+ sprintf(sdst, "%s", "exec_hi");
+ else if (insn->vop3b.sdst < GFX10_VOP3A_SRC_INTEGER_MINUS_1)
+ sprintf(sdst, "0x%x",
+ insn->vop3b.sdst - GFX10_VOP3A_SRC_INTEGER_0);
+ else if (insn->vop3b.sdst < GFX10_VOP3A_SRC_SHARED_BASE)
+ sprintf(sdst, "-0x%x",
+ insn->vop3b.sdst - GFX10_VOP3A_SRC_INTEGER_MINUS_1);
+ else if (insn->vop3b.sdst == GFX10_VOP3A_SRC_VCCZ)
+ sprintf(sdst, "%s", "vcc");
+ else if (insn->vop3b.sdst == GFX10_VOP3A_SRC_EXECZ)
+ sprintf(sdst, "%s", "exec");
+ else if (insn->vop3b.sdst == GFX10_VOP3A_SRC_SCC)
+ sprintf(sdst, "%s", "scc");
+ else if (insn->vop3b.sdst == GFX10_VOP3A_SRC_LITERAL_CONST)
+ sprintf(sdst, "0x%x", insn->vop3b.literal);
+ else if (insn->vop3b.sdst >= GFX10_VOP3A_SRC_VGPR_BASE)
+ sprintf(sdst, "v%d",
+ insn->vop3b.sdst - GFX10_VOP3A_SRC_VGPR_BASE);
+
+ if (insn->vop3b.src0 < GFX10_VOP3A_SRC_VCC_LO)
+ sprintf(src0, "s%d", insn->vop3b.src0);
+ else if (insn->vop3b.src0 == GFX10_VOP3A_SRC_VCC_LO)
+ sprintf(src0, "%s", "vcc_lo");
+ else if (insn->vop3b.src0 == GFX10_VOP3A_SRC_VCC_HI)
+ sprintf(src0, "%s", "vcc_hi");
+ else if (insn->vop3b.src0 == GFX10_VOP3A_SRC_NULL)
+ sprintf(src0, "%s", "null");
+ else if (insn->vop3b.src0 == GFX10_VOP3A_SRC_EXEC_LO)
+ sprintf(src0, "%s", "exec_lo");
+ else if (insn->vop3b.src0 == GFX10_VOP3A_SRC_EXEC_HI)
+ sprintf(src0, "%s", "exec_hi");
+ else if (insn->vop3b.src0 < GFX10_VOP3A_SRC_INTEGER_MINUS_1)
+ sprintf(src0, "0x%x",
+ insn->vop3b.src0 - GFX10_VOP3A_SRC_INTEGER_0);
+ else if (insn->vop3b.src0 < GFX10_VOP3A_SRC_SHARED_BASE)
+ sprintf(src0, "-0x%x",
+ insn->vop3b.src0 - GFX10_VOP3A_SRC_INTEGER_MINUS_1);
+ else if (insn->vop3b.src0 == GFX10_VOP3A_SRC_VCCZ)
+ sprintf(src0, "%s", "vcc");
+ else if (insn->vop3b.src0 == GFX10_VOP3A_SRC_EXECZ)
+ sprintf(src0, "%s", "exec");
+ else if (insn->vop3b.src0 == GFX10_VOP3A_SRC_SCC)
+ sprintf(src0, "%s", "scc");
+ else if (insn->vop3b.src0 == GFX10_VOP3A_SRC_LITERAL_CONST)
+ sprintf(src0, "0x%x", insn->vop3b.literal);
+ else if (insn->vop3b.src0 >= GFX10_VOP3A_SRC_VGPR_BASE)
+ sprintf(src0, "v%d",
+ insn->vop3b.src0 - GFX10_VOP3A_SRC_VGPR_BASE);
+
+ if (insn->vop3b.src1 < GFX10_VOP3A_SRC_VCC_LO)
+ sprintf(src1, "s%d", insn->vop3b.src1);
+ else if (insn->vop3b.src1 == GFX10_VOP3A_SRC_VCC_LO)
+ sprintf(src1, "%s", "vcc_lo");
+ else if (insn->vop3b.src1 == GFX10_VOP3A_SRC_VCC_HI)
+ sprintf(src1, "%s", "vcc_hi");
+ else if (insn->vop3b.src1 == GFX10_VOP3A_SRC_NULL)
+ sprintf(src1, "%s", "null");
+ else if (insn->vop3b.src1 == GFX10_VOP3A_SRC_EXEC_LO)
+ sprintf(src1, "%s", "exec_lo");
+ else if (insn->vop3b.src1 == GFX10_VOP3A_SRC_EXEC_HI)
+ sprintf(src1, "%s", "exec_hi");
+ else if (insn->vop3b.src1 < GFX10_VOP3A_SRC_INTEGER_MINUS_1)
+ sprintf(src1, "0x%x",
+ insn->vop3b.src1 - GFX10_VOP3A_SRC_INTEGER_0);
+ else if (insn->vop3b.src1 < GFX10_VOP3A_SRC_SHARED_BASE)
+ sprintf(src1, "-0x%x",
+ insn->vop3b.src1 - GFX10_VOP3A_SRC_INTEGER_MINUS_1);
+ else if (insn->vop3b.src1 == GFX10_VOP3A_SRC_VCCZ)
+ sprintf(src1, "%s", "vcc");
+ else if (insn->vop3b.src1 == GFX10_VOP3A_SRC_EXECZ)
+ sprintf(src1, "%s", "exec");
+ else if (insn->vop3b.src1 == GFX10_VOP3A_SRC_SCC)
+ sprintf(src1, "%s", "scc");
+ else if (insn->vop3b.src1 == GFX10_VOP3A_SRC_LITERAL_CONST)
+ sprintf(src1, "0x%x", insn->vop3b.literal);
+ else if (insn->vop3b.src1 >= GFX10_VOP3A_SRC_VGPR_BASE)
+ sprintf(src1, "v%d",
+ insn->vop3b.src1 - GFX10_VOP3A_SRC_VGPR_BASE);
+
+ if (insn->vop3b.src2 < GFX10_VOP3A_SRC_VCC_LO)
+ sprintf(src2, "s%d", insn->vop3b.src2);
+ else if (insn->vop3b.src2 == GFX10_VOP3A_SRC_VCC_LO)
+ sprintf(src2, "%s", "vcc_lo");
+ else if (insn->vop3b.src2 == GFX10_VOP3A_SRC_VCC_HI)
+ sprintf(src2, "%s", "vcc_hi");
+ else if (insn->vop3b.src2 == GFX10_VOP3A_SRC_NULL)
+ sprintf(src2, "%s", "null");
+ else if (insn->vop3b.src2 == GFX10_VOP3A_SRC_EXEC_LO)
+ sprintf(src2, "%s", "exec_lo");
+ else if (insn->vop3b.src2 == GFX10_VOP3A_SRC_EXEC_HI)
+ sprintf(src2, "%s", "exec_hi");
+ else if (insn->vop3b.src2 < GFX10_VOP3A_SRC_INTEGER_MINUS_1)
+ sprintf(src2, "0x%x",
+ insn->vop3b.src2 - GFX10_VOP3A_SRC_INTEGER_0);
+ else if (insn->vop3b.src2 < GFX10_VOP3A_SRC_SHARED_BASE)
+ sprintf(src2, "-0x%x",
+ insn->vop3b.src2 - GFX10_VOP3A_SRC_INTEGER_MINUS_1);
+ else if (insn->vop3b.src2 == GFX10_VOP3A_SRC_VCCZ)
+ sprintf(src2, "%s", "vcc");
+ else if (insn->vop3b.src2 == GFX10_VOP3A_SRC_EXECZ)
+ sprintf(src2, "%s", "exec");
+ else if (insn->vop3b.src2 == GFX10_VOP3A_SRC_SCC)
+ sprintf(src2, "%s", "scc");
+ else if (insn->vop3b.src2 == GFX10_VOP3A_SRC_LITERAL_CONST)
+ sprintf(src2, "0x%x", insn->vop3b.literal);
+ else if (insn->vop3b.src2 >= GFX10_VOP3A_SRC_VGPR_BASE)
+ sprintf(src2, "v%d",
+ insn->vop3b.src2 - GFX10_VOP3A_SRC_VGPR_BASE);
+
+ pr_debug("knod_asm %s v%d, %s, %s, %s, %s\n",
+ opnames_gfx10[AMDGCN_INSN_TYPE_VOP3B][insn->vop3b.op],
+ insn->vop3b.vdst, sdst, src0, src1, src2);
+}
+
+static inline void gfx10_debug_vop1(union amdgcn_gfx10_insn *insn)
+{
+ char src0[20];
+
+ if (insn->vop1.src0 < GFX10_VOP1_SRC_VCC_LO)
+ sprintf(src0, "s%d", insn->vop1.src0);
+ else if (insn->vop1.src0 == GFX10_VOP1_SRC_VCC_LO)
+ sprintf(src0, "%s", "vcc_lo");
+ else if (insn->vop1.src0 == GFX10_VOP1_SRC_VCC_HI)
+ sprintf(src0, "%s", "vcc_hi");
+ else if (insn->vop1.src0 == GFX10_VOP1_SRC_NULL)
+ sprintf(src0, "%s", "null");
+ else if (insn->vop1.src0 == GFX10_VOP1_SRC_EXEC_LO)
+ sprintf(src0, "%s", "exec_lo");
+ else if (insn->vop1.src0 == GFX10_VOP1_SRC_EXEC_HI)
+ sprintf(src0, "%s", "exec_hi");
+ else if (insn->vop1.src0 < GFX10_VOP1_SRC_INTEGER_MINUS_1)
+ sprintf(src0, "0x%x",
+ insn->vop1.src0 - GFX10_VOP1_SRC_INTEGER_0);
+ else if (insn->vop1.src0 < GFX10_VOP1_SRC_SHARED_BASE)
+ sprintf(src0, "-0x%x",
+ insn->vop1.src0 - GFX10_VOP1_SRC_INTEGER_MINUS_1);
+ else if (insn->vop1.src0 == GFX10_VOP1_SRC_VCCZ)
+ sprintf(src0, "%s", "vcc");
+ else if (insn->vop1.src0 == GFX10_VOP1_SRC_EXECZ)
+ sprintf(src0, "%s", "exec");
+ else if (insn->vop1.src0 == GFX10_VOP1_SRC_SCC)
+ sprintf(src0, "%s", "scc");
+ else if (insn->vop1.src0 == GFX10_VOP1_SRC_LITERAL_CONST)
+ sprintf(src0, "0x%x", insn->vop1.literal);
+ else if (insn->vop1.src0 >= GFX10_VOP1_SRC_VGPR_BASE)
+ sprintf(src0, "v%d",
+ insn->vop1.src0 - GFX10_VOP1_SRC_VGPR_BASE);
+
+ pr_debug("knod_asm %s v%d, %s\n",
+ opnames_gfx10[AMDGCN_INSN_TYPE_VOP1][insn->vop1.op],
+ insn->vop1.vdst, src0);
+}
+
+static inline void gfx10_debug_vopc(union amdgcn_gfx10_insn *insn)
+{
+ char src0[20];
+
+ if (insn->vopc.src0 < GFX10_VOPC_SRC_VCC_LO)
+ sprintf(src0, "s%d", insn->vopc.src0);
+ else if (insn->vopc.src0 == GFX10_VOPC_SRC_VCC_LO)
+ sprintf(src0, "%s", "vcc_lo");
+ else if (insn->vopc.src0 == GFX10_VOPC_SRC_VCC_HI)
+ sprintf(src0, "%s", "vcc_hi");
+ else if (insn->vopc.src0 == GFX10_VOPC_SRC_NULL)
+ sprintf(src0, "%s", "null");
+ else if (insn->vopc.src0 == GFX10_VOPC_SRC_EXEC_LO)
+ sprintf(src0, "%s", "exec_lo");
+ else if (insn->vopc.src0 == GFX10_VOPC_SRC_EXEC_HI)
+ sprintf(src0, "%s", "exec_hi");
+ else if (insn->vopc.src0 < GFX10_VOPC_SRC_INTEGER_MINUS_1)
+ sprintf(src0, "0x%x",
+ insn->vopc.src0 - GFX10_VOPC_SRC_INTEGER_0);
+ else if (insn->vopc.src0 < GFX10_VOPC_SRC_SHARED_BASE)
+ sprintf(src0, "-0x%x",
+ insn->vopc.src0 - GFX10_VOPC_SRC_INTEGER_MINUS_1);
+ else if (insn->vopc.src0 == GFX10_VOPC_SRC_VCCZ)
+ sprintf(src0, "%s", "vcc");
+ else if (insn->vopc.src0 == GFX10_VOPC_SRC_EXECZ)
+ sprintf(src0, "%s", "exec");
+ else if (insn->vopc.src0 == GFX10_VOPC_SRC_SCC)
+ sprintf(src0, "%s", "scc");
+ else if (insn->vopc.src0 == GFX10_VOPC_SRC_LITERAL_CONST)
+ sprintf(src0, "0x%x", insn->vopc.literal);
+ else if (insn->vopc.src0 >= GFX10_VOPC_SRC_VGPR_BASE)
+ sprintf(src0, "v%d",
+ insn->vopc.src0 - GFX10_VOPC_SRC_VGPR_BASE);
+
+ pr_debug("knod_asm %s %s, v%d\n",
+ opnames_gfx10[AMDGCN_INSN_TYPE_VOPC][insn->vopc.op],
+ src0, insn->vopc.vsrc1);
+}
+
+static inline void gfx10_debug_vop2(union amdgcn_gfx10_insn *insn)
+{
+ char src0[20];
+
+ if (insn->vop2.src0 < GFX10_VOP2_SRC_VCC_LO)
+ sprintf(src0, "s%d", insn->vop2.src0);
+ else if (insn->vop2.src0 == GFX10_VOP2_SRC_VCC_LO)
+ sprintf(src0, "%s", "vcc_lo");
+ else if (insn->vop2.src0 == GFX10_VOP2_SRC_VCC_HI)
+ sprintf(src0, "%s", "vcc_hi");
+ else if (insn->vop2.src0 == GFX10_VOP2_SRC_NULL)
+ sprintf(src0, "%s", "null");
+ else if (insn->vop2.src0 == GFX10_VOP2_SRC_EXEC_LO)
+ sprintf(src0, "%s", "exec_lo");
+ else if (insn->vop2.src0 == GFX10_VOP2_SRC_EXEC_HI)
+ sprintf(src0, "%s", "exec_hi");
+ else if (insn->vop2.src0 < GFX10_VOP2_SRC_INTEGER_MINUS_1)
+ sprintf(src0, "0x%x",
+ insn->vop2.src0 - GFX10_VOP2_SRC_INTEGER_0);
+ else if (insn->vop2.src0 < GFX10_VOP2_SRC_SHARED_BASE)
+ sprintf(src0, "-0x%x",
+ insn->vop2.src0 - GFX10_VOP2_SRC_INTEGER_MINUS_1);
+ else if (insn->vop2.src0 == GFX10_VOP2_SRC_VCCZ)
+ sprintf(src0, "%s", "vcc");
+ else if (insn->vop2.src0 == GFX10_VOP2_SRC_EXECZ)
+ sprintf(src0, "%s", "exec");
+ else if (insn->vop2.src0 == GFX10_VOP2_SRC_SCC)
+ sprintf(src0, "%s", "scc");
+ else if (insn->vop2.src0 == GFX10_VOP2_SRC_LITERAL_CONST)
+ sprintf(src0, "0x%x", insn->vop2.literal);
+ else if (insn->vop2.src0 >= GFX10_VOP2_SRC_VGPR_BASE)
+ sprintf(src0, "v%d",
+ insn->vop2.src0 - GFX10_VOP2_SRC_VGPR_BASE);
+
+ pr_debug("knod_asm %s v%d, %s v%d\n",
+ opnames_gfx10[AMDGCN_INSN_TYPE_VOP2][insn->vop2.op],
+ insn->vop2.vdst,
+ src0,
+ insn->vop2.vsrc1);
+}
+
+static inline void gfx10_debug_sop1(union amdgcn_gfx10_insn *insn)
+{
+ char ssrc0[20];
+
+ if (insn->sop1.ssrc0 < GFX10_SOP1_SSRC_VCC_LO)
+ sprintf(ssrc0, "s%d", insn->sop1.ssrc0);
+ else if (insn->sop1.ssrc0 == GFX10_SOP1_SSRC_VCC_LO)
+ sprintf(ssrc0, "%s", "vcc_lo");
+ else if (insn->sop1.ssrc0 == GFX10_SOP1_SSRC_VCC_HI)
+ sprintf(ssrc0, "%s", "vcc_hi");
+ else if (insn->sop1.ssrc0 == GFX10_SOP1_SSRC_NULL)
+ sprintf(ssrc0, "%s", "null");
+ else if (insn->sop1.ssrc0 == GFX10_SOP1_SSRC_EXEC_LO)
+ sprintf(ssrc0, "%s", "exec_lo");
+ else if (insn->sop1.ssrc0 == GFX10_SOP1_SSRC_EXEC_HI)
+ sprintf(ssrc0, "%s", "exec_hi");
+ else if (insn->sop1.ssrc0 < GFX10_SOP1_SSRC_INTEGER_MINUS_1)
+ sprintf(ssrc0, "0x%x",
+ insn->sop1.ssrc0 - GFX10_SOP1_SSRC_INTEGER_0);
+ else if (insn->sop1.ssrc0 < GFX10_SOP1_SSRC_SHARED_BASE)
+ sprintf(ssrc0, "-0x%x",
+ insn->sop1.ssrc0 - GFX10_SOP1_SSRC_INTEGER_MINUS_1);
+ else if (insn->sop1.ssrc0 == GFX10_SOP1_SSRC_VCCZ)
+ sprintf(ssrc0, "%s", "vcc");
+ else if (insn->sop1.ssrc0 == GFX10_SOP1_SSRC_EXECZ)
+ sprintf(ssrc0, "%s", "exec");
+ else if (insn->sop1.ssrc0 == GFX10_SOP1_SSRC_SCC)
+ sprintf(ssrc0, "%s", "scc");
+ else if (insn->sop1.ssrc0 == GFX10_SOP1_SSRC_LITERAL_CONST)
+ sprintf(ssrc0, "0x%x", insn->sop1.literal);
+
+ pr_debug("knod_asm %s s%d, %s\n",
+ opnames_gfx10[AMDGCN_INSN_TYPE_SOP1][insn->sop1.op],
+ insn->sop1.sdst,
+ ssrc0);
+}
+
+static inline void gfx10_debug_sopp(union amdgcn_gfx10_insn *insn)
+{
+ pr_debug("knod_asm %s 0x%x\n",
+ opnames_gfx10[AMDGCN_INSN_TYPE_SOPP][insn->sopp.op],
+ insn->sopp.simm16);
+}
+
+static inline void gfx10_debug_smem(union amdgcn_gfx10_insn *insn)
+{
+ pr_debug("%s s[%d:%d], s[%d:%d], 0x%x\n",
+ opnames_gfx10[AMDGCN_INSN_TYPE_SMEM][insn->smem.op],
+ insn->smem.sdata,
+ insn->smem.sdata + 1,
+ (insn->smem.sbase * 2),
+ (insn->smem.sbase * 2) + 1,
+ insn->smem.offset);
+}
+
+static inline void gfx10_debug_mubuf(union amdgcn_gfx10_insn *insn)
+{
+ pr_debug("knod_asm %s v%d, v%d, s[%d:%d], 0x%x offen offset:%d\n",
+ opnames_gfx10[AMDGCN_INSN_TYPE_MUBUF][insn->mubuf.op],
+ insn->mubuf.vdata,
+ insn->mubuf.vaddr,
+ insn->mubuf.srsrc,
+ insn->mubuf.srsrc + 3,
+ insn->mubuf.soffset,
+ insn->mubuf.offset);
+}
+
+static inline void gfx10_debug_global(union amdgcn_gfx10_insn *insn)
+{
+ if (insn->flat.op == GFX9_GLOBAL_STORE_BYTE) {
+ pr_debug("knod_asm %s v[%d:%d], v%d, off offset:%d\n",
+ opnames_gfx10[AMDGCN_INSN_TYPE_FLAT][insn->flat.op],
+ insn->flat.addr,
+ insn->flat.addr + 1,
+ insn->flat.data,
+ insn->flat.offset);
+ } else if (insn->flat.op == GFX9_GLOBAL_STORE_SHORT) {
+ pr_debug("knod_asm %s v[%d:%d], v%d, off offset:%d\n",
+ opnames_gfx10[AMDGCN_INSN_TYPE_FLAT][insn->flat.op],
+ insn->flat.addr,
+ insn->flat.addr + 1,
+ insn->flat.data,
+ insn->flat.offset);
+ } else if (insn->flat.op == GFX9_GLOBAL_STORE_DWORD) {
+ pr_debug("knod_asm %s v[%d:%d], v%d, off offset:%d\n",
+ opnames_gfx10[AMDGCN_INSN_TYPE_FLAT][insn->flat.op],
+ insn->flat.addr,
+ insn->flat.addr + 1,
+ insn->flat.data,
+ insn->flat.offset);
+ } else if (insn->flat.op == GFX9_GLOBAL_STORE_DWORDX2) {
+ pr_debug("knod_asm %s v[%d:%d], v[%d:%d], off offset:%d\n",
+ opnames_gfx10[AMDGCN_INSN_TYPE_FLAT][insn->flat.op],
+ insn->flat.addr,
+ insn->flat.addr + 1,
+ insn->flat.data,
+ insn->flat.data + 1,
+ insn->flat.offset);
+ } else {
+ pr_debug("knod_asm %s v[%d:%d], v[%d:%d], off offset:%d\n",
+ opnames_gfx10[AMDGCN_INSN_TYPE_FLAT][insn->flat.op],
+ insn->flat.vdst,
+ insn->flat.vdst + 1,
+ insn->flat.addr,
+ insn->flat.addr + 1,
+ insn->flat.offset);
+ }
+}
+
+static inline void decode_ssrc8(int ssrc, u32 literal, char *buf)
+{
+ if (ssrc < 106)
+ sprintf(buf, "s%d", ssrc);
+ else if (ssrc == 106)
+ sprintf(buf, "vcc_lo");
+ else if (ssrc == 107)
+ sprintf(buf, "vcc_hi");
+ else if (ssrc == 125)
+ sprintf(buf, "null");
+ else if (ssrc == 126)
+ sprintf(buf, "exec_lo");
+ else if (ssrc == 127)
+ sprintf(buf, "exec_hi");
+ else if (ssrc < 193)
+ sprintf(buf, "0x%x", ssrc - 128);
+ else if (ssrc < 235)
+ sprintf(buf, "-0x%x", ssrc - 193);
+ else if (ssrc == 251)
+ sprintf(buf, "vcc");
+ else if (ssrc == 252)
+ sprintf(buf, "exec");
+ else if (ssrc == 253)
+ sprintf(buf, "scc");
+ else if (ssrc == 255)
+ sprintf(buf, "0x%x", literal);
+ else
+ sprintf(buf, "?%d", ssrc);
+}
+
+static inline void decode_vsrc9(int src, u32 literal, char *buf)
+{
+ if (src >= 256)
+ sprintf(buf, "v%d", src - 256);
+ else
+ decode_ssrc8(src, literal, buf);
+}
+
+static inline void gfx10_debug_sop2(union amdgcn_gfx10_insn *insn)
+{
+ char ssrc0[20], ssrc1[20];
+
+ decode_ssrc8(insn->sop2.ssrc0, insn->sop2.literal, ssrc0);
+ decode_ssrc8(insn->sop2.ssrc1, insn->sop2.literal, ssrc1);
+ pr_debug("knod_asm %s s%d, %s, %s\n",
+ opnames_gfx10[AMDGCN_INSN_TYPE_SOP2][insn->sop2.op],
+ insn->sop2.sdst, ssrc0, ssrc1);
+}
+
+static inline void gfx10_debug_sopk(union amdgcn_gfx10_insn *insn)
+{
+ pr_debug("knod_asm %s s%d, 0x%x\n",
+ opnames_gfx10[AMDGCN_INSN_TYPE_SOPK][insn->sopk.op],
+ insn->sopk.sdst, insn->sopk.simm16);
+}
+
+static inline void gfx10_debug_sopc(union amdgcn_gfx10_insn *insn)
+{
+ char ssrc0[20], ssrc1[20];
+
+ decode_ssrc8(insn->sopc.ssrc0, insn->sopc.literal, ssrc0);
+ decode_ssrc8(insn->sopc.ssrc1, insn->sopc.literal, ssrc1);
+ pr_debug("knod_asm %s %s, %s\n",
+ opnames_gfx10[AMDGCN_INSN_TYPE_SOPC][insn->sopc.op],
+ ssrc0, ssrc1);
+}
+
+static inline void gfx10_debug_vop3p(union amdgcn_gfx10_insn *insn)
+{
+ char src0[20], src1[20], src2[20];
+
+ decode_vsrc9(insn->vop3p.src0, insn->vop3p.literal, src0);
+ decode_vsrc9(insn->vop3p.src1, insn->vop3p.literal, src1);
+ decode_vsrc9(insn->vop3p.src2, insn->vop3p.literal, src2);
+ pr_debug("knod_asm %s v%d, %s, %s, %s\n",
+ opnames_gfx10[AMDGCN_INSN_TYPE_VOP3P][insn->vop3p.op],
+ (int)insn->vop3p.vdst, src0, src1, src2);
+}
+
+static inline void gfx10_debug_sdwa(union amdgcn_gfx10_insn *insn)
+{
+ pr_debug("knod_asm sdwa src0:%d dst_sel:%d src0_sel:%d src1_sel:%d\n",
+ insn->sdwa.src0, insn->sdwa.dst_sel,
+ insn->sdwa.src0_sel, insn->sdwa.src1_sel);
+}
+
+static inline void gfx10_debug_sdwab(union amdgcn_gfx10_insn *insn)
+{
+ pr_debug("knod_asm sdwab src0:%d sdst:s%d src0_sel:%d src1_sel:%d\n",
+ insn->sdwab.src0, insn->sdwab.sdst,
+ insn->sdwab.src0_sel, insn->sdwab.src1_sel);
+}
+
+static inline void gfx10_debug_dpp16(union amdgcn_gfx10_insn *insn)
+{
+ pr_debug("knod_asm dpp16 (not decoded)\n");
+}
+
+static inline void gfx10_debug_dpp8(union amdgcn_gfx10_insn *insn)
+{
+ pr_debug("knod_asm dpp8 (not decoded)\n");
+}
+
+static inline void gfx10_debug_vintrp(union amdgcn_gfx10_insn *insn)
+{
+ pr_debug("knod_asm vintrp (not decoded)\n");
+}
+
+static inline void gfx10_debug_ds(union amdgcn_gfx10_insn *insn)
+{
+ pr_debug("knod_asm %s v%d, v%d, v%d, v%d offset0:%d offset1:%d%s\n",
+ opnames_gfx10[AMDGCN_INSN_TYPE_DS][insn->ds.op],
+ (int)insn->ds.vdst, (int)insn->ds.addr,
+ (int)insn->ds.data0, (int)insn->ds.data1,
+ (int)insn->ds.offset0, (int)insn->ds.offset1,
+ insn->ds.gds ? " gds" : "");
+}
+
+static inline void gfx10_debug_mtbuf(union amdgcn_gfx10_insn *insn)
+{
+ pr_debug("knod_asm %s v%d, v%d, s[%d:%d], s%d format:%d offset:%d\n",
+ opnames_gfx10[AMDGCN_INSN_TYPE_MTBUF][insn->mtbuf.op],
+ (int)insn->mtbuf.vdata, (int)insn->mtbuf.vaddr,
+ (int)insn->mtbuf.srsrc * 4, (int)insn->mtbuf.srsrc * 4 + 3,
+ (int)insn->mtbuf.soffset, (int)insn->mtbuf.foamat,
+ (int)insn->mtbuf.offset);
+}
+
+static inline void gfx10_debug_mimg(union amdgcn_gfx10_insn *insn)
+{
+ pr_debug("knod_asm mimg (not decoded)\n");
+}
+
+static inline void gfx10_debug_exp(union amdgcn_gfx10_insn *insn)
+{
+ pr_debug("knod_asm exp (not decoded)\n");
+}
+
+static inline void gfx10_debug_insn(struct amdgcn_insn *insn)
+{
+ u32 *ptr;
+
+ switch (insn->type) {
+ case AMDGCN_INSN_TYPE_SOP2:
+ gfx10_debug_sop2(&insn->gfx10);
+ break;
+ case AMDGCN_INSN_TYPE_SOPK:
+ gfx10_debug_sopk(&insn->gfx10);
+ break;
+ case AMDGCN_INSN_TYPE_SOP1:
+ gfx10_debug_sop1(&insn->gfx10);
+ break;
+ case AMDGCN_INSN_TYPE_SOPC:
+ gfx10_debug_sopc(&insn->gfx10);
+ break;
+ case AMDGCN_INSN_TYPE_SOPP:
+ gfx10_debug_sopp(&insn->gfx10);
+ break;
+ case AMDGCN_INSN_TYPE_SMEM:
+ gfx10_debug_smem(&insn->gfx10);
+ break;
+ case AMDGCN_INSN_TYPE_VOP2:
+ gfx10_debug_vop2(&insn->gfx10);
+ break;
+ case AMDGCN_INSN_TYPE_VOP1:
+ gfx10_debug_vop1(&insn->gfx10);
+ break;
+ case AMDGCN_INSN_TYPE_VOPC:
+ gfx10_debug_vopc(&insn->gfx10);
+ break;
+ case AMDGCN_INSN_TYPE_VOP3A:
+ gfx10_debug_vop3a(&insn->gfx10);
+ break;
+ case AMDGCN_INSN_TYPE_VOP3B:
+ gfx10_debug_vop3b(&insn->gfx10);
+ break;
+ case AMDGCN_INSN_TYPE_VOP3P:
+ gfx10_debug_vop3p(&insn->gfx10);
+ break;
+ case AMDGCN_INSN_TYPE_SDWA:
+ gfx10_debug_sdwa(&insn->gfx10);
+ break;
+ case AMDGCN_INSN_TYPE_SDWAB:
+ gfx10_debug_sdwab(&insn->gfx10);
+ break;
+ case AMDGCN_INSN_TYPE_DPP16:
+ gfx10_debug_dpp16(&insn->gfx10);
+ break;
+ case AMDGCN_INSN_TYPE_DPP8:
+ gfx10_debug_dpp8(&insn->gfx10);
+ break;
+ case AMDGCN_INSN_TYPE_VINTRP:
+ gfx10_debug_vintrp(&insn->gfx10);
+ break;
+ case AMDGCN_INSN_TYPE_DS:
+ gfx10_debug_ds(&insn->gfx10);
+ break;
+ case AMDGCN_INSN_TYPE_MTBUF:
+ gfx10_debug_mtbuf(&insn->gfx10);
+ break;
+ case AMDGCN_INSN_TYPE_MUBUF:
+ gfx10_debug_mubuf(&insn->gfx10);
+ break;
+ case AMDGCN_INSN_TYPE_MIMG:
+ gfx10_debug_mimg(&insn->gfx10);
+ break;
+ case AMDGCN_INSN_TYPE_FLAT:
+ gfx10_debug_global(&insn->gfx10);
+ break;
+ case AMDGCN_INSN_TYPE_EXP:
+ gfx10_debug_exp(&insn->gfx10);
+ break;
+ default:
+ WARN_ON_ONCE(1);
+ break;
+ }
+
+ ptr = (u32 *)&insn->gfx10;
+ if (insn->size == 4) {
+ pr_debug("knod_asm %s %u %.8X\n", __func__, __LINE__, ptr[0]);
+ } else if (insn->size == 8) {
+ pr_debug("knod_asm %s %u %.8X %.8X\n",
+ __func__, __LINE__, ptr[0], ptr[1]);
+ } else if (insn->size == 12) {
+ pr_debug("knod_asm %s %u %.8X %.8X %.8X\n",
+ __func__, __LINE__, ptr[0], ptr[1], ptr[2]);
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void gfx9_debug_vop3a(union amdgcn_gfx9_insn *insn)
+{
+ char src0[20];
+ char src1[20];
+ char src2[20];
+
+ if (insn->vop3a.src0 < GFX9_VOP3A_SRC_VCC_LO)
+ sprintf(src0, "s%d", insn->vop3a.src0);
+ else if (insn->vop3a.src0 == GFX9_VOP3A_SRC_VCC_LO)
+ sprintf(src0, "%s", "vcc_lo");
+ else if (insn->vop3a.src0 == GFX9_VOP3A_SRC_VCC_HI)
+ sprintf(src0, "%s", "vcc_hi");
+ else if (insn->vop3a.src0 == GFX9_VOP3A_SRC_NULL)
+ sprintf(src0, "%s", "null");
+ else if (insn->vop3a.src0 == GFX9_VOP3A_SRC_EXEC_LO)
+ sprintf(src0, "%s", "exec_lo");
+ else if (insn->vop3a.src0 == GFX9_VOP3A_SRC_EXEC_HI)
+ sprintf(src0, "%s", "exec_hi");
+ else if (insn->vop3a.src0 < GFX9_VOP3A_SRC_INTEGER_MINUS_1)
+ sprintf(src0, "0x%x",
+ insn->vop3a.src0 - GFX9_VOP3A_SRC_INTEGER_0);
+ else if (insn->vop3a.src0 < GFX9_VOP3A_SRC_SHARED_BASE)
+ sprintf(src0, "-0x%x",
+ insn->vop3a.src0 - GFX9_VOP3A_SRC_INTEGER_MINUS_1);
+ else if (insn->vop3a.src0 == GFX9_VOP3A_SRC_VCCZ)
+ sprintf(src0, "%s", "vcc");
+ else if (insn->vop3a.src0 == GFX9_VOP3A_SRC_EXECZ)
+ sprintf(src0, "%s", "exec");
+ else if (insn->vop3a.src0 == GFX9_VOP3A_SRC_SCC)
+ sprintf(src0, "%s", "scc");
+ else if (insn->vop3a.src0 == GFX9_VOP3A_SRC_LITERAL_CONST)
+ sprintf(src0, "0x%x", insn->vop3a.literal);
+ else if (insn->vop3a.src0 >= GFX9_VOP3A_SRC_VGPR_BASE)
+ sprintf(src0, "v%d",
+ insn->vop3a.src0 - GFX9_VOP3A_SRC_VGPR_BASE);
+
+ if (insn->vop3a.src1 < GFX9_VOP3A_SRC_VCC_LO)
+ sprintf(src1, "s%d", insn->vop3a.src1);
+ else if (insn->vop3a.src1 == GFX9_VOP3A_SRC_VCC_LO)
+ sprintf(src1, "%s", "vcc_lo");
+ else if (insn->vop3a.src1 == GFX9_VOP3A_SRC_VCC_HI)
+ sprintf(src1, "%s", "vcc_hi");
+ else if (insn->vop3a.src1 == GFX9_VOP3A_SRC_NULL)
+ sprintf(src1, "%s", "null");
+ else if (insn->vop3a.src1 == GFX9_VOP3A_SRC_EXEC_LO)
+ sprintf(src1, "%s", "exec_lo");
+ else if (insn->vop3a.src1 == GFX9_VOP3A_SRC_EXEC_HI)
+ sprintf(src1, "%s", "exec_hi");
+ else if (insn->vop3a.src1 < GFX9_VOP3A_SRC_INTEGER_MINUS_1)
+ sprintf(src1, "0x%x",
+ insn->vop3a.src1 - GFX9_VOP3A_SRC_INTEGER_0);
+ else if (insn->vop3a.src1 < GFX9_VOP3A_SRC_SHARED_BASE)
+ sprintf(src1, "-0x%x",
+ insn->vop3a.src1 - GFX9_VOP3A_SRC_INTEGER_MINUS_1);
+ else if (insn->vop3a.src1 == GFX9_VOP3A_SRC_VCCZ)
+ sprintf(src1, "%s", "vcc");
+ else if (insn->vop3a.src1 == GFX9_VOP3A_SRC_EXECZ)
+ sprintf(src1, "%s", "exec");
+ else if (insn->vop3a.src1 == GFX9_VOP3A_SRC_SCC)
+ sprintf(src1, "%s", "scc");
+ else if (insn->vop3a.src1 == GFX9_VOP3A_SRC_LITERAL_CONST)
+ sprintf(src1, "0x%x", insn->vop3a.literal);
+ else if (insn->vop3a.src1 >= GFX9_VOP3A_SRC_VGPR_BASE)
+ sprintf(src1, "v%d",
+ insn->vop3a.src1 - GFX9_VOP3A_SRC_VGPR_BASE);
+
+ if (insn->vop3a.src2 < GFX9_VOP3A_SRC_VCC_LO)
+ sprintf(src2, "s%d", insn->vop3a.src2);
+ else if (insn->vop3a.src2 == GFX9_VOP3A_SRC_VCC_LO)
+ sprintf(src2, "%s", "vcc_lo");
+ else if (insn->vop3a.src2 == GFX9_VOP3A_SRC_VCC_HI)
+ sprintf(src2, "%s", "vcc_hi");
+ else if (insn->vop3a.src2 == GFX9_VOP3A_SRC_NULL)
+ sprintf(src2, "%s", "null");
+ else if (insn->vop3a.src2 == GFX9_VOP3A_SRC_EXEC_LO)
+ sprintf(src2, "%s", "exec_lo");
+ else if (insn->vop3a.src2 == GFX9_VOP3A_SRC_EXEC_HI)
+ sprintf(src2, "%s", "exec_hi");
+ else if (insn->vop3a.src2 < GFX9_VOP3A_SRC_INTEGER_MINUS_1)
+ sprintf(src2, "0x%x",
+ insn->vop3a.src2 - GFX9_VOP3A_SRC_INTEGER_0);
+ else if (insn->vop3a.src2 < GFX9_VOP3A_SRC_SHARED_BASE)
+ sprintf(src2, "-0x%x",
+ insn->vop3a.src2 - GFX9_VOP3A_SRC_INTEGER_MINUS_1);
+ else if (insn->vop3a.src2 == GFX9_VOP3A_SRC_VCCZ)
+ sprintf(src2, "%s", "vcc");
+ else if (insn->vop3a.src2 == GFX9_VOP3A_SRC_EXECZ)
+ sprintf(src2, "%s", "exec");
+ else if (insn->vop3a.src2 == GFX9_VOP3A_SRC_SCC)
+ sprintf(src2, "%s", "scc");
+ else if (insn->vop3a.src2 == GFX9_VOP3A_SRC_LITERAL_CONST)
+ sprintf(src2, "0x%x", insn->vop3a.literal);
+ else if (insn->vop3a.src2 >= GFX9_VOP3A_SRC_VGPR_BASE)
+ sprintf(src2, "v%d",
+ insn->vop3a.src2 - GFX9_VOP3A_SRC_VGPR_BASE);
+
+ pr_debug("knod_asm %s v%d, %s, %s, %s\n",
+ opnames_gfx9[AMDGCN_INSN_TYPE_VOP3A][insn->vop3a.op],
+ insn->vop3a.vdst, src0, src1, src2);
+}
+
+static inline void gfx9_debug_vop3b(union amdgcn_gfx9_insn *insn)
+{
+ char src0[20];
+ char src1[20];
+ char src2[20];
+ char sdst[20];
+
+ if (insn->vop3b.sdst < GFX9_VOP3B_SRC_VCC_LO)
+ sprintf(sdst, "s%d", insn->vop3b.sdst);
+ else if (insn->vop3b.sdst == GFX9_VOP3B_SRC_VCC_LO)
+ sprintf(sdst, "%s", "vcc_lo");
+ else if (insn->vop3b.sdst == GFX9_VOP3B_SRC_VCC_HI)
+ sprintf(sdst, "%s", "vcc_hi");
+ else if (insn->vop3b.sdst == GFX9_VOP3B_SRC_NULL)
+ sprintf(sdst, "%s", "null");
+ else if (insn->vop3b.sdst == GFX9_VOP3B_SRC_EXEC_LO)
+ sprintf(sdst, "%s", "exec_lo");
+ else if (insn->vop3b.sdst == GFX9_VOP3B_SRC_EXEC_HI)
+ sprintf(sdst, "%s", "exec_hi");
+ else if (insn->vop3b.sdst < GFX9_VOP3B_SRC_INTEGER_MINUS_1)
+ sprintf(sdst, "0x%x",
+ insn->vop3b.sdst - GFX9_VOP3B_SRC_INTEGER_0);
+ else if (insn->vop3b.sdst < GFX9_VOP3B_SRC_SHARED_BASE)
+ sprintf(sdst, "-0x%x",
+ insn->vop3b.sdst - GFX9_VOP3B_SRC_INTEGER_MINUS_1);
+ else if (insn->vop3b.sdst == GFX9_VOP3B_SRC_VCCZ)
+ sprintf(sdst, "%s", "vcc");
+ else if (insn->vop3b.sdst == GFX9_VOP3B_SRC_EXECZ)
+ sprintf(sdst, "%s", "exec");
+ else if (insn->vop3b.sdst == GFX9_VOP3B_SRC_SCC)
+ sprintf(sdst, "%s", "scc");
+ else if (insn->vop3b.sdst == GFX9_VOP3B_SRC_LITERAL_CONST)
+ sprintf(sdst, "0x%x", insn->vop3b.literal);
+ else if (insn->vop3b.sdst >= GFX9_VOP3B_SRC_VGPR_BASE)
+ sprintf(sdst, "v%d",
+ insn->vop3b.sdst - GFX9_VOP3B_SRC_VGPR_BASE);
+
+ if (insn->vop3b.src0 < GFX9_VOP3B_SRC_VCC_LO)
+ sprintf(src0, "s%d", insn->vop3b.src0);
+ else if (insn->vop3b.src0 == GFX9_VOP3B_SRC_VCC_LO)
+ sprintf(src0, "%s", "vcc_lo");
+ else if (insn->vop3b.src0 == GFX9_VOP3B_SRC_VCC_HI)
+ sprintf(src0, "%s", "vcc_hi");
+ else if (insn->vop3b.src0 == GFX9_VOP3B_SRC_NULL)
+ sprintf(src0, "%s", "null");
+ else if (insn->vop3b.src0 == GFX9_VOP3B_SRC_EXEC_LO)
+ sprintf(src0, "%s", "exec_lo");
+ else if (insn->vop3b.src0 == GFX9_VOP3B_SRC_EXEC_HI)
+ sprintf(src0, "%s", "exec_hi");
+ else if (insn->vop3b.src0 < GFX9_VOP3B_SRC_INTEGER_MINUS_1)
+ sprintf(src0, "0x%x",
+ insn->vop3b.src0 - GFX9_VOP3B_SRC_INTEGER_0);
+ else if (insn->vop3b.src0 < GFX9_VOP3B_SRC_SHARED_BASE)
+ sprintf(src0, "-0x%x",
+ insn->vop3b.src0 - GFX9_VOP3B_SRC_INTEGER_MINUS_1);
+ else if (insn->vop3b.src0 == GFX9_VOP3B_SRC_VCCZ)
+ sprintf(src0, "%s", "vcc");
+ else if (insn->vop3b.src0 == GFX9_VOP3B_SRC_EXECZ)
+ sprintf(src0, "%s", "exec");
+ else if (insn->vop3b.src0 == GFX9_VOP3B_SRC_SCC)
+ sprintf(src0, "%s", "scc");
+ else if (insn->vop3b.src0 == GFX9_VOP3B_SRC_LITERAL_CONST)
+ sprintf(src0, "0x%x", insn->vop3b.literal);
+ else if (insn->vop3b.src0 >= GFX9_VOP3B_SRC_VGPR_BASE)
+ sprintf(src0, "v%d",
+ insn->vop3b.src0 - GFX9_VOP3B_SRC_VGPR_BASE);
+
+ if (insn->vop3b.src1 < GFX9_VOP3B_SRC_VCC_LO)
+ sprintf(src1, "s%d", insn->vop3b.src1);
+ else if (insn->vop3b.src1 == GFX9_VOP3B_SRC_VCC_LO)
+ sprintf(src1, "%s", "vcc_lo");
+ else if (insn->vop3b.src1 == GFX9_VOP3B_SRC_VCC_HI)
+ sprintf(src1, "%s", "vcc_hi");
+ else if (insn->vop3b.src1 == GFX9_VOP3B_SRC_NULL)
+ sprintf(src1, "%s", "null");
+ else if (insn->vop3b.src1 == GFX9_VOP3B_SRC_EXEC_LO)
+ sprintf(src1, "%s", "exec_lo");
+ else if (insn->vop3b.src1 == GFX9_VOP3B_SRC_EXEC_HI)
+ sprintf(src1, "%s", "exec_hi");
+ else if (insn->vop3b.src1 < GFX9_VOP3B_SRC_INTEGER_MINUS_1)
+ sprintf(src1, "0x%x",
+ insn->vop3b.src1 - GFX9_VOP3B_SRC_INTEGER_0);
+ else if (insn->vop3b.src1 < GFX9_VOP3B_SRC_SHARED_BASE)
+ sprintf(src1, "-0x%x",
+ insn->vop3b.src1 - GFX9_VOP3B_SRC_INTEGER_MINUS_1);
+ else if (insn->vop3b.src1 == GFX9_VOP3B_SRC_VCCZ)
+ sprintf(src1, "%s", "vcc");
+ else if (insn->vop3b.src1 == GFX9_VOP3B_SRC_EXECZ)
+ sprintf(src1, "%s", "exec");
+ else if (insn->vop3b.src1 == GFX9_VOP3B_SRC_SCC)
+ sprintf(src1, "%s", "scc");
+ else if (insn->vop3b.src1 == GFX9_VOP3B_SRC_LITERAL_CONST)
+ sprintf(src1, "0x%x", insn->vop3b.literal);
+ else if (insn->vop3b.src1 >= GFX9_VOP3B_SRC_VGPR_BASE)
+ sprintf(src1, "v%d",
+ insn->vop3b.src1 - GFX9_VOP3B_SRC_VGPR_BASE);
+
+ if (insn->vop3b.src2 < GFX9_VOP3B_SRC_VCC_LO)
+ sprintf(src2, "s%d", insn->vop3b.src2);
+ else if (insn->vop3b.src2 == GFX9_VOP3B_SRC_VCC_LO)
+ sprintf(src2, "%s", "vcc_lo");
+ else if (insn->vop3b.src2 == GFX9_VOP3B_SRC_VCC_HI)
+ sprintf(src2, "%s", "vcc_hi");
+ else if (insn->vop3b.src2 == GFX9_VOP3B_SRC_NULL)
+ sprintf(src2, "%s", "null");
+ else if (insn->vop3b.src2 == GFX9_VOP3B_SRC_EXEC_LO)
+ sprintf(src2, "%s", "exec_lo");
+ else if (insn->vop3b.src2 == GFX9_VOP3B_SRC_EXEC_HI)
+ sprintf(src2, "%s", "exec_hi");
+ else if (insn->vop3b.src2 < GFX9_VOP3B_SRC_INTEGER_MINUS_1)
+ sprintf(src2, "0x%x",
+ insn->vop3b.src2 - GFX9_VOP3B_SRC_INTEGER_0);
+ else if (insn->vop3b.src2 < GFX9_VOP3B_SRC_SHARED_BASE)
+ sprintf(src2, "-0x%x",
+ insn->vop3b.src2 - GFX9_VOP3B_SRC_INTEGER_MINUS_1);
+ else if (insn->vop3b.src2 == GFX9_VOP3B_SRC_VCCZ)
+ sprintf(src2, "%s", "vcc");
+ else if (insn->vop3b.src2 == GFX9_VOP3B_SRC_EXECZ)
+ sprintf(src2, "%s", "exec");
+ else if (insn->vop3b.src2 == GFX9_VOP3B_SRC_SCC)
+ sprintf(src2, "%s", "scc");
+ else if (insn->vop3b.src2 == GFX9_VOP3B_SRC_LITERAL_CONST)
+ sprintf(src2, "0x%x", insn->vop3b.literal);
+ else if (insn->vop3b.src2 >= GFX9_VOP3B_SRC_VGPR_BASE)
+ sprintf(src2, "v%d",
+ insn->vop3b.src2 - GFX9_VOP3B_SRC_VGPR_BASE);
+
+ pr_debug("knod_asm %s v%d, %s, %s, %s, %s\n",
+ opnames_gfx9[AMDGCN_INSN_TYPE_VOP3B][insn->vop3b.op],
+ insn->vop3b.vdst, sdst, src0, src1, src2);
+}
+
+static inline void gfx9_debug_vop1(union amdgcn_gfx9_insn *insn)
+{
+ char src0[20];
+
+ if (insn->vop1.src0 < GFX9_VOP1_SRC_VCC_LO)
+ sprintf(src0, "s%d", insn->vop1.src0);
+ else if (insn->vop1.src0 == GFX9_VOP1_SRC_VCC_LO)
+ sprintf(src0, "%s", "vcc_lo");
+ else if (insn->vop1.src0 == GFX9_VOP1_SRC_VCC_HI)
+ sprintf(src0, "%s", "vcc_hi");
+ else if (insn->vop1.src0 == GFX9_VOP1_SRC_NULL)
+ sprintf(src0, "%s", "null");
+ else if (insn->vop1.src0 == GFX9_VOP1_SRC_EXEC_LO)
+ sprintf(src0, "%s", "exec_lo");
+ else if (insn->vop1.src0 == GFX9_VOP1_SRC_EXEC_HI)
+ sprintf(src0, "%s", "exec_hi");
+ else if (insn->vop1.src0 < GFX9_VOP1_SRC_INTEGER_MINUS_1)
+ sprintf(src0, "0x%x",
+ insn->vop1.src0 - GFX9_VOP1_SRC_INTEGER_0);
+ else if (insn->vop1.src0 < GFX9_VOP1_SRC_SHARED_BASE)
+ sprintf(src0, "-0x%x",
+ insn->vop1.src0 - GFX9_VOP1_SRC_INTEGER_MINUS_1);
+ else if (insn->vop1.src0 == GFX9_VOP1_SRC_VCCZ)
+ sprintf(src0, "%s", "vcc");
+ else if (insn->vop1.src0 == GFX9_VOP1_SRC_EXECZ)
+ sprintf(src0, "%s", "exec");
+ else if (insn->vop1.src0 == GFX9_VOP1_SRC_SCC)
+ sprintf(src0, "%s", "scc");
+ else if (insn->vop1.src0 == GFX9_VOP1_SRC_LITERAL_CONST)
+ sprintf(src0, "0x%x", insn->vop1.literal);
+ else if (insn->vop1.src0 >= GFX9_VOP1_SRC_VGPR_BASE)
+ sprintf(src0, "v%d", insn->vop1.src0 - GFX9_VOP1_SRC_VGPR_BASE);
+
+ pr_debug("knod_asm %s v%d, %s\n",
+ opnames_gfx9[AMDGCN_INSN_TYPE_VOP1][insn->vop1.op],
+ insn->vop1.vdst, src0);
+}
+
+static inline void gfx9_debug_vopc(union amdgcn_gfx9_insn *insn)
+{
+ char src0[20];
+
+ if (insn->vopc.src0 < GFX9_VOPC_SRC_VCC_LO)
+ sprintf(src0, "s%d", insn->vopc.src0);
+ else if (insn->vopc.src0 == GFX9_VOPC_SRC_VCC_LO)
+ sprintf(src0, "%s", "vcc_lo");
+ else if (insn->vopc.src0 == GFX9_VOPC_SRC_VCC_HI)
+ sprintf(src0, "%s", "vcc_hi");
+ else if (insn->vopc.src0 == GFX9_VOPC_SRC_NULL)
+ sprintf(src0, "%s", "null");
+ else if (insn->vopc.src0 == GFX9_VOPC_SRC_EXEC_LO)
+ sprintf(src0, "%s", "exec_lo");
+ else if (insn->vopc.src0 == GFX9_VOPC_SRC_EXEC_HI)
+ sprintf(src0, "%s", "exec_hi");
+ else if (insn->vopc.src0 < GFX9_VOPC_SRC_INTEGER_MINUS_1)
+ sprintf(src0, "0x%x",
+ insn->vopc.src0 - GFX9_VOPC_SRC_INTEGER_0);
+ else if (insn->vopc.src0 < GFX9_VOPC_SRC_SHARED_BASE)
+ sprintf(src0, "-0x%x",
+ insn->vopc.src0 - GFX9_VOPC_SRC_INTEGER_MINUS_1);
+ else if (insn->vopc.src0 == GFX9_VOPC_SRC_VCCZ)
+ sprintf(src0, "%s", "vcc");
+ else if (insn->vopc.src0 == GFX9_VOPC_SRC_EXECZ)
+ sprintf(src0, "%s", "exec");
+ else if (insn->vopc.src0 == GFX9_VOPC_SRC_SCC)
+ sprintf(src0, "%s", "scc");
+ else if (insn->vopc.src0 == GFX9_VOPC_SRC_LITERAL_CONST)
+ sprintf(src0, "0x%x", insn->vopc.literal);
+ else if (insn->vopc.src0 >= GFX9_VOPC_SRC_VGPR_BASE)
+ sprintf(src0, "v%d", insn->vopc.src0 - GFX9_VOPC_SRC_VGPR_BASE);
+
+ pr_debug("knod_asm %s %s, v%d\n",
+ opnames_gfx9[AMDGCN_INSN_TYPE_VOPC][insn->vopc.op],
+ src0,
+ insn->vopc.vsrc1);
+}
+
+static inline void gfx9_debug_vop2(union amdgcn_gfx9_insn *insn)
+{
+ char src0[20];
+
+ if (insn->vop2.src0 < GFX9_VOP2_SRC_VCC_LO)
+ sprintf(src0, "s%d", insn->vop2.src0);
+ else if (insn->vop2.src0 == GFX9_VOP2_SRC_VCC_LO)
+ sprintf(src0, "%s", "vcc_lo");
+ else if (insn->vop2.src0 == GFX9_VOP2_SRC_VCC_HI)
+ sprintf(src0, "%s", "vcc_hi");
+ else if (insn->vop2.src0 == GFX9_VOP2_SRC_NULL)
+ sprintf(src0, "%s", "null");
+ else if (insn->vop2.src0 == GFX9_VOP2_SRC_EXEC_LO)
+ sprintf(src0, "%s", "exec_lo");
+ else if (insn->vop2.src0 == GFX9_VOP2_SRC_EXEC_HI)
+ sprintf(src0, "%s", "exec_hi");
+ else if (insn->vop2.src0 < GFX9_VOP2_SRC_INTEGER_MINUS_1)
+ sprintf(src0, "0x%x",
+ insn->vop2.src0 - GFX9_VOP2_SRC_INTEGER_0);
+ else if (insn->vop2.src0 < GFX9_VOP2_SRC_SHARED_BASE)
+ sprintf(src0, "-0x%x",
+ insn->vop2.src0 - GFX9_VOP2_SRC_INTEGER_MINUS_1);
+ else if (insn->vop2.src0 == GFX9_VOP2_SRC_VCCZ)
+ sprintf(src0, "%s", "vcc");
+ else if (insn->vop2.src0 == GFX9_VOP2_SRC_EXECZ)
+ sprintf(src0, "%s", "exec");
+ else if (insn->vop2.src0 == GFX9_VOP2_SRC_SCC)
+ sprintf(src0, "%s", "scc");
+ else if (insn->vop2.src0 == GFX9_VOP2_SRC_LITERAL_CONST)
+ sprintf(src0, "0x%x", insn->vop2.literal);
+ else if (insn->vop2.src0 >= GFX9_VOP2_SRC_VGPR_BASE)
+ sprintf(src0, "v%d", insn->vop2.src0 - GFX9_VOP2_SRC_VGPR_BASE);
+
+ pr_debug("knod_asm %s v%d, %s v%d\n",
+ opnames_gfx9[AMDGCN_INSN_TYPE_VOP2][insn->vop2.op],
+ insn->vop2.vdst,
+ src0,
+ insn->vop2.vsrc1);
+}
+
+static inline void gfx9_debug_sop1(union amdgcn_gfx9_insn *insn)
+{
+ char ssrc0[20];
+
+ if (insn->sop1.ssrc0 < GFX9_SOP1_SSRC_VCC_LO)
+ sprintf(ssrc0, "s%d", insn->sop1.ssrc0);
+ else if (insn->sop1.ssrc0 == GFX9_SOP1_SSRC_VCC_LO)
+ sprintf(ssrc0, "%s", "vcc_lo");
+ else if (insn->sop1.ssrc0 == GFX9_SOP1_SSRC_VCC_HI)
+ sprintf(ssrc0, "%s", "vcc_hi");
+ else if (insn->sop1.ssrc0 == GFX9_SOP1_SSRC_NULL)
+ sprintf(ssrc0, "%s", "null");
+ else if (insn->sop1.ssrc0 == GFX9_SOP1_SSRC_EXEC_LO)
+ sprintf(ssrc0, "%s", "exec_lo");
+ else if (insn->sop1.ssrc0 == GFX9_SOP1_SSRC_EXEC_HI)
+ sprintf(ssrc0, "%s", "exec_hi");
+ else if (insn->sop1.ssrc0 < GFX9_SOP1_SSRC_INTEGER_MINUS_1)
+ sprintf(ssrc0, "0x%x",
+ insn->sop1.ssrc0 - GFX9_SOP1_SSRC_INTEGER_0);
+ else if (insn->sop1.ssrc0 < GFX9_SOP1_SSRC_SHARED_BASE)
+ sprintf(ssrc0, "-0x%x",
+ insn->sop1.ssrc0 - GFX9_SOP1_SSRC_INTEGER_MINUS_1);
+ else if (insn->sop1.ssrc0 == GFX9_SOP1_SSRC_VCCZ)
+ sprintf(ssrc0, "%s", "vcc");
+ else if (insn->sop1.ssrc0 == GFX9_SOP1_SSRC_EXECZ)
+ sprintf(ssrc0, "%s", "exec");
+ else if (insn->sop1.ssrc0 == GFX9_SOP1_SSRC_SCC)
+ sprintf(ssrc0, "%s", "scc");
+ else if (insn->sop1.ssrc0 == GFX9_SOP1_SSRC_LITERAL_CONST)
+ sprintf(ssrc0, "0x%x", insn->sop1.literal);
+
+ pr_debug("knod_asm %s s%d, %s\n",
+ opnames_gfx9[AMDGCN_INSN_TYPE_SOP1][insn->sop1.op],
+ insn->sop1.sdst,
+ ssrc0);
+}
+
+static inline void gfx9_debug_sopp(union amdgcn_gfx9_insn *insn)
+{
+ pr_debug("knod_asm %s 0x%x\n",
+ opnames_gfx9[AMDGCN_INSN_TYPE_SOPP][insn->sopp.op],
+ insn->sopp.simm16);
+}
+
+static inline void gfx9_debug_smem(union amdgcn_gfx9_insn *insn)
+{
+ pr_debug("%s s[%d:%d], s[%d:%d], 0x%x\n",
+ opnames_gfx9[AMDGCN_INSN_TYPE_SMEM][insn->smem.op],
+ insn->smem.sdata,
+ insn->smem.sdata + 1,
+ (insn->smem.sbase * 2),
+ (insn->smem.sbase * 2) + 1,
+ insn->smem.offset);
+}
+
+static inline void gfx9_debug_mubuf(union amdgcn_gfx9_insn *insn)
+{
+ pr_debug("knod_asm %s v%d, v%d, s[%d:%d], 0x%x offen offset:%d\n",
+ opnames_gfx9[AMDGCN_INSN_TYPE_MUBUF][insn->mubuf.op],
+ insn->mubuf.vdata,
+ insn->mubuf.vaddr,
+ insn->mubuf.srsrc,
+ insn->mubuf.srsrc + 3,
+ insn->mubuf.soffset,
+ insn->mubuf.offset);
+}
+
+static inline void gfx9_debug_global(union amdgcn_gfx9_insn *insn)
+{
+ if (insn->flat.op == GFX9_GLOBAL_STORE_BYTE) {
+ pr_debug("knod_asm %s v[%d:%d], v%d, off offset:%d\n",
+ opnames_gfx9[AMDGCN_INSN_TYPE_FLAT][insn->flat.op],
+ insn->flat.addr,
+ insn->flat.addr + 1,
+ insn->flat.data,
+ insn->flat.offset);
+ } else if (insn->flat.op == GFX9_GLOBAL_STORE_SHORT) {
+ pr_debug("knod_asm %s v[%d:%d], v%d, off offset:%d\n",
+ opnames_gfx9[AMDGCN_INSN_TYPE_FLAT][insn->flat.op],
+ insn->flat.addr,
+ insn->flat.addr + 1,
+ insn->flat.data,
+ insn->flat.offset);
+ } else if (insn->flat.op == GFX9_GLOBAL_STORE_DWORD) {
+ pr_debug("knod_asm %s v[%d:%d], v%d, off offset:%d\n",
+ opnames_gfx9[AMDGCN_INSN_TYPE_FLAT][insn->flat.op],
+ insn->flat.addr,
+ insn->flat.addr + 1,
+ insn->flat.data,
+ insn->flat.offset);
+ } else if (insn->flat.op == GFX9_GLOBAL_STORE_DWORDX2) {
+ pr_debug("knod_asm %s v[%d:%d], v[%d:%d], off offset:%d\n",
+ opnames_gfx9[AMDGCN_INSN_TYPE_FLAT][insn->flat.op],
+ insn->flat.addr,
+ insn->flat.addr + 1,
+ insn->flat.data,
+ insn->flat.data + 1,
+ insn->flat.offset);
+ } else {
+ pr_debug("knod_asm %s v[%d:%d], v[%d:%d], off offset:%d\n",
+ opnames_gfx9[AMDGCN_INSN_TYPE_FLAT][insn->flat.op],
+ insn->flat.vdst,
+ insn->flat.vdst + 1,
+ insn->flat.addr,
+ insn->flat.addr + 1,
+ insn->flat.offset);
+ }
+}
+
+static inline void gfx9_debug_sop2(union amdgcn_gfx9_insn *insn)
+{
+ char ssrc0[20], ssrc1[20];
+
+ decode_ssrc8(insn->sop2.ssrc0, insn->sop2.literal, ssrc0);
+ decode_ssrc8(insn->sop2.ssrc1, insn->sop2.literal, ssrc1);
+ pr_debug("knod_asm %s s%d, %s, %s\n",
+ opnames_gfx9[AMDGCN_INSN_TYPE_SOP2][insn->sop2.op],
+ insn->sop2.sdst, ssrc0, ssrc1);
+}
+
+static inline void gfx9_debug_sopk(union amdgcn_gfx9_insn *insn)
+{
+ pr_debug("knod_asm %s s%d, 0x%x\n",
+ opnames_gfx9[AMDGCN_INSN_TYPE_SOPK][insn->sopk.op],
+ insn->sopk.sdst, insn->sopk.simm16);
+}
+
+static inline void gfx9_debug_sopc(union amdgcn_gfx9_insn *insn)
+{
+ char ssrc0[20], ssrc1[20];
+
+ decode_ssrc8(insn->sopc.ssrc0, insn->sopc.literal, ssrc0);
+ decode_ssrc8(insn->sopc.ssrc1, insn->sopc.literal, ssrc1);
+ pr_debug("knod_asm %s %s, %s\n",
+ opnames_gfx9[AMDGCN_INSN_TYPE_SOPC][insn->sopc.op],
+ ssrc0, ssrc1);
+}
+
+static inline void gfx9_debug_vop3p(union amdgcn_gfx9_insn *insn)
+{
+ char src0[20], src1[20], src2[20];
+
+ decode_vsrc9(insn->vop3p.src0, insn->vop3p.literal, src0);
+ decode_vsrc9(insn->vop3p.src1, insn->vop3p.literal, src1);
+ decode_vsrc9(insn->vop3p.src2, insn->vop3p.literal, src2);
+ pr_debug("knod_asm %s v%d, %s, %s, %s\n",
+ opnames_gfx9[AMDGCN_INSN_TYPE_VOP3P][insn->vop3p.op],
+ (int)insn->vop3p.vdst, src0, src1, src2);
+}
+
+static inline void gfx9_debug_sdwa(union amdgcn_gfx9_insn *insn)
+{
+ pr_debug("knod_asm sdwa src0:%d dst_sel:%d src0_sel:%d src1_sel:%d\n",
+ insn->sdwa.src0, insn->sdwa.dst_sel,
+ insn->sdwa.src0_sel, insn->sdwa.src1_sel);
+}
+
+static inline void gfx9_debug_sdwab(union amdgcn_gfx9_insn *insn)
+{
+ pr_debug("knod_asm sdwab src0:%d sdst:s%d src0_sel:%d src1_sel:%d\n",
+ insn->sdwab.src0, insn->sdwab.sdst,
+ insn->sdwab.src0_sel, insn->sdwab.src1_sel);
+}
+
+static inline void gfx9_debug_dpp16(union amdgcn_gfx9_insn *insn)
+{
+ pr_debug("knod_asm dpp16 (not decoded)\n");
+}
+
+static inline void gfx9_debug_dpp8(union amdgcn_gfx9_insn *insn)
+{
+ pr_debug("knod_asm dpp8 (not decoded)\n");
+}
+
+static inline void gfx9_debug_vintrp(union amdgcn_gfx9_insn *insn)
+{
+ pr_debug("knod_asm vintrp (not decoded)\n");
+}
+
+static inline void gfx9_debug_ds(union amdgcn_gfx9_insn *insn)
+{
+ pr_debug("knod_asm %s v%d, v%d, v%d, v%d offset0:%d offset1:%d%s\n",
+ opnames_gfx9[AMDGCN_INSN_TYPE_DS][insn->ds.op],
+ (int)insn->ds.vdst, (int)insn->ds.addr,
+ (int)insn->ds.data0, (int)insn->ds.data1,
+ (int)insn->ds.offset0, (int)insn->ds.offset1,
+ insn->ds.gds ? " gds" : "");
+}
+
+static inline void gfx9_debug_mtbuf(union amdgcn_gfx9_insn *insn)
+{
+ pr_debug("knod_asm %s v%d, v%d, s[%d:%d], s%d dfmt:%d nfmt:%d offset:%d\n",
+ opnames_gfx9[AMDGCN_INSN_TYPE_MTBUF][insn->mtbuf.op],
+ (int)insn->mtbuf.vdata, (int)insn->mtbuf.vaddr,
+ (int)insn->mtbuf.srsrc * 4, (int)insn->mtbuf.srsrc * 4 + 3,
+ (int)insn->mtbuf.soffset, (int)insn->mtbuf.dfmt,
+ (int)insn->mtbuf.nfmt, (int)insn->mtbuf.offset);
+}
+
+static inline void gfx9_debug_mimg(union amdgcn_gfx9_insn *insn)
+{
+ pr_debug("knod_asm mimg (not decoded)\n");
+}
+
+static inline void gfx9_debug_exp(union amdgcn_gfx9_insn *insn)
+{
+ pr_debug("knod_asm exp (not decoded)\n");
+}
+
+static inline void gfx9_debug_insn(struct amdgcn_insn *insn)
+{
+ u32 *ptr;
+
+ switch (insn->type) {
+ case AMDGCN_INSN_TYPE_SOP2:
+ gfx9_debug_sop2(&insn->gfx9);
+ break;
+ case AMDGCN_INSN_TYPE_SOPK:
+ gfx9_debug_sopk(&insn->gfx9);
+ break;
+ case AMDGCN_INSN_TYPE_SOP1:
+ gfx9_debug_sop1(&insn->gfx9);
+ break;
+ case AMDGCN_INSN_TYPE_SOPC:
+ gfx9_debug_sopc(&insn->gfx9);
+ break;
+ case AMDGCN_INSN_TYPE_SOPP:
+ gfx9_debug_sopp(&insn->gfx9);
+ break;
+ case AMDGCN_INSN_TYPE_SMEM:
+ gfx9_debug_smem(&insn->gfx9);
+ break;
+ case AMDGCN_INSN_TYPE_VOP2:
+ gfx9_debug_vop2(&insn->gfx9);
+ break;
+ case AMDGCN_INSN_TYPE_VOP1:
+ gfx9_debug_vop1(&insn->gfx9);
+ break;
+ case AMDGCN_INSN_TYPE_VOPC:
+ gfx9_debug_vopc(&insn->gfx9);
+ break;
+ case AMDGCN_INSN_TYPE_VOP3A:
+ gfx9_debug_vop3a(&insn->gfx9);
+ break;
+ case AMDGCN_INSN_TYPE_VOP3B:
+ gfx9_debug_vop3b(&insn->gfx9);
+ break;
+ case AMDGCN_INSN_TYPE_VOP3P:
+ gfx9_debug_vop3p(&insn->gfx9);
+ break;
+ case AMDGCN_INSN_TYPE_SDWA:
+ gfx9_debug_sdwa(&insn->gfx9);
+ break;
+ case AMDGCN_INSN_TYPE_SDWAB:
+ gfx9_debug_sdwab(&insn->gfx9);
+ break;
+ case AMDGCN_INSN_TYPE_DPP16:
+ gfx9_debug_dpp16(&insn->gfx9);
+ break;
+ case AMDGCN_INSN_TYPE_DPP8:
+ gfx9_debug_dpp8(&insn->gfx9);
+ break;
+ case AMDGCN_INSN_TYPE_VINTRP:
+ gfx9_debug_vintrp(&insn->gfx9);
+ break;
+ case AMDGCN_INSN_TYPE_DS:
+ gfx9_debug_ds(&insn->gfx9);
+ break;
+ case AMDGCN_INSN_TYPE_MTBUF:
+ gfx9_debug_mtbuf(&insn->gfx9);
+ break;
+ case AMDGCN_INSN_TYPE_MUBUF:
+ gfx9_debug_mubuf(&insn->gfx9);
+ break;
+ case AMDGCN_INSN_TYPE_MIMG:
+ gfx9_debug_mimg(&insn->gfx9);
+ break;
+ case AMDGCN_INSN_TYPE_FLAT:
+ gfx9_debug_global(&insn->gfx9);
+ break;
+ case AMDGCN_INSN_TYPE_EXP:
+ gfx9_debug_exp(&insn->gfx9);
+ break;
+ default:
+ WARN_ON_ONCE(1);
+ break;
+ }
+
+ ptr = (u32 *)&insn->gfx9;
+ if (insn->size == 4) {
+ pr_debug("knod_asm %s %u %.8X\n", __func__, __LINE__, ptr[0]);
+ } else if (insn->size == 8) {
+ pr_debug("knod_asm %s %u %.8X %.8X\n",
+ __func__, __LINE__, ptr[0], ptr[1]);
+ } else if (insn->size == 12) {
+ pr_debug("knod_asm %s %u %.8X %.8X %.8X\n",
+ __func__, __LINE__, ptr[0], ptr[1], ptr[2]);
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static inline void debug_insn(int version, struct amdgcn_insn *insn)
+{
+ if (version == 10)
+ gfx10_debug_insn(insn);
+ else if (version == 9)
+ gfx9_debug_insn(insn);
+ else
+ WARN_ON_ONCE(1);
+}
+
+static inline void gfx10_debugfs_vop3a(union amdgcn_gfx10_insn *insn,
+ struct seq_file *m)
+{
+ char src0[20];
+ char src1[20];
+ char src2[20];
+
+ if (insn->vop3a.src0 < GFX10_VOP3A_SRC_VCC_LO)
+ sprintf(src0, "s%d", insn->vop3a.src0);
+ else if (insn->vop3a.src0 == GFX10_VOP3A_SRC_VCC_LO)
+ sprintf(src0, "%s", "vcc_lo");
+ else if (insn->vop3a.src0 == GFX10_VOP3A_SRC_VCC_HI)
+ sprintf(src0, "%s", "vcc_hi");
+ else if (insn->vop3a.src0 == GFX10_VOP3A_SRC_NULL)
+ sprintf(src0, "%s", "null");
+ else if (insn->vop3a.src0 == GFX10_VOP3A_SRC_EXEC_LO)
+ sprintf(src0, "%s", "exec_lo");
+ else if (insn->vop3a.src0 == GFX10_VOP3A_SRC_EXEC_HI)
+ sprintf(src0, "%s", "exec_hi");
+ else if (insn->vop3a.src0 < GFX10_VOP3A_SRC_INTEGER_MINUS_1)
+ sprintf(src0, "0x%x",
+ insn->vop3a.src0 - GFX10_VOP3A_SRC_INTEGER_0);
+ else if (insn->vop3a.src0 < GFX10_VOP3A_SRC_SHARED_BASE)
+ sprintf(src0, "-0x%x",
+ insn->vop3a.src0 - GFX10_VOP3A_SRC_INTEGER_MINUS_1);
+ else if (insn->vop3a.src0 == GFX10_VOP3A_SRC_VCCZ)
+ sprintf(src0, "%s", "vcc");
+ else if (insn->vop3a.src0 == GFX10_VOP3A_SRC_EXECZ)
+ sprintf(src0, "%s", "exec");
+ else if (insn->vop3a.src0 == GFX10_VOP3A_SRC_SCC)
+ sprintf(src0, "%s", "scc");
+ else if (insn->vop3a.src0 == GFX10_VOP3A_SRC_LITERAL_CONST)
+ sprintf(src0, "0x%x", insn->vop3a.literal);
+ else if (insn->vop3a.src0 >= GFX10_VOP3A_SRC_VGPR_BASE)
+ sprintf(src0, "v%d",
+ insn->vop3a.src0 - GFX10_VOP3A_SRC_VGPR_BASE);
+
+ if (insn->vop3a.src1 < GFX10_VOP3A_SRC_VCC_LO)
+ sprintf(src1, "s%d", insn->vop3a.src1);
+ else if (insn->vop3a.src1 == GFX10_VOP3A_SRC_VCC_LO)
+ sprintf(src1, "%s", "vcc_lo");
+ else if (insn->vop3a.src1 == GFX10_VOP3A_SRC_VCC_HI)
+ sprintf(src1, "%s", "vcc_hi");
+ else if (insn->vop3a.src1 == GFX10_VOP3A_SRC_NULL)
+ sprintf(src1, "%s", "null");
+ else if (insn->vop3a.src1 == GFX10_VOP3A_SRC_EXEC_LO)
+ sprintf(src1, "%s", "exec_lo");
+ else if (insn->vop3a.src1 == GFX10_VOP3A_SRC_EXEC_HI)
+ sprintf(src1, "%s", "exec_hi");
+ else if (insn->vop3a.src1 < GFX10_VOP3A_SRC_INTEGER_MINUS_1)
+ sprintf(src1, "0x%x",
+ insn->vop3a.src1 - GFX10_VOP3A_SRC_INTEGER_0);
+ else if (insn->vop3a.src1 < GFX10_VOP3A_SRC_SHARED_BASE)
+ sprintf(src1, "-0x%x",
+ insn->vop3a.src1 - GFX10_VOP3A_SRC_INTEGER_MINUS_1);
+ else if (insn->vop3a.src1 == GFX10_VOP3A_SRC_VCCZ)
+ sprintf(src1, "%s", "vcc");
+ else if (insn->vop3a.src1 == GFX10_VOP3A_SRC_EXECZ)
+ sprintf(src1, "%s", "exec");
+ else if (insn->vop3a.src1 == GFX10_VOP3A_SRC_SCC)
+ sprintf(src1, "%s", "scc");
+ else if (insn->vop3a.src1 == GFX10_VOP3A_SRC_LITERAL_CONST)
+ sprintf(src1, "0x%x", insn->vop3a.literal);
+ else if (insn->vop3a.src1 >= GFX10_VOP3A_SRC_VGPR_BASE)
+ sprintf(src1, "v%d",
+ insn->vop3a.src1 - GFX10_VOP3A_SRC_VGPR_BASE);
+
+ if (insn->vop3a.src2 < GFX10_VOP3A_SRC_VCC_LO)
+ sprintf(src2, "s%d", insn->vop3a.src2);
+ else if (insn->vop3a.src2 == GFX10_VOP3A_SRC_VCC_LO)
+ sprintf(src2, "%s", "vcc_lo");
+ else if (insn->vop3a.src2 == GFX10_VOP3A_SRC_VCC_HI)
+ sprintf(src2, "%s", "vcc_hi");
+ else if (insn->vop3a.src2 == GFX10_VOP3A_SRC_NULL)
+ sprintf(src2, "%s", "null");
+ else if (insn->vop3a.src2 == GFX10_VOP3A_SRC_EXEC_LO)
+ sprintf(src2, "%s", "exec_lo");
+ else if (insn->vop3a.src2 == GFX10_VOP3A_SRC_EXEC_HI)
+ sprintf(src2, "%s", "exec_hi");
+ else if (insn->vop3a.src2 < GFX10_VOP3A_SRC_INTEGER_MINUS_1)
+ sprintf(src2, "0x%x",
+ insn->vop3a.src2 - GFX10_VOP3A_SRC_INTEGER_0);
+ else if (insn->vop3a.src2 < GFX10_VOP3A_SRC_SHARED_BASE)
+ sprintf(src2, "-0x%x",
+ insn->vop3a.src2 - GFX10_VOP3A_SRC_INTEGER_MINUS_1);
+ else if (insn->vop3a.src2 == GFX10_VOP3A_SRC_VCCZ)
+ sprintf(src2, "%s", "vcc");
+ else if (insn->vop3a.src2 == GFX10_VOP3A_SRC_EXECZ)
+ sprintf(src2, "%s", "exec");
+ else if (insn->vop3a.src2 == GFX10_VOP3A_SRC_SCC)
+ sprintf(src2, "%s", "scc");
+ else if (insn->vop3a.src2 == GFX10_VOP3A_SRC_LITERAL_CONST)
+ sprintf(src2, "0x%x", insn->vop3a.literal);
+ else if (insn->vop3a.src2 >= GFX10_VOP3A_SRC_VGPR_BASE)
+ sprintf(src2, "v%d",
+ insn->vop3a.src2 - GFX10_VOP3A_SRC_VGPR_BASE);
+
+ seq_printf(m, "%s v%d, %s, %s, %s\n",
+ opnames_gfx10[AMDGCN_INSN_TYPE_VOP3A][insn->vop3a.op],
+ insn->vop3a.vdst, src0, src1, src2);
+}
+
+static inline void gfx10_debugfs_vop3b(union amdgcn_gfx10_insn *insn,
+ struct seq_file *m)
+{
+ char src0[20];
+ char src1[20];
+ char src2[20];
+ char sdst[20];
+
+ if (insn->vop3b.sdst < GFX10_VOP3A_SRC_VCC_LO)
+ sprintf(sdst, "s%d", insn->vop3b.sdst);
+ else if (insn->vop3b.sdst == GFX10_VOP3A_SRC_VCC_LO)
+ sprintf(sdst, "%s", "vcc_lo");
+ else if (insn->vop3b.sdst == GFX10_VOP3A_SRC_VCC_HI)
+ sprintf(sdst, "%s", "vcc_hi");
+ else if (insn->vop3b.sdst == GFX10_VOP3A_SRC_NULL)
+ sprintf(sdst, "%s", "null");
+ else if (insn->vop3b.sdst == GFX10_VOP3A_SRC_EXEC_LO)
+ sprintf(sdst, "%s", "exec_lo");
+ else if (insn->vop3b.sdst == GFX10_VOP3A_SRC_EXEC_HI)
+ sprintf(sdst, "%s", "exec_hi");
+ else if (insn->vop3b.sdst < GFX10_VOP3A_SRC_INTEGER_MINUS_1)
+ sprintf(sdst, "0x%x",
+ insn->vop3b.sdst - GFX10_VOP3A_SRC_INTEGER_0);
+ else if (insn->vop3b.sdst < GFX10_VOP3A_SRC_SHARED_BASE)
+ sprintf(sdst, "-0x%x",
+ insn->vop3b.sdst - GFX10_VOP3A_SRC_INTEGER_MINUS_1);
+ else if (insn->vop3b.sdst == GFX10_VOP3A_SRC_VCCZ)
+ sprintf(sdst, "%s", "vcc");
+ else if (insn->vop3b.sdst == GFX10_VOP3A_SRC_EXECZ)
+ sprintf(sdst, "%s", "exec");
+ else if (insn->vop3b.sdst == GFX10_VOP3A_SRC_SCC)
+ sprintf(sdst, "%s", "scc");
+ else if (insn->vop3b.sdst == GFX10_VOP3A_SRC_LITERAL_CONST)
+ sprintf(sdst, "0x%x", insn->vop3b.literal);
+ else if (insn->vop3b.sdst >= GFX10_VOP3A_SRC_VGPR_BASE)
+ sprintf(sdst, "v%d",
+ insn->vop3b.sdst - GFX10_VOP3A_SRC_VGPR_BASE);
+
+ if (insn->vop3b.src0 < GFX10_VOP3A_SRC_VCC_LO)
+ sprintf(src0, "s%d", insn->vop3b.src0);
+ else if (insn->vop3b.src0 == GFX10_VOP3A_SRC_VCC_LO)
+ sprintf(src0, "%s", "vcc_lo");
+ else if (insn->vop3b.src0 == GFX10_VOP3A_SRC_VCC_HI)
+ sprintf(src0, "%s", "vcc_hi");
+ else if (insn->vop3b.src0 == GFX10_VOP3A_SRC_NULL)
+ sprintf(src0, "%s", "null");
+ else if (insn->vop3b.src0 == GFX10_VOP3A_SRC_EXEC_LO)
+ sprintf(src0, "%s", "exec_lo");
+ else if (insn->vop3b.src0 == GFX10_VOP3A_SRC_EXEC_HI)
+ sprintf(src0, "%s", "exec_hi");
+ else if (insn->vop3b.src0 < GFX10_VOP3A_SRC_INTEGER_MINUS_1)
+ sprintf(src0, "0x%x",
+ insn->vop3b.src0 - GFX10_VOP3A_SRC_INTEGER_0);
+ else if (insn->vop3b.src0 < GFX10_VOP3A_SRC_SHARED_BASE)
+ sprintf(src0, "-0x%x",
+ insn->vop3b.src0 - GFX10_VOP3A_SRC_INTEGER_MINUS_1);
+ else if (insn->vop3b.src0 == GFX10_VOP3A_SRC_VCCZ)
+ sprintf(src0, "%s", "vcc");
+ else if (insn->vop3b.src0 == GFX10_VOP3A_SRC_EXECZ)
+ sprintf(src0, "%s", "exec");
+ else if (insn->vop3b.src0 == GFX10_VOP3A_SRC_SCC)
+ sprintf(src0, "%s", "scc");
+ else if (insn->vop3b.src0 == GFX10_VOP3A_SRC_LITERAL_CONST)
+ sprintf(src0, "0x%x", insn->vop3b.literal);
+ else if (insn->vop3b.src0 >= GFX10_VOP3A_SRC_VGPR_BASE)
+ sprintf(src0, "v%d",
+ insn->vop3b.src0 - GFX10_VOP3A_SRC_VGPR_BASE);
+
+ if (insn->vop3b.src1 < GFX10_VOP3A_SRC_VCC_LO)
+ sprintf(src1, "s%d", insn->vop3b.src1);
+ else if (insn->vop3b.src1 == GFX10_VOP3A_SRC_VCC_LO)
+ sprintf(src1, "%s", "vcc_lo");
+ else if (insn->vop3b.src1 == GFX10_VOP3A_SRC_VCC_HI)
+ sprintf(src1, "%s", "vcc_hi");
+ else if (insn->vop3b.src1 == GFX10_VOP3A_SRC_NULL)
+ sprintf(src1, "%s", "null");
+ else if (insn->vop3b.src1 == GFX10_VOP3A_SRC_EXEC_LO)
+ sprintf(src1, "%s", "exec_lo");
+ else if (insn->vop3b.src1 == GFX10_VOP3A_SRC_EXEC_HI)
+ sprintf(src1, "%s", "exec_hi");
+ else if (insn->vop3b.src1 < GFX10_VOP3A_SRC_INTEGER_MINUS_1)
+ sprintf(src1, "0x%x",
+ insn->vop3b.src1 - GFX10_VOP3A_SRC_INTEGER_0);
+ else if (insn->vop3b.src1 < GFX10_VOP3A_SRC_SHARED_BASE)
+ sprintf(src1, "-0x%x",
+ insn->vop3b.src1 - GFX10_VOP3A_SRC_INTEGER_MINUS_1);
+ else if (insn->vop3b.src1 == GFX10_VOP3A_SRC_VCCZ)
+ sprintf(src1, "%s", "vcc");
+ else if (insn->vop3b.src1 == GFX10_VOP3A_SRC_EXECZ)
+ sprintf(src1, "%s", "exec");
+ else if (insn->vop3b.src1 == GFX10_VOP3A_SRC_SCC)
+ sprintf(src1, "%s", "scc");
+ else if (insn->vop3b.src1 == GFX10_VOP3A_SRC_LITERAL_CONST)
+ sprintf(src1, "0x%x", insn->vop3b.literal);
+ else if (insn->vop3b.src1 >= GFX10_VOP3A_SRC_VGPR_BASE)
+ sprintf(src1, "v%d",
+ insn->vop3b.src1 - GFX10_VOP3A_SRC_VGPR_BASE);
+
+ if (insn->vop3b.src2 < GFX10_VOP3A_SRC_VCC_LO)
+ sprintf(src2, "s%d", insn->vop3b.src2);
+ else if (insn->vop3b.src2 == GFX10_VOP3A_SRC_VCC_LO)
+ sprintf(src2, "%s", "vcc_lo");
+ else if (insn->vop3b.src2 == GFX10_VOP3A_SRC_VCC_HI)
+ sprintf(src2, "%s", "vcc_hi");
+ else if (insn->vop3b.src2 == GFX10_VOP3A_SRC_NULL)
+ sprintf(src2, "%s", "null");
+ else if (insn->vop3b.src2 == GFX10_VOP3A_SRC_EXEC_LO)
+ sprintf(src2, "%s", "exec_lo");
+ else if (insn->vop3b.src2 == GFX10_VOP3A_SRC_EXEC_HI)
+ sprintf(src2, "%s", "exec_hi");
+ else if (insn->vop3b.src2 < GFX10_VOP3A_SRC_INTEGER_MINUS_1)
+ sprintf(src2, "0x%x",
+ insn->vop3b.src2 - GFX10_VOP3A_SRC_INTEGER_0);
+ else if (insn->vop3b.src2 < GFX10_VOP3A_SRC_SHARED_BASE)
+ sprintf(src2, "-0x%x",
+ insn->vop3b.src2 - GFX10_VOP3A_SRC_INTEGER_MINUS_1);
+ else if (insn->vop3b.src2 == GFX10_VOP3A_SRC_VCCZ)
+ sprintf(src2, "%s", "vcc");
+ else if (insn->vop3b.src2 == GFX10_VOP3A_SRC_EXECZ)
+ sprintf(src2, "%s", "exec");
+ else if (insn->vop3b.src2 == GFX10_VOP3A_SRC_SCC)
+ sprintf(src2, "%s", "scc");
+ else if (insn->vop3b.src2 == GFX10_VOP3A_SRC_LITERAL_CONST)
+ sprintf(src2, "0x%x", insn->vop3b.literal);
+ else if (insn->vop3b.src2 >= GFX10_VOP3A_SRC_VGPR_BASE)
+ sprintf(src2, "v%d",
+ insn->vop3b.src2 - GFX10_VOP3A_SRC_VGPR_BASE);
+
+ seq_printf(m, "%s v%d, %s, %s, %s, %s\n",
+ opnames_gfx10[AMDGCN_INSN_TYPE_VOP3B][insn->vop3b.op],
+ insn->vop3b.vdst, sdst, src0, src1, src2);
+}
+
+static inline void gfx10_debugfs_vop1(union amdgcn_gfx10_insn *insn,
+ struct seq_file *m)
+{
+ char src0[20];
+
+ if (insn->vop1.src0 < GFX10_VOP1_SRC_VCC_LO)
+ sprintf(src0, "s%d", insn->vop1.src0);
+ else if (insn->vop1.src0 == GFX10_VOP1_SRC_VCC_LO)
+ sprintf(src0, "%s", "vcc_lo");
+ else if (insn->vop1.src0 == GFX10_VOP1_SRC_VCC_HI)
+ sprintf(src0, "%s", "vcc_hi");
+ else if (insn->vop1.src0 == GFX10_VOP1_SRC_NULL)
+ sprintf(src0, "%s", "null");
+ else if (insn->vop1.src0 == GFX10_VOP1_SRC_EXEC_LO)
+ sprintf(src0, "%s", "exec_lo");
+ else if (insn->vop1.src0 == GFX10_VOP1_SRC_EXEC_HI)
+ sprintf(src0, "%s", "exec_hi");
+ else if (insn->vop1.src0 < GFX10_VOP1_SRC_INTEGER_MINUS_1)
+ sprintf(src0, "0x%x",
+ insn->vop1.src0 - GFX10_VOP1_SRC_INTEGER_0);
+ else if (insn->vop1.src0 < GFX10_VOP1_SRC_SHARED_BASE)
+ sprintf(src0, "-0x%x",
+ insn->vop1.src0 - GFX10_VOP1_SRC_INTEGER_MINUS_1);
+ else if (insn->vop1.src0 == GFX10_VOP1_SRC_VCCZ)
+ sprintf(src0, "%s", "vcc");
+ else if (insn->vop1.src0 == GFX10_VOP1_SRC_EXECZ)
+ sprintf(src0, "%s", "exec");
+ else if (insn->vop1.src0 == GFX10_VOP1_SRC_SCC)
+ sprintf(src0, "%s", "scc");
+ else if (insn->vop1.src0 == GFX10_VOP1_SRC_LITERAL_CONST)
+ sprintf(src0, "0x%x", insn->vop1.literal);
+ else if (insn->vop1.src0 >= GFX10_VOP1_SRC_VGPR_BASE)
+ sprintf(src0, "v%d",
+ insn->vop1.src0 - GFX10_VOP1_SRC_VGPR_BASE);
+
+ seq_printf(m, "%s v%d, %s\n",
+ opnames_gfx10[AMDGCN_INSN_TYPE_VOP1][insn->vop1.op],
+ insn->vop1.vdst, src0);
+}
+
+static inline void gfx10_debugfs_vopc(union amdgcn_gfx10_insn *insn,
+ struct seq_file *m)
+{
+ char src0[20];
+
+ if (insn->vopc.src0 < GFX10_VOPC_SRC_VCC_LO)
+ sprintf(src0, "s%d", insn->vopc.src0);
+ else if (insn->vopc.src0 == GFX10_VOPC_SRC_VCC_LO)
+ sprintf(src0, "%s", "vcc_lo");
+ else if (insn->vopc.src0 == GFX10_VOPC_SRC_VCC_HI)
+ sprintf(src0, "%s", "vcc_hi");
+ else if (insn->vopc.src0 == GFX10_VOPC_SRC_NULL)
+ sprintf(src0, "%s", "null");
+ else if (insn->vopc.src0 == GFX10_VOPC_SRC_EXEC_LO)
+ sprintf(src0, "%s", "exec_lo");
+ else if (insn->vopc.src0 == GFX10_VOPC_SRC_EXEC_HI)
+ sprintf(src0, "%s", "exec_hi");
+ else if (insn->vopc.src0 < GFX10_VOPC_SRC_INTEGER_MINUS_1)
+ sprintf(src0, "0x%x",
+ insn->vopc.src0 - GFX10_VOPC_SRC_INTEGER_0);
+ else if (insn->vopc.src0 < GFX10_VOPC_SRC_SHARED_BASE)
+ sprintf(src0, "-0x%x",
+ insn->vopc.src0 - GFX10_VOPC_SRC_INTEGER_MINUS_1);
+ else if (insn->vopc.src0 == GFX10_VOPC_SRC_VCCZ)
+ sprintf(src0, "%s", "vcc");
+ else if (insn->vopc.src0 == GFX10_VOPC_SRC_EXECZ)
+ sprintf(src0, "%s", "exec");
+ else if (insn->vopc.src0 == GFX10_VOPC_SRC_SCC)
+ sprintf(src0, "%s", "scc");
+ else if (insn->vopc.src0 == GFX10_VOPC_SRC_LITERAL_CONST)
+ sprintf(src0, "0x%x", insn->vopc.literal);
+ else if (insn->vopc.src0 >= GFX10_VOPC_SRC_VGPR_BASE)
+ sprintf(src0, "v%d",
+ insn->vopc.src0 - GFX10_VOPC_SRC_VGPR_BASE);
+
+ seq_printf(m, "%s %s, v%d\n",
+ opnames_gfx10[AMDGCN_INSN_TYPE_VOPC][insn->vopc.op],
+ src0, insn->vopc.vsrc1);
+}
+
+static inline void gfx10_debugfs_vop2(union amdgcn_gfx10_insn *insn,
+ struct seq_file *m)
+{
+ char src0[20];
+
+ if (insn->vop2.src0 < GFX10_VOP2_SRC_VCC_LO)
+ sprintf(src0, "s%d", insn->vop2.src0);
+ else if (insn->vop2.src0 == GFX10_VOP2_SRC_VCC_LO)
+ sprintf(src0, "%s", "vcc_lo");
+ else if (insn->vop2.src0 == GFX10_VOP2_SRC_VCC_HI)
+ sprintf(src0, "%s", "vcc_hi");
+ else if (insn->vop2.src0 == GFX10_VOP2_SRC_NULL)
+ sprintf(src0, "%s", "null");
+ else if (insn->vop2.src0 == GFX10_VOP2_SRC_EXEC_LO)
+ sprintf(src0, "%s", "exec_lo");
+ else if (insn->vop2.src0 == GFX10_VOP2_SRC_EXEC_HI)
+ sprintf(src0, "%s", "exec_hi");
+ else if (insn->vop2.src0 < GFX10_VOP2_SRC_INTEGER_MINUS_1)
+ sprintf(src0, "0x%x",
+ insn->vop2.src0 - GFX10_VOP2_SRC_INTEGER_0);
+ else if (insn->vop2.src0 < GFX10_VOP2_SRC_SHARED_BASE)
+ sprintf(src0, "-0x%x",
+ insn->vop2.src0 - GFX10_VOP2_SRC_INTEGER_MINUS_1);
+ else if (insn->vop2.src0 == GFX10_VOP2_SRC_VCCZ)
+ sprintf(src0, "%s", "vcc");
+ else if (insn->vop2.src0 == GFX10_VOP2_SRC_EXECZ)
+ sprintf(src0, "%s", "exec");
+ else if (insn->vop2.src0 == GFX10_VOP2_SRC_SCC)
+ sprintf(src0, "%s", "scc");
+ else if (insn->vop2.src0 == GFX10_VOP2_SRC_LITERAL_CONST)
+ sprintf(src0, "0x%x", insn->vop2.literal);
+ else if (insn->vop2.src0 >= GFX10_VOP2_SRC_VGPR_BASE)
+ sprintf(src0, "v%d",
+ insn->vop2.src0 - GFX10_VOP2_SRC_VGPR_BASE);
+
+ seq_printf(m, "%s v%d, %s v%d\n",
+ opnames_gfx10[AMDGCN_INSN_TYPE_VOP2][insn->vop2.op],
+ insn->vop2.vdst,
+ src0,
+ insn->vop2.vsrc1);
+}
+
+static inline void gfx10_debugfs_sop1(union amdgcn_gfx10_insn *insn,
+ struct seq_file *m)
+{
+ char ssrc0[20];
+
+ if (insn->sop1.ssrc0 < GFX10_SOP1_SSRC_VCC_LO)
+ sprintf(ssrc0, "s%d", insn->sop1.ssrc0);
+ else if (insn->sop1.ssrc0 == GFX10_SOP1_SSRC_VCC_LO)
+ sprintf(ssrc0, "%s", "vcc_lo");
+ else if (insn->sop1.ssrc0 == GFX10_SOP1_SSRC_VCC_HI)
+ sprintf(ssrc0, "%s", "vcc_hi");
+ else if (insn->sop1.ssrc0 == GFX10_SOP1_SSRC_NULL)
+ sprintf(ssrc0, "%s", "null");
+ else if (insn->sop1.ssrc0 == GFX10_SOP1_SSRC_EXEC_LO)
+ sprintf(ssrc0, "%s", "exec_lo");
+ else if (insn->sop1.ssrc0 == GFX10_SOP1_SSRC_EXEC_HI)
+ sprintf(ssrc0, "%s", "exec_hi");
+ else if (insn->sop1.ssrc0 < GFX10_SOP1_SSRC_INTEGER_MINUS_1)
+ sprintf(ssrc0, "0x%x",
+ insn->sop1.ssrc0 - GFX10_SOP1_SSRC_INTEGER_0);
+ else if (insn->sop1.ssrc0 < GFX10_SOP1_SSRC_SHARED_BASE)
+ sprintf(ssrc0, "-0x%x",
+ insn->sop1.ssrc0 - GFX10_SOP1_SSRC_INTEGER_MINUS_1);
+ else if (insn->sop1.ssrc0 == GFX10_SOP1_SSRC_VCCZ)
+ sprintf(ssrc0, "%s", "vcc");
+ else if (insn->sop1.ssrc0 == GFX10_SOP1_SSRC_EXECZ)
+ sprintf(ssrc0, "%s", "exec");
+ else if (insn->sop1.ssrc0 == GFX10_SOP1_SSRC_SCC)
+ sprintf(ssrc0, "%s", "scc");
+ else if (insn->sop1.ssrc0 == GFX10_SOP1_SSRC_LITERAL_CONST)
+ sprintf(ssrc0, "0x%x", insn->sop1.literal);
+
+ seq_printf(m, "%s s%d, %s\n",
+ opnames_gfx10[AMDGCN_INSN_TYPE_SOP1][insn->sop1.op],
+ insn->sop1.sdst,
+ ssrc0);
+}
+
+static inline void gfx10_debugfs_sopp(union amdgcn_gfx10_insn *insn,
+ struct seq_file *m)
+{
+ seq_printf(m, "%s 0x%x\n",
+ opnames_gfx10[AMDGCN_INSN_TYPE_SOPP][insn->sopp.op],
+ insn->sopp.simm16);
+}
+
+static inline void gfx10_debugfs_smem(union amdgcn_gfx10_insn *insn,
+ struct seq_file *m)
+{
+ seq_printf(m, "%s s[%d:%d], s[%d:%d], 0x%x\n",
+ opnames_gfx10[AMDGCN_INSN_TYPE_SMEM][insn->smem.op],
+ insn->smem.sdata,
+ insn->smem.sdata + 1,
+ (insn->smem.sbase * 2),
+ (insn->smem.sbase * 2) + 1,
+ insn->smem.offset);
+}
+
+static inline void gfx10_debugfs_mubuf(union amdgcn_gfx10_insn *insn,
+ struct seq_file *m)
+{
+ seq_printf(m, "%s v%d, v%d, s[%d:%d], 0x%x offen offset:%d\n",
+ opnames_gfx10[AMDGCN_INSN_TYPE_MUBUF][insn->mubuf.op],
+ insn->mubuf.vdata,
+ insn->mubuf.vaddr,
+ insn->mubuf.srsrc,
+ insn->mubuf.srsrc + 3,
+ insn->mubuf.soffset,
+ insn->mubuf.offset);
+}
+
+static inline void gfx10_debugfs_global(union amdgcn_gfx10_insn *insn,
+ struct seq_file *m)
+{
+ if (insn->flat.op == GFX9_GLOBAL_STORE_BYTE) {
+ seq_printf(m, "%s v[%d:%d], v%d, off offset:%d\n",
+ opnames_gfx10[AMDGCN_INSN_TYPE_FLAT][insn->flat.op],
+ insn->flat.addr,
+ insn->flat.addr + 1,
+ insn->flat.data,
+ insn->flat.offset);
+ } else if (insn->flat.op == GFX9_GLOBAL_STORE_SHORT) {
+ seq_printf(m, "%s v[%d:%d], v%d, off offset:%d\n",
+ opnames_gfx10[AMDGCN_INSN_TYPE_FLAT][insn->flat.op],
+ insn->flat.addr,
+ insn->flat.addr + 1,
+ insn->flat.data,
+ insn->flat.offset);
+ } else if (insn->flat.op == GFX9_GLOBAL_STORE_DWORD) {
+ seq_printf(m, "%s v[%d:%d], v%d, off offset:%d\n",
+ opnames_gfx10[AMDGCN_INSN_TYPE_FLAT][insn->flat.op],
+ insn->flat.addr,
+ insn->flat.addr + 1,
+ insn->flat.data,
+ insn->flat.offset);
+ } else if (insn->flat.op == GFX9_GLOBAL_STORE_DWORDX2) {
+ seq_printf(m, "%s v[%d:%d], v[%d:%d], off offset:%d\n",
+ opnames_gfx10[AMDGCN_INSN_TYPE_FLAT][insn->flat.op],
+ insn->flat.addr,
+ insn->flat.addr + 1,
+ insn->flat.data,
+ insn->flat.data + 1,
+ insn->flat.offset);
+ } else {
+ seq_printf(m, "%s v[%d:%d], v[%d:%d], off offset:%d\n",
+ opnames_gfx10[AMDGCN_INSN_TYPE_FLAT][insn->flat.op],
+ insn->flat.vdst,
+ insn->flat.vdst + 1,
+ insn->flat.addr,
+ insn->flat.addr + 1,
+ insn->flat.offset);
+ }
+}
+
+static inline void gfx10_debugfs_sop2(union amdgcn_gfx10_insn *insn,
+ struct seq_file *m)
+{
+ char ssrc0[20], ssrc1[20];
+
+ decode_ssrc8(insn->sop2.ssrc0, insn->sop2.literal, ssrc0);
+ decode_ssrc8(insn->sop2.ssrc1, insn->sop2.literal, ssrc1);
+ seq_printf(m, "%s s%d, %s, %s\n",
+ opnames_gfx10[AMDGCN_INSN_TYPE_SOP2][insn->sop2.op],
+ insn->sop2.sdst, ssrc0, ssrc1);
+}
+
+static inline void gfx10_debugfs_sopk(union amdgcn_gfx10_insn *insn,
+ struct seq_file *m)
+{
+ seq_printf(m, "%s s%d, 0x%x\n",
+ opnames_gfx10[AMDGCN_INSN_TYPE_SOPK][insn->sopk.op],
+ insn->sopk.sdst, insn->sopk.simm16);
+}
+
+static inline void gfx10_debugfs_sopc(union amdgcn_gfx10_insn *insn,
+ struct seq_file *m)
+{
+ char ssrc0[20], ssrc1[20];
+
+ decode_ssrc8(insn->sopc.ssrc0, insn->sopc.literal, ssrc0);
+ decode_ssrc8(insn->sopc.ssrc1, insn->sopc.literal, ssrc1);
+ seq_printf(m, "%s %s, %s\n",
+ opnames_gfx10[AMDGCN_INSN_TYPE_SOPC][insn->sopc.op],
+ ssrc0, ssrc1);
+}
+
+static inline void gfx10_debugfs_vop3p(union amdgcn_gfx10_insn *insn,
+ struct seq_file *m)
+{
+ char src0[20], src1[20], src2[20];
+
+ decode_vsrc9(insn->vop3p.src0, insn->vop3p.literal, src0);
+ decode_vsrc9(insn->vop3p.src1, insn->vop3p.literal, src1);
+ decode_vsrc9(insn->vop3p.src2, insn->vop3p.literal, src2);
+ seq_printf(m, "%s v%d, %s, %s, %s\n",
+ opnames_gfx10[AMDGCN_INSN_TYPE_VOP3P][insn->vop3p.op],
+ (int)insn->vop3p.vdst, src0, src1, src2);
+}
+
+static inline void gfx10_debugfs_sdwa(union amdgcn_gfx10_insn *insn,
+ struct seq_file *m)
+{
+ seq_printf(m, "sdwa src0:%d dst_sel:%d src0_sel:%d src1_sel:%d\n",
+ insn->sdwa.src0, insn->sdwa.dst_sel,
+ insn->sdwa.src0_sel, insn->sdwa.src1_sel);
+}
+
+static inline void gfx10_debugfs_sdwab(union amdgcn_gfx10_insn *insn,
+ struct seq_file *m)
+{
+ seq_printf(m, "sdwab src0:%d sdst:s%d src0_sel:%d src1_sel:%d\n",
+ insn->sdwab.src0, insn->sdwab.sdst,
+ insn->sdwab.src0_sel, insn->sdwab.src1_sel);
+}
+
+static inline void gfx10_debugfs_dpp16(union amdgcn_gfx10_insn *insn,
+ struct seq_file *m)
+{
+ seq_puts(m, "dpp16 (not decoded)\n");
+}
+
+static inline void gfx10_debugfs_dpp8(union amdgcn_gfx10_insn *insn,
+ struct seq_file *m)
+{
+ seq_puts(m, "dpp8 (not decoded)\n");
+}
+
+static inline void gfx10_debugfs_vintrp(union amdgcn_gfx10_insn *insn,
+ struct seq_file *m)
+{
+ seq_puts(m, "vintrp (not decoded)\n");
+}
+
+static inline void gfx10_debugfs_ds(union amdgcn_gfx10_insn *insn,
+ struct seq_file *m)
+{
+ seq_printf(m, "%s v%d, v%d, v%d, v%d offset0:%d offset1:%d%s\n",
+ opnames_gfx10[AMDGCN_INSN_TYPE_DS][insn->ds.op],
+ (int)insn->ds.vdst, (int)insn->ds.addr,
+ (int)insn->ds.data0, (int)insn->ds.data1,
+ (int)insn->ds.offset0, (int)insn->ds.offset1,
+ insn->ds.gds ? " gds" : "");
+}
+
+static inline void gfx10_debugfs_mtbuf(union amdgcn_gfx10_insn *insn,
+ struct seq_file *m)
+{
+ seq_printf(m, "%s v%d, v%d, s[%d:%d], s%d format:%d offset:%d\n",
+ opnames_gfx10[AMDGCN_INSN_TYPE_MTBUF][insn->mtbuf.op],
+ (int)insn->mtbuf.vdata, (int)insn->mtbuf.vaddr,
+ (int)insn->mtbuf.srsrc * 4, (int)insn->mtbuf.srsrc * 4 + 3,
+ (int)insn->mtbuf.soffset, (int)insn->mtbuf.foamat,
+ (int)insn->mtbuf.offset);
+}
+
+static inline void gfx10_debugfs_mimg(union amdgcn_gfx10_insn *insn,
+ struct seq_file *m)
+{
+ seq_puts(m, "mimg (not decoded)\n");
+}
+
+static inline void gfx10_debugfs_exp(union amdgcn_gfx10_insn *insn,
+ struct seq_file *m)
+{
+ seq_puts(m, "exp (not decoded)\n");
+}
+
+static inline void gfx10_debugfs_insn(struct amdgcn_insn *insn,
+ struct seq_file *m)
+{
+ u32 *ptr = (u32 *)&insn->gfx10;
+
+ if (insn->size == 4)
+ seq_printf(m, "%.8X\t\t\t", ptr[0]);
+ else if (insn->size == 8)
+ seq_printf(m, "%.8X %.8X\t\t", ptr[0], ptr[1]);
+ else if (insn->size == 12)
+ seq_printf(m, "%.8X %.8X %.8X\t\t\t", ptr[0], ptr[1], ptr[2]);
+ else
+ WARN_ON_ONCE(1);
+
+ switch (insn->type) {
+ case AMDGCN_INSN_TYPE_SOP2:
+ gfx10_debugfs_sop2(&insn->gfx10, m);
+ break;
+ case AMDGCN_INSN_TYPE_SOPK:
+ gfx10_debugfs_sopk(&insn->gfx10, m);
+ break;
+ case AMDGCN_INSN_TYPE_SOP1:
+ gfx10_debugfs_sop1(&insn->gfx10, m);
+ break;
+ case AMDGCN_INSN_TYPE_SOPC:
+ gfx10_debugfs_sopc(&insn->gfx10, m);
+ break;
+ case AMDGCN_INSN_TYPE_SOPP:
+ gfx10_debugfs_sopp(&insn->gfx10, m);
+ break;
+ case AMDGCN_INSN_TYPE_SMEM:
+ gfx10_debugfs_smem(&insn->gfx10, m);
+ break;
+ case AMDGCN_INSN_TYPE_VOP2:
+ gfx10_debugfs_vop2(&insn->gfx10, m);
+ break;
+ case AMDGCN_INSN_TYPE_VOP1:
+ gfx10_debugfs_vop1(&insn->gfx10, m);
+ break;
+ case AMDGCN_INSN_TYPE_VOPC:
+ gfx10_debugfs_vopc(&insn->gfx10, m);
+ break;
+ case AMDGCN_INSN_TYPE_VOP3A:
+ gfx10_debugfs_vop3a(&insn->gfx10, m);
+ break;
+ case AMDGCN_INSN_TYPE_VOP3B:
+ gfx10_debugfs_vop3b(&insn->gfx10, m);
+ break;
+ case AMDGCN_INSN_TYPE_VOP3P:
+ gfx10_debugfs_vop3p(&insn->gfx10, m);
+ break;
+ case AMDGCN_INSN_TYPE_SDWA:
+ gfx10_debugfs_sdwa(&insn->gfx10, m);
+ break;
+ case AMDGCN_INSN_TYPE_SDWAB:
+ gfx10_debugfs_sdwab(&insn->gfx10, m);
+ break;
+ case AMDGCN_INSN_TYPE_DPP16:
+ gfx10_debugfs_dpp16(&insn->gfx10, m);
+ break;
+ case AMDGCN_INSN_TYPE_DPP8:
+ gfx10_debugfs_dpp8(&insn->gfx10, m);
+ break;
+ case AMDGCN_INSN_TYPE_VINTRP:
+ gfx10_debugfs_vintrp(&insn->gfx10, m);
+ break;
+ case AMDGCN_INSN_TYPE_DS:
+ gfx10_debugfs_ds(&insn->gfx10, m);
+ break;
+ case AMDGCN_INSN_TYPE_MTBUF:
+ gfx10_debugfs_mtbuf(&insn->gfx10, m);
+ break;
+ case AMDGCN_INSN_TYPE_MUBUF:
+ gfx10_debugfs_mubuf(&insn->gfx10, m);
+ break;
+ case AMDGCN_INSN_TYPE_MIMG:
+ gfx10_debugfs_mimg(&insn->gfx10, m);
+ break;
+ case AMDGCN_INSN_TYPE_FLAT:
+ gfx10_debugfs_global(&insn->gfx10, m);
+ break;
+ case AMDGCN_INSN_TYPE_EXP:
+ gfx10_debugfs_exp(&insn->gfx10, m);
+ break;
+ default:
+ WARN_ON_ONCE(1);
+ break;
+ }
+}
+
+static inline void gfx9_debugfs_vop3a(union amdgcn_gfx9_insn *insn,
+ struct seq_file *m)
+{
+ char src0[20];
+ char src1[20];
+ char src2[20];
+
+ if (insn->vop3a.src0 < GFX9_VOP3A_SRC_VCC_LO)
+ sprintf(src0, "s%d", insn->vop3a.src0);
+ else if (insn->vop3a.src0 == GFX9_VOP3A_SRC_VCC_LO)
+ sprintf(src0, "%s", "vcc_lo");
+ else if (insn->vop3a.src0 == GFX9_VOP3A_SRC_VCC_HI)
+ sprintf(src0, "%s", "vcc_hi");
+ else if (insn->vop3a.src0 == GFX9_VOP3A_SRC_NULL)
+ sprintf(src0, "%s", "null");
+ else if (insn->vop3a.src0 == GFX9_VOP3A_SRC_EXEC_LO)
+ sprintf(src0, "%s", "exec_lo");
+ else if (insn->vop3a.src0 == GFX9_VOP3A_SRC_EXEC_HI)
+ sprintf(src0, "%s", "exec_hi");
+ else if (insn->vop3a.src0 < GFX9_VOP3A_SRC_INTEGER_MINUS_1)
+ sprintf(src0, "0x%x",
+ insn->vop3a.src0 - GFX9_VOP3A_SRC_INTEGER_0);
+ else if (insn->vop3a.src0 < GFX9_VOP3A_SRC_SHARED_BASE)
+ sprintf(src0, "-0x%x",
+ insn->vop3a.src0 - GFX9_VOP3A_SRC_INTEGER_MINUS_1);
+ else if (insn->vop3a.src0 == GFX9_VOP3A_SRC_VCCZ)
+ sprintf(src0, "%s", "vcc");
+ else if (insn->vop3a.src0 == GFX9_VOP3A_SRC_EXECZ)
+ sprintf(src0, "%s", "exec");
+ else if (insn->vop3a.src0 == GFX9_VOP3A_SRC_SCC)
+ sprintf(src0, "%s", "scc");
+ else if (insn->vop3a.src0 == GFX9_VOP3A_SRC_LITERAL_CONST)
+ sprintf(src0, "0x%x", insn->vop3a.literal);
+ else if (insn->vop3a.src0 >= GFX9_VOP3A_SRC_VGPR_BASE)
+ sprintf(src0, "v%d",
+ insn->vop3a.src0 - GFX9_VOP3A_SRC_VGPR_BASE);
+
+ if (insn->vop3a.src1 < GFX9_VOP3A_SRC_VCC_LO)
+ sprintf(src1, "s%d", insn->vop3a.src1);
+ else if (insn->vop3a.src1 == GFX9_VOP3A_SRC_VCC_LO)
+ sprintf(src1, "%s", "vcc_lo");
+ else if (insn->vop3a.src1 == GFX9_VOP3A_SRC_VCC_HI)
+ sprintf(src1, "%s", "vcc_hi");
+ else if (insn->vop3a.src1 == GFX9_VOP3A_SRC_NULL)
+ sprintf(src1, "%s", "null");
+ else if (insn->vop3a.src1 == GFX9_VOP3A_SRC_EXEC_LO)
+ sprintf(src1, "%s", "exec_lo");
+ else if (insn->vop3a.src1 == GFX9_VOP3A_SRC_EXEC_HI)
+ sprintf(src1, "%s", "exec_hi");
+ else if (insn->vop3a.src1 < GFX9_VOP3A_SRC_INTEGER_MINUS_1)
+ sprintf(src1, "0x%x",
+ insn->vop3a.src1 - GFX9_VOP3A_SRC_INTEGER_0);
+ else if (insn->vop3a.src1 < GFX9_VOP3A_SRC_SHARED_BASE)
+ sprintf(src1, "-0x%x",
+ insn->vop3a.src1 - GFX9_VOP3A_SRC_INTEGER_MINUS_1);
+ else if (insn->vop3a.src1 == GFX9_VOP3A_SRC_VCCZ)
+ sprintf(src1, "%s", "vcc");
+ else if (insn->vop3a.src1 == GFX9_VOP3A_SRC_EXECZ)
+ sprintf(src1, "%s", "exec");
+ else if (insn->vop3a.src1 == GFX9_VOP3A_SRC_SCC)
+ sprintf(src1, "%s", "scc");
+ else if (insn->vop3a.src1 == GFX9_VOP3A_SRC_LITERAL_CONST)
+ sprintf(src1, "0x%x", insn->vop3a.literal);
+ else if (insn->vop3a.src1 >= GFX9_VOP3A_SRC_VGPR_BASE)
+ sprintf(src1, "v%d",
+ insn->vop3a.src1 - GFX9_VOP3A_SRC_VGPR_BASE);
+
+ if (insn->vop3a.src2 < GFX9_VOP3A_SRC_VCC_LO)
+ sprintf(src2, "s%d", insn->vop3a.src2);
+ else if (insn->vop3a.src2 == GFX9_VOP3A_SRC_VCC_LO)
+ sprintf(src2, "%s", "vcc_lo");
+ else if (insn->vop3a.src2 == GFX9_VOP3A_SRC_VCC_HI)
+ sprintf(src2, "%s", "vcc_hi");
+ else if (insn->vop3a.src2 == GFX9_VOP3A_SRC_NULL)
+ sprintf(src2, "%s", "null");
+ else if (insn->vop3a.src2 == GFX9_VOP3A_SRC_EXEC_LO)
+ sprintf(src2, "%s", "exec_lo");
+ else if (insn->vop3a.src2 == GFX9_VOP3A_SRC_EXEC_HI)
+ sprintf(src2, "%s", "exec_hi");
+ else if (insn->vop3a.src2 < GFX9_VOP3A_SRC_INTEGER_MINUS_1)
+ sprintf(src2, "0x%x",
+ insn->vop3a.src2 - GFX9_VOP3A_SRC_INTEGER_0);
+ else if (insn->vop3a.src2 < GFX9_VOP3A_SRC_SHARED_BASE)
+ sprintf(src2, "-0x%x",
+ insn->vop3a.src2 - GFX9_VOP3A_SRC_INTEGER_MINUS_1);
+ else if (insn->vop3a.src2 == GFX9_VOP3A_SRC_VCCZ)
+ sprintf(src2, "%s", "vcc");
+ else if (insn->vop3a.src2 == GFX9_VOP3A_SRC_EXECZ)
+ sprintf(src2, "%s", "exec");
+ else if (insn->vop3a.src2 == GFX9_VOP3A_SRC_SCC)
+ sprintf(src2, "%s", "scc");
+ else if (insn->vop3a.src2 == GFX9_VOP3A_SRC_LITERAL_CONST)
+ sprintf(src2, "0x%x", insn->vop3a.literal);
+ else if (insn->vop3a.src2 >= GFX9_VOP3A_SRC_VGPR_BASE)
+ sprintf(src2, "v%d",
+ insn->vop3a.src2 - GFX9_VOP3A_SRC_VGPR_BASE);
+
+ seq_printf(m, "%s v%d, %s, %s, %s\n",
+ opnames_gfx9[AMDGCN_INSN_TYPE_VOP3A][insn->vop3a.op],
+ insn->vop3a.vdst, src0, src1, src2);
+}
+
+static inline void gfx9_debugfs_vop3b(union amdgcn_gfx9_insn *insn,
+ struct seq_file *m)
+{
+ char src0[20];
+ char src1[20];
+ char src2[20];
+ char sdst[20];
+
+ if (insn->vop3b.sdst < GFX9_VOP3B_SRC_VCC_LO)
+ sprintf(sdst, "s%d", insn->vop3b.sdst);
+ else if (insn->vop3b.sdst == GFX9_VOP3B_SRC_VCC_LO)
+ sprintf(sdst, "%s", "vcc_lo");
+ else if (insn->vop3b.sdst == GFX9_VOP3B_SRC_VCC_HI)
+ sprintf(sdst, "%s", "vcc_hi");
+ else if (insn->vop3b.sdst == GFX9_VOP3B_SRC_NULL)
+ sprintf(sdst, "%s", "null");
+ else if (insn->vop3b.sdst == GFX9_VOP3B_SRC_EXEC_LO)
+ sprintf(sdst, "%s", "exec_lo");
+ else if (insn->vop3b.sdst == GFX9_VOP3B_SRC_EXEC_HI)
+ sprintf(sdst, "%s", "exec_hi");
+ else if (insn->vop3b.sdst < GFX9_VOP3B_SRC_INTEGER_MINUS_1)
+ sprintf(sdst, "0x%x",
+ insn->vop3b.sdst - GFX9_VOP3B_SRC_INTEGER_0);
+ else if (insn->vop3b.sdst < GFX9_VOP3B_SRC_SHARED_BASE)
+ sprintf(sdst, "-0x%x",
+ insn->vop3b.sdst - GFX9_VOP3B_SRC_INTEGER_MINUS_1);
+ else if (insn->vop3b.sdst == GFX9_VOP3B_SRC_VCCZ)
+ sprintf(sdst, "%s", "vcc");
+ else if (insn->vop3b.sdst == GFX9_VOP3B_SRC_EXECZ)
+ sprintf(sdst, "%s", "exec");
+ else if (insn->vop3b.sdst == GFX9_VOP3B_SRC_SCC)
+ sprintf(sdst, "%s", "scc");
+ else if (insn->vop3b.sdst == GFX9_VOP3B_SRC_LITERAL_CONST)
+ sprintf(sdst, "0x%x", insn->vop3b.literal);
+ else if (insn->vop3b.sdst >= GFX9_VOP3B_SRC_VGPR_BASE)
+ sprintf(sdst, "v%d",
+ insn->vop3b.sdst - GFX9_VOP3B_SRC_VGPR_BASE);
+
+ if (insn->vop3b.src0 < GFX9_VOP3B_SRC_VCC_LO)
+ sprintf(src0, "s%d", insn->vop3b.src0);
+ else if (insn->vop3b.src0 == GFX9_VOP3B_SRC_VCC_LO)
+ sprintf(src0, "%s", "vcc_lo");
+ else if (insn->vop3b.src0 == GFX9_VOP3B_SRC_VCC_HI)
+ sprintf(src0, "%s", "vcc_hi");
+ else if (insn->vop3b.src0 == GFX9_VOP3B_SRC_NULL)
+ sprintf(src0, "%s", "null");
+ else if (insn->vop3b.src0 == GFX9_VOP3B_SRC_EXEC_LO)
+ sprintf(src0, "%s", "exec_lo");
+ else if (insn->vop3b.src0 == GFX9_VOP3B_SRC_EXEC_HI)
+ sprintf(src0, "%s", "exec_hi");
+ else if (insn->vop3b.src0 < GFX9_VOP3B_SRC_INTEGER_MINUS_1)
+ sprintf(src0, "0x%x",
+ insn->vop3b.src0 - GFX9_VOP3B_SRC_INTEGER_0);
+ else if (insn->vop3b.src0 < GFX9_VOP3B_SRC_SHARED_BASE)
+ sprintf(src0, "-0x%x",
+ insn->vop3b.src0 - GFX9_VOP3B_SRC_INTEGER_MINUS_1);
+ else if (insn->vop3b.src0 == GFX9_VOP3B_SRC_VCCZ)
+ sprintf(src0, "%s", "vcc");
+ else if (insn->vop3b.src0 == GFX9_VOP3B_SRC_EXECZ)
+ sprintf(src0, "%s", "exec");
+ else if (insn->vop3b.src0 == GFX9_VOP3B_SRC_SCC)
+ sprintf(src0, "%s", "scc");
+ else if (insn->vop3b.src0 == GFX9_VOP3B_SRC_LITERAL_CONST)
+ sprintf(src0, "0x%x", insn->vop3b.literal);
+ else if (insn->vop3b.src0 >= GFX9_VOP3B_SRC_VGPR_BASE)
+ sprintf(src0, "v%d",
+ insn->vop3b.src0 - GFX9_VOP3B_SRC_VGPR_BASE);
+
+ if (insn->vop3b.src1 < GFX9_VOP3B_SRC_VCC_LO)
+ sprintf(src1, "s%d", insn->vop3b.src1);
+ else if (insn->vop3b.src1 == GFX9_VOP3B_SRC_VCC_LO)
+ sprintf(src1, "%s", "vcc_lo");
+ else if (insn->vop3b.src1 == GFX9_VOP3B_SRC_VCC_HI)
+ sprintf(src1, "%s", "vcc_hi");
+ else if (insn->vop3b.src1 == GFX9_VOP3B_SRC_NULL)
+ sprintf(src1, "%s", "null");
+ else if (insn->vop3b.src1 == GFX9_VOP3B_SRC_EXEC_LO)
+ sprintf(src1, "%s", "exec_lo");
+ else if (insn->vop3b.src1 == GFX9_VOP3B_SRC_EXEC_HI)
+ sprintf(src1, "%s", "exec_hi");
+ else if (insn->vop3b.src1 < GFX9_VOP3B_SRC_INTEGER_MINUS_1)
+ sprintf(src1, "0x%x",
+ insn->vop3b.src1 - GFX9_VOP3B_SRC_INTEGER_0);
+ else if (insn->vop3b.src1 < GFX9_VOP3B_SRC_SHARED_BASE)
+ sprintf(src1, "-0x%x",
+ insn->vop3b.src1 - GFX9_VOP3B_SRC_INTEGER_MINUS_1);
+ else if (insn->vop3b.src1 == GFX9_VOP3B_SRC_VCCZ)
+ sprintf(src1, "%s", "vcc");
+ else if (insn->vop3b.src1 == GFX9_VOP3B_SRC_EXECZ)
+ sprintf(src1, "%s", "exec");
+ else if (insn->vop3b.src1 == GFX9_VOP3B_SRC_SCC)
+ sprintf(src1, "%s", "scc");
+ else if (insn->vop3b.src1 == GFX9_VOP3B_SRC_LITERAL_CONST)
+ sprintf(src1, "0x%x", insn->vop3b.literal);
+ else if (insn->vop3b.src1 >= GFX9_VOP3B_SRC_VGPR_BASE)
+ sprintf(src1, "v%d",
+ insn->vop3b.src1 - GFX9_VOP3B_SRC_VGPR_BASE);
+
+ if (insn->vop3b.src2 < GFX9_VOP3B_SRC_VCC_LO)
+ sprintf(src2, "s%d", insn->vop3b.src2);
+ else if (insn->vop3b.src2 == GFX9_VOP3B_SRC_VCC_LO)
+ sprintf(src2, "%s", "vcc_lo");
+ else if (insn->vop3b.src2 == GFX9_VOP3B_SRC_VCC_HI)
+ sprintf(src2, "%s", "vcc_hi");
+ else if (insn->vop3b.src2 == GFX9_VOP3B_SRC_NULL)
+ sprintf(src2, "%s", "null");
+ else if (insn->vop3b.src2 == GFX9_VOP3B_SRC_EXEC_LO)
+ sprintf(src2, "%s", "exec_lo");
+ else if (insn->vop3b.src2 == GFX9_VOP3B_SRC_EXEC_HI)
+ sprintf(src2, "%s", "exec_hi");
+ else if (insn->vop3b.src2 < GFX9_VOP3B_SRC_INTEGER_MINUS_1)
+ sprintf(src2, "0x%x",
+ insn->vop3b.src2 - GFX9_VOP3B_SRC_INTEGER_0);
+ else if (insn->vop3b.src2 < GFX9_VOP3B_SRC_SHARED_BASE)
+ sprintf(src2, "-0x%x",
+ insn->vop3b.src2 - GFX9_VOP3B_SRC_INTEGER_MINUS_1);
+ else if (insn->vop3b.src2 == GFX9_VOP3B_SRC_VCCZ)
+ sprintf(src2, "%s", "vcc");
+ else if (insn->vop3b.src2 == GFX9_VOP3B_SRC_EXECZ)
+ sprintf(src2, "%s", "exec");
+ else if (insn->vop3b.src2 == GFX9_VOP3B_SRC_SCC)
+ sprintf(src2, "%s", "scc");
+ else if (insn->vop3b.src2 == GFX9_VOP3B_SRC_LITERAL_CONST)
+ sprintf(src2, "0x%x", insn->vop3b.literal);
+ else if (insn->vop3b.src2 >= GFX9_VOP3B_SRC_VGPR_BASE)
+ sprintf(src2, "v%d",
+ insn->vop3b.src2 - GFX9_VOP3B_SRC_VGPR_BASE);
+
+ seq_printf(m, "%s v%d, %s, %s, %s, %s\n",
+ opnames_gfx9[AMDGCN_INSN_TYPE_VOP3B][insn->vop3b.op],
+ insn->vop3b.vdst, sdst, src0, src1, src2);
+}
+
+static inline void gfx9_debugfs_vop1(union amdgcn_gfx9_insn *insn,
+ struct seq_file *m)
+{
+ char src0[20];
+
+ if (insn->vop1.src0 < GFX9_VOP1_SRC_VCC_LO)
+ sprintf(src0, "s%d", insn->vop1.src0);
+ else if (insn->vop1.src0 == GFX9_VOP1_SRC_VCC_LO)
+ sprintf(src0, "%s", "vcc_lo");
+ else if (insn->vop1.src0 == GFX9_VOP1_SRC_VCC_HI)
+ sprintf(src0, "%s", "vcc_hi");
+ else if (insn->vop1.src0 == GFX9_VOP1_SRC_NULL)
+ sprintf(src0, "%s", "null");
+ else if (insn->vop1.src0 == GFX9_VOP1_SRC_EXEC_LO)
+ sprintf(src0, "%s", "exec_lo");
+ else if (insn->vop1.src0 == GFX9_VOP1_SRC_EXEC_HI)
+ sprintf(src0, "%s", "exec_hi");
+ else if (insn->vop1.src0 < GFX9_VOP1_SRC_INTEGER_MINUS_1)
+ sprintf(src0, "0x%x",
+ insn->vop1.src0 - GFX9_VOP1_SRC_INTEGER_0);
+ else if (insn->vop1.src0 < GFX9_VOP1_SRC_SHARED_BASE)
+ sprintf(src0, "-0x%x",
+ insn->vop1.src0 - GFX9_VOP1_SRC_INTEGER_MINUS_1);
+ else if (insn->vop1.src0 == GFX9_VOP1_SRC_VCCZ)
+ sprintf(src0, "%s", "vcc");
+ else if (insn->vop1.src0 == GFX9_VOP1_SRC_EXECZ)
+ sprintf(src0, "%s", "exec");
+ else if (insn->vop1.src0 == GFX9_VOP1_SRC_SCC)
+ sprintf(src0, "%s", "scc");
+ else if (insn->vop1.src0 == GFX9_VOP1_SRC_LITERAL_CONST)
+ sprintf(src0, "0x%x", insn->vop1.literal);
+ else if (insn->vop1.src0 >= GFX9_VOP1_SRC_VGPR_BASE)
+ sprintf(src0, "v%d", insn->vop1.src0 - GFX9_VOP1_SRC_VGPR_BASE);
+
+ seq_printf(m, "%s v%d, %s\n",
+ opnames_gfx9[AMDGCN_INSN_TYPE_VOP1][insn->vop1.op],
+ insn->vop1.vdst, src0);
+}
+
+static inline void gfx9_debugfs_vopc(union amdgcn_gfx9_insn *insn,
+ struct seq_file *m)
+{
+ char src0[20];
+
+ if (insn->vopc.src0 < GFX9_VOPC_SRC_VCC_LO)
+ sprintf(src0, "s%d", insn->vopc.src0);
+ else if (insn->vopc.src0 == GFX9_VOPC_SRC_VCC_LO)
+ sprintf(src0, "%s", "vcc_lo");
+ else if (insn->vopc.src0 == GFX9_VOPC_SRC_VCC_HI)
+ sprintf(src0, "%s", "vcc_hi");
+ else if (insn->vopc.src0 == GFX9_VOPC_SRC_NULL)
+ sprintf(src0, "%s", "null");
+ else if (insn->vopc.src0 == GFX9_VOPC_SRC_EXEC_LO)
+ sprintf(src0, "%s", "exec_lo");
+ else if (insn->vopc.src0 == GFX9_VOPC_SRC_EXEC_HI)
+ sprintf(src0, "%s", "exec_hi");
+ else if (insn->vopc.src0 < GFX9_VOPC_SRC_INTEGER_MINUS_1)
+ sprintf(src0, "0x%x",
+ insn->vopc.src0 - GFX9_VOPC_SRC_INTEGER_0);
+ else if (insn->vopc.src0 < GFX9_VOPC_SRC_SHARED_BASE)
+ sprintf(src0, "-0x%x",
+ insn->vopc.src0 - GFX9_VOPC_SRC_INTEGER_MINUS_1);
+ else if (insn->vopc.src0 == GFX9_VOPC_SRC_VCCZ)
+ sprintf(src0, "%s", "vcc");
+ else if (insn->vopc.src0 == GFX9_VOPC_SRC_EXECZ)
+ sprintf(src0, "%s", "exec");
+ else if (insn->vopc.src0 == GFX9_VOPC_SRC_SCC)
+ sprintf(src0, "%s", "scc");
+ else if (insn->vopc.src0 == GFX9_VOPC_SRC_LITERAL_CONST)
+ sprintf(src0, "0x%x", insn->vopc.literal);
+ else if (insn->vopc.src0 >= GFX9_VOPC_SRC_VGPR_BASE)
+ sprintf(src0, "v%d", insn->vopc.src0 - GFX9_VOPC_SRC_VGPR_BASE);
+
+ seq_printf(m, "%s %s, v%d\n",
+ opnames_gfx9[AMDGCN_INSN_TYPE_VOPC][insn->vopc.op],
+ src0,
+ insn->vopc.vsrc1);
+}
+
+static inline void gfx9_debugfs_vop2(union amdgcn_gfx9_insn *insn,
+ struct seq_file *m)
+{
+ char src0[20];
+
+ if (insn->vop2.src0 < GFX9_VOP2_SRC_VCC_LO)
+ sprintf(src0, "s%d", insn->vop2.src0);
+ else if (insn->vop2.src0 == GFX9_VOP2_SRC_VCC_LO)
+ sprintf(src0, "%s", "vcc_lo");
+ else if (insn->vop2.src0 == GFX9_VOP2_SRC_VCC_HI)
+ sprintf(src0, "%s", "vcc_hi");
+ else if (insn->vop2.src0 == GFX9_VOP2_SRC_NULL)
+ sprintf(src0, "%s", "null");
+ else if (insn->vop2.src0 == GFX9_VOP2_SRC_EXEC_LO)
+ sprintf(src0, "%s", "exec_lo");
+ else if (insn->vop2.src0 == GFX9_VOP2_SRC_EXEC_HI)
+ sprintf(src0, "%s", "exec_hi");
+ else if (insn->vop2.src0 < GFX9_VOP2_SRC_INTEGER_MINUS_1)
+ sprintf(src0, "0x%x",
+ insn->vop2.src0 - GFX9_VOP2_SRC_INTEGER_0);
+ else if (insn->vop2.src0 < GFX9_VOP2_SRC_SHARED_BASE)
+ sprintf(src0, "-0x%x",
+ insn->vop2.src0 - GFX9_VOP2_SRC_INTEGER_MINUS_1);
+ else if (insn->vop2.src0 == GFX9_VOP2_SRC_VCCZ)
+ sprintf(src0, "%s", "vcc");
+ else if (insn->vop2.src0 == GFX9_VOP2_SRC_EXECZ)
+ sprintf(src0, "%s", "exec");
+ else if (insn->vop2.src0 == GFX9_VOP2_SRC_SCC)
+ sprintf(src0, "%s", "scc");
+ else if (insn->vop2.src0 == GFX9_VOP2_SRC_LITERAL_CONST)
+ sprintf(src0, "0x%x", insn->vop2.literal);
+ else if (insn->vop2.src0 >= GFX9_VOP2_SRC_VGPR_BASE)
+ sprintf(src0, "v%d", insn->vop2.src0 - GFX9_VOP2_SRC_VGPR_BASE);
+
+ seq_printf(m, "%s v%d, %s v%d\n",
+ opnames_gfx9[AMDGCN_INSN_TYPE_VOP2][insn->vop2.op],
+ insn->vop2.vdst,
+ src0,
+ insn->vop2.vsrc1);
+}
+
+static inline void gfx9_debugfs_sop1(union amdgcn_gfx9_insn *insn,
+ struct seq_file *m)
+{
+ char ssrc0[20];
+
+ if (insn->sop1.ssrc0 < GFX9_SOP1_SSRC_VCC_LO)
+ sprintf(ssrc0, "s%d", insn->sop1.ssrc0);
+ else if (insn->sop1.ssrc0 == GFX9_SOP1_SSRC_VCC_LO)
+ sprintf(ssrc0, "%s", "vcc_lo");
+ else if (insn->sop1.ssrc0 == GFX9_SOP1_SSRC_VCC_HI)
+ sprintf(ssrc0, "%s", "vcc_hi");
+ else if (insn->sop1.ssrc0 == GFX9_SOP1_SSRC_NULL)
+ sprintf(ssrc0, "%s", "null");
+ else if (insn->sop1.ssrc0 == GFX9_SOP1_SSRC_EXEC_LO)
+ sprintf(ssrc0, "%s", "exec_lo");
+ else if (insn->sop1.ssrc0 == GFX9_SOP1_SSRC_EXEC_HI)
+ sprintf(ssrc0, "%s", "exec_hi");
+ else if (insn->sop1.ssrc0 < GFX9_SOP1_SSRC_INTEGER_MINUS_1)
+ sprintf(ssrc0, "0x%x",
+ insn->sop1.ssrc0 - GFX9_SOP1_SSRC_INTEGER_0);
+ else if (insn->sop1.ssrc0 < GFX9_SOP1_SSRC_SHARED_BASE)
+ sprintf(ssrc0, "-0x%x",
+ insn->sop1.ssrc0 - GFX9_SOP1_SSRC_INTEGER_MINUS_1);
+ else if (insn->sop1.ssrc0 == GFX9_SOP1_SSRC_VCCZ)
+ sprintf(ssrc0, "%s", "vcc");
+ else if (insn->sop1.ssrc0 == GFX9_SOP1_SSRC_EXECZ)
+ sprintf(ssrc0, "%s", "exec");
+ else if (insn->sop1.ssrc0 == GFX9_SOP1_SSRC_SCC)
+ sprintf(ssrc0, "%s", "scc");
+ else if (insn->sop1.ssrc0 == GFX9_SOP1_SSRC_LITERAL_CONST)
+ sprintf(ssrc0, "0x%x", insn->sop1.literal);
+
+ seq_printf(m, "%s s%d, %s\n",
+ opnames_gfx9[AMDGCN_INSN_TYPE_SOP1][insn->sop1.op],
+ insn->sop1.sdst,
+ ssrc0);
+}
+
+static inline void gfx9_debugfs_sopp(union amdgcn_gfx9_insn *insn,
+ struct seq_file *m)
+{
+ seq_printf(m, "%s 0x%x\n",
+ opnames_gfx9[AMDGCN_INSN_TYPE_SOPP][insn->sopp.op],
+ insn->sopp.simm16);
+}
+
+static inline void gfx9_debugfs_smem(union amdgcn_gfx9_insn *insn,
+ struct seq_file *m)
+{
+ seq_printf(m, "%s s[%d:%d], s[%d:%d], 0x%x\n",
+ opnames_gfx9[AMDGCN_INSN_TYPE_SMEM][insn->smem.op],
+ insn->smem.sdata,
+ insn->smem.sdata + 1,
+ (insn->smem.sbase * 2),
+ (insn->smem.sbase * 2) + 1,
+ insn->smem.offset);
+}
+
+static inline void gfx9_debugfs_mubuf(union amdgcn_gfx9_insn *insn,
+ struct seq_file *m)
+{
+ seq_printf(m, "%s v%d, v%d, s[%d:%d], 0x%x offen offset:%d\n",
+ opnames_gfx9[AMDGCN_INSN_TYPE_MUBUF][insn->mubuf.op],
+ insn->mubuf.vdata,
+ insn->mubuf.vaddr,
+ insn->mubuf.srsrc,
+ insn->mubuf.srsrc + 3,
+ insn->mubuf.soffset,
+ insn->mubuf.offset);
+}
+
+static inline void gfx9_debugfs_global(union amdgcn_gfx9_insn *insn,
+ struct seq_file *m)
+{
+ if (insn->flat.op == GFX9_GLOBAL_STORE_BYTE) {
+ seq_printf(m, "%s v[%d:%d], v%d, off offset:%d\n",
+ opnames_gfx9[AMDGCN_INSN_TYPE_FLAT][insn->flat.op],
+ insn->flat.addr,
+ insn->flat.addr + 1,
+ insn->flat.data,
+ insn->flat.offset);
+ } else if (insn->flat.op == GFX9_GLOBAL_STORE_SHORT) {
+ seq_printf(m, "%s v[%d:%d], v%d, off offset:%d\n",
+ opnames_gfx9[AMDGCN_INSN_TYPE_FLAT][insn->flat.op],
+ insn->flat.addr,
+ insn->flat.addr + 1,
+ insn->flat.data,
+ insn->flat.offset);
+ } else if (insn->flat.op == GFX9_GLOBAL_STORE_DWORD) {
+ seq_printf(m, "%s v[%d:%d], v%d, off offset:%d\n",
+ opnames_gfx9[AMDGCN_INSN_TYPE_FLAT][insn->flat.op],
+ insn->flat.addr,
+ insn->flat.addr + 1,
+ insn->flat.data,
+ insn->flat.offset);
+ } else if (insn->flat.op == GFX9_GLOBAL_STORE_DWORDX2) {
+ seq_printf(m, "%s v[%d:%d], v[%d:%d], off offset:%d\n",
+ opnames_gfx9[AMDGCN_INSN_TYPE_FLAT][insn->flat.op],
+ insn->flat.addr,
+ insn->flat.addr + 1,
+ insn->flat.data,
+ insn->flat.data + 1,
+ insn->flat.offset);
+ } else {
+ seq_printf(m, "%s v[%d:%d], v[%d:%d], off offset:%d\n",
+ opnames_gfx9[AMDGCN_INSN_TYPE_FLAT][insn->flat.op],
+ insn->flat.vdst,
+ insn->flat.vdst + 1,
+ insn->flat.addr,
+ insn->flat.addr + 1,
+ insn->flat.offset);
+ }
+}
+
+static inline void gfx9_debugfs_sop2(union amdgcn_gfx9_insn *insn,
+ struct seq_file *m)
+{
+ char ssrc0[20], ssrc1[20];
+
+ decode_ssrc8(insn->sop2.ssrc0, insn->sop2.literal, ssrc0);
+ decode_ssrc8(insn->sop2.ssrc1, insn->sop2.literal, ssrc1);
+ seq_printf(m, "%s s%d, %s, %s\n",
+ opnames_gfx9[AMDGCN_INSN_TYPE_SOP2][insn->sop2.op],
+ insn->sop2.sdst, ssrc0, ssrc1);
+}
+
+static inline void gfx9_debugfs_sopk(union amdgcn_gfx9_insn *insn,
+ struct seq_file *m)
+{
+ seq_printf(m, "%s s%d, 0x%x\n",
+ opnames_gfx9[AMDGCN_INSN_TYPE_SOPK][insn->sopk.op],
+ insn->sopk.sdst, insn->sopk.simm16);
+}
+
+static inline void gfx9_debugfs_sopc(union amdgcn_gfx9_insn *insn,
+ struct seq_file *m)
+{
+ char ssrc0[20], ssrc1[20];
+
+ decode_ssrc8(insn->sopc.ssrc0, insn->sopc.literal, ssrc0);
+ decode_ssrc8(insn->sopc.ssrc1, insn->sopc.literal, ssrc1);
+ seq_printf(m, "%s %s, %s\n",
+ opnames_gfx9[AMDGCN_INSN_TYPE_SOPC][insn->sopc.op],
+ ssrc0, ssrc1);
+}
+
+static inline void gfx9_debugfs_vop3p(union amdgcn_gfx9_insn *insn,
+ struct seq_file *m)
+{
+ char src0[20], src1[20], src2[20];
+
+ decode_vsrc9(insn->vop3p.src0, insn->vop3p.literal, src0);
+ decode_vsrc9(insn->vop3p.src1, insn->vop3p.literal, src1);
+ decode_vsrc9(insn->vop3p.src2, insn->vop3p.literal, src2);
+ seq_printf(m, "%s v%d, %s, %s, %s\n",
+ opnames_gfx9[AMDGCN_INSN_TYPE_VOP3P][insn->vop3p.op],
+ (int)insn->vop3p.vdst, src0, src1, src2);
+}
+
+static inline void gfx9_debugfs_sdwa(union amdgcn_gfx9_insn *insn,
+ struct seq_file *m)
+{
+ seq_printf(m, "sdwa src0:%d dst_sel:%d src0_sel:%d src1_sel:%d\n",
+ insn->sdwa.src0, insn->sdwa.dst_sel,
+ insn->sdwa.src0_sel, insn->sdwa.src1_sel);
+}
+
+static inline void gfx9_debugfs_sdwab(union amdgcn_gfx9_insn *insn,
+ struct seq_file *m)
+{
+ seq_printf(m, "sdwab src0:%d sdst:s%d src0_sel:%d src1_sel:%d\n",
+ insn->sdwab.src0, insn->sdwab.sdst,
+ insn->sdwab.src0_sel, insn->sdwab.src1_sel);
+}
+
+static inline void gfx9_debugfs_dpp16(union amdgcn_gfx9_insn *insn,
+ struct seq_file *m)
+{
+ seq_puts(m, "dpp16 (not decoded)\n");
+}
+
+static inline void gfx9_debugfs_dpp8(union amdgcn_gfx9_insn *insn,
+ struct seq_file *m)
+{
+ seq_puts(m, "dpp8 (not decoded)\n");
+}
+
+static inline void gfx9_debugfs_vintrp(union amdgcn_gfx9_insn *insn,
+ struct seq_file *m)
+{
+ seq_puts(m, "vintrp (not decoded)\n");
+}
+
+static inline void gfx9_debugfs_ds(union amdgcn_gfx9_insn *insn,
+ struct seq_file *m)
+{
+ seq_printf(m, "%s v%d, v%d, v%d, v%d offset0:%d offset1:%d%s\n",
+ opnames_gfx9[AMDGCN_INSN_TYPE_DS][insn->ds.op],
+ (int)insn->ds.vdst, (int)insn->ds.addr,
+ (int)insn->ds.data0, (int)insn->ds.data1,
+ (int)insn->ds.offset0, (int)insn->ds.offset1,
+ insn->ds.gds ? " gds" : "");
+}
+
+static inline void gfx9_debugfs_mtbuf(union amdgcn_gfx9_insn *insn,
+ struct seq_file *m)
+{
+ seq_printf(m, "%s v%d, v%d, s[%d:%d], s%d dfmt:%d nfmt:%d offset:%d\n",
+ opnames_gfx9[AMDGCN_INSN_TYPE_MTBUF][insn->mtbuf.op],
+ (int)insn->mtbuf.vdata, (int)insn->mtbuf.vaddr,
+ (int)insn->mtbuf.srsrc * 4, (int)insn->mtbuf.srsrc * 4 + 3,
+ (int)insn->mtbuf.soffset, (int)insn->mtbuf.dfmt,
+ (int)insn->mtbuf.nfmt, (int)insn->mtbuf.offset);
+}
+
+static inline void gfx9_debugfs_mimg(union amdgcn_gfx9_insn *insn,
+ struct seq_file *m)
+{
+ seq_puts(m, "mimg (not decoded)\n");
+}
+
+static inline void gfx9_debugfs_exp(union amdgcn_gfx9_insn *insn,
+ struct seq_file *m)
+{
+ seq_puts(m, "exp (not decoded)\n");
+}
+
+static inline void gfx9_debugfs_insn(struct amdgcn_insn *insn,
+ struct seq_file *m)
+{
+ u32 *ptr = (u32 *)&insn->gfx9;
+
+ if (insn->size == 4)
+ seq_printf(m, "%.8X\t\t\t", ptr[0]);
+ else if (insn->size == 8)
+ seq_printf(m, "%.8X %.8X\t\t", ptr[0], ptr[1]);
+ else if (insn->size == 12)
+ seq_printf(m, "%.8X %.8X %.8X\t", ptr[0], ptr[1], ptr[2]);
+ else
+ WARN_ON_ONCE(1);
+
+ switch (insn->type) {
+ case AMDGCN_INSN_TYPE_SOP2:
+ gfx9_debugfs_sop2(&insn->gfx9, m);
+ break;
+ case AMDGCN_INSN_TYPE_SOPK:
+ gfx9_debugfs_sopk(&insn->gfx9, m);
+ break;
+ case AMDGCN_INSN_TYPE_SOP1:
+ gfx9_debugfs_sop1(&insn->gfx9, m);
+ break;
+ case AMDGCN_INSN_TYPE_SOPC:
+ gfx9_debugfs_sopc(&insn->gfx9, m);
+ break;
+ case AMDGCN_INSN_TYPE_SOPP:
+ gfx9_debugfs_sopp(&insn->gfx9, m);
+ break;
+ case AMDGCN_INSN_TYPE_SMEM:
+ gfx9_debugfs_smem(&insn->gfx9, m);
+ break;
+ case AMDGCN_INSN_TYPE_VOP2:
+ gfx9_debugfs_vop2(&insn->gfx9, m);
+ break;
+ case AMDGCN_INSN_TYPE_VOP1:
+ gfx9_debugfs_vop1(&insn->gfx9, m);
+ break;
+ case AMDGCN_INSN_TYPE_VOPC:
+ gfx9_debugfs_vopc(&insn->gfx9, m);
+ break;
+ case AMDGCN_INSN_TYPE_VOP3A:
+ gfx9_debugfs_vop3a(&insn->gfx9, m);
+ break;
+ case AMDGCN_INSN_TYPE_VOP3B:
+ gfx9_debugfs_vop3b(&insn->gfx9, m);
+ break;
+ case AMDGCN_INSN_TYPE_VOP3P:
+ gfx9_debugfs_vop3p(&insn->gfx9, m);
+ break;
+ case AMDGCN_INSN_TYPE_SDWA:
+ gfx9_debugfs_sdwa(&insn->gfx9, m);
+ break;
+ case AMDGCN_INSN_TYPE_SDWAB:
+ gfx9_debugfs_sdwab(&insn->gfx9, m);
+ break;
+ case AMDGCN_INSN_TYPE_DPP16:
+ gfx9_debugfs_dpp16(&insn->gfx9, m);
+ break;
+ case AMDGCN_INSN_TYPE_DPP8:
+ gfx9_debugfs_dpp8(&insn->gfx9, m);
+ break;
+ case AMDGCN_INSN_TYPE_VINTRP:
+ gfx9_debugfs_vintrp(&insn->gfx9, m);
+ break;
+ case AMDGCN_INSN_TYPE_DS:
+ gfx9_debugfs_ds(&insn->gfx9, m);
+ break;
+ case AMDGCN_INSN_TYPE_MTBUF:
+ gfx9_debugfs_mtbuf(&insn->gfx9, m);
+ break;
+ case AMDGCN_INSN_TYPE_MUBUF:
+ gfx9_debugfs_mubuf(&insn->gfx9, m);
+ break;
+ case AMDGCN_INSN_TYPE_MIMG:
+ gfx9_debugfs_mimg(&insn->gfx9, m);
+ break;
+ case AMDGCN_INSN_TYPE_FLAT:
+ gfx9_debugfs_global(&insn->gfx9, m);
+ break;
+ case AMDGCN_INSN_TYPE_EXP:
+ gfx9_debugfs_exp(&insn->gfx9, m);
+ break;
+ default:
+ WARN_ON_ONCE(1);
+ break;
+ }
+}
+
+static inline void debugfs_insn(int version, struct amdgcn_insn *insn,
+ struct seq_file *m)
+{
+ if (version == 10)
+ gfx10_debugfs_insn(insn, m);
+ else if (version == 9)
+ gfx9_debugfs_insn(insn, m);
+ else
+ WARN_ON_ONCE(1);
+}
+
+/*
+ * Recover instruction type + size from a raw machine-code word stream into a
+ * struct amdgcn_insn, so code holding only the emitted u32 blob (the feature
+ * shaders emit via the low-level emit_gfxN_* helpers straight into the GPU
+ * code buffer, discarding type/size) can be fed to the shared disassembler
+ * debugfs_insn() instead of a per-feature hand-rolled hex re-parser.
+ *
+ * This mirrors the proven encoding-detection that used to live in the
+ * per-feature disassemblers. The checks are ordered widest
+ * fixed-prefix first: SOPP/SOP1/SOPC (9-bit enc) before SOPK (4-bit) before
+ * SOP2 (2-bit), and every bit31==1 class (VOP1/VOPC/SMEM/DS/FLAT/MUBUF/VOP3)
+ * before the VOP2 (bit31==0) catch-all. Version-specific encodings: SMEM
+ * (gfx9 0x30 / gfx10 0x3d), SOPC literal (gfx10 only) and VOP3 (gfx9 0x34 /
+ * gfx10 0x35). size is in bytes (4/8/12) to match what emit_* stamps in.
+ */
+static inline void amdgcn_classify(int version, const u32 *code,
+ struct amdgcn_insn *out)
+{
+ enum amdgcn_insn_type type;
+ u32 w0 = code[0];
+ bool is_b;
+ u32 dwords;
+ u32 enc;
+ u32 op;
+
+ enc = (w0 >> 23) & 0x1ff;
+
+ if (enc == 0x17f) { /* SOPP */
+ type = AMDGCN_INSN_TYPE_SOPP;
+ dwords = 1;
+ } else if (enc == 0x17d) { /* SOP1 */
+ type = AMDGCN_INSN_TYPE_SOP1;
+ dwords = ((w0 & 0xff) == 0xff) ? 2 : 1;
+ } else if (enc == 0x17e) { /* SOPC */
+ type = AMDGCN_INSN_TYPE_SOPC;
+ dwords = (version == 10 &&
+ ((w0 & 0xff) == 0xff || ((w0 >> 8) & 0xff) == 0xff))
+ ? 2 : 1;
+ } else if (((w0 >> 28) & 0xf) == 0xb) { /* SOPK */
+ type = AMDGCN_INSN_TYPE_SOPK;
+ dwords = 1;
+ } else if (((w0 >> 30) & 0x3) == 0x2) { /* SOP2 */
+ type = AMDGCN_INSN_TYPE_SOP2;
+ dwords = ((w0 & 0xff) == 0xff ||
+ ((w0 >> 8) & 0xff) == 0xff) ? 2 : 1;
+ } else if (((w0 >> 26) & 0x3f) ==
+ (version == 10 ? 0x3d : 0x30)) { /* SMEM */
+ type = AMDGCN_INSN_TYPE_SMEM;
+ dwords = 2;
+ } else if (((w0 >> 25) & 0x7f) == 0x3f) { /* VOP1 */
+ type = AMDGCN_INSN_TYPE_VOP1;
+ dwords = ((w0 & 0xff) == 0xff) ? 2 : 1;
+ } else if (((w0 >> 25) & 0x7f) == 0x3e) { /* VOPC */
+ type = AMDGCN_INSN_TYPE_VOPC;
+ dwords = 1;
+ } else if (((w0 >> 26) & 0x3f) == 0x36) { /* DS */
+ type = AMDGCN_INSN_TYPE_DS;
+ dwords = 2;
+ /* FLAT/GLOBAL/SCRATCH */
+ } else if (((w0 >> 26) & 0x3f) == 0x37) {
+ type = AMDGCN_INSN_TYPE_FLAT;
+ dwords = 2;
+ } else if (((w0 >> 26) & 0x3f) == 0x38) { /* MUBUF */
+ type = AMDGCN_INSN_TYPE_MUBUF;
+ dwords = 2;
+ } else if (((w0 >> 26) & 0x3f) ==
+ (version == 10 ? 0x35 : 0x34)) { /* VOP3A / VOP3B */
+ op = (w0 >> 16) & 0x3ff;
+ if (version == 10)
+ is_b = (op == GFX10_V_ADD_CO_U32 ||
+ op == GFX10_V_SUB_CO_U32 ||
+ op == GFX10_V_SUBREV_CO_U32 ||
+ op == GFX10_V_MAD_U64_U32 ||
+ op == GFX10_V_MAD_I64_I32 ||
+ op == GFX10_V_DIV_SCALE_F32 ||
+ op == GFX10_V_DIV_SCALE_F64);
+ else
+ is_b = (op == GFX9_V_MAD_U64_U32 ||
+ op == GFX9_V_MAD_I64_I32 ||
+ op == GFX9_V_DIV_SCALE_F64);
+ type = is_b ? AMDGCN_INSN_TYPE_VOP3B : AMDGCN_INSN_TYPE_VOP3A;
+ dwords = 2;
+ } else if (((w0 >> 31) & 0x1) == 0) { /* VOP2 */
+ type = AMDGCN_INSN_TYPE_VOP2;
+ dwords = ((w0 & 0xff) == 0xff) ? 2 : 1;
+ } else { /* unknown: 1 dword */
+ type = AMDGCN_INSN_TYPE_SOPP;
+ dwords = 1;
+ }
+
+ memset(out, 0, sizeof(*out));
+ out->type = type;
+ out->size = dwords * 4;
+ memcpy(&out->gfx10, code, dwords * 4);
+}
+
+/*
+ * Drop-in replacement for the old per-feature hex re-parsers: classify
+ * one raw instruction at @code, print it through the
+ * shared (complete-opnames) disassembler, and return the number of dwords
+ * consumed so the caller can advance. @max_dwords guards against reading a
+ * second dword past the end of the buffer.
+ */
+static inline int amdgcn_disasm_raw(int version, const u32 *code,
+ int max_dwords, struct seq_file *m)
+{
+ struct amdgcn_insn insn;
+
+ if (max_dwords <= 0)
+ return 1;
+ amdgcn_classify(version, code, &insn);
+ if ((int)(insn.size / 4) > max_dwords) {
+ seq_printf(m, "%.8X\t\t\t(truncated)\n", code[0]);
+ return 1;
+ }
+ debugfs_insn(version, &insn, m);
+ return insn.size / 4;
+}
+
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdkfd/knod/knod_gfx10_insn.h b/drivers/gpu/drm/amd/amdkfd/knod/knod_gfx10_insn.h
new file mode 100644
index 000000000000..f2699adb6c7d
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdkfd/knod/knod_gfx10_insn.h
@@ -0,0 +1,3978 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/* Copyright (c) 2021 Taehee Yoo <ap420073@gmail.com>
+ * Copyright (c) 2021 Hoyeon Lee <hoyeon.rhee@gmail.com>
+ */
+
+#ifndef KFD_AMDGPU_GFX10_INSN_H_INCLUDED
+#define KFD_AMDGPU_GFX10_INSN_H_INCLUDED
+
+#include "knod_amdgpu.h"
+
+/* See knod_gfx9_insn.h for rationale on not including knod_amdgpu_insn.h */
+
+#define GFX10_SRC_SGPR_BASE 0
+#define GFX10_SRC_VCC_LO 106
+#define GFX10_SRC_VCC_HI 107
+#define GFX10_SRC_TTPM_BASE 108
+#define GFX10_SRC_M0 124
+#define GFX10_SRC_NULL 125
+#define GFX10_SRC_EXEC_LO 126
+#define GFX10_SRC_EXEC_HI 127
+#define GFX10_SRC_INTEGER_0 128
+#define GFX10_SRC_INTEGER_MINUS_1 193
+#define GFX10_SRC_SHARED_BASE 235
+#define GFX10_SRC_SHARED_LIMIT 236
+#define GFX10_SRC_PRIVATE_BASE 237
+#define GFX10_SRC_PRIVATE_LIMIT 238
+#define GFX10_SRC_POPS_EXITING_WAVE_ID 239
+#define GFX10_SRC_SDWA 249
+#define GFX10_SRC_DDP16 250
+#define GFX10_SRC_VCCZ 251
+#define GFX10_SRC_EXECZ 252
+#define GFX10_SRC_SCC 253
+#define GFX10_SRC_LITERAL_CONST 255
+#define GFX10_SRC_VGPR_BASE 256
+
+static int gfx10_param_base[__AMDGCN_PARAM_TYPE_MAX] = {
+ GFX10_SRC_SGPR_BASE,
+ GFX10_SRC_VCC_LO,
+ GFX10_SRC_VCC_HI,
+ GFX10_SRC_TTPM_BASE,
+ GFX10_SRC_M0,
+ GFX10_SRC_NULL,
+ GFX10_SRC_EXEC_LO,
+ GFX10_SRC_EXEC_HI,
+ GFX10_SRC_INTEGER_0,
+ GFX10_SRC_INTEGER_MINUS_1,
+ GFX10_SRC_SHARED_BASE,
+ GFX10_SRC_SHARED_LIMIT,
+ GFX10_SRC_PRIVATE_BASE,
+ GFX10_SRC_PRIVATE_LIMIT,
+ GFX10_SRC_POPS_EXITING_WAVE_ID,
+ GFX10_SRC_SDWA,
+ GFX10_SRC_DDP16,
+ GFX10_SRC_VCCZ,
+ GFX10_SRC_EXECZ,
+ GFX10_SRC_SCC,
+ GFX10_SRC_LITERAL_CONST,
+ GFX10_SRC_VGPR_BASE,
+};
+
+/* Scalar ALU and Control Format */
+struct amdgcn_gfx10_sop2 {
+ u32 ssrc0:8;
+ u32 ssrc1:8;
+ u32 sdst:7;
+ u32 op:7;
+ u32 encoding:2;
+ u32 literal;
+};
+
+enum amdgcn_gfx10_sop2_opcode {
+ GFX10_S_ADD_U32 = 0,
+ GFX10_S_SUB_U32 = 1,
+ GFX10_S_ADD_I32 = 2,
+ GFX10_S_SUB_I32 = 3,
+ GFX10_S_ADDC_U32 = 4,
+ GFX10_S_SUBB_U32 = 5,
+ GFX10_S_MIN_I32 = 6,
+ GFX10_S_MIN_U32 = 7,
+ GFX10_S_MAX_I32 = 8,
+ GFX10_S_MAX_U32 = 9,
+ GFX10_S_CSELECT_B32 = 10,
+ GFX10_S_CELECT_B64 = 11,
+ /* 12-13: reserved */
+ GFX10_S_AND_B32 = 14,
+ GFX10_S_AND_B64 = 15,
+ GFX10_S_OR_B32 = 16,
+ GFX10_S_OR_B64 = 17,
+ GFX10_S_XOR_B32 = 18,
+ GFX10_S_XOR_B64 = 19,
+ GFX10_S_ANDN2_B32 = 20,
+ GFX10_S_ANDN2_B64 = 21,
+ GFX10_S_ORN2_B32 = 22,
+ GFX10_S_ORN2_B64 = 23,
+ GFX10_S_NAND_B32 = 24,
+ GFX10_S_NAND_B64 = 25,
+ GFX10_S_NOR_B32 = 26,
+ GFX10_S_NOR_B64 = 27,
+ GFX10_S_XNOR_B32 = 28,
+ GFX10_S_XNOR_B64 = 29,
+ GFX10_S_LSHL_B32 = 30,
+ GFX10_S_LSHL_B64 = 31,
+ GFX10_S_LSHR_B32 = 32,
+ GFX10_S_LSHR_B64 = 33,
+ GFX10_S_ASHR_I32 = 34,
+ GFX10_S_ASHR_I64 = 35,
+ GFX10_S_BFM_B32 = 36,
+ GFX10_S_BFM_B64 = 37,
+ GFX10_S_MUL_I32 = 38,
+ GFX10_S_MUL_I64 = 39,
+ GFX10_S_BFE_U32 = 40,
+ GFX10_S_BFE_I32 = 41,
+ GFX10_S_BFE_U64 = 42,
+ GFX10_S_ABSDIFF_I32 = 44,
+ GFX10_S_LSHL1_ADD_U32 = 46,
+ GFX10_S_LSHL2_ADD_U32 = 47,
+ GFX10_S_LSHL3_ADD_U32 = 48,
+ GFX10_S_LSHL4_ADD_U32 = 49,
+ GFX10_S_PACK_LL_B32_B16 = 50,
+ GFX10_S_PACK_LH_B32_B16 = 51,
+ GFX10_S_PACK_HH_B32_B16 = 52,
+ GFX10_S_MUL_HI_U32 = 53,
+ GFX10_S_MUL_HI_I32 = 54,
+};
+
+#define GFX10_SOP2_ENCODING 0x2
+#define GFX10_SOP2_SSRC_SGPR_BASE 0
+#define GFX10_SOP2_SSRC_VCC_LO 106
+#define GFX10_SOP2_SSRC_VCC_HI 107
+#define GFX10_SOP2_SSRC_TTPM_BASE 108
+#define GFX10_SOP2_SSRC_M0 124
+#define GFX10_SOP2_SSRC_NULL 125
+#define GFX10_SOP2_SSRC_EXEC_LO 126
+#define GFX10_SOP2_SSRC_EXEC_HI 127
+#define GFX10_SOP2_SSRC_INTEGER_0 128
+#define GFX10_SOP2_SSRC_INTEGER_MINUS_1 193
+#define GFX10_SOP2_SSRC_SHARED_BASE 235
+#define GFX10_SOP2_SSRC_SHARED_LIMIT 236
+#define GFX10_SOP2_SSRC_PRIVATE_BASE 237
+#define GFX10_SOP2_SSRC_PRIVATE_LIMIT 238
+#define GFX10_SOP2_SSRC_POPS_EXITING_WAVE_ID 239
+#define GFX10_SOP2_SSRC_VCCZ 251
+#define GFX10_SOP2_SSRC_EXECZ 252
+#define GFX10_SOP2_SSRC_SCC 253
+/* SDST
+ * Same codes as SSRC0, above except only codes 0-127 are valid.
+ */
+
+struct amdgcn_gfx10_sopk {
+ u32 simm16:16;
+ u32 sdst:7;
+ u32 op:5;
+ u32 encoding:4;
+};
+
+enum amdgcn_gfx10_sopk_opcode {
+ GFX10_S_MOVK_I32 = 0,
+ GFX10_S_VERSION = 1,
+ GFX10_S_CMOVK_I32 = 2,
+ GFX10_S_CMPK_EQ_I32 = 3,
+ GFX10_S_CMPK_LG_I32 = 4,
+ GFX10_S_CMPK_GT_I32 = 5,
+ GFX10_S_CMPK_GE_I32 = 6,
+ GFX10_S_CMPK_LT_I32 = 7,
+ GFX10_S_CMPK_LE_I32 = 8,
+ GFX10_S_CMPK_EQ_U32 = 9,
+ GFX10_S_CMPK_LG_U32 = 10,
+ GFX10_S_CMPK_GT_U32 = 11,
+ GFX10_S_CMPK_GE_U32 = 12,
+ GFX10_S_CMPK_LT_U32 = 13,
+ GFX10_S_CMPK_LE_U32 = 14,
+ GFX10_S_ADDK_I32 = 15,
+ GFX10_S_MULK_I32 = 16,
+ /* 17: reserved */
+ GFX10_S_GETREG_B32 = 18,
+ GFX10_S_SETREG_B32 = 19,
+ /* 20: reserved */
+ GFX10_S_SETREG_IMM32_B32 = 21,
+ GFX10_S_CALL_B64 = 22,
+ GFX10_S_WAITCNT_VSCNT = 23,
+ GFX10_S_WAITCNT_VMCNT = 24,
+ GFX10_S_WAITCNT_EXPCNT = 25,
+ GFX10_S_WAITCNT_LGKMCNT = 26,
+ GFX10_S_SUBVECTOR_LOOP_BEGIN = 27,
+ GFX10_S_SUBVECTOR_LOOP_END = 28,
+};
+
+#define GFX10_SOPK_ENCODING 0xb
+#define GFX10_SOPK_SDST_SGPR0_BASE 0
+#define GFX10_SOPK_SDST_VCC_LO 106
+#define GFX10_SOPK_SDST_VCC_HI 107
+#define GFX10_SOPK_SDST_TTPM_BASE 108
+#define GFX10_SOPK_SDST_M0 124
+#define GFX10_SOPK_SDST_NULL 125
+#define GFX10_SOPK_SDST_EXEC_LO 126
+#define GFX10_SOPK_SDST_EXEC_HI 127
+
+struct amdgcn_gfx10_sop1 {
+ u32 ssrc0:8;
+ u32 op:8;
+ u32 sdst:7;
+ u32 encoding:9;
+ u32 literal;
+};
+
+enum amdgcn_gfx10_sop1_opcode {
+ GFX10_S_MOV_B32 = 3,
+ GFX10_S_MOV_B64 = 4,
+ GFX10_S_CMOV_B32 = 5,
+ GFX10_S_CMOV_B64 = 6,
+ GFX10_S_NOT_B32 = 7,
+ GFX10_S_NOT_B64 = 8,
+ GFX10_S_WQM_B32 = 9,
+ GFX10_S_WQM_B64 = 10,
+ GFX10_S_BREV_B32 = 11,
+ GFX10_S_BREV_B64 = 12,
+ GFX10_S_BCNT0_I32_B32 = 13,
+ GFX10_S_BCNT0_I32_B64 = 14,
+ GFX10_S_BCNT1_I32_B32 = 15,
+ GFX10_S_BCNT1_I32_B64 = 16,
+ GFX10_S_FF0_I32_B32 = 17,
+ GFX10_S_FF0_I32_B64 = 18,
+ GFX10_S_FF1_I32_B32 = 19,
+ GFX10_S_FF1_I32_B64 = 20,
+ GFX10_S_FLBIT_I32_B32 = 21,
+ GFX10_S_FLBIT_I32_B64 = 22,
+ GFX10_S_FLBIT_I32 = 23,
+ GFX10_S_FLBIT_I32_I64 = 24,
+ GFX10_S_SEXT_I32_I8 = 25,
+ GFX10_S_SEXT_I32_I16 = 26,
+ GFX10_S_BITSET0_B32 = 27,
+ GFX10_S_BITSET0_B64 = 28,
+ GFX10_S_BITSET1_B32 = 29,
+ GFX10_S_BITSET1_B64 = 30,
+ GFX10_S_GETPC_B64 = 31,
+ GFX10_S_SETPC_B64 = 32,
+ GFX10_S_SWAPPC_B64 = 33,
+ GFX10_S_RFE_B64 = 34,
+ /* 35: reserved */
+ GFX10_S_AND_SAVEEXEC_B64 = 36,
+ /* 37-42: OR/XOR/ANDN2/ORN2/NAND/NOR_SAVEEXEC_B64 (unused) */
+ GFX10_S_XNOR_SAVEEXEC_B64 = 43,
+ GFX10_S_QUADMASK_B32 = 44,
+ GFX10_S_QUADMASK_B64 = 45,
+ GFX10_S_MOVRELS_B32 = 46,
+ GFX10_S_MOVRELS_B64 = 47,
+ GFX10_S_MOVRELD_B32 = 48,
+ GFX10_S_MOVRELD_B64 = 49,
+ /* 50-51: reserved */
+ GFX10_S_ABS_I32 = 52,
+ /* 53-54: reserved */
+ GFX10_S_ANDN1_SAVEEXEC_B64 = 55,
+ GFX10_S_ORN1_SAVEEXEC_B64 = 56,
+ GFX10_S_ANDN1_WREXEC_B64 = 57,
+ GFX10_S_ANDN2_WREXEC_B64 = 58,
+ GFX10_S_BITREPLICATE_B64_B32 = 59,
+ GFX10_S_AND_SAVEEXEC_B32 = 60,
+ GFX10_S_OR_SAVEEXEC_B32 = 61,
+ GFX10_S_XOR_SAVEEXEC_B32 = 62,
+ GFX10_S_ANDN2_SAVEEXEC_B32 = 63,
+ GFX10_S_ORN2_SAVEEXEC_B32 = 64,
+ GFX10_S_NAND_SAVEEXEC_B32 = 65,
+ GFX10_S_NOR_SAVEEXEC_B32 = 66,
+ GFX10_S_XNOR_SAVEEXEC_B32 = 67,
+ GFX10_S_ANDN1_SAVEEXEC_B32 = 68,
+ GFX10_S_ORN1_SAVEEXEC_B32 = 69,
+ GFX10_S_ANDN1_WREXEC_B32 = 70,
+ GFX10_S_ANDN2_WREXEC_B32 = 71,
+ /* 72: reserved */
+ GFX10_S_MOVRELSD_2_B32 = 73,
+};
+
+#define GFX10_SOP1_ENCODING 0x17d
+#define GFX10_SOP1_SSRC_SGPR_BASE 0
+#define GFX10_SOP1_SSRC_VCC_LO 106
+#define GFX10_SOP1_SSRC_VCC_HI 107
+#define GFX10_SOP1_SSRC_TTPM_BASE 108
+#define GFX10_SOP1_SSRC_M0 124
+#define GFX10_SOP1_SSRC_NULL 125
+#define GFX10_SOP1_SSRC_EXEC_LO 126
+#define GFX10_SOP1_SSRC_EXEC_HI 127
+#define GFX10_SOP1_SSRC_INTEGER_0 128
+#define GFX10_SOP1_SSRC_INTEGER_MINUS_1 193
+#define GFX10_SOP1_SSRC_SHARED_BASE 235
+#define GFX10_SOP1_SSRC_SHARED_LIMIT 236
+#define GFX10_SOP1_SSRC_PRIVATE_BASE 237
+#define GFX10_SOP1_SSRC_PRIVATE_LIMIT 238
+#define GFX10_SOP1_SSRC_POPS_EXITING_WAVE_ID 239
+#define GFX10_SOP1_SSRC_VCCZ 251
+#define GFX10_SOP1_SSRC_EXECZ 252
+#define GFX10_SOP1_SSRC_SCC 253
+#define GFX10_SOP1_SSRC_LITERAL_CONST 255
+/* SDST
+ * Same codes as SSRC0, above except only codes 0-127 are valid.
+ */
+
+struct amdgcn_gfx10_sopc {
+ u32 ssrc0:8;
+ u32 ssrc1:8;
+ u32 op:7;
+ u32 encoding:9;
+ u32 literal;
+};
+
+enum amdgcn_gfx10_sopc_opcode {
+ GFX10_S_CMP_EQ_I32 = 0,
+ GFX10_S_CMP_LG_I32 = 1,
+ GFX10_S_CMP_GT_I32 = 2,
+ GFX10_S_CMP_GE_I32 = 3,
+ GFX10_S_CMP_LT_I32 = 4,
+ GFX10_S_CMP_LE_I32 = 5,
+ GFX10_S_CMP_EQ_U32 = 6,
+ GFX10_S_CMP_LG_U32 = 7,
+ GFX10_S_CMP_GT_U32 = 8,
+ GFX10_S_CMP_GE_U32 = 9,
+ GFX10_S_CMP_LT_U32 = 10,
+ GFX10_S_CMP_LE_U32 = 11,
+ GFX10_S_BITCMP0_B32 = 12,
+ GFX10_S_BITCMP1_B32 = 13,
+ GFX10_S_BITCMP0_B64 = 14,
+ GFX10_S_BITCMP1_B64 = 15,
+ /* 16-17: reserved */
+ GFX10_S_CMP_EQ_U64 = 18,
+ GFX10_S_CMP_LG_U64 = 19,
+};
+
+#define GFX10_SOPC_ENCODING 0x17e
+#define GFX10_SOPC_SSRC_SGPR_BASE 0
+#define GFX10_SOPC_SSRC_VCC_LO 106
+#define GFX10_SOPC_SSRC_VCC_HI 107
+#define GFX10_SOPC_SSRC_TTPM_BASE 108
+#define GFX10_SOPC_SSRC_M0 124
+#define GFX10_SOPC_SSRC_NULL 125
+#define GFX10_SOPC_SSRC_EXEC_LO 126
+#define GFX10_SOPC_SSRC_EXEC_HI 127
+#define GFX10_SOPC_SSRC_INTEGER_0 128
+#define GFX10_SOPC_SSRC_INTEGER_MINUS_1 193
+#define GFX10_SOPC_SSRC_SHARED_BASE 235
+#define GFX10_SOPC_SSRC_SHARED_LIMIT 236
+#define GFX10_SOPC_SSRC_PRIVATE_BASE 237
+#define GFX10_SOPC_SSRC_PRIVATE_LIMIT 238
+#define GFX10_SOPC_SSRC_POPS_EXITING_WAVE_ID 239
+#define GFX10_SOPC_SSRC_VCCZ 251
+#define GFX10_SOPC_SSRC_EXECZ 252
+#define GFX10_SOPC_SSRC_SCC 253
+/* SDST
+ * Same codes as SSRC0, above except only codes 0-127 are valid.
+ */
+
+struct amdgcn_gfx10_sopp_vmcnt {
+ u16 vmcnt1:4;
+ u16 expcnt:3;
+ u16 dummy:1;
+ u16 lgkmcnt:6;
+ u16 vmcnt2:2;
+};
+
+struct amdgcn_gfx10_sopp {
+ u32 simm16:16;
+ u32 op:7;
+ u32 encoding:9;
+};
+
+enum amdgcn_gfx10_sopp_opcode {
+ GFX10_S_NOP = 0,
+ GFX10_S_ENDPGM = 1,
+ GFX10_S_BRANCH = 2,
+ GFX10_S_WAKEUP = 3,
+ GFX10_S_CBRANCH_SCC0 = 4,
+ GFX10_S_CBRANCH_SCC1 = 5,
+ GFX10_S_CBRANCH_VCCZ = 6,
+ GFX10_S_CBRANCH_VCCNZ = 7,
+ GFX10_S_CBRANCH_EXECZ = 8,
+ GFX10_S_CBRANCH_EXECNZ = 9,
+ GFX10_S_BARRIER = 10,
+ GFX10_S_SETKILL = 11,
+ GFX10_S_WAITCNT = 12,
+ GFX10_S_SETHALT = 13,
+ GFX10_S_SLEEP = 14,
+ GFX10_S_SETPRIO = 15,
+ GFX10_S_SENDMSG = 16,
+ GFX10_S_SENDMSGHALT = 17,
+ GFX10_S_TRAP = 18,
+ GFX10_S_ICACHE_INV = 19,
+ GFX10_S_INCPERFLEVEL = 20,
+ GFX10_S_DECPERFLEVEL = 21,
+ GFX10_S_TTRACEDATA = 22,
+ GFX10_S_CBRANCH_CDBGSYS = 23,
+ GFX10_S_CBRANCH_CDBGUSER = 24,
+ GFX10_S_CBRANCH_CDBGSYS_OR_USER = 25,
+ GFX10_S_CBRANCH_CDBGSYS_AND_USER = 26,
+ GFX10_S_ENDPGM_SAVED = 27,
+ /* 28-29: reserved */
+ GFX10_S_ENDPGM_ORDERED_PS_DONE = 30,
+ GFX10_S_CODE_END = 31,
+ GFX10_S_INST_PREFETCH = 32,
+ GFX10_S_CLAUSE = 33,
+ /* 34: reserved */
+ GFX10_S_WAITCNT_DEPCTR = 35,
+ GFX10_S_ROUND_MODE = 36,
+ GFX10_S_DENORM_MODE = 37,
+ /* 38-39: reserved */
+ GFX10_S_TTRACEDATA_IMM = 40,
+};
+
+#define GFX10_SOPP_ENCODING 0x17f
+
+/* Scalar Memory Format */
+struct amdgcn_gfx10_smem {
+ u64 sbase:6;
+ u64 sdata:7;
+ u64 dummy1:1;
+ u64 dlc:1;
+ u64 dummy2:1;
+ u64 glc:1;
+ u64 dummy3:1;
+ u64 op:8;
+ u64 encoding:6;
+ u64 offset:21;
+ u64 dummy4:4;
+ u64 soffset:7;
+};
+
+enum amdgcn_gfx10_smem_opcode {
+ GFX10_S_LOAD_DWORD = 0,
+ GFX10_S_LOAD_DWORDX2 = 1,
+ GFX10_S_LOAD_DWORDX4 = 2,
+ GFX10_S_LOAD_DWORDX8 = 3,
+ GFX10_S_LOAD_DWORDX16 = 4,
+ /* 5-7: reserved */
+ GFX10_S_BUFFER_LOAD_DWORD = 8,
+ GFX10_S_BUFFER_LOAD_DWORDX2 = 9,
+ GFX10_S_BUFFER_LOAD_DWORDX4 = 10,
+ GFX10_S_BUFFER_LOAD_DWORDX8 = 11,
+ GFX10_S_BUFFER_LOAD_DWORDX16 = 12,
+ /* 13-30: reserved */
+ GFX10_S_GL1_INV = 31,
+ GFX10_S_DCACHE_INV = 32,
+ /* 33-35: reserved */
+ GFX10_S_MEMTIME = 36,
+ GFX10_S_MEMREALTIME = 37,
+ GFX10_S_ATC_PROBE = 38,
+ GFX10_S_ATC_PROBE_BUFFER = 39,
+};
+
+#define GFX10_SMEM_ENCODING 0x3d
+#define GFX10_SMEM_SOFFSET_NULL 125
+
+/* Vector ALU Format */
+struct amdgcn_gfx10_vop2 {
+ u32 src0:9;
+ u32 vsrc1:8;
+ u32 vdst:8;
+ u32 op:6;
+ u32 encoding:1;
+ u32 literal;
+};
+
+enum amdgcn_gfx10_vop2_opcode {
+ /* 0: reserved */
+ GFX10_V_CNDMASK_B32 = 1,
+ GFX10_V_DOT2C_F32_F16 = 2,
+ GFX10_V_ADD_F32 = 3,
+ GFX10_V_SUB_F32 = 4,
+ GFX10_V_SUBREV_F32 = 5,
+ GFX10_V_FMAC_LEGACY_F32 = 6,
+ GFX10_V_MUL_LEGACY_F32 = 7,
+ GFX10_V_MUL_F32 = 8,
+ GFX10_V_MUL_I32_I24 = 9,
+ GFX10_V_MUL_HI_I32_I24 = 10,
+ GFX10_V_MUL_U32_U24 = 11,
+ GFX10_V_MUL_HI_U32_U24 = 12,
+ GFX10_V_DOT4C_I32_I8 = 13,
+ /* 14: reserved */
+ GFX10_V_MIN_F32 = 15,
+ GFX10_V_MAX_F32 = 16,
+ GFX10_V_MIN_I32 = 17,
+ GFX10_V_MAX_I32 = 18,
+ GFX10_V_MIN_U32 = 19,
+ GFX10_V_MAX_U32 = 20,
+ /* 21: reserved */
+ GFX10_V_LSHRREV_B32 = 22,
+ /* 23: reserved */
+ GFX10_V_ASHRREV_I32 = 24,
+ /* 25: reserved */
+ GFX10_V_LSHLREV_B32 = 26,
+ GFX10_V_AND_B32 = 27,
+ GFX10_V_OR_B32 = 28,
+ GFX10_V_XOR_B32 = 29,
+ GFX10_V_XNOR_B32 = 30,
+ /* 31-36: reserved */
+ GFX10_V_ADD_NC_U32 = 37,
+ GFX10_V_SUB_NC_U32 = 38,
+ GFX10_V_SUBREV_NC_U32 = 39,
+ GFX10_V_ADD_CO_CI_U32 = 40,
+ GFX10_V_SUB_CO_CI_U32 = 41,
+ GFX10_V_SUBREV_CO_CI_U32 = 42,
+ GFX10_V_FMAC_F32 = 43,
+ GFX10_V_FMAMK_F32 = 44,
+ GFX10_V_FMAAK_F32 = 45,
+ /* 46: reserved */
+ GFX10_V_CVT_PKRTZ_F16_F32 = 47,
+ /* 48-49: reserved */
+ GFX10_V_ADD_F16 = 50,
+ GFX10_V_SUB_F16 = 51,
+ GFX10_V_SUBREV_F16 = 52,
+ GFX10_V_MUL_F16 = 53,
+ GFX10_V_FMAC_F16 = 54,
+ GFX10_V_FMAMK_F16 = 55,
+ GFX10_V_FMAAK_F16 = 56,
+ GFX10_V_MAX_F16 = 57,
+ GFX10_V_MIN_F16 = 58,
+ GFX10_V_LDEXP_F16 = 59,
+ GFX10_V_PK_FMAC_F16 = 60,
+};
+
+#define GFX10_VOP2_ENCODING 0x0
+#define GFX10_VOP2_SRC_SGPR_BASE 0
+#define GFX10_VOP2_SRC_VCC_LO 106
+#define GFX10_VOP2_SRC_VCC_HI 107
+#define GFX10_VOP2_SRC_TTPM_BASE 108
+#define GFX10_VOP2_SRC_M0 124
+#define GFX10_VOP2_SRC_NULL 125
+#define GFX10_VOP2_SRC_EXEC_LO 126
+#define GFX10_VOP2_SRC_EXEC_HI 127
+#define GFX10_VOP2_SRC_INTEGER_0 128
+#define GFX10_VOP2_SRC_INTEGER_MINUS_1 193
+#define GFX10_VOP2_SRC_SHARED_BASE 235
+#define GFX10_VOP2_SRC_SHARED_LIMIT 236
+#define GFX10_VOP2_SRC_PRIVATE_BASE 237
+#define GFX10_VOP2_SRC_PRIVATE_LIMIT 238
+#define GFX10_VOP2_SRC_POPS_EXITING_WAVE_ID 239
+#define GFX10_VOP2_SRC_SDWA 249
+#define GFX10_VOP2_SRC_DDP16 250
+#define GFX10_VOP2_SRC_VCCZ 251
+#define GFX10_VOP2_SRC_EXECZ 252
+#define GFX10_VOP2_SRC_SCC 253
+#define GFX10_VOP2_SRC_LITERAL_CONST 255
+#define GFX10_VOP2_SRC_VGPR_BASE 256
+
+struct amdgcn_gfx10_vop1 {
+ u32 src0:9;
+ u32 op:8;
+ u32 vdst:8;
+ u32 encoding:7;
+ u32 literal;
+};
+
+enum amdgcn_gfx10_vop1_opcode {
+ GFX10_V_NOP = 0,
+ GFX10_V_MOV_B32 = 1,
+ GFX10_V_READFIRSTLANE_B32 = 2,
+ GFX10_V_CVT_I32_F64 = 3,
+ GFX10_V_CVT_F64_I32 = 4,
+ GFX10_V_CVT_F32_I32 = 5,
+ GFX10_V_CVT_F32_U32 = 6,
+ GFX10_V_CVT_U32_F32 = 7,
+ GFX10_V_CVT_I32_F32 = 8,
+ /* 9: reserved */
+ GFX10_V_CVT_F16_F32 = 10,
+ GFX10_V_CVT_F32_F16 = 11,
+ GFX10_V_CVT_RPI_I32_F32 = 12,
+ GFX10_V_CVT_FLR_I32_F32 = 13,
+ GFX10_V_CVT_OFF_F32_I4 = 14,
+ GFX10_V_CVT_F32_F64 = 15,
+ GFX10_V_CVT_F64_F32 = 16,
+ GFX10_V_CVT_F32_UBYTE0 = 17,
+ GFX10_V_CVT_F32_UBYTE1 = 18,
+ GFX10_V_CVT_F32_UBYTE2 = 19,
+ GFX10_V_CVT_F32_UBYTE3 = 20,
+ GFX10_V_CVT_U32_F64 = 21,
+ GFX10_V_CVT_F64_U32 = 22,
+ GFX10_V_TRUNC_F64 = 23,
+ GFX10_V_CEIL_F64 = 24,
+ GFX10_V_RNDNE_F64 = 25,
+ GFX10_V_FLOOR_F64 = 26,
+ GFX10_V_PIPEFLUSH = 27,
+ /* 28-31: reserved */
+ GFX10_V_FRACT_F32 = 32,
+ GFX10_V_TRUNC_F32 = 33,
+ GFX10_V_CEIL_F32 = 34,
+ GFX10_V_RNDNE_F32 = 35,
+ GFX10_V_FLOOR_F32 = 36,
+ GFX10_V_EXP_F32 = 37,
+ /* 38: reserved */
+ GFX10_V_LOG_F32 = 39,
+ /* 40-41: reserved */
+ GFX10_V_RCP_F32 = 42,
+ GFX10_V_RCP_IFLAG_F32 = 43,
+ /* 44-45: reserved */
+ GFX10_V_RSQ_F32 = 46,
+ GFX10_V_RCP_F64 = 47,
+ /* 48: reserved */
+ GFX10_V_RSQ_F64 = 49,
+ /* 50: reserved */
+ GFX10_V_SQRT_F32 = 51,
+ GFX10_V_SQRT_F64 = 52,
+ GFX10_V_SIN_F32 = 53,
+ GFX10_V_COS_F32 = 54,
+ GFX10_V_NOT_B32 = 55,
+ GFX10_V_BFREV_B32 = 56,
+ GFX10_V_FFBH_U32 = 57,
+ GFX10_V_FFBL_B32 = 58,
+ GFX10_V_FFBH_I32 = 59,
+ GFX10_V_FREXP_EXP_I32_F64 = 60,
+ GFX10_V_FREXP_MANT_F64 = 61,
+ GFX10_V_FRACT_F64 = 62,
+ GFX10_V_FREXP_EXP_I32_F32 = 63,
+ GFX10_V_FREXP_MANT_F32 = 64,
+ GFX10_V_CLREXCP = 65,
+ GFX10_V_MOVRELD_B32 = 66,
+ GFX10_V_MOVRELS_B32 = 67,
+ GFX10_V_MOVRELSD_B32 = 68,
+ /* 69-71: reserved */
+ GFX10_V_MOVRELSD_2_B32 = 72,
+ /* 73-79: reserved */
+ GFX10_V_CVT_F16_U16 = 80,
+ GFX10_V_CVT_F16_I16 = 81,
+ GFX10_V_CVT_U16_F16 = 82,
+ GFX10_V_CVT_I16_F16 = 83,
+ GFX10_V_RCP_F16 = 84,
+ GFX10_V_SQRT_F16 = 85,
+ GFX10_V_RSQ_F16 = 86,
+ GFX10_V_LOG_F16 = 87,
+ GFX10_V_EXP_F16 = 88,
+ GFX10_V_FREXP_MANT_F16 = 89,
+ GFX10_V_FREXP_EXP_I16_F16 = 90,
+ GFX10_V_FLOOR_F16 = 91,
+ GFX10_V_CEIL_F16 = 92,
+ GFX10_V_TRUNC_F16 = 93,
+ GFX10_V_RNDNE_F16 = 94,
+ GFX10_V_FRACT_F16 = 95,
+ GFX10_V_SIN_F16 = 96,
+ GFX10_V_COS_F16 = 97,
+ GFX10_V_SAT_PK_U8_I16 = 98,
+ GFX10_V_CVT_NORM_I16_F16 = 99,
+ GFX10_V_CVT_NORM_U16_F16 = 100,
+ GFX10_V_SWAP_B32 = 101,
+ /* 102-103: reserved */
+ GFX10_V_SWAPREL_B32 = 104,
+};
+
+#define GFX10_VOP1_ENCODING 0x3f
+#define GFX10_VOP1_SRC_SGPR_BASE 0
+#define GFX10_VOP1_SRC_VCC_LO 106
+#define GFX10_VOP1_SRC_VCC_HI 107
+#define GFX10_VOP1_SRC_TTPM_BASE 108
+#define GFX10_VOP1_SRC_M0 124
+#define GFX10_VOP1_SRC_NULL 125
+#define GFX10_VOP1_SRC_EXEC_LO 126
+#define GFX10_VOP1_SRC_EXEC_HI 127
+#define GFX10_VOP1_SRC_INTEGER_0 128
+#define GFX10_VOP1_SRC_INTEGER_MINUS_1 193
+#define GFX10_VOP1_SRC_SHARED_BASE 235
+#define GFX10_VOP1_SRC_SHARED_LIMIT 236
+#define GFX10_VOP1_SRC_PRIVATE_BASE 237
+#define GFX10_VOP1_SRC_PRIVATE_LIMIT 238
+#define GFX10_VOP1_SRC_POPS_EXITING_WAVE_ID 239
+#define GFX10_VOP1_SRC_SDWA 249
+#define GFX10_VOP1_SRC_DDP16 250
+#define GFX10_VOP1_SRC_VCCZ 251
+#define GFX10_VOP1_SRC_EXECZ 252
+#define GFX10_VOP1_SRC_SCC 253
+#define GFX10_VOP1_SRC_LITERAL_CONST 255
+#define GFX10_VOP1_SRC_VGPR_BASE 256
+
+struct amdgcn_gfx10_vopc {
+ u32 src0:9;
+ u32 vsrc1:8;
+ u32 op:8;
+ u32 encoding:7;
+ u32 literal;
+};
+
+enum amdgcn_gfx10_vopc_compare_offset16 {
+ GFX10_VOPC16_F = 0,
+ GFX10_VOPC16_LT = 1,
+ GFX10_VOPC16_EQ = 2,
+ GFX10_VOPC16_LE = 3,
+ GFX10_VOPC16_GT = 4,
+ GFX10_VOPC16_LG = 5,
+ GFX10_VOPC16_GE = 6,
+ GFX10_VOPC16_O = 7,
+ GFX10_VOPC16_U = 8,
+ GFX10_VOPC16_NGE = 9,
+ GFX10_VOPC16_NLG = 10,
+ GFX10_VOPC16_NGT = 11,
+ GFX10_VOPC16_NLE = 12,
+ GFX10_VOPC16_NEQ = 13,
+ GFX10_VOPC16_NLT = 14,
+ GFX10_VOPC16_TRU = 15,
+};
+
+enum amdgcn_gfx10_vopc_compare_offset8 {
+ GFX10_VOPC8_F = 0,
+ GFX10_VOPC8_LT = 1,
+ GFX10_VOPC8_EQ = 2,
+ GFX10_VOPC8_LE = 3,
+ GFX10_VOPC8_GT = 4,
+ GFX10_VOPC8_LG = 5,
+ GFX10_VOPC8_GE = 6,
+ GFX10_VOPC8_TRU = 7,
+};
+
+enum amdgcn_gfx10_vopc_opcode {
+ GFX10_V_CMP_F_F32 = 0,
+ GFX10_V_CMP_LT_F32 = 1,
+ GFX10_V_CMP_EQ_F32 = 2,
+ GFX10_V_CMP_LE_F32 = 3,
+ GFX10_V_CMP_GT_F32 = 4,
+ GFX10_V_CMP_LG_F32 = 5,
+ GFX10_V_CMP_GE_F32 = 6,
+ GFX10_V_CMP_O_F32 = 7,
+ GFX10_V_CMP_U_F32 = 8,
+ GFX10_V_CMP_NGE_F32 = 9,
+ GFX10_V_CMP_NLG_F32 = 10,
+ GFX10_V_CMP_NGT_F32 = 11,
+ GFX10_V_CMP_NLE_F32 = 12,
+ GFX10_V_CMP_NEQ_F32 = 13,
+ GFX10_V_CMP_NLT_F32 = 14,
+ GFX10_V_CMP_TRU_F32 = 15,
+ GFX10_V_CMPX_F_F32 = 16,
+ GFX10_V_CMPX_LT_F32 = 17,
+ GFX10_V_CMPX_EQ_F32 = 18,
+ GFX10_V_CMPX_LE_F32 = 19,
+ GFX10_V_CMPX_GT_F32 = 20,
+ GFX10_V_CMPX_LG_F32 = 21,
+ GFX10_V_CMPX_GE_F32 = 22,
+ GFX10_V_CMPX_O_F32 = 23,
+ GFX10_V_CMPX_U_F32 = 24,
+ GFX10_V_CMPX_NGE_F32 = 25,
+ GFX10_V_CMPX_NLG_F32 = 26,
+ GFX10_V_CMPX_NGT_F32 = 27,
+ GFX10_V_CMPX_NLE_F32 = 28,
+ GFX10_V_CMPX_NEQ_F32 = 29,
+ GFX10_V_CMPX_NLT_F32 = 30,
+ GFX10_V_CMPX_TRU_F32 = 31,
+ GFX10_V_CMP_F_F64 = 32,
+ GFX10_V_CMP_LT_F64 = 33,
+ GFX10_V_CMP_EQ_F64 = 34,
+ GFX10_V_CMP_LE_F64 = 35,
+ GFX10_V_CMP_GT_F64 = 36,
+ GFX10_V_CMP_LG_F64 = 37,
+ GFX10_V_CMP_GE_F64 = 38,
+ GFX10_V_CMP_O_F64 = 39,
+ GFX10_V_CMP_U_F64 = 40,
+ GFX10_V_CMP_NGE_F64 = 41,
+ GFX10_V_CMP_NLG_F64 = 42,
+ GFX10_V_CMP_NGT_F64 = 43,
+ GFX10_V_CMP_NLE_F64 = 44,
+ GFX10_V_CMP_NEQ_F64 = 45,
+ GFX10_V_CMP_NLT_F64 = 46,
+ GFX10_V_CMP_TRU_F64 = 47,
+ GFX10_V_CMPX_F_F64 = 48,
+ GFX10_V_CMPX_LT_F64 = 49,
+ GFX10_V_CMPX_EQ_F64 = 50,
+ GFX10_V_CMPX_LE_F64 = 51,
+ GFX10_V_CMPX_GT_F64 = 52,
+ GFX10_V_CMPX_LG_F64 = 53,
+ GFX10_V_CMPX_GE_F64 = 54,
+ GFX10_V_CMPX_O_F64 = 55,
+ GFX10_V_CMPX_U_F64 = 56,
+ GFX10_V_CMPX_NGE_F64 = 57,
+ GFX10_V_CMPX_NLG_F64 = 58,
+ GFX10_V_CMPX_NGT_F64 = 59,
+ GFX10_V_CMPX_NLE_F64 = 60,
+ GFX10_V_CMPX_NEQ_F64 = 61,
+ GFX10_V_CMPX_NLT_F64 = 62,
+ GFX10_V_CMPX_TRU_F64 = 63,
+ /* 64-127: reserved */
+ GFX10_V_CMP_F_I32 = 128,
+ GFX10_V_CMP_LT_I32 = 129,
+ GFX10_V_CMP_EQ_I32 = 130,
+ GFX10_V_CMP_LE_I32 = 131,
+ GFX10_V_CMP_GT_I32 = 132,
+ GFX10_V_CMP_NE_I32 = 133,
+ GFX10_V_CMP_GE_I32 = 134,
+ GFX10_V_CMP_T_I32 = 135,
+ GFX10_V_CMP_CLASS_F32 = 136,
+ GFX10_V_CMP_LT_I16 = 137,
+ GFX10_V_CMP_EQ_I16 = 138,
+ GFX10_V_CMP_LE_I16 = 139,
+ GFX10_V_CMP_GT_I16 = 140,
+ GFX10_V_CMP_NE_I16 = 141,
+ GFX10_V_CMP_GE_I16 = 142,
+ GFX10_V_CMP_CLASS_F16 = 143,
+ GFX10_V_CMPX_F_I32 = 144,
+ GFX10_V_CMPX_LT_I32 = 145,
+ GFX10_V_CMPX_EQ_I32 = 146,
+ GFX10_V_CMPX_LE_I32 = 147,
+ GFX10_V_CMPX_GT_I32 = 148,
+ GFX10_V_CMPX_NE_I32 = 149,
+ GFX10_V_CMPX_GE_I32 = 150,
+ GFX10_V_CMPX_T_I32 = 151,
+ GFX10_V_CMPX_CLASS_F32 = 152,
+ GFX10_V_CMPX_LT_I16 = 153,
+ GFX10_V_CMPX_EQ_I16 = 154,
+ GFX10_V_CMPX_LE_I16 = 155,
+ GFX10_V_CMPX_GT_I16 = 156,
+ GFX10_V_CMPX_NE_I16 = 157,
+ GFX10_V_CMPX_GE_I16 = 158,
+ GFX10_V_CMPX_CLASS_F16 = 159,
+ GFX10_V_CMP_F_I64 = 160,
+ GFX10_V_CMP_LT_I64 = 161,
+ GFX10_V_CMP_EQ_I64 = 162,
+ GFX10_V_CMP_LE_I64 = 163,
+ GFX10_V_CMP_GT_I64 = 164,
+ GFX10_V_CMP_NE_I64 = 165,
+ GFX10_V_CMP_GE_I64 = 166,
+ GFX10_V_CMP_T_I64 = 167,
+ GFX10_V_CMP_CLASS_F64 = 168,
+ GFX10_V_CMP_LT_U16 = 169,
+ GFX10_V_CMP_EQ_U16 = 170,
+ GFX10_V_CMP_LE_U16 = 171,
+ GFX10_V_CMP_GT_U16 = 172,
+ GFX10_V_CMP_NE_U16 = 173,
+ GFX10_V_CMP_GE_U16 = 174,
+ /* 175: reserved */
+ GFX10_V_CMPX_F_I64 = 176,
+ GFX10_V_CMPX_LT_I64 = 177,
+ GFX10_V_CMPX_EQ_I64 = 178,
+ GFX10_V_CMPX_LE_I64 = 179,
+ GFX10_V_CMPX_GT_I64 = 180,
+ GFX10_V_CMPX_NE_I64 = 181,
+ GFX10_V_CMPX_GE_I64 = 182,
+ GFX10_V_CMPX_T_I64 = 183,
+ GFX10_V_CMPX_CLASS_F64 = 184,
+ GFX10_V_CMPX_LT_U16 = 185,
+ GFX10_V_CMPX_EQ_U16 = 186,
+ GFX10_V_CMPX_LE_U16 = 187,
+ GFX10_V_CMPX_GT_U16 = 188,
+ GFX10_V_CMPX_NE_U16 = 189,
+ GFX10_V_CMPX_GE_U16 = 190,
+ /* 191: reserved */
+ GFX10_V_CMP_F_U32 = 192,
+ GFX10_V_CMP_LT_U32 = 193,
+ GFX10_V_CMP_EQ_U32 = 194,
+ GFX10_V_CMP_LE_U32 = 195,
+ GFX10_V_CMP_GT_U32 = 196,
+ GFX10_V_CMP_NE_U32 = 197,
+ GFX10_V_CMP_GE_U32 = 198,
+ GFX10_V_CMP_T_U32 = 199,
+ GFX10_V_CMP_F_F16 = 200,
+ GFX10_V_CMP_LT_F16 = 201,
+ GFX10_V_CMP_EQ_F16 = 202,
+ GFX10_V_CMP_LE_F16 = 203,
+ GFX10_V_CMP_GT_F16 = 204,
+ GFX10_V_CMP_LG_F16 = 205,
+ GFX10_V_CMP_GE_F16 = 206,
+ GFX10_V_CMP_O_F16 = 207,
+ GFX10_V_CMPX_F_U32 = 208,
+ GFX10_V_CMPX_LT_U32 = 209,
+ GFX10_V_CMPX_EQ_U32 = 210,
+ GFX10_V_CMPX_LE_U32 = 211,
+ GFX10_V_CMPX_GT_U32 = 212,
+ GFX10_V_CMPX_NE_U32 = 213,
+ GFX10_V_CMPX_GE_U32 = 214,
+ GFX10_V_CMPX_T_U32 = 215,
+ GFX10_V_CMPX_F_F16 = 216,
+ GFX10_V_CMPX_LT_F16 = 217,
+ GFX10_V_CMPX_EQ_F16 = 218,
+ GFX10_V_CMPX_LE_F16 = 219,
+ GFX10_V_CMPX_GT_F16 = 220,
+ GFX10_V_CMPX_LG_F16 = 221,
+ GFX10_V_CMPX_GE_F16 = 222,
+ GFX10_V_CMPX_O_F16 = 223,
+ GFX10_V_CMP_F_U64 = 224,
+ GFX10_V_CMP_LT_U64 = 225,
+ GFX10_V_CMP_EQ_U64 = 226,
+ GFX10_V_CMP_LE_U64 = 227,
+ GFX10_V_CMP_GT_U64 = 228,
+ GFX10_V_CMP_NE_U64 = 229,
+ GFX10_V_CMP_GE_U64 = 230,
+ GFX10_V_CMP_T_U64 = 231,
+ GFX10_V_CMP_U_F16 = 232,
+ GFX10_V_CMP_NGE_F16 = 233,
+ GFX10_V_CMP_NLG_F16 = 234,
+ GFX10_V_CMP_NGT_F16 = 235,
+ GFX10_V_CMP_NLE_F16 = 236,
+ GFX10_V_CMP_NEQ_F16 = 237,
+ GFX10_V_CMP_NLT_F16 = 238,
+ GFX10_V_CMP_TRU_F16 = 239,
+ GFX10_V_CMPX_F_U64 = 240,
+ GFX10_V_CMPX_LT_U64 = 241,
+ GFX10_V_CMPX_EQ_U64 = 242,
+ GFX10_V_CMPX_LE_U64 = 243,
+ GFX10_V_CMPX_GT_U64 = 244,
+ GFX10_V_CMPX_NE_U64 = 245,
+ GFX10_V_CMPX_GE_U64 = 246,
+ GFX10_V_CMPX_T_U64 = 247,
+ GFX10_V_CMPX_U_F16 = 248,
+ GFX10_V_CMPX_NGE_F16 = 249,
+ GFX10_V_CMPX_NLG_F16 = 250,
+ GFX10_V_CMPX_NGT_F16 = 251,
+ GFX10_V_CMPX_NLE_F16 = 252,
+ GFX10_V_CMPX_NEQ_F16 = 253,
+ GFX10_V_CMPX_NLT_F16 = 254,
+ GFX10_V_CMPX_TRU_F16 = 255,
+};
+
+#define GFX10_VOPC_ENCODING 0x3e
+#define GFX10_VOPC_SRC_SGPR_BASE 0
+#define GFX10_VOPC_SRC_VCC_LO 106
+#define GFX10_VOPC_SRC_VCC_HI 107
+#define GFX10_VOPC_SRC_TTPM_BASE 108
+#define GFX10_VOPC_SRC_M0 124
+#define GFX10_VOPC_SRC_NULL 125
+#define GFX10_VOPC_SRC_EXEC_LO 126
+#define GFX10_VOPC_SRC_EXEC_HI 127
+#define GFX10_VOPC_SRC_INTEGER_0 128
+#define GFX10_VOPC_SRC_INTEGER_MINUS_1 193
+#define GFX10_VOPC_SRC_SHARED_BASE 235
+#define GFX10_VOPC_SRC_SHARED_LIMIT 236
+#define GFX10_VOPC_SRC_PRIVATE_BASE 237
+#define GFX10_VOPC_SRC_PRIVATE_LIMIT 238
+#define GFX10_VOPC_SRC_POPS_EXITING_WAVE_ID 239
+#define GFX10_VOPC_SRC_SDWA 249
+#define GFX10_VOPC_SRC_DDP16 250
+#define GFX10_VOPC_SRC_VCCZ 251
+#define GFX10_VOPC_SRC_EXECZ 252
+#define GFX10_VOPC_SRC_SCC 253
+#define GFX10_VOPC_SRC_LITERAL_CONST 255
+#define GFX10_VOPC_SRC_VGPR_BASE 256
+
+struct amdgcn_gfx10_vop3a {
+ u64 vdst:8;
+ u64 abs:3;
+ u64 op_sel:4;
+ u64 clmp:1;
+ u64 op:10;
+ u64 encoding:6;
+ u64 src0:9;
+ u64 src1:9;
+ u64 src2:9;
+ u64 omod:2;
+ u64 neg:3;
+ u32 literal;
+};
+
+enum amdgcn_gfx10_vop3a_opcode {
+ GFX10_V_FMA_LEGACY_F32 = 320,
+ /* 321: reserved */
+ GFX10_V_MAD_I32_I24 = 322,
+ GFX10_V_MAD_U32_U24 = 323,
+ GFX10_V_CUBEID_F32 = 324,
+ GFX10_V_CUBESC_F32 = 325,
+ GFX10_V_CUBETC_F32 = 326,
+ GFX10_V_CUBEMA_F32 = 327,
+ GFX10_V_BFE_U32 = 328,
+ GFX10_V_BFE_I32 = 329,
+ GFX10_V_BFI_B32 = 330,
+ GFX10_V_FMA_F32 = 331,
+ GFX10_V_FMA_F64 = 332,
+ GFX10_V_LERP_U8 = 333,
+ GFX10_V_ALIGNBIT_B32 = 334,
+ GFX10_V_ALIGNBYTE_B32 = 335,
+ GFX10_V_MULLIT_F32 = 336,
+ GFX10_V_MIN3_F32 = 337,
+ GFX10_V_MIN3_I32 = 338,
+ GFX10_V_MIN3_U32 = 339,
+ GFX10_V_MAX3_F32 = 340,
+ GFX10_V_MAX3_I32 = 341,
+ GFX10_V_MAX3_U32 = 342,
+ GFX10_V_MED3_F32 = 343,
+ GFX10_V_MED3_I32 = 344,
+ GFX10_V_MED3_U32 = 345,
+ GFX10_V_SAD_U8 = 346,
+ GFX10_V_SAD_HI_U8 = 347,
+ GFX10_V_SAD_U16 = 348,
+ GFX10_V_SAD_U32 = 349,
+ GFX10_V_CVT_PK_U8_F32 = 350,
+ GFX10_V_DIV_FIXUP_F32 = 351,
+ GFX10_V_DIV_FIXUP_F64 = 352,
+ /* 353-355: reserved */
+ GFX10_V_ADD_F64 = 356,
+ GFX10_V_MUL_F64 = 357,
+ GFX10_V_MIN_F64 = 358,
+ GFX10_V_MAX_F64 = 359,
+ GFX10_V_LDEXP_F64 = 360,
+ GFX10_V_MUL_LO_U32 = 361,
+ GFX10_V_MUL_HI_U32 = 362,
+ /* 363: reserved */
+ GFX10_V_MUL_HI_I32 = 364,
+ /* 365-366: VOP3B (V_DIV_SCALE_F32/F64) */
+ GFX10_V_DIV_FMAS_F32 = 367,
+ GFX10_V_DIV_FMAS_F64 = 368,
+ GFX10_V_MSAD_U8 = 369,
+ GFX10_V_QSAD_PK_U16_U8 = 370,
+ GFX10_V_MQSAD_PK_U16_U8 = 371,
+ GFX10_V_TRIG_PREOP_F64 = 372,
+ GFX10_V_MQSAD_U32_U8 = 373,
+ /* 374-375: VOP3B (V_MAD_U64_U32/I64_I32) */
+ GFX10_V_XOR3_B32 = 376,
+ /* 377-766: reserved */
+ GFX10_V_LSHLREV_B64 = 767,
+ GFX10_V_LSHRREV_B64 = 768,
+ GFX10_V_ASHRREV_I64 = 769,
+ /* 770: reserved */
+ GFX10_V_ADD_NC_U16 = 771,
+ GFX10_V_SUB_NC_U16 = 772,
+ GFX10_V_MUL_LO_U16 = 773,
+ /* 774: reserved */
+ GFX10_V_LSHRREV_B16 = 775,
+ GFX10_V_ASHRREV_I16 = 776,
+ GFX10_V_MAX_U16 = 777,
+ GFX10_V_MAX_I16 = 778,
+ GFX10_V_MIN_U16 = 779,
+ GFX10_V_MIN_I16 = 780,
+ GFX10_V_ADD_NC_I16 = 781,
+ GFX10_V_SUB_NC_I16 = 782,
+ /* 783-784: VOP3B (V_ADD_CO_U32/V_SUB_CO_U32) */
+ GFX10_V_PACK_B32_F16 = 785,
+ GFX10_V_CVT_PKNORM_I16_F16 = 786,
+ GFX10_V_CVT_PKNORM_U16_F16 = 787,
+ GFX10_V_LSHLREV_B16 = 788,
+ /* 789-792: reserved; 793: VOP3B (V_SUBREV_CO_U32) */
+ GFX10_V_MAD_U16 = 832,
+ /* 833: reserved */
+ GFX10_V_INTERP_P1LL_F16 = 834,
+ GFX10_V_INTERP_P1LV_F16 = 835,
+ GFX10_V_PERM_B32 = 836,
+ GFX10_V_XAD_U32 = 837,
+ GFX10_V_LSHL_ADD_U32 = 838,
+ GFX10_V_ADD_LSHL_U32 = 839,
+ /* 840-842: reserved */
+ GFX10_V_FMA_F16 = 843,
+ /* 844-848: reserved */
+ GFX10_V_MIN3_F16 = 849,
+ GFX10_V_MIN3_I16 = 850,
+ GFX10_V_MIN3_U16 = 851,
+ GFX10_V_MAX3_F16 = 852,
+ GFX10_V_MAX3_I16 = 853,
+ GFX10_V_MAX3_U16 = 854,
+ GFX10_V_MED3_F16 = 855,
+ GFX10_V_MED3_I16 = 856,
+ GFX10_V_MED3_U16 = 857,
+ GFX10_V_INTERP_P2_F16 = 858,
+ /* 859-861: reserved */
+ GFX10_V_MAD_I16 = 862,
+ GFX10_V_DIV_FIXUP_F16 = 863,
+ GFX10_V_READLANE_B32 = 864,
+ GFX10_V_WRITELANE_B32 = 865,
+ GFX10_V_LDEXP_F32 = 866,
+ GFX10_V_BFM_B32 = 867,
+ GFX10_V_BCNT_U32_B32 = 868,
+ GFX10_V_MBCNT_LO_U32_B32 = 869,
+ GFX10_V_MBCNT_HI_U32_B32 = 870,
+ /* 871: reserved */
+ GFX10_V_CVT_PKNORM_I16_F32 = 872,
+ GFX10_V_CVT_PKNORM_U16_F32 = 873,
+ GFX10_V_CVT_PK_U16_U32 = 874,
+ GFX10_V_CVT_PK_I16_I32 = 875,
+ /* 876: reserved */
+ GFX10_V_ADD3_U32 = 877,
+ /* 878: reserved */
+ GFX10_V_LSHL_OR_B32 = 879,
+ /* 880: reserved */
+ GFX10_V_AND_OR_B32 = 881,
+ GFX10_V_OR3_B32 = 882,
+ GFX10_V_MAD_U32_U16 = 883,
+ /* 884: reserved */
+ GFX10_V_MAD_I32_I16 = 885,
+ GFX10_V_SUB_NC_I32 = 886,
+ GFX10_V_PERMLANE16_B32 = 887,
+ GFX10_V_PERMLANEX16_B32 = 888,
+ /* 889-894: reserved */
+ GFX10_V_ADD_NC_I32 = 895,
+};
+
+enum amdgcn_gfx10_vop3a_abs {
+ GFX10_VOP3A_ABS_SRC0,
+ GFX10_VOP3A_ABS_SRC1,
+ GFX10_VOP3A_ABS_SRC2,
+};
+
+#define GFX10_VOP3A_ENCODING 0x35
+#define GFX10_VOP3A_SRC_SGPR_BASE 0
+#define GFX10_VOP3A_SRC_VCC_LO 106
+#define GFX10_VOP3A_SRC_VCC_HI 107
+#define GFX10_VOP3A_SRC_TTPM_BASE 108
+#define GFX10_VOP3A_SRC_M0 124
+#define GFX10_VOP3A_SRC_NULL 125
+#define GFX10_VOP3A_SRC_EXEC_LO 126
+#define GFX10_VOP3A_SRC_EXEC_HI 127
+#define GFX10_VOP3A_SRC_INTEGER_0 128
+#define GFX10_VOP3A_SRC_INTEGER_MINUS_1 193
+#define GFX10_VOP3A_SRC_SHARED_BASE 235
+#define GFX10_VOP3A_SRC_SHARED_LIMIT 236
+#define GFX10_VOP3A_SRC_PRIVATE_BASE 237
+#define GFX10_VOP3A_SRC_PRIVATE_LIMIT 238
+#define GFX10_VOP3A_SRC_POPS_EXITING_WAVE_ID 239
+#define GFX10_VOP3A_SRC_SDWA 249
+#define GFX10_VOP3A_SRC_DDP16 250
+#define GFX10_VOP3A_SRC_VCCZ 251
+#define GFX10_VOP3A_SRC_EXECZ 252
+#define GFX10_VOP3A_SRC_SCC 253
+#define GFX10_VOP3A_SRC_LITERAL_CONST 255
+#define GFX10_VOP3A_SRC_VGPR_BASE 256
+
+/*
+ * Two-source VOP3 forms have reserved third-source bits on gfx10. Encoding
+ * an inline integer zero there executes, but LLVM/RGP reject the instruction.
+ */
+#define GFX10_VOP3_UNUSED_SRC 0
+
+struct amdgcn_gfx10_vop3b {
+ u64 vdst:8;
+ u64 sdst:7;
+ u64 clmp:1;
+ u64 op:10;
+ u64 encoding:6;
+ u64 src0:9;
+ u64 src1:9;
+ u64 src2:9;
+ u64 omod:2;
+ u64 neg:3;
+ u32 literal;
+};
+
+enum amdgcn_gfx10_vop3b_opcode {
+ GFX10_V_DIV_SCALE_F32 = 365,
+ GFX10_V_DIV_SCALE_F64 = 366,
+ GFX10_V_MAD_U64_U32 = 374,
+ GFX10_V_MAD_I64_I32 = 375,
+ GFX10_V_ADD_CO_U32 = 783,
+ GFX10_V_SUB_CO_U32 = 784,
+ GFX10_V_SUBREV_CO_U32 = 793,
+};
+
+#define GFX10_VOP3B_ENCODING 0x35
+#define GFX10_VOP3B_SRC_SGPR_BASE 0
+#define GFX10_VOP3B_SRC_VCC_LO 106
+#define GFX10_VOP3B_SRC_VCC_HI 107
+#define GFX10_VOP3B_SRC_TTPM_BASE 108
+#define GFX10_VOP3B_SRC_M0 124
+#define GFX10_VOP3B_SRC_NULL 125
+#define GFX10_VOP3B_SRC_EXEC_LO 126
+#define GFX10_VOP3B_SRC_EXEC_HI 127
+#define GFX10_VOP3B_SRC_INTEGER_0 128
+#define GFX10_VOP3B_SRC_INTEGER_MINUS_1 193
+#define GFX10_VOP3B_SRC_SHARED_BASE 235
+#define GFX10_VOP3B_SRC_SHARED_LIMIT 236
+#define GFX10_VOP3B_SRC_PRIVATE_BASE 237
+#define GFX10_VOP3B_SRC_PRIVATE_LIMIT 238
+#define GFX10_VOP3B_SRC_POPS_EXITING_WAVE_ID 239
+#define GFX10_VOP3B_SRC_SDWA 249
+#define GFX10_VOP3B_SRC_DDP16 250
+#define GFX10_VOP3B_SRC_VCCZ 251
+#define GFX10_VOP3B_SRC_EXECZ 252
+#define GFX10_VOP3B_SRC_SCC 253
+#define GFX10_VOP3B_SRC_LITERAL_CONST 255
+#define GFX10_VOP3B_SRC_VGPR_BASE 256
+
+struct amdgcn_gfx10_vop3p {
+ u64 vdst:8;
+ u64 neg_hi:3;
+ u64 op_sel:3;
+ u64 op_sel_hi2:1;
+ u64 clmp:1;
+ u64 op:7;
+ u64 dummy:3;
+ u64 encoding:6;
+ u64 src0:9;
+ u64 src1:9;
+ u64 src2:9;
+ u64 op_sel_hi:2;
+ u64 neg:3;
+ u32 literal;
+};
+
+enum amdgcn_gfx10_vop3p_opcode {
+ GFX10_V_PK_MAD_I16 = 0,
+ GFX10_V_PK_MUL_LO_U16 = 1,
+ GFX10_V_PK_ADD_I16 = 2,
+ GFX10_V_PK_SUB_I16 = 3,
+ GFX10_V_PK_LSHLREV_B16 = 4,
+ GFX10_V_PK_LSHRREV_B16 = 5,
+ GFX10_V_PK_ASHRREV_I16 = 6,
+ GFX10_V_PK_MAX_I16 = 7,
+ GFX10_V_PK_MIN_I16 = 8,
+ GFX10_V_PK_MAD_U16 = 9,
+ GFX10_V_PK_ADD_U16 = 10,
+ GFX10_V_PK_SUB_U16 = 11,
+ GFX10_V_PK_MAX_U16 = 12,
+ GFX10_V_PK_MIN_U16 = 13,
+ GFX10_V_PK_FMA_F16 = 14,
+ GFX10_V_PK_ADD_F16 = 15,
+ GFX10_V_PK_MUL_F16 = 16,
+ GFX10_V_PK_MIN_F16 = 17,
+ GFX10_V_PK_MAX_F16 = 18,
+ GFX10_V_DOT2_F32_F16 = 19,
+ GFX10_V_DOT2_I32_I16 = 20,
+ GFX10_V_DOT2_U32_U16 = 21,
+ GFX10_V_DOT4_I32_I8 = 22,
+ GFX10_V_DOT4_U32_U8 = 23,
+ GFX10_V_DOT8_I32_I4 = 24,
+ GFX10_V_DOT8_U32_U4 = 25,
+ /* 26-31: reserved */
+ GFX10_V_FMA_MIX_F32 = 32,
+ GFX10_V_FMA_MIXLO_F16 = 33,
+ GFX10_V_FMA_MIXHI_F16 = 34,
+};
+
+#define GFX10_VOP3P_ENCODING 0x33
+#define GFX10_VOP3P_SRC_SGPR_BASE 0
+#define GFX10_VOP3P_SRC_VCC_LO 106
+#define GFX10_VOP3P_SRC_VCC_HI 107
+#define GFX10_VOP3P_SRC_TTPM_BASE 108
+#define GFX10_VOP3P_SRC_M0 124
+#define GFX10_VOP3P_SRC_NULL 125
+#define GFX10_VOP3P_SRC_EXEC_LO 126
+#define GFX10_VOP3P_SRC_EXEC_HI 127
+#define GFX10_VOP3P_SRC_INTEGER_0 128
+#define GFX10_VOP3P_SRC_INTEGER_MINUS_1 193
+#define GFX10_VOP3P_SRC_SHARED_BASE 235
+#define GFX10_VOP3P_SRC_SHARED_LIMIT 236
+#define GFX10_VOP3P_SRC_PRIVATE_BASE 237
+#define GFX10_VOP3P_SRC_PRIVATE_LIMIT 238
+#define GFX10_VOP3P_SRC_POPS_EXITING_WAVE_ID 239
+#define GFX10_VOP3P_SRC_SDWA 249
+#define GFX10_VOP3P_SRC_DDP16 250
+#define GFX10_VOP3P_SRC_VCCZ 251
+#define GFX10_VOP3P_SRC_EXECZ 252
+#define GFX10_VOP3P_SRC_SCC 253
+#define GFX10_VOP3P_SRC_VGPR_BASE 256
+
+struct amdgcn_gfx10_sdwa {
+ u32 src0:8;
+ u32 dst_sel:3;
+ u32 dst_u:2;
+ u32 clmp:1;
+ u32 omod:2;
+ u32 src0_sel:3;
+ u32 src0_sext:1;
+ u32 src0_neg:1;
+ u32 src0_abs:1;
+ u32 dummy1:1;
+ u32 s0:1;
+ u32 src1_sel:3;
+ u32 src1_sext:1;
+ u32 src1_neg:1;
+ u32 src1_abs:1;
+ u32 dummy2:1;
+ u32 s1:1;
+};
+
+struct amdgcn_gfx10_sdwab {
+ u32 src0:8;
+ u32 sdst:7;
+ u32 sd:1;
+ u32 src0_sel:3;
+ u32 src0_sext:1;
+ u32 src0_neg:1;
+ u32 src0_abs:1;
+ u32 dummy1:1;
+ u32 s0:1;
+ u32 src1_sel:3;
+ u32 src1_sext:1;
+ u32 src1_neg:1;
+ u32 src1_abs:1;
+ u32 dummy2:1;
+ u32 s1:1;
+};
+
+struct amdgcn_gfx10_dpp16 {
+};
+
+struct amdgcn_gfx10_dpp8 {
+};
+
+/* Vector Parameter Interpolation Format */
+struct amdgcn_gfx10_vintrp {
+};
+
+/* LDS and GDS Format */
+struct amdgcn_gfx10_ds {
+ u64 offset0:8;
+ u64 offset1:8;
+ u64 dummy:1;
+ u64 gds:1;
+ u64 op:8;
+ u64 encoding:6;
+ u64 addr:8;
+ u64 data0:8;
+ u64 data1:8;
+ u64 vdst:8;
+};
+
+enum amdgcn_gfx10_ds_opcode {
+ GFX10_DS_ADD_U32 = 0,
+ GFX10_DS_SUB_U32 = 1,
+ GFX10_DS_RSUB_U32 = 2,
+ GFX10_DS_INC_U32 = 3,
+ GFX10_DS_DEC_U32 = 4,
+ GFX10_DS_MIN_I32 = 5,
+ GFX10_DS_MAX_I32 = 6,
+ GFX10_DS_MIN_U32 = 7,
+ GFX10_DS_MAX_U32 = 8,
+ GFX10_DS_AND_B32 = 9,
+ GFX10_DS_OR_B32 = 10,
+ GFX10_DS_XOR_B32 = 11,
+ GFX10_DS_MSKOR_B32 = 12,
+ GFX10_DS_WRITE_B32 = 13,
+ GFX10_DS_WRITE2_B32 = 14,
+ GFX10_DS_WRITE2ST64_B32 = 15,
+ GFX10_DS_CMPST_B32 = 16,
+ GFX10_DS_CMPST_F32 = 17,
+ GFX10_DS_MIN_F32 = 18,
+ GFX10_DS_MAX_F32 = 19,
+ GFX10_DS_NOP = 20,
+ GFX10_DS_ADD_F32 = 21,
+ /* 22-23: reserved */
+ GFX10_DS_GWS_SEMA_RELEASE_ALL = 24,
+ GFX10_DS_GWS_INIT = 25,
+ GFX10_DS_GWS_SEMA_V = 26,
+ GFX10_DS_GWS_SEMA_BR = 27,
+ GFX10_DS_GWS_SEMA_P = 28,
+ GFX10_DS_GWS_BARRIER = 29,
+ GFX10_DS_WRITE_B8 = 30,
+ GFX10_DS_WRITE_B16 = 31,
+ GFX10_DS_ADD_RTN_U32 = 32,
+ GFX10_DS_SUB_RTN_U32 = 33,
+ GFX10_DS_RSUB_RTN_U32 = 34,
+ GFX10_DS_INC_RTN_U32 = 35,
+ GFX10_DS_DEC_RTN_U32 = 36,
+ GFX10_DS_MIN_RTN_I32 = 37,
+ GFX10_DS_MAX_RTN_I32 = 38,
+ GFX10_DS_MIN_RTN_U32 = 39,
+ GFX10_DS_MAX_RTN_U32 = 40,
+ GFX10_DS_AND_RTN_B32 = 41,
+ GFX10_DS_OR_RTN_B32 = 42,
+ GFX10_DS_XOR_RTN_B32 = 43,
+ GFX10_DS_MSKOR_RTN_B32 = 44,
+ GFX10_DS_WRXCHG_RTN_B32 = 45,
+ GFX10_DS_WRXCHG2_RTN_B32 = 46,
+ GFX10_DS_WRXCHG2ST64_RTN_B32 = 47,
+ GFX10_DS_CMPST_RTN_B32 = 48,
+ GFX10_DS_CMPST_RTN_F32 = 49,
+ GFX10_DS_MIN_RTN_F32 = 50,
+ GFX10_DS_MAX_RTN_F32 = 51,
+ GFX10_DS_WRAP_RTN_B32 = 52,
+ GFX10_DS_SWIZZLE_B32 = 53,
+ GFX10_DS_READ_B32 = 54,
+ GFX10_DS_READ2_B32 = 55,
+ GFX10_DS_READ2ST64_B32 = 56,
+ GFX10_DS_READ_I8 = 57,
+ GFX10_DS_READ_U8 = 58,
+ GFX10_DS_READ_I16 = 59,
+ GFX10_DS_READ_U16 = 60,
+ GFX10_DS_CONSUME = 61,
+ GFX10_DS_APPEND = 62,
+ GFX10_DS_ORDERED_COUNT = 63,
+ GFX10_DS_ADD_U64 = 64,
+ GFX10_DS_SUB_U64 = 65,
+ GFX10_DS_RSUB_U64 = 66,
+ GFX10_DS_INC_U64 = 67,
+ GFX10_DS_DEC_U64 = 68,
+ GFX10_DS_MIN_I64 = 69,
+ GFX10_DS_MAX_I64 = 70,
+ GFX10_DS_MIN_U64 = 71,
+ GFX10_DS_MAX_U64 = 72,
+ GFX10_DS_AND_B64 = 73,
+ GFX10_DS_OR_B64 = 74,
+ GFX10_DS_XOR_B64 = 75,
+ GFX10_DS_MSKOR_B64 = 76,
+ GFX10_DS_WRITE_B64 = 77,
+ GFX10_DS_WRITE2_B64 = 78,
+ GFX10_DS_WRITE2ST64_B64 = 79,
+ GFX10_DS_CMPST_B64 = 80,
+ GFX10_DS_CMPST_F64 = 81,
+ GFX10_DS_MIN_F64 = 82,
+ GFX10_DS_MAX_F64 = 83,
+ /* 84: reserved */
+ GFX10_DS_ADD_RTN_F32 = 85,
+ /* 86-95: reserved */
+ GFX10_DS_ADD_RTN_U64 = 96,
+ GFX10_DS_SUB_RTN_U64 = 97,
+ GFX10_DS_RSUB_RTN_U64 = 98,
+ GFX10_DS_INC_RTN_U64 = 99,
+ GFX10_DS_DEC_RTN_U64 = 100,
+ GFX10_DS_MIN_RTN_I64 = 101,
+ GFX10_DS_MAX_RTN_I64 = 102,
+ GFX10_DS_MIN_RTN_U64 = 103,
+ GFX10_DS_MAX_RTN_U64 = 104,
+ GFX10_DS_AND_RTN_B64 = 105,
+ GFX10_DS_OR_RTN_B64 = 106,
+ GFX10_DS_XOR_RTN_B64 = 107,
+ GFX10_DS_MSKOR_RTN_B64 = 108,
+ GFX10_DS_WRXCHG_RTN_B64 = 109,
+ GFX10_DS_WRXCHG2_RTN_B64 = 110,
+ GFX10_DS_WRXCHG2ST64_RTN_B64 = 111,
+ GFX10_DS_CMPST_RTN_B64 = 112,
+ GFX10_DS_CMPST_RTN_F64 = 113,
+ GFX10_DS_MIN_RTN_F64 = 114,
+ GFX10_DS_MAX_RTN_F64 = 115,
+ /* 116-117: reserved */
+ GFX10_DS_READ_B64 = 118,
+ GFX10_DS_READ2_B64 = 119,
+ GFX10_DS_READ2ST64_B64 = 120,
+ /* 121-125: reserved */
+ GFX10_DS_CONDXCHG32_RTN_B64 = 126,
+ /* 127-159: reserved */
+ GFX10_DS_WRITE_B8_D16_HI = 160,
+ GFX10_DS_WRITE_B16_D16_HI = 161,
+ GFX10_DS_READ_U8_D16 = 162,
+ GFX10_DS_READ_U8_D16_HI = 163,
+ GFX10_DS_READ_I8_D16 = 164,
+ GFX10_DS_READ_I8_D16_HI = 165,
+ GFX10_DS_READ_U16_D16 = 166,
+ GFX10_DS_READ_U16_D16_HI = 167,
+ /* 168-175: reserved */
+ GFX10_DS_WRITE_ADDTID_B32 = 176,
+ GFX10_DS_READ_ADDTID_B32 = 177,
+ GFX10_DS_PERMUTE_B32 = 178,
+ GFX10_DS_BPERMUTE_B32 = 179,
+ /* 180-221: reserved */
+ GFX10_DS_WRITE_B96 = 222,
+ GFX10_DS_WRITE_B128 = 223,
+ /* 224-253: reserved */
+ GFX10_DS_READ_B96 = 254,
+ GFX10_DS_READ_B128 = 255,
+};
+
+#define GFX10_DS_ENCODING 0x36
+#define GFX10_DS_GDS 1
+#define GFX10_DS_LDS 0
+
+/* Vector Memory Buffer Formats */
+struct amdgcn_gfx10_mtbuf {
+ u64 offset:12;
+ u64 offen:1;
+ u64 idxen:1;
+ u64 glc:1;
+ u64 dlc:1;
+ u64 op:3;
+ u64 foamat:7;
+ u64 encoding:6;
+ u64 vaddr:8;
+ u64 vdata:8;
+ u64 srsrc:5;
+ u64 opm:1;
+ u64 slc:1;
+ u64 tfe:1;
+ u64 soffset:8;
+};
+
+enum amdgcn_gfx10_mtbuf_opcode {
+ GFX10_TBUFFER_LOAD_FORMAT_X = 0,
+ GFX10_TBUFFER_LOAD_FORMAT_XY = 1,
+ GFX10_TBUFFER_LOAD_FORMAT_XYZ = 2,
+ GFX10_TBUFFER_LOAD_FORMAT_XYZW = 3,
+ GFX10_TBUFFER_STORE_FORMAT_X = 4,
+ GFX10_TBUFFER_STORE_FORMAT_XY = 5,
+ GFX10_TBUFFER_STORE_FORMAT_XYZ = 6,
+ GFX10_TBUFFER_STORE_FORMAT_XYZW = 7,
+ GFX10_TBUFFER_LOAD_FORMAT_D16_X = 8,
+ GFX10_TBUFFER_LOAD_FORMAT_D16_XY = 9,
+ GFX10_TBUFFER_LOAD_FORMAT_D16_XYZ = 10,
+ GFX10_TBUFFER_LOAD_FORMAT_D16_XYZW = 11,
+ GFX10_TBUFFER_STORE_FORMAT_D16_X = 12,
+ GFX10_TBUFFER_STORE_FORMAT_D16_XY = 13,
+ GFX10_TBUFFER_STORE_FORMAT_D16_XYZ = 14,
+ GFX10_TBUFFER_STORE_FORMAT_D16_XYZW = 15,
+};
+
+#define GFX10_MUBUF_ENCODING 0x38
+#define GFX10_MUBUF_SOFFSET_SGPR_BASE 0
+#define GFX10_MUBUF_SOFFSET_VCC_LO 106
+#define GFX10_MUBUF_SOFFSET_VCC_HI 107
+#define GFX10_MUBUF_SOFFSET_TTPM_BASE 108
+#define GFX10_MUBUF_SOFFSET_M0 124
+#define GFX10_MUBUF_SOFFSET_NULL 125
+#define GFX10_MUBUF_SOFFSET_EXEC_LO 126
+#define GFX10_MUBUF_SOFFSET_EXEC_HI 127
+#define GFX10_MUBUF_SOFFSET_INTEGER_0 128
+#define GFX10_MUBUF_SOFFSET_INTEGER_MINUS_1 193
+#define GFX10_MUBUF_SOFFSET_SHARED_BASE 235
+#define GFX10_MUBUF_SOFFSET_SHARED_LIMIT 236
+#define GFX10_MUBUF_SOFFSET_PRIVATE_BASE 237
+#define GFX10_MUBUF_SOFFSET_PRIVATE_LIMIT 238
+#define GFX10_MUBUF_SOFFSET_POPS_EXITING_WAVE_ID 239
+#define GFX10_MUBUF_SOFFSET_VCCZ 251
+#define GFX10_MUBUF_SOFFSET_EXECZ 252
+#define GFX10_MUBUF_SOFFSET_SCC 253
+
+struct amdgcn_gfx10_mubuf {
+ u64 offset:12;
+ u64 offen:1;
+ u64 idxen:1;
+ u64 glc:1;
+ u64 dlc:1;
+ u64 lds:1;
+ u64 dummy1:1;
+ u64 op:7;
+ u64 opm:1;
+ u64 encoding:6;
+ u64 vaddr:8;
+ u64 vdata:8;
+ u64 srsrc:5;
+ u64 dummy2:1;
+ u64 slc:1;
+ u64 tfe:1;
+ u64 soffset:8;
+};
+
+enum amdgcn_gfx10_mubuf_opcode {
+ GFX10_BUFFER_LOAD_FORMAT_X = 0,
+ GFX10_BUFFER_LOAD_FORMAT_XY = 1,
+ GFX10_BUFFER_LOAD_FORMAT_XYZ = 2,
+ GFX10_BUFFER_LOAD_FORMAT_XYZW = 3,
+ GFX10_BUFFER_STORE_FORMAT_X = 4,
+ GFX10_BUFFER_STORE_FORMAT_XY = 5,
+ GFX10_BUFFER_STORE_FORMAT_XYZ = 6,
+ GFX10_BUFFER_STORE_FORMAT_XYZW = 7,
+ GFX10_BUFFER_LOAD_UBYTE = 8,
+ GFX10_BUFFER_LOAD_SBYTE = 9,
+ GFX10_BUFFER_LOAD_USHORT = 10,
+ GFX10_BUFFER_LOAD_SSHORT = 11,
+ GFX10_BUFFER_LOAD_DWORD = 12,
+ GFX10_BUFFER_LOAD_DWORDX2 = 13,
+ GFX10_BUFFER_LOAD_DWORDX4 = 14,
+ GFX10_BUFFER_LOAD_DWORDX3 = 15,
+ /* 16-23: reserved */
+ GFX10_BUFFER_STORE_BYTE = 24,
+ GFX10_BUFFER_STORE_BYTE_D16_HI = 25,
+ GFX10_BUFFER_STORE_SHORT = 26,
+ GFX10_BUFFER_STORE_SHORT_D16_HI = 27,
+ GFX10_BUFFER_STORE_DWORD = 28,
+ GFX10_BUFFER_STORE_DWORDX2 = 29,
+ GFX10_BUFFER_STORE_DWORDX4 = 30,
+ GFX10_BUFFER_STORE_DWORDX3 = 31,
+ GFX10_BUFFER_LOAD_UBYTE_D16 = 32,
+ GFX10_BUFFER_LOAD_UBYTE_D16_HI = 33,
+ GFX10_BUFFER_LOAD_SBYTE_D16 = 34,
+ GFX10_BUFFER_LOAD_SBYTE_D16_HI = 35,
+ GFX10_BUFFER_LOAD_SHORT_D16 = 36,
+ GFX10_BUFFER_LOAD_SHORT_D16_HI = 37,
+ GFX10_BUFFER_LOAD_FORMAT_D16_HI_X = 38,
+ GFX10_BUFFER_STORE_FORMAT_D16_HI_X = 39,
+ /* 40-47: reserved */
+ GFX10_BUFFER_ATOMIC_SWAP = 48,
+ GFX10_BUFFER_ATOMIC_CMPSWAP = 49,
+ GFX10_BUFFER_ATOMIC_ADD = 50,
+ GFX10_BUFFER_ATOMIC_SUB = 51,
+ GFX10_BUFFER_ATOMIC_CSUB = 52,
+ GFX10_BUFFER_ATOMIC_SMIN = 53,
+ GFX10_BUFFER_ATOMIC_UMIN = 54, /* was 58 - BUG FIX (ISA p.212) */
+ GFX10_BUFFER_ATOMIC_SMAX = 55,
+ GFX10_BUFFER_ATOMIC_UMAX = 56,
+ GFX10_BUFFER_ATOMIC_AND = 57,
+ GFX10_BUFFER_ATOMIC_OR = 58,
+ GFX10_BUFFER_ATOMIC_XOR = 59,
+ GFX10_BUFFER_ATOMIC_INC = 60,
+ GFX10_BUFFER_ATOMIC_DEC = 61,
+ GFX10_BUFFER_ATOMIC_FCMPSWAP = 62,
+ GFX10_BUFFER_ATOMIC_FMIN = 63,
+ GFX10_BUFFER_ATOMIC_FMAX = 64,
+ /* 65-79: reserved */
+ GFX10_BUFFER_ATOMIC_SWAP_X2 = 80,
+ GFX10_BUFFER_ATOMIC_CMPSWAP_X2 = 81,
+ GFX10_BUFFER_ATOMIC_ADD_X2 = 82,
+ GFX10_BUFFER_ATOMIC_SUB_X2 = 83,
+ /* 84: reserved */
+ GFX10_BUFFER_ATOMIC_SMIN_X2 = 85,
+ GFX10_BUFFER_ATOMIC_UMIN_X2 = 86,
+ GFX10_BUFFER_ATOMIC_SMAX_X2 = 87,
+ GFX10_BUFFER_ATOMIC_UMAX_X2 = 88,
+ GFX10_BUFFER_ATOMIC_AND_X2 = 89,
+ GFX10_BUFFER_ATOMIC_OR_X2 = 90,
+ GFX10_BUFFER_ATOMIC_XOR_X2 = 91,
+ GFX10_BUFFER_ATOMIC_INC_X2 = 92,
+ GFX10_BUFFER_ATOMIC_DEC_X2 = 93,
+ GFX10_BUFFER_ATOMIC_FCMPSWAP_X2 = 94,
+ GFX10_BUFFER_ATOMIC_FMIN_X2 = 95,
+ GFX10_BUFFER_ATOMIC_FMAX_X2 = 96,
+ /* 97-112: reserved */
+ GFX10_BUFFER_GL0_INV = 113,
+ GFX10_BUFFER_GL1_INV = 114,
+ /* 115-127: reserved */
+ GFX10_BUFFER_LOAD_FORMAT_D16_X = 128,
+ GFX10_BUFFER_LOAD_FORMAT_D16_XY = 129,
+ GFX10_BUFFER_LOAD_FORMAT_D16_XYZ = 130,
+ GFX10_BUFFER_LOAD_FORMAT_D16_XYZW = 131,
+ GFX10_BUFFER_STORE_FORMAT_D16_X = 132,
+ GFX10_BUFFER_STORE_FORMAT_D16_XY = 133,
+ GFX10_BUFFER_STORE_FORMAT_D16_XYZ = 134,
+ GFX10_BUFFER_STORE_FORMAT_D16_XYZW = 135,
+};
+
+/* Vector Memory Image Format */
+struct amdgcn_gfx10_mimg {
+};
+
+#define GFX10_FLAT_ENCODING 0x37
+#define GFX10_FLAT_SADDR_SGPR_BASE 0
+#define GFX10_FLAT_SADDR_VCC_LO 106
+#define GFX10_FLAT_SADDR_VCC_HI 107
+#define GFX10_FLAT_SADDR_TTPM_BASE 108
+#define GFX10_FLAT_SADDR_M0 124
+#define GFX10_FLAT_SADDR_NULL 125
+#define GFX10_FLAT_SADDR_EXEC_LO 126
+#define GFX10_FLAT_SADDR_EXEC_HI 127
+#define GFX10_FLAT_SADDR_INTEGER_0 128
+#define GFX10_FLAT_SADDR_INTEGER_MINUS_1 193
+#define GFX10_FLAT_SADDR_SHARED_BASE 235
+#define GFX10_FLAT_SADDR_SHARED_LIMIT 236
+#define GFX10_FLAT_SADDR_PRIVATE_BASE 237
+#define GFX10_FLAT_SADDR_PRIVATE_LIMIT 238
+#define GFX10_FLAT_SADDR_POPS_EXITING_WAVE_ID 239
+#define GFX10_FLAT_SADDR_VCCZ 251
+#define GFX10_FLAT_SADDR_EXECZ 252
+#define GFX10_FLAT_SADDR_SCC 253
+
+/*
+ * Scalar SGPR that provides an offset address. Use NULL for disabled global
+ * addressing; the 0x7f encoding is not accepted by LLVM/RGP for gfx10 global
+ * memory instructions.
+ * Meaning of this field is different for Scratch and Global:
+ * Flat: Unused.
+ * Scratch: Use an SGPR (instead of VGPR) for the address.
+ * Global: Use the SGPR to provide a base address; the VGPR provides a 32-bit
+ * offset per lane.
+ */
+#define GFX10_FLAT_SADDR_DISABLE GFX10_FLAT_SADDR_NULL
+#define GFX10_FLAT_SEG_FLAT 0
+#define GFX10_FLAT_SEG_SCRATCH 1
+#define GFX10_FLAT_SEG_GLOBAL 2
+
+/* Flat Formats */
+struct amdgcn_gfx10_flat {
+ u64 offset:12;
+ u64 dlc:1;
+ u64 lds:1;
+ u64 seg:2;
+ u64 glc:1;
+ u64 slc:1;
+ u64 op:7;
+ u64 dummy1:1;
+ u64 encoding:6;
+ u64 addr:8;
+ u64 data:8;
+ u64 saddr:7;
+ u64 dummy2:1;
+ u64 vdst:8;
+};
+
+enum amdgcn_gfx10_flat_opcode {
+ /* 0-7: reserved */
+ GFX10_FLAT_LOAD_UBYTE = 8,
+ GFX10_FLAT_LOAD_SBYTE = 9,
+ GFX10_FLAT_LOAD_USHORT = 10,
+ GFX10_FLAT_LOAD_SSHORT = 11,
+ GFX10_FLAT_LOAD_DWORD = 12,
+ GFX10_FLAT_LOAD_DWORDX2 = 13,
+ GFX10_FLAT_LOAD_DWORDX4 = 14,
+ GFX10_FLAT_LOAD_DWORDX3 = 15,
+ /* 16-23: reserved */
+ GFX10_FLAT_STORE_BYTE = 24,
+ GFX10_FLAT_STORE_BYTE_D16_HI = 25,
+ GFX10_FLAT_STORE_SHORT = 26,
+ GFX10_FLAT_STORE_SHORT_D16_HI = 27,
+ GFX10_FLAT_STORE_DWORD = 28,
+ GFX10_FLAT_STORE_DWORDX2 = 29,
+ GFX10_FLAT_STORE_DWORDX4 = 30,
+ GFX10_FLAT_STORE_DWORDX3 = 31,
+ GFX10_FLAT_LOAD_UBYTE_D16 = 32,
+ GFX10_FLAT_LOAD_UBYTE_D16_HI = 33,
+ GFX10_FLAT_LOAD_SBYTE_D16 = 34,
+ GFX10_FLAT_LOAD_SBYTE_D16_HI = 35,
+ GFX10_FLAT_LOAD_SHORT_D16 = 36,
+ GFX10_FLAT_LOAD_SHORT_D16_HI = 37,
+ /* 38-47: reserved */
+ GFX10_FLAT_ATOMIC_SWAP = 48,
+ GFX10_FLAT_ATOMIC_CMPSWAP = 49,
+ GFX10_FLAT_ATOMIC_ADD = 50,
+ GFX10_FLAT_ATOMIC_SUB = 51,
+ /* 52: reserved (CSUB is GLOBAL/MUBUF only) */
+ GFX10_FLAT_ATOMIC_SMIN = 53,
+ GFX10_FLAT_ATOMIC_UMIN = 54,
+ GFX10_FLAT_ATOMIC_SMAX = 55,
+ GFX10_FLAT_ATOMIC_UMAX = 56,
+ GFX10_FLAT_ATOMIC_AND = 57,
+ GFX10_FLAT_ATOMIC_OR = 58,
+ GFX10_FLAT_ATOMIC_XOR = 59,
+ GFX10_FLAT_ATOMIC_INC = 60,
+ GFX10_FLAT_ATOMIC_DEC = 61,
+ GFX10_FLAT_ATOMIC_FCMPSWAP = 62,
+ GFX10_FLAT_ATOMIC_FMIN = 63,
+ GFX10_FLAT_ATOMIC_FMAX = 64,
+ /* 65-79: reserved */
+ GFX10_FLAT_ATOMIC_SWAP_X2 = 80,
+ GFX10_FLAT_ATOMIC_CMPSWAP_X2 = 81,
+ GFX10_FLAT_ATOMIC_ADD_X2 = 82,
+ GFX10_FLAT_ATOMIC_SUB_X2 = 83,
+ /* 84: reserved */
+ GFX10_FLAT_ATOMIC_SMIN_X2 = 85,
+ GFX10_FLAT_ATOMIC_UMIN_X2 = 86,
+ GFX10_FLAT_ATOMIC_SMAX_X2 = 87,
+ GFX10_FLAT_ATOMIC_UMAX_X2 = 88,
+ GFX10_FLAT_ATOMIC_AND_X2 = 89,
+ GFX10_FLAT_ATOMIC_OR_X2 = 90,
+ GFX10_FLAT_ATOMIC_XOR_X2 = 91,
+ GFX10_FLAT_ATOMIC_INC_X2 = 92,
+ GFX10_FLAT_ATOMIC_DEC_X2 = 93,
+ GFX10_FLAT_ATOMIC_FCMPSWAP_X2 = 94,
+ GFX10_FLAT_ATOMIC_FMIN_X2 = 95,
+ GFX10_FLAT_ATOMIC_FMAX_X2 = 96,
+};
+
+enum amdgcn_gfx10_global_opcode {
+ /* 0-7: reserved */
+ GFX10_GLOBAL_LOAD_UBYTE = 8,
+ GFX10_GLOBAL_LOAD_SBYTE = 9,
+ GFX10_GLOBAL_LOAD_USHORT = 10,
+ GFX10_GLOBAL_LOAD_SSHORT = 11,
+ GFX10_GLOBAL_LOAD_DWORD = 12,
+ GFX10_GLOBAL_LOAD_DWORDX2 = 13,
+ GFX10_GLOBAL_LOAD_DWORDX4 = 14,
+ GFX10_GLOBAL_LOAD_DWORDX3 = 15,
+ /* 16-21: reserved */
+ GFX10_GLOBAL_LOAD_DWORD_ADDTID = 22,
+ GFX10_GLOBAL_STORE_DWORD_ADDTID = 23,
+ GFX10_GLOBAL_STORE_BYTE = 24,
+ GFX10_GLOBAL_STORE_BYTE_D16_HI = 25,
+ GFX10_GLOBAL_STORE_SHORT = 26,
+ GFX10_GLOBAL_STORE_SHORT_D16_HI = 27,
+ GFX10_GLOBAL_STORE_DWORD = 28,
+ GFX10_GLOBAL_STORE_DWORDX2 = 29,
+ GFX10_GLOBAL_STORE_DWORDX4 = 30,
+ GFX10_GLOBAL_STORE_DWORDX3 = 31,
+ GFX10_GLOBAL_LOAD_UBYTE_D16 = 32,
+ GFX10_GLOBAL_LOAD_UBYTE_D16_HI = 33,
+ GFX10_GLOBAL_LOAD_SBYTE_D16 = 34,
+ GFX10_GLOBAL_LOAD_SBYTE_D16_HI = 35,
+ GFX10_GLOBAL_LOAD_SHORT_D16 = 36,
+ GFX10_GLOBAL_LOAD_SHORT_D16_HI = 37,
+ /* 38-47: reserved */
+ GFX10_GLOBAL_ATOMIC_SWAP = 48,
+ GFX10_GLOBAL_ATOMIC_CMPSWAP = 49,
+ GFX10_GLOBAL_ATOMIC_ADD = 50,
+ GFX10_GLOBAL_ATOMIC_SUB = 51,
+ GFX10_GLOBAL_ATOMIC_CSUB = 52,
+ GFX10_GLOBAL_ATOMIC_SMIN = 53,
+ GFX10_GLOBAL_ATOMIC_UMIN = 54,
+ GFX10_GLOBAL_ATOMIC_SMAX = 55,
+ GFX10_GLOBAL_ATOMIC_UMAX = 56,
+ GFX10_GLOBAL_ATOMIC_AND = 57,
+ GFX10_GLOBAL_ATOMIC_OR = 58,
+ GFX10_GLOBAL_ATOMIC_XOR = 59,
+ GFX10_GLOBAL_ATOMIC_INC = 60,
+ GFX10_GLOBAL_ATOMIC_DEC = 61,
+ GFX10_GLOBAL_ATOMIC_FCMPSWAP = 62,
+ GFX10_GLOBAL_ATOMIC_FMIN = 63,
+ GFX10_GLOBAL_ATOMIC_FMAX = 64,
+ /* 65-79: reserved */
+ GFX10_GLOBAL_ATOMIC_SWAP_X2 = 80,
+ GFX10_GLOBAL_ATOMIC_CMPSWAP_X2 = 81,
+ GFX10_GLOBAL_ATOMIC_ADD_X2 = 82,
+ GFX10_GLOBAL_ATOMIC_SUB_X2 = 83,
+ /* 84: reserved */
+ GFX10_GLOBAL_ATOMIC_SMIN_X2 = 85,
+ GFX10_GLOBAL_ATOMIC_UMIN_X2 = 86,
+ GFX10_GLOBAL_ATOMIC_SMAX_X2 = 87,
+ GFX10_GLOBAL_ATOMIC_UMAX_X2 = 88,
+ GFX10_GLOBAL_ATOMIC_AND_X2 = 89,
+ GFX10_GLOBAL_ATOMIC_OR_X2 = 90,
+ GFX10_GLOBAL_ATOMIC_XOR_X2 = 91,
+ GFX10_GLOBAL_ATOMIC_INC_X2 = 92,
+ GFX10_GLOBAL_ATOMIC_DEC_X2 = 93,
+ GFX10_GLOBAL_ATOMIC_FCMPSWAP_X2 = 94,
+ GFX10_GLOBAL_ATOMIC_FMIN_X2 = 95,
+ GFX10_GLOBAL_ATOMIC_FMAX_X2 = 96,
+};
+
+enum amdgcn_gfx10_scratch_opcode {
+ /* 0-7: reserved */
+ GFX10_SCRATCH_LOAD_UBYTE = 8,
+ GFX10_SCRATCH_LOAD_SBYTE = 9,
+ GFX10_SCRATCH_LOAD_USHORT = 10,
+ GFX10_SCRATCH_LOAD_SSHORT = 11,
+ GFX10_SCRATCH_LOAD_DWORD = 12,
+ GFX10_SCRATCH_LOAD_DWORDX2 = 13,
+ GFX10_SCRATCH_LOAD_DWORDX4 = 14,
+ GFX10_SCRATCH_LOAD_DWORDX3 = 15,
+ /* 16-23: reserved */
+ GFX10_SCRATCH_STORE_BYTE = 24,
+ GFX10_SCRATCH_STORE_BYTE_D16_HI = 25,
+ GFX10_SCRATCH_STORE_SHORT = 26,
+ GFX10_SCRATCH_STORE_SHORT_D16_HI = 27,
+ GFX10_SCRATCH_STORE_DWORD = 28,
+ GFX10_SCRATCH_STORE_DWORDX2 = 29,
+ GFX10_SCRATCH_STORE_DWORDX4 = 30,
+ GFX10_SCRATCH_STORE_DWORDX3 = 31,
+ GFX10_SCRATCH_LOAD_UBYTE_D16 = 32,
+ GFX10_SCRATCH_LOAD_UBYTE_D16_HI = 33,
+ GFX10_SCRATCH_LOAD_SBYTE_D16 = 34,
+ GFX10_SCRATCH_LOAD_SBYTE_D16_HI = 35,
+ GFX10_SCRATCH_LOAD_SHORT_D16 = 36,
+ GFX10_SCRATCH_LOAD_SHORT_D16_HI = 37,
+};
+
+/* Export Format */
+struct amdgcn_gfx10_exp {
+};
+
+union amdgcn_gfx10_insn {
+ struct amdgcn_gfx10_sop2 sop2;
+ struct amdgcn_gfx10_sopk sopk;
+ struct amdgcn_gfx10_sop1 sop1;
+ struct amdgcn_gfx10_sopc sopc;
+ struct amdgcn_gfx10_sopp sopp;
+ struct amdgcn_gfx10_smem smem;
+ struct amdgcn_gfx10_vop2 vop2;
+ struct amdgcn_gfx10_vop1 vop1;
+ struct amdgcn_gfx10_vopc vopc;
+ struct amdgcn_gfx10_vop3a vop3a;
+ struct amdgcn_gfx10_vop3b vop3b;
+ struct amdgcn_gfx10_vop3p vop3p;
+ struct amdgcn_gfx10_sdwa sdwa;
+ struct amdgcn_gfx10_sdwab sdwab;
+ struct amdgcn_gfx10_dpp16 dpp16;
+ struct amdgcn_gfx10_dpp8 dpp8;
+ struct amdgcn_gfx10_vintrp vintrp;
+ struct amdgcn_gfx10_ds ds;
+ struct amdgcn_gfx10_mtbuf mtbuf;
+ struct amdgcn_gfx10_mubuf mubuf;
+ struct amdgcn_gfx10_mimg mimg;
+ struct amdgcn_gfx10_flat flat;
+ struct amdgcn_gfx10_exp exp;
+};
+
+/* Cast macro - convert u32 *buf position to GFX10 insn union pointer. */
+#define I10(buf, n) ((union amdgcn_gfx10_insn *)&(buf)[(n)])
+
+inline u32 gfx10_get_param_base(struct amdgcn_param32 param)
+{
+ return gfx10_param_base[param.type];
+}
+
+inline u32 emit_gfx10_s_load_dwordx2(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src, int offset)
+{
+ /* s_load_dwordx2 <dst/sdata>, <src/sbase>, <offset>
+ * sdata: register, load to.
+ * sbase: sgpr-pair, load from.
+ * offset: offset of kernarg_address in the hsa_kernel_dispatch_packet.
+ *
+ * sdata = *(sbase + offset);
+ * param = (__global struct _knod_bpf_param *)pkt.kernarg_address;
+ */
+
+ insn->smem.sbase = KNOD_AMDGPU_REG_PAIR(src.v);
+ insn->smem.sdata = dst.v;
+ insn->smem.dummy1 = 0;
+ insn->smem.dlc = 0;
+ insn->smem.dummy2 = 0;
+ insn->smem.glc = 0;
+ insn->smem.dummy3 = 0;
+ insn->smem.op = GFX10_S_LOAD_DWORDX2;
+ insn->smem.encoding = GFX10_SMEM_ENCODING;
+ insn->smem.offset = offset;
+ insn->smem.dummy4 = 0;
+ insn->smem.soffset = GFX10_SMEM_SOFFSET_NULL; /* no SGPR offset */
+
+ return 8;
+}
+
+inline u32 emit_gfx10_v_bfe_i32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1,
+ struct amdgcn_param32 src2)
+{
+ WARN_ON(dst.type != AMDGCN_PARAM_TYPE_VGPR);
+ WARN_ON(src1.type == AMDGCN_PARAM_TYPE_LITERAL_CONST ||
+ src2.type == AMDGCN_PARAM_TYPE_LITERAL_CONST);
+ insn->vop3a.vdst = dst.v;
+ insn->vop3a.abs = 0;
+ insn->vop3a.op_sel = 0;
+ insn->vop3a.clmp = 0;
+ insn->vop3a.op = GFX10_V_BFE_I32;
+ insn->vop3a.encoding = GFX10_VOP3A_ENCODING;
+ insn->vop3a.src1 = gfx10_get_param_base(src1) + src1.v;
+ insn->vop3a.src2 = gfx10_get_param_base(src2) + src2.v;
+ insn->vop3a.omod = 0;
+ insn->vop3a.neg = 0;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop3a.src0 = gfx10_get_param_base(src0);
+ insn->vop3a.literal = src0.v;
+ return 12;
+ }
+ insn->vop3a.src0 = gfx10_get_param_base(src0) + src0.v;
+
+ return 8;
+}
+
+inline u32 emit_gfx10_v_bfe_u32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1,
+ struct amdgcn_param32 src2)
+{
+ WARN_ON(dst.type != AMDGCN_PARAM_TYPE_VGPR);
+ WARN_ON(src1.type == AMDGCN_PARAM_TYPE_LITERAL_CONST ||
+ src2.type == AMDGCN_PARAM_TYPE_LITERAL_CONST);
+ insn->vop3a.vdst = dst.v;
+ insn->vop3a.abs = 0;
+ insn->vop3a.op_sel = 0;
+ insn->vop3a.clmp = 0;
+ insn->vop3a.op = GFX10_V_BFE_U32;
+ insn->vop3a.encoding = GFX10_VOP3A_ENCODING;
+ insn->vop3a.src1 = gfx10_get_param_base(src1) + src1.v;
+ insn->vop3a.src2 = gfx10_get_param_base(src2) + src2.v;
+ insn->vop3a.omod = 0;
+ insn->vop3a.neg = 0;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop3a.src0 = gfx10_get_param_base(src0);
+ insn->vop3a.literal = src0.v;
+ return 12;
+ }
+ insn->vop3a.src0 = gfx10_get_param_base(src0) + src0.v;
+
+ return 8;
+}
+
+inline u32 emit_gfx10_v_bfi_b32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1,
+ struct amdgcn_param32 src2)
+{
+ WARN_ON(dst.type != AMDGCN_PARAM_TYPE_VGPR);
+ WARN_ON(src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST ||
+ src1.type == AMDGCN_PARAM_TYPE_LITERAL_CONST ||
+ src2.type == AMDGCN_PARAM_TYPE_LITERAL_CONST);
+ insn->vop3a.vdst = dst.v;
+ insn->vop3a.abs = 0;
+ insn->vop3a.op_sel = 0;
+ insn->vop3a.clmp = 0;
+ insn->vop3a.op = GFX10_V_BFI_B32;
+ insn->vop3a.encoding = GFX10_VOP3A_ENCODING;
+ insn->vop3a.src1 = gfx10_get_param_base(src1) + src1.v;
+ insn->vop3a.src2 = gfx10_get_param_base(src2) + src2.v;
+ insn->vop3a.omod = 0;
+ insn->vop3a.neg = 0;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop3a.src0 = gfx10_get_param_base(src0);
+ insn->vop3a.literal = src0.v;
+ return 12;
+ }
+ insn->vop3a.src0 = gfx10_get_param_base(src0) + src0.v;
+
+ return 8;
+}
+
+inline u32 emit_gfx10_v_lshl_add_u32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1,
+ struct amdgcn_param32 src2)
+{
+ /* v_lshl_add_u32 <dst>, <src0>, <src1>, <src2>
+ *
+ * <dst> = (<src0> << <src1>) + <src2>;
+ */
+
+ insn->vop3a.vdst = dst.v;
+ insn->vop3a.abs = 0;
+ insn->vop3a.op_sel = 0;
+ insn->vop3a.clmp = 0;
+ insn->vop3a.op = GFX10_V_LSHL_ADD_U32;
+ insn->vop3a.encoding = GFX10_VOP3A_ENCODING;
+ insn->vop3a.src1 = gfx10_get_param_base(src1) + src1.v;
+ insn->vop3a.src2 = gfx10_get_param_base(src2) + src2.v;
+ insn->vop3a.omod = 0;
+ insn->vop3a.neg = 0;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop3a.src0 = gfx10_get_param_base(src0);
+ insn->vop3a.literal = src0.v;
+ return 12;
+ }
+ insn->vop3a.src0 = gfx10_get_param_base(src0) + src0.v;
+
+ return 8;
+}
+
+inline u32 emit_gfx10_v_lshl_or_b32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1,
+ struct amdgcn_param32 src2)
+{
+ /* v_lshl_or_b32 <dst>, <src0>, <src1>, <src2>
+ *
+ * D.u = (S0.u << S1.u[4:0]) | S2.u
+ */
+
+ WARN_ON(src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST ||
+ src1.type == AMDGCN_PARAM_TYPE_LITERAL_CONST ||
+ src2.type == AMDGCN_PARAM_TYPE_LITERAL_CONST ||
+ src2.type == AMDGCN_PARAM_TYPE_LITERAL_CONST);
+
+ insn->vop3a.vdst = dst.v;
+ insn->vop3a.abs = 0;
+ insn->vop3a.op_sel = 0;
+ insn->vop3a.clmp = 0;
+ insn->vop3a.op = GFX10_V_LSHL_OR_B32;
+ insn->vop3a.encoding = GFX10_VOP3A_ENCODING;
+ insn->vop3a.src1 = gfx10_get_param_base(src1) + src1.v;
+ insn->vop3a.src2 = gfx10_get_param_base(src2) + src2.v;
+ insn->vop3a.omod = 0;
+ insn->vop3a.neg = 0;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop3a.src0 = gfx10_get_param_base(src0);
+ insn->vop3a.literal = src0.v;
+ return 12;
+ }
+ insn->vop3a.src0 = gfx10_get_param_base(src0) + src0.v;
+
+ return 8;
+}
+
+inline u32 emit_gfx10_v_mad_u64_u32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param64 vdst,
+ struct amdgcn_param32 sdst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1,
+ struct amdgcn_param64 src2)
+{
+ /* v_mad_u64_u32 <vdst>, <sdst>, <src0_imm>, <src1>, <src2>
+ * <vdst>: destination vgpr.
+ * <sdst>: destination sgpr.
+ * <src0>: source vgpr
+ * <src1>: source vgpr
+ * <src2>: source vgpr
+ *
+ * {vcc_out,D.u64} = S0.u32 * S1.u32 + S2.u64.
+ * ctx = ¶m->sub[idx].ctx;
+ */
+ WARN_ON(src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST ||
+ src1.type == AMDGCN_PARAM_TYPE_LITERAL_CONST ||
+ src2.lo.type == AMDGCN_PARAM_TYPE_LITERAL_CONST ||
+ src2.hi.type == AMDGCN_PARAM_TYPE_LITERAL_CONST);
+ insn->vop3b.vdst = vdst.lo.v;
+ insn->vop3b.sdst = sdst.v;
+ insn->vop3b.clmp = 0;
+ insn->vop3b.op = GFX10_V_MAD_U64_U32;
+ insn->vop3b.encoding = GFX10_VOP3B_ENCODING;
+ insn->vop3b.src1 = gfx10_get_param_base(src1) + src1.v;
+ insn->vop3b.src2 = gfx10_get_param_base(src2.lo) + src2.lo.v;
+ insn->vop3b.omod = 0;
+ insn->vop3b.neg = 0;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop3b.src0 = gfx10_get_param_base(src0);
+ insn->vop3b.literal = src0.v;
+ return 12;
+ }
+ insn->vop3b.src0 = gfx10_get_param_base(src0) + src0.v;
+
+ return 8;
+}
+
+inline u32 emit_gfx10_s_mov_b32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst, struct amdgcn_param32 src)
+{
+ if (src.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->sop1.ssrc0 = gfx10_get_param_base(src);
+ insn->sop1.literal = src.v;
+ } else {
+ insn->sop1.ssrc0 = gfx10_get_param_base(src) + src.v;
+ }
+ insn->sop1.op = GFX10_S_MOV_B32;
+ insn->sop1.sdst = gfx10_get_param_base(dst) + dst.v;
+ insn->sop1.encoding = GFX10_SOP1_ENCODING;
+
+ if (src.type == AMDGCN_PARAM_TYPE_LITERAL_CONST)
+ return 8;
+ return 4;
+}
+
+inline u32 emit_gfx10_v_mov_b32_e32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src)
+{
+ if (dst.type != AMDGCN_PARAM_TYPE_VGPR)
+ WARN_ON_ONCE(1);
+ insn->vop1.encoding = GFX10_VOP1_ENCODING;
+ insn->vop1.vdst = dst.v;
+ insn->vop1.op = GFX10_V_MOV_B32;
+ if (src.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop1.src0 = GFX10_VOP1_SRC_LITERAL_CONST;
+ insn->vop1.literal = src.v;
+
+ return 8;
+ }
+ insn->vop1.src0 = gfx10_get_param_base(src) + src.v;
+
+ return 4;
+}
+
+inline u32 emit_gfx10_v_add_co_u32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ insn->vop3b.vdst = dst.v;
+ insn->vop3b.sdst = GFX10_VOP3B_SRC_VCC_LO;
+ insn->vop3b.clmp = 0;
+ insn->vop3b.op = GFX10_V_ADD_CO_U32;
+ insn->vop3b.encoding = GFX10_VOP3B_ENCODING;
+ insn->vop3b.src1 = gfx10_get_param_base(src1) + src1.v;
+ insn->vop3b.src2 = GFX10_VOP3_UNUSED_SRC;
+ insn->vop3b.omod = 0;
+ insn->vop3b.neg = 0;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop3b.src0 = gfx10_get_param_base(src0);
+ insn->vop3b.literal = src0.v;
+ return 12;
+ }
+ insn->vop3b.src0 = gfx10_get_param_base(src0) + src0.v;
+
+ return 8;
+}
+
+inline u32 emit_gfx10_v_add_co_ci_u32_e32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ insn->vop2.vsrc1 = src1.v;
+ insn->vop2.vdst = dst.v;
+ insn->vop2.op = GFX10_V_ADD_CO_CI_U32;
+ insn->vop2.encoding = GFX10_VOP2_ENCODING;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop2.src0 = gfx10_get_param_base(src0);
+ insn->vop2.literal = src0.v;
+ return 8;
+ }
+ insn->vop2.src0 = gfx10_get_param_base(src0) + src0.v;
+ return 4;
+}
+
+inline u32 emit_gfx10_v_xor_b32_e32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ insn->vop2.vsrc1 = src1.v;
+ insn->vop2.vdst = dst.v;
+ insn->vop2.op = GFX10_V_XOR_B32;
+ insn->vop2.encoding = GFX10_VOP2_ENCODING;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop2.src0 = gfx10_get_param_base(src0);
+ insn->vop2.literal = src0.v;
+ return 8;
+ }
+ insn->vop2.src0 = gfx10_get_param_base(src0) + src0.v;
+
+ return 4;
+}
+
+inline u32 emit_gfx10_v_or_b32_e32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ insn->vop2.vsrc1 = src1.v;
+ insn->vop2.vdst = dst.v;
+ insn->vop2.op = GFX10_V_OR_B32;
+ insn->vop2.encoding = GFX10_VOP2_ENCODING;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop2.src0 = gfx10_get_param_base(src0);
+ insn->vop2.literal = src0.v;
+ return 8;
+ }
+ insn->vop2.src0 = gfx10_get_param_base(src0) + src0.v;
+
+ return 4;
+}
+
+inline u32 emit_gfx10_v_cndmask_b32_e32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ insn->vop2.vsrc1 = src1.v;
+ insn->vop2.vdst = dst.v;
+ insn->vop2.op = GFX10_V_CNDMASK_B32;
+ insn->vop2.encoding = GFX10_VOP2_ENCODING;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop2.src0 = gfx10_get_param_base(src0);
+ insn->vop2.literal = src0.v;
+ return 8;
+ }
+ insn->vop2.src0 = gfx10_get_param_base(src0) + src0.v;
+
+ return 4;
+}
+
+inline u32 emit_gfx10_v_and_b32_e32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ insn->vop2.vsrc1 = src1.v;
+ insn->vop2.vdst = dst.v;
+ insn->vop2.op = GFX10_V_AND_B32;
+ insn->vop2.encoding = GFX10_VOP2_ENCODING;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop2.src0 = gfx10_get_param_base(src0);
+ insn->vop2.literal = src0.v;
+ return 8;
+ }
+ insn->vop2.src0 = gfx10_get_param_base(src0) + src0.v;
+
+ return 4;
+}
+
+inline u32 emit_gfx10_v_sub_co_ci_u32_e32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ /* vdst = src0 - vsrc1 - vcc */
+ insn->vop2.vsrc1 = src1.v;
+ insn->vop2.vdst = dst.v;
+ insn->vop2.op = GFX10_V_SUB_CO_CI_U32;
+ insn->vop2.encoding = GFX10_VOP2_ENCODING;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop2.src0 = gfx10_get_param_base(src0);
+ insn->vop2.literal = src0.v;
+ return 8;
+ }
+ insn->vop2.src0 = gfx10_get_param_base(src0) + src0.v;
+ return 4;
+}
+
+inline u32 emit_gfx10_v_subrev_co_ci_u32_e32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ /* vdst = src0 - vsrc1 - vcc */
+ insn->vop2.vsrc1 = src1.v;
+ insn->vop2.vdst = dst.v;
+ insn->vop2.op = GFX10_V_SUBREV_CO_CI_U32;
+ insn->vop2.encoding = GFX10_VOP2_ENCODING;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop2.src0 = gfx10_get_param_base(src0);
+ insn->vop2.literal = src0.v;
+ return 8;
+ }
+ insn->vop2.src0 = gfx10_get_param_base(src0) + src0.v;
+ return 4;
+}
+
+inline u32 emit_gfx10_v_sub_co_u32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ /* dst = src0 - src1 */
+ insn->vop3b.vdst = dst.v;
+ insn->vop3b.sdst = GFX10_VOP3B_SRC_VCC_LO;
+ insn->vop3b.clmp = 0;
+ insn->vop3b.op = GFX10_V_SUB_CO_U32;
+ insn->vop3b.encoding = GFX10_VOP3B_ENCODING;
+ insn->vop3b.src1 = gfx10_get_param_base(src1) + src1.v;
+ insn->vop3b.src2 = GFX10_VOP3_UNUSED_SRC;
+ insn->vop3b.omod = 0;
+ insn->vop3b.neg = 0;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop3b.src0 = gfx10_get_param_base(src0);
+ insn->vop3b.literal = src0.v;
+ return 12;
+ }
+ insn->vop3b.src0 = gfx10_get_param_base(src0) + src0.v;
+
+ return 8;
+}
+
+inline u32 emit_gfx10_v_subrev_co_u32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ /* dst = src0 - src1 */
+ insn->vop3b.vdst = dst.v;
+ insn->vop3b.sdst = GFX10_VOP3B_SRC_VCC_LO;
+ insn->vop3b.clmp = 0;
+ insn->vop3b.op = GFX10_V_SUBREV_CO_U32;
+ insn->vop3b.encoding = GFX10_VOP3B_ENCODING;
+ insn->vop3b.src1 = gfx10_get_param_base(src1) + src1.v;
+ insn->vop3b.src2 = GFX10_VOP3_UNUSED_SRC;
+ insn->vop3b.omod = 0;
+ insn->vop3b.neg = 0;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop3b.src0 = gfx10_get_param_base(src0);
+ insn->vop3b.literal = src0.v;
+ return 12;
+ }
+ insn->vop3b.src0 = gfx10_get_param_base(src0) + src0.v;
+
+ return 8;
+}
+
+inline u32 emit_gfx10_v_mul_lo_u32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ insn->vop3a.vdst = dst.v;
+ insn->vop3a.abs = 0;
+ insn->vop3a.op_sel = 0;
+ insn->vop3a.clmp = 0;
+ insn->vop3a.op = GFX10_V_MUL_LO_U32;
+ insn->vop3a.encoding = GFX10_VOP3A_ENCODING;
+ insn->vop3a.src1 = gfx10_get_param_base(src1) + src1.v;
+ insn->vop3a.src2 = GFX10_VOP3_UNUSED_SRC;
+ insn->vop3a.omod = 0;
+ insn->vop3a.neg = 0;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop3a.src0 = gfx10_get_param_base(src0);
+ insn->vop3a.literal = src0.v;
+ return 12;
+ }
+ insn->vop3a.src0 = gfx10_get_param_base(src0) + src0.v;
+
+ return 8;
+}
+
+/* v_mbcnt_lo_u32_b32 vdst, src0, vsrc1 */
+inline u32 emit_gfx10_v_mbcnt_lo_u32_b32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ insn->vop3a.vdst = dst.v;
+ insn->vop3a.abs = 0;
+ insn->vop3a.op_sel = 0;
+ insn->vop3a.clmp = 0;
+ insn->vop3a.op = GFX10_V_MBCNT_LO_U32_B32;
+ insn->vop3a.encoding = GFX10_VOP3A_ENCODING;
+ insn->vop3a.src0 = gfx10_get_param_base(src0) + src0.v;
+ insn->vop3a.src1 = gfx10_get_param_base(src1) + src1.v;
+ insn->vop3a.src2 = GFX10_VOP3_UNUSED_SRC;
+ insn->vop3a.omod = 0;
+ insn->vop3a.neg = 0;
+
+ return 8;
+}
+
+/* v_mbcnt_hi_u32_b32 vdst, src0, vsrc1 */
+inline u32 emit_gfx10_v_mbcnt_hi_u32_b32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ insn->vop3a.vdst = dst.v;
+ insn->vop3a.abs = 0;
+ insn->vop3a.op_sel = 0;
+ insn->vop3a.clmp = 0;
+ insn->vop3a.op = GFX10_V_MBCNT_HI_U32_B32;
+ insn->vop3a.encoding = GFX10_VOP3A_ENCODING;
+ insn->vop3a.src0 = gfx10_get_param_base(src0) + src0.v;
+ insn->vop3a.src1 = gfx10_get_param_base(src1) + src1.v;
+ insn->vop3a.src2 = GFX10_VOP3_UNUSED_SRC;
+ insn->vop3a.omod = 0;
+ insn->vop3a.neg = 0;
+
+ return 8;
+}
+
+inline u32 emit_gfx10_v_mul_hi_u32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ insn->vop3a.vdst = dst.v;
+ insn->vop3a.abs = 0;
+ insn->vop3a.op_sel = 0;
+ insn->vop3a.clmp = 0;
+ insn->vop3a.op = GFX10_V_MUL_HI_U32;
+ insn->vop3a.encoding = GFX10_VOP3A_ENCODING;
+ insn->vop3a.src1 = gfx10_get_param_base(src1) + src1.v;
+ insn->vop3a.src2 = GFX10_VOP3_UNUSED_SRC;
+ insn->vop3a.omod = 0;
+ insn->vop3a.neg = 0;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop3a.src0 = gfx10_get_param_base(src0);
+ insn->vop3a.literal = src0.v;
+ return 12;
+ }
+ insn->vop3a.src0 = gfx10_get_param_base(src0) + src0.v;
+
+ return 8;
+}
+
+inline u32 emit_gfx10_v_lshlrev_b64(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src0,
+ struct amdgcn_param64 src1)
+{
+ /* dst = s1 << s0 */
+ insn->vop3a.vdst = dst.lo.v;
+ insn->vop3a.abs = 0;
+ insn->vop3a.op_sel = 0;
+ insn->vop3a.clmp = 0;
+ insn->vop3a.op = GFX10_V_LSHLREV_B64;
+ insn->vop3a.encoding = GFX10_VOP3A_ENCODING;
+ insn->vop3a.src1 = gfx10_get_param_base(src1.lo) + src1.lo.v;
+ insn->vop3a.src2 = GFX10_VOP3_UNUSED_SRC;
+ insn->vop3a.omod = 0;
+ insn->vop3a.neg = 0;
+ if (src0.lo.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop3a.src0 = gfx10_get_param_base(src0.lo);
+ insn->vop3a.literal = src0.lo.v;
+ return 12;
+ }
+ insn->vop3a.src0 = gfx10_get_param_base(src0.lo) + src0.lo.v;
+
+ return 8;
+}
+
+inline u32 emit_gfx10_v_lshrrev_b64(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src0,
+ struct amdgcn_param64 src1)
+{
+ /* dst = s1 >> s0 */
+ insn->vop3a.vdst = dst.lo.v;
+ insn->vop3a.abs = 0;
+ insn->vop3a.op_sel = 0;
+ insn->vop3a.clmp = 0;
+ insn->vop3a.op = GFX10_V_LSHRREV_B64;
+ insn->vop3a.encoding = GFX10_VOP3A_ENCODING;
+ insn->vop3a.src1 = gfx10_get_param_base(src1.lo) + src1.lo.v;
+ insn->vop3a.src2 = GFX10_VOP3_UNUSED_SRC;
+ insn->vop3a.omod = 0;
+ insn->vop3a.neg = 0;
+ if (src0.lo.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop3a.src0 = gfx10_get_param_base(src0.lo);
+ insn->vop3a.literal = src0.lo.v;
+ return 12;
+ }
+ insn->vop3a.src0 = gfx10_get_param_base(src0.lo) + src0.lo.v;
+
+ return 8;
+}
+
+inline u32 emit_gfx10_v_lshlrev_b32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ /* dst = s1 << s0 */
+ WARN_ON(dst.type != AMDGCN_PARAM_TYPE_VGPR);
+ insn->vop2.vsrc1 = src1.v;
+ insn->vop2.vdst = dst.v;
+ insn->vop2.op = GFX10_V_LSHLREV_B32;
+ insn->vop2.encoding = GFX10_VOP2_ENCODING;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop2.src0 = gfx10_get_param_base(src0);
+ insn->vop2.literal = src0.v;
+ return 8;
+ }
+ insn->vop2.src0 = gfx10_get_param_base(src0) + src0.v;
+
+ return 4;
+}
+
+inline u32 emit_gfx10_v_lshrrev_b32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ /* dst = s1 << s0 */
+ WARN_ON(dst.type != AMDGCN_PARAM_TYPE_VGPR);
+ insn->vop2.vsrc1 = src1.v;
+ insn->vop2.vdst = dst.v;
+ insn->vop2.op = GFX10_V_LSHRREV_B32;
+ insn->vop2.encoding = GFX10_VOP2_ENCODING;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop2.src0 = gfx10_get_param_base(src0);
+ insn->vop2.literal = src0.v;
+ return 8;
+ }
+ insn->vop2.src0 = gfx10_get_param_base(src0) + src0.v;
+
+ return 4;
+}
+
+inline u32 emit_gfx10_v_add_nc_u32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ insn->vop2.vsrc1 = src1.v;
+ insn->vop2.vdst = dst.v;
+ insn->vop2.op = GFX10_V_ADD_NC_U32;
+ insn->vop2.encoding = GFX10_VOP2_ENCODING;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop2.src0 = gfx10_get_param_base(src0);
+ insn->vop2.literal = src0.v;
+ return 8;
+ }
+ insn->vop2.src0 = gfx10_get_param_base(src0) + src0.v;
+ return 4;
+}
+
+inline u32 emit_gfx10_v_sub_nc_u32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ insn->vop2.vsrc1 = src1.v;
+ insn->vop2.vdst = dst.v;
+ insn->vop2.op = GFX10_V_SUB_NC_U32;
+ insn->vop2.encoding = GFX10_VOP2_ENCODING;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop2.src0 = gfx10_get_param_base(src0);
+ insn->vop2.literal = src0.v;
+ return 8;
+ }
+ insn->vop2.src0 = gfx10_get_param_base(src0) + src0.v;
+ return 4;
+}
+
+inline u32 emit_gfx10_v_ashrrev_i64(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src0,
+ struct amdgcn_param64 src1)
+{
+ /* dst = s1 >> s0 */
+ insn->vop3a.vdst = dst.lo.v;
+ insn->vop3a.abs = 0;
+ insn->vop3a.op_sel = 0;
+ insn->vop3a.clmp = 0;
+ insn->vop3a.op = GFX10_V_ASHRREV_I64;
+ insn->vop3a.encoding = GFX10_VOP3A_ENCODING;
+ insn->vop3a.src1 = gfx10_get_param_base(src1.lo) + src1.lo.v;
+ insn->vop3a.src2 = GFX10_VOP3_UNUSED_SRC;
+ insn->vop3a.omod = 0;
+ insn->vop3a.neg = 0;
+ if (src0.lo.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop3a.src0 = gfx10_get_param_base(src0.lo);
+ insn->vop3a.literal = src0.lo.v;
+ return 12;
+ }
+ insn->vop3a.src0 = gfx10_get_param_base(src0.lo) + src0.lo.v;
+
+ return 8;
+}
+
+inline u32 emit_gfx10_v_ashrrev_i32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ /* dst = s1 >> s0 (arithmetic) */
+ WARN_ON(dst.type != AMDGCN_PARAM_TYPE_VGPR);
+ insn->vop2.vsrc1 = src1.v;
+ insn->vop2.vdst = dst.v;
+ insn->vop2.op = GFX10_V_ASHRREV_I32;
+ insn->vop2.encoding = GFX10_VOP2_ENCODING;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop2.src0 = gfx10_get_param_base(src0);
+ insn->vop2.literal = src0.v;
+ return 8;
+ }
+ insn->vop2.src0 = gfx10_get_param_base(src0) + src0.v;
+
+ return 4;
+}
+
+inline u32 emit_gfx10_v_alignbit_b32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1,
+ struct amdgcn_param32 src2)
+{
+ /* dst = s1 >> s0 */
+ WARN_ON(dst.type != AMDGCN_PARAM_TYPE_VGPR);
+ insn->vop3a.vdst = dst.v;
+ insn->vop3a.abs = 0;
+ insn->vop3a.op_sel = 0;
+ insn->vop3a.clmp = 0;
+ insn->vop3a.op = GFX10_V_ALIGNBIT_B32;
+ insn->vop3a.encoding = GFX10_VOP3A_ENCODING;
+ insn->vop3a.src1 = gfx10_get_param_base(src1) + src1.v;
+ insn->vop3a.src2 = gfx10_get_param_base(src2) + src2.v;
+ insn->vop3a.omod = 0;
+ insn->vop3a.neg = 0;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop3a.src0 = gfx10_get_param_base(src0);
+ insn->vop3a.literal = src0.v;
+ return 12;
+ }
+ insn->vop3a.src0 = gfx10_get_param_base(src0) + src0.v;
+
+ return 8;
+}
+
+inline u32 emit_gfx10_v_cmp_eq_u64(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src)
+{
+ /* VCC = S0 == S1 */
+ insn->vopc.src0 = gfx10_get_param_base(dst) + dst.v;
+ insn->vopc.vsrc1 = src.v;
+ insn->vopc.op = GFX10_V_CMP_EQ_U64;
+ insn->vopc.encoding = GFX10_VOPC_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx10_v_cmp_eq_u32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src)
+{
+ /* VCC = S0 == S1 */
+ insn->vopc.src0 = gfx10_get_param_base(dst) + dst.v;
+ insn->vopc.vsrc1 = src.v;
+ insn->vopc.op = GFX10_V_CMP_EQ_U32;
+ insn->vopc.encoding = GFX10_VOPC_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx10_v_cmp_gt_u64(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src)
+{
+ /* VCC = S0 > S1 */
+ insn->vopc.src0 = gfx10_get_param_base(dst.lo) + dst.lo.v;
+ insn->vopc.vsrc1 = src.lo.v;
+ insn->vopc.op = GFX10_V_CMP_GT_U64;
+ insn->vopc.encoding = GFX10_VOPC_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx10_v_cmp_gt_i64(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src)
+{
+ /* VCC = S0 > S1 (signed) */
+ insn->vopc.src0 = gfx10_get_param_base(dst.lo) + dst.lo.v;
+ insn->vopc.vsrc1 = src.lo.v;
+ insn->vopc.op = GFX10_V_CMP_GT_I64;
+ insn->vopc.encoding = GFX10_VOPC_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx10_v_cmp_ge_u32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src)
+{
+ /* VCC = S0 >= S1 */
+ insn->vopc.src0 = gfx10_get_param_base(dst) + dst.v;
+ insn->vopc.vsrc1 = src.v;
+ insn->vopc.op = GFX10_V_CMP_GE_U32;
+ insn->vopc.encoding = GFX10_VOPC_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx10_v_cmp_gt_u32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src)
+{
+ /* VCC = S0 > S1 */
+ insn->vopc.src0 = gfx10_get_param_base(dst) + dst.v;
+ insn->vopc.vsrc1 = src.v;
+ insn->vopc.op = GFX10_V_CMP_GT_U32;
+ insn->vopc.encoding = GFX10_VOPC_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx10_v_cmp_lt_u32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src)
+{
+ /* VCC = S0 < S1 */
+ insn->vopc.src0 = gfx10_get_param_base(dst) + dst.v;
+ insn->vopc.vsrc1 = src.v;
+ insn->vopc.op = GFX10_V_CMP_LT_U32;
+ insn->vopc.encoding = GFX10_VOPC_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx10_v_cmp_le_u32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src)
+{
+ /* VCC = S0 <= S1 */
+ insn->vopc.src0 = gfx10_get_param_base(dst) + dst.v;
+ insn->vopc.vsrc1 = src.v;
+ insn->vopc.op = GFX10_V_CMP_LE_U32;
+ insn->vopc.encoding = GFX10_VOPC_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx10_v_cmp_gt_i32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src)
+{
+ /* VCC = S0 > S1 (signed) */
+ insn->vopc.src0 = gfx10_get_param_base(dst) + dst.v;
+ insn->vopc.vsrc1 = src.v;
+ insn->vopc.op = GFX10_V_CMP_GT_I32;
+ insn->vopc.encoding = GFX10_VOPC_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx10_v_cmp_ge_i32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src)
+{
+ /* VCC = S0 >= S1 (signed) */
+ insn->vopc.src0 = gfx10_get_param_base(dst) + dst.v;
+ insn->vopc.vsrc1 = src.v;
+ insn->vopc.op = GFX10_V_CMP_GE_I32;
+ insn->vopc.encoding = GFX10_VOPC_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx10_v_cmp_lt_i32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src)
+{
+ /* VCC = S0 < S1 (signed) */
+ insn->vopc.src0 = gfx10_get_param_base(dst) + dst.v;
+ insn->vopc.vsrc1 = src.v;
+ insn->vopc.op = GFX10_V_CMP_LT_I32;
+ insn->vopc.encoding = GFX10_VOPC_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx10_v_cmp_le_i32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src)
+{
+ /* VCC = S0 <= S1 (signed) */
+ insn->vopc.src0 = gfx10_get_param_base(dst) + dst.v;
+ insn->vopc.vsrc1 = src.v;
+ insn->vopc.op = GFX10_V_CMP_LE_I32;
+ insn->vopc.encoding = GFX10_VOPC_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx10_v_cmpx_lt_u32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src)
+{
+ /* EXEC &= (S0 < S1) */
+ insn->vopc.src0 = gfx10_get_param_base(dst) + dst.v;
+ insn->vopc.vsrc1 = src.v;
+ insn->vopc.op = GFX10_V_CMPX_LT_U32;
+ insn->vopc.encoding = GFX10_VOPC_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx10_v_cmp_ge_u64(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src)
+{
+ /* VCC = S0 >= S1 */
+ insn->vopc.src0 = gfx10_get_param_base(dst.lo) + dst.lo.v;
+ insn->vopc.vsrc1 = src.lo.v;
+ insn->vopc.op = GFX10_V_CMP_GE_U64;
+ insn->vopc.encoding = GFX10_VOPC_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx10_v_cmp_ge_i64(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src)
+{
+ /* VCC = S0 >= S1 (signed) */
+ insn->vopc.src0 = gfx10_get_param_base(dst.lo) + dst.lo.v;
+ insn->vopc.vsrc1 = src.lo.v;
+ insn->vopc.op = GFX10_V_CMP_GE_I64;
+ insn->vopc.encoding = GFX10_VOPC_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx10_v_cmp_lt_u64(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src)
+{
+ /* VCC = S0 < S1 */
+ insn->vopc.src0 = gfx10_get_param_base(dst.lo) + dst.lo.v;
+ insn->vopc.vsrc1 = src.lo.v;
+ insn->vopc.op = GFX10_V_CMP_LT_U64;
+ insn->vopc.encoding = GFX10_VOPC_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx10_v_cmp_lt_i64(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src)
+{
+ /* VCC = S0 < S1 (signed) */
+ insn->vopc.src0 = gfx10_get_param_base(dst.lo) + dst.lo.v;
+ insn->vopc.vsrc1 = src.lo.v;
+ insn->vopc.op = GFX10_V_CMP_LT_I64;
+ insn->vopc.encoding = GFX10_VOPC_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx10_v_cmp_le_u64(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src)
+{
+ /* VCC = S0 <= S1 */
+ insn->vopc.src0 = gfx10_get_param_base(dst.lo) + dst.lo.v;
+ insn->vopc.vsrc1 = src.lo.v;
+ insn->vopc.op = GFX10_V_CMP_LE_U64;
+ insn->vopc.encoding = GFX10_VOPC_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx10_v_cmp_le_i64(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src)
+{
+ /* VCC = S0 <= S1 (signed) */
+ insn->vopc.src0 = gfx10_get_param_base(dst.lo) + dst.lo.v;
+ insn->vopc.vsrc1 = src.lo.v;
+ insn->vopc.op = GFX10_V_CMP_LE_I64;
+ insn->vopc.encoding = GFX10_VOPC_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx10_buffer_load_ubyte(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src,
+ short off)
+{
+ /* buffer_load_ubyte <dst>, <src>, s[0:3], 0 offen offset:<off> */
+ insn->mubuf.offset = off;
+ insn->mubuf.offen = 1;
+ insn->mubuf.idxen = 0;
+ insn->mubuf.glc = 0;
+ insn->mubuf.dlc = 0;
+ insn->mubuf.lds = 0;
+ insn->mubuf.dummy1 = 0;
+ insn->mubuf.op = GFX10_BUFFER_LOAD_UBYTE;
+ insn->mubuf.opm = 0;
+ insn->mubuf.encoding = GFX10_MUBUF_ENCODING;
+ insn->mubuf.vaddr = src.v;
+ insn->mubuf.vdata = dst.v;
+ insn->mubuf.srsrc = 0; /* s[0:3] */
+ insn->mubuf.dummy2 = 0;
+ insn->mubuf.slc = 0;
+ insn->mubuf.tfe = 0;
+ insn->mubuf.soffset = GFX10_MUBUF_SOFFSET_INTEGER_0;
+
+ return 8;
+}
+
+inline u32 emit_gfx10_buffer_load_ushort(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src,
+ short off)
+{
+ /* buffer_load_ushort <dst>, <src>, s[0:3], 0 offen offset:<off> */
+ insn->mubuf.offset = off;
+ insn->mubuf.offen = 1;
+ insn->mubuf.idxen = 0;
+ insn->mubuf.glc = 0;
+ insn->mubuf.dlc = 0;
+ insn->mubuf.lds = 0;
+ insn->mubuf.dummy1 = 0;
+ insn->mubuf.op = GFX10_BUFFER_LOAD_USHORT;
+ insn->mubuf.opm = 0;
+ insn->mubuf.encoding = GFX10_MUBUF_ENCODING;
+ insn->mubuf.vaddr = src.v;
+ insn->mubuf.vdata = dst.v;
+ insn->mubuf.srsrc = 0; /* s[0:3] */
+ insn->mubuf.dummy2 = 0;
+ insn->mubuf.slc = 0;
+ insn->mubuf.tfe = 0;
+ insn->mubuf.soffset = GFX10_MUBUF_SOFFSET_INTEGER_0;
+
+ return 8;
+}
+
+inline u32 emit_gfx10_buffer_load_dword(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src,
+ short off)
+{
+ /* buffer_load_dword <dst>, <src>, s[0:3], 0 offen offset:<off> */
+ insn->mubuf.offset = off;
+ insn->mubuf.offen = 1;
+ insn->mubuf.idxen = 0;
+ insn->mubuf.glc = 0;
+ insn->mubuf.dlc = 0;
+ insn->mubuf.lds = 0;
+ insn->mubuf.dummy1 = 0;
+ insn->mubuf.op = GFX10_BUFFER_LOAD_DWORD;
+ insn->mubuf.opm = 0;
+ insn->mubuf.encoding = GFX10_MUBUF_ENCODING;
+ insn->mubuf.vaddr = src.v;
+ insn->mubuf.vdata = dst.v;
+ insn->mubuf.srsrc = 0; /* s[0:3] */
+ insn->mubuf.dummy2 = 0;
+ insn->mubuf.slc = 0;
+ insn->mubuf.tfe = 0;
+ insn->mubuf.soffset = GFX10_MUBUF_SOFFSET_INTEGER_0;
+
+ return 8;
+}
+
+inline u32 emit_gfx10_buffer_load_dwordx2(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src,
+ short off)
+{
+ /* buffer_load_dwordx2 <dst>, <src>, s[0:3], 0 offen offset:<off> */
+ insn->mubuf.offset = off;
+ insn->mubuf.offen = 1;
+ insn->mubuf.idxen = 0;
+ insn->mubuf.glc = 0;
+ insn->mubuf.dlc = 0;
+ insn->mubuf.lds = 0;
+ insn->mubuf.dummy1 = 0;
+ insn->mubuf.op = GFX10_BUFFER_LOAD_DWORDX2;
+ insn->mubuf.opm = 0;
+ insn->mubuf.encoding = GFX10_MUBUF_ENCODING;
+ insn->mubuf.vaddr = src.v;
+ insn->mubuf.vdata = dst.v;
+ insn->mubuf.srsrc = 0; /* s[0:3] */
+ insn->mubuf.dummy2 = 0;
+ insn->mubuf.slc = 0;
+ insn->mubuf.tfe = 0;
+ insn->mubuf.soffset = GFX10_MUBUF_SOFFSET_INTEGER_0;
+
+ return 8;
+}
+
+inline u32 emit_gfx10_buffer_load_dwordx4(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src,
+ short off)
+{
+ /* buffer_load_dwordx4 <dst>, <src>, s[0:3], 0 offen offset:<off> */
+ insn->mubuf.offset = off;
+ insn->mubuf.offen = 1;
+ insn->mubuf.idxen = 0;
+ insn->mubuf.glc = 0;
+ insn->mubuf.dlc = 0;
+ insn->mubuf.lds = 0;
+ insn->mubuf.dummy1 = 0;
+ insn->mubuf.op = GFX10_BUFFER_LOAD_DWORDX4;
+ insn->mubuf.opm = 0;
+ insn->mubuf.encoding = GFX10_MUBUF_ENCODING;
+ insn->mubuf.vaddr = src.v;
+ insn->mubuf.vdata = dst.v;
+ insn->mubuf.srsrc = 0; /* s[0:3] */
+ insn->mubuf.dummy2 = 0;
+ insn->mubuf.slc = 0;
+ insn->mubuf.tfe = 0;
+ insn->mubuf.soffset = GFX10_MUBUF_SOFFSET_INTEGER_0;
+
+ return 8;
+}
+
+inline u32 emit_gfx10_global_load_ubyte(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src,
+ short off)
+{
+ /* global_load_ubyte <dst>, <srcs>, off offset:<off> */
+ insn->flat.offset = off;
+ insn->flat.dlc = 0;
+ insn->flat.lds = 0;
+ insn->flat.seg = GFX10_FLAT_SEG_GLOBAL;
+ insn->flat.glc = 0;
+ insn->flat.slc = 0;
+ insn->flat.op = GFX10_GLOBAL_LOAD_UBYTE;
+ insn->flat.encoding = GFX10_FLAT_ENCODING;
+ insn->flat.addr = src.v;
+ insn->flat.data = 0;
+ insn->flat.saddr = GFX10_FLAT_SADDR_DISABLE;
+ insn->flat.dummy1 = 0;
+ insn->flat.dummy2 = 0;
+ insn->flat.vdst = dst.v;
+
+ return 8;
+}
+
+inline u32 emit_gfx10_global_load_ushort(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src,
+ short off)
+{
+ /* global_load_ushort <dst>, <srcs>, off offset:<off> */
+ insn->flat.offset = off;
+ insn->flat.dlc = 0;
+ insn->flat.lds = 0;
+ insn->flat.seg = GFX10_FLAT_SEG_GLOBAL;
+ insn->flat.glc = 0;
+ insn->flat.slc = 0;
+ insn->flat.op = GFX10_GLOBAL_LOAD_USHORT;
+ insn->flat.encoding = GFX10_FLAT_ENCODING;
+ insn->flat.addr = src.v;
+ insn->flat.data = 0;
+ insn->flat.saddr = GFX10_FLAT_SADDR_DISABLE;
+ insn->flat.dummy1 = 0;
+ insn->flat.dummy2 = 0;
+ insn->flat.vdst = dst.v;
+
+ return 8;
+}
+
+inline u32 emit_gfx10_global_load_dword(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src,
+ short off)
+{
+ /* global_load_dword <dst>, <srcs>, off offset:<off> */
+ insn->flat.offset = off;
+ insn->flat.dlc = 0;
+ insn->flat.lds = 0;
+ insn->flat.seg = GFX10_FLAT_SEG_GLOBAL;
+ insn->flat.glc = 0;
+ insn->flat.slc = 0;
+ insn->flat.op = GFX10_GLOBAL_LOAD_DWORD;
+ insn->flat.encoding = GFX10_FLAT_ENCODING;
+ insn->flat.addr = src.v;
+ insn->flat.data = 0;
+ insn->flat.saddr = GFX10_FLAT_SADDR_DISABLE;
+ insn->flat.dummy1 = 0;
+ insn->flat.dummy2 = 0;
+ insn->flat.vdst = dst.v;
+
+ return 8;
+}
+
+inline u32 emit_gfx10_global_load_dwordx2(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src,
+ short off)
+{
+ /* global_load_dwordx2 <dst>, <srcs>, off offset:<off> */
+ insn->flat.offset = off;
+ insn->flat.dlc = 0;
+ insn->flat.lds = 0;
+ insn->flat.seg = GFX10_FLAT_SEG_GLOBAL;
+ insn->flat.glc = 0;
+ insn->flat.slc = 0;
+ insn->flat.op = GFX10_GLOBAL_LOAD_DWORDX2;
+ insn->flat.encoding = GFX10_FLAT_ENCODING;
+ insn->flat.addr = src.v;
+ insn->flat.data = 0;
+ insn->flat.saddr = GFX10_FLAT_SADDR_DISABLE;
+ insn->flat.dummy1 = 0;
+ insn->flat.dummy2 = 0;
+ insn->flat.vdst = dst.v;
+
+ return 8;
+}
+
+inline u32 emit_gfx10_global_load_dwordx4(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src,
+ short off)
+{
+ /* global_load_dwordx4 <dst>, <srcs>, off offset:<off> */
+ insn->flat.offset = off;
+ insn->flat.dlc = 0;
+ insn->flat.lds = 0;
+ insn->flat.seg = GFX10_FLAT_SEG_GLOBAL;
+ insn->flat.glc = 0;
+ insn->flat.slc = 0;
+ insn->flat.op = GFX10_GLOBAL_LOAD_DWORDX4;
+ insn->flat.encoding = GFX10_FLAT_ENCODING;
+ insn->flat.addr = src.v;
+ insn->flat.data = 0;
+ insn->flat.saddr = GFX10_FLAT_SADDR_DISABLE;
+ insn->flat.dummy1 = 0;
+ insn->flat.dummy2 = 0;
+ insn->flat.vdst = dst.v;
+
+ return 8;
+}
+
+inline u32 emit_gfx10_global_store_byte(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src, int off)
+{
+ /* global_store_byte <src>, <dst>, off offset:<off>
+ * *(char *)(dst + off) = src;
+ */
+ insn->flat.offset = off;
+ insn->flat.dlc = 0;
+ insn->flat.lds = 0;
+ insn->flat.seg = GFX10_FLAT_SEG_GLOBAL;
+ insn->flat.glc = 1;
+ insn->flat.slc = 1;
+ insn->flat.op = GFX10_GLOBAL_STORE_BYTE;
+ insn->flat.encoding = GFX10_FLAT_ENCODING;
+ insn->flat.addr = dst.v;
+ insn->flat.data = src.v;
+ insn->flat.saddr = GFX10_FLAT_SADDR_DISABLE;
+ insn->flat.dummy1 = 0;
+ insn->flat.dummy2 = 0;
+ insn->flat.vdst = 0;
+
+ return 8;
+}
+
+inline u32 emit_gfx10_global_store_short(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src, int off)
+{
+ /* global_store_short <src>, <dst>, off offset:<off>
+ * *(char *)(dst + off) = src;
+ */
+ insn->flat.offset = off;
+ insn->flat.dlc = 0;
+ insn->flat.lds = 0;
+ insn->flat.seg = GFX10_FLAT_SEG_GLOBAL;
+ insn->flat.glc = 1;
+ insn->flat.slc = 1;
+ insn->flat.op = GFX10_GLOBAL_STORE_SHORT;
+ insn->flat.encoding = GFX10_FLAT_ENCODING;
+ insn->flat.addr = dst.v;
+ insn->flat.data = src.v;
+ insn->flat.saddr = GFX10_FLAT_SADDR_DISABLE;
+ insn->flat.dummy1 = 0;
+ insn->flat.dummy2 = 0;
+ insn->flat.vdst = 0;
+
+ return 8;
+}
+
+inline u32 emit_gfx10_global_store_dword(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src, int off)
+{
+ /* global_store_dword <src>, <dst>, off offset:<off>
+ * *(char *)(dst + off) = src;
+ */
+ insn->flat.offset = off;
+ insn->flat.dlc = 0;
+ insn->flat.lds = 0;
+ insn->flat.seg = GFX10_FLAT_SEG_GLOBAL;
+ insn->flat.glc = 1;
+ insn->flat.slc = 1;
+ insn->flat.op = GFX10_GLOBAL_STORE_DWORD;
+ insn->flat.encoding = GFX10_FLAT_ENCODING;
+ insn->flat.addr = dst.v;
+ insn->flat.data = src.v;
+ insn->flat.saddr = GFX10_FLAT_SADDR_DISABLE;
+ insn->flat.dummy1 = 0;
+ insn->flat.dummy2 = 0;
+ insn->flat.vdst = 0;
+
+ return 8;
+}
+
+/* global_atomic_add vdst, addr, data, off (GLC=1: return old value)
+ * old_val = *(u32 *)(addr + off); *(u32 *)(addr + off) += data; vdst = old_val
+ */
+/* GFX10 global atomic: opcode only differs, all else identical */
+#define DEFINE_GFX10_GLOBAL_ATOMIC(name, opcode) \
+inline u32 emit_gfx10_global_atomic_##name(union amdgcn_gfx10_insn *insn,\
+ struct amdgcn_param32 vdst, \
+ struct amdgcn_param32 addr, \
+ struct amdgcn_param32 data, \
+ int off, int glc) \
+{ \
+ insn->flat.offset = off; \
+ insn->flat.dlc = 0; \
+ insn->flat.lds = 0; \
+ insn->flat.seg = GFX10_FLAT_SEG_GLOBAL; \
+ insn->flat.glc = glc; \
+ insn->flat.slc = 0; \
+ insn->flat.op = (opcode); \
+ insn->flat.encoding = GFX10_FLAT_ENCODING; \
+ insn->flat.addr = addr.v; \
+ insn->flat.data = data.v; \
+ insn->flat.saddr = GFX10_FLAT_SADDR_DISABLE; \
+ insn->flat.dummy1 = 0; \
+ insn->flat.dummy2 = 0; \
+ insn->flat.vdst = vdst.v; \
+ return 8; \
+}
+
+DEFINE_GFX10_GLOBAL_ATOMIC(add, GFX10_GLOBAL_ATOMIC_ADD)
+DEFINE_GFX10_GLOBAL_ATOMIC(and, GFX10_GLOBAL_ATOMIC_AND)
+DEFINE_GFX10_GLOBAL_ATOMIC(or, GFX10_GLOBAL_ATOMIC_OR)
+DEFINE_GFX10_GLOBAL_ATOMIC(xor, GFX10_GLOBAL_ATOMIC_XOR)
+DEFINE_GFX10_GLOBAL_ATOMIC(swap, GFX10_GLOBAL_ATOMIC_SWAP)
+DEFINE_GFX10_GLOBAL_ATOMIC(cmpswap, GFX10_GLOBAL_ATOMIC_CMPSWAP)
+DEFINE_GFX10_GLOBAL_ATOMIC(add_x2, GFX10_GLOBAL_ATOMIC_ADD_X2)
+DEFINE_GFX10_GLOBAL_ATOMIC(and_x2, GFX10_GLOBAL_ATOMIC_AND_X2)
+DEFINE_GFX10_GLOBAL_ATOMIC(or_x2, GFX10_GLOBAL_ATOMIC_OR_X2)
+DEFINE_GFX10_GLOBAL_ATOMIC(xor_x2, GFX10_GLOBAL_ATOMIC_XOR_X2)
+DEFINE_GFX10_GLOBAL_ATOMIC(swap_x2, GFX10_GLOBAL_ATOMIC_SWAP_X2)
+DEFINE_GFX10_GLOBAL_ATOMIC(cmpswap_x2, GFX10_GLOBAL_ATOMIC_CMPSWAP_X2)
+
+inline u32 emit_gfx10_global_store_dwordx2(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src, int off)
+{
+ /* global_store_dwordx2 <src>, <dst>, off offset:<off>
+ * *(char *)(dst + off) = src;
+ */
+ insn->flat.offset = off;
+ insn->flat.dlc = 0;
+ insn->flat.lds = 0;
+ insn->flat.seg = GFX10_FLAT_SEG_GLOBAL;
+ insn->flat.glc = 1;
+ insn->flat.slc = 1;
+ insn->flat.op = GFX10_GLOBAL_STORE_DWORDX2;
+ insn->flat.encoding = GFX10_FLAT_ENCODING;
+ insn->flat.addr = dst.v;
+ insn->flat.data = src.v;
+ insn->flat.saddr = GFX10_FLAT_SADDR_DISABLE;
+ insn->flat.dummy1 = 0;
+ insn->flat.dummy2 = 0;
+ insn->flat.vdst = 0;
+
+ return 8;
+}
+
+inline u32 emit_gfx10_global_store_dwordx4(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src, int off)
+{
+ /* global_store_dwordx4 <src>, <dst>, off offset:<off>
+ * *(char *)(dst + off) = src;
+ */
+ insn->flat.offset = off;
+ insn->flat.dlc = 0;
+ insn->flat.lds = 0;
+ insn->flat.seg = GFX10_FLAT_SEG_GLOBAL;
+ insn->flat.glc = 1;
+ insn->flat.slc = 1;
+ insn->flat.op = GFX10_GLOBAL_STORE_DWORDX4;
+ insn->flat.encoding = GFX10_FLAT_ENCODING;
+ insn->flat.addr = dst.v;
+ insn->flat.data = src.v;
+ insn->flat.saddr = GFX10_FLAT_SADDR_DISABLE;
+ insn->flat.dummy1 = 0;
+ insn->flat.dummy2 = 0;
+ insn->flat.vdst = 0;
+
+ return 8;
+}
+
+inline u32 emit_gfx10_s_branch(union amdgcn_gfx10_insn *insn,
+ short off)
+{
+ insn->sopp.simm16 = off;
+ insn->sopp.op = GFX10_S_BRANCH;
+ insn->sopp.encoding = GFX10_SOPP_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx10_s_cbranch_vccz(union amdgcn_gfx10_insn *insn,
+ short off)
+{
+ insn->sopp.simm16 = off;
+ insn->sopp.op = GFX10_S_CBRANCH_VCCZ;
+ insn->sopp.encoding = GFX10_SOPP_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx10_s_cbranch_vccnz(union amdgcn_gfx10_insn *insn,
+ short off)
+{
+ insn->sopp.simm16 = off;
+ insn->sopp.op = GFX10_S_CBRANCH_VCCNZ;
+ insn->sopp.encoding = GFX10_SOPP_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx10_branch_fixup(union amdgcn_gfx10_insn *insn,
+ short off)
+{
+ WARN_ON(insn->sopp.op != GFX10_S_BRANCH &&
+ insn->sopp.op != GFX10_S_CBRANCH_VCCZ &&
+ insn->sopp.op != GFX10_S_CBRANCH_VCCNZ &&
+ insn->sopp.op != GFX10_S_CBRANCH_EXECZ &&
+ insn->sopp.op != GFX10_S_CBRANCH_EXECNZ);
+ insn->sopp.simm16 = off;
+
+ return 4;
+}
+
+inline u32 emit_gfx10_s_waitcnt_lgkmcnt(union amdgcn_gfx10_insn *insn)
+{
+ struct amdgcn_gfx10_sopp_vmcnt vmcnt;
+ u16 simm16;
+ /* s_waitcnt lgkmcnt (0)
+ * wait for memory
+ */
+ vmcnt.vmcnt1 = -1;
+ vmcnt.vmcnt2 = -1;
+ vmcnt.expcnt = -1;
+ vmcnt.dummy = 0;
+ vmcnt.lgkmcnt = 0;
+ memcpy(&simm16, &vmcnt, sizeof(u16));
+ insn->sopp.simm16 = simm16;
+ insn->sopp.op = GFX10_S_WAITCNT;
+ insn->sopp.encoding = GFX10_SOPP_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx10_s_waitcnt_vmcnt(union amdgcn_gfx10_insn *insn)
+{
+ struct amdgcn_gfx10_sopp_vmcnt vmcnt;
+ u16 simm16;
+ /* s_waitcnt lgkmcnt (0)
+ * wait for memory
+ */
+ vmcnt.vmcnt1 = 0;
+ vmcnt.vmcnt2 = 0;
+ vmcnt.expcnt = -1;
+ vmcnt.dummy = 0;
+ vmcnt.lgkmcnt = -1;
+ memcpy(&simm16, &vmcnt, sizeof(u16));
+ insn->sopp.simm16 = simm16;
+ insn->sopp.op = GFX10_S_WAITCNT;
+ insn->sopp.encoding = GFX10_SOPP_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx10_s_waitcnt_vmcnt_lgkmcnt(union amdgcn_gfx10_insn *insn)
+{
+ struct amdgcn_gfx10_sopp_vmcnt vmcnt;
+ u16 simm16;
+ /* s_waitcnt lgkmcnt (0)
+ * wait for memory
+ */
+ vmcnt.vmcnt1 = 0;
+ vmcnt.vmcnt2 = 0;
+ vmcnt.expcnt = -1;
+ vmcnt.dummy = 0;
+ vmcnt.lgkmcnt = 0;
+ memcpy(&simm16, &vmcnt, sizeof(u16));
+ insn->sopp.simm16 = simm16;
+ insn->sopp.op = GFX10_S_WAITCNT;
+ insn->sopp.encoding = GFX10_SOPP_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx10_s_nop(union amdgcn_gfx10_insn *insn)
+{
+ insn->sopp.simm16 = 0;
+ insn->sopp.op = GFX10_S_NOP;
+ insn->sopp.encoding = GFX10_SOPP_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx10_s_endpgm(union amdgcn_gfx10_insn *insn)
+{
+ insn->sopp.simm16 = 0;
+ insn->sopp.op = GFX10_S_ENDPGM;
+ insn->sopp.encoding = GFX10_SOPP_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx10_s_code_end(union amdgcn_gfx10_insn *insn)
+{
+ insn->sopp.simm16 = 0;
+ insn->sopp.op = GFX10_S_CODE_END;
+ insn->sopp.encoding = GFX10_SOPP_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx10_s_icache_inv(union amdgcn_gfx10_insn *insn)
+{
+ insn->sopp.simm16 = 0;
+ insn->sopp.op = GFX10_S_ICACHE_INV;
+ insn->sopp.encoding = GFX10_SOPP_ENCODING;
+
+ return 4;
+}
+
+/* Structurized CFG instructions.
+ * These use raw SGPR indices for 64-bit pair operations involving
+ * EXEC (126) and VCC (106) special registers.
+ */
+
+/* s_and_saveexec_b64 s[sdst:sdst+1], s[ssrc:ssrc+1]
+ * sdst = EXEC; EXEC &= ssrc; SCC = (EXEC != 0)
+ */
+inline u32 emit_gfx10_s_and_saveexec_b64(union amdgcn_gfx10_insn *insn,
+ u8 sdst, u8 ssrc)
+{
+ insn->sop1.ssrc0 = ssrc;
+ insn->sop1.op = GFX10_S_AND_SAVEEXEC_B64;
+ insn->sop1.sdst = sdst;
+ insn->sop1.encoding = GFX10_SOP1_ENCODING;
+
+ return 4;
+}
+
+/* s_bcnt1_i32_b64 sdst, s[ssrc:ssrc+1]
+ * sdst = popcount(ssrc). SCC = (sdst != 0)
+ */
+inline u32 emit_gfx10_s_bcnt1_i32_b64(union amdgcn_gfx10_insn *insn,
+ u8 sdst, u8 ssrc)
+{
+ insn->sop1.ssrc0 = ssrc;
+ insn->sop1.op = GFX10_S_BCNT1_I32_B64;
+ insn->sop1.sdst = sdst;
+ insn->sop1.encoding = GFX10_SOP1_ENCODING;
+
+ return 4;
+}
+
+/* s_mov_b64 s[sdst:sdst+1], s[ssrc:ssrc+1] (or special src like 0) */
+inline u32 emit_gfx10_s_mov_b64(union amdgcn_gfx10_insn *insn,
+ u8 sdst, u8 ssrc)
+{
+ insn->sop1.ssrc0 = ssrc;
+ insn->sop1.op = GFX10_S_MOV_B64;
+ insn->sop1.sdst = sdst;
+ insn->sop1.encoding = GFX10_SOP1_ENCODING;
+
+ return 4;
+}
+
+/* s_and_b64 s[sdst:sdst+1], s[ssrc0:ssrc0+1], s[ssrc1:ssrc1+1] */
+inline u32 emit_gfx10_s_and_b64(union amdgcn_gfx10_insn *insn,
+ u8 sdst, u8 ssrc0, u8 ssrc1)
+{
+ insn->sop2.ssrc0 = ssrc0;
+ insn->sop2.ssrc1 = ssrc1;
+ insn->sop2.sdst = sdst;
+ insn->sop2.op = GFX10_S_AND_B64;
+ insn->sop2.encoding = GFX10_SOP2_ENCODING;
+
+ return 4;
+}
+
+/* s_or_b64 s[sdst:sdst+1], s[ssrc0:ssrc0+1], s[ssrc1:ssrc1+1] */
+inline u32 emit_gfx10_s_or_b64(union amdgcn_gfx10_insn *insn,
+ u8 sdst, u8 ssrc0, u8 ssrc1)
+{
+ insn->sop2.ssrc0 = ssrc0;
+ insn->sop2.ssrc1 = ssrc1;
+ insn->sop2.sdst = sdst;
+ insn->sop2.op = GFX10_S_OR_B64;
+ insn->sop2.encoding = GFX10_SOP2_ENCODING;
+
+ return 4;
+}
+
+/* s_andn2_b64 s[sdst:sdst+1], s[ssrc0:ssrc0+1], s[ssrc1:ssrc1+1]
+ * sdst = ssrc0 & ~ssrc1
+ */
+inline u32 emit_gfx10_s_andn2_b64(union amdgcn_gfx10_insn *insn,
+ u8 sdst, u8 ssrc0, u8 ssrc1)
+{
+ insn->sop2.ssrc0 = ssrc0;
+ insn->sop2.ssrc1 = ssrc1;
+ insn->sop2.sdst = sdst;
+ insn->sop2.op = GFX10_S_ANDN2_B64;
+ insn->sop2.encoding = GFX10_SOP2_ENCODING;
+
+ return 4;
+}
+
+/* s_cbranch_execz off - branch if EXEC == 0 */
+inline u32 emit_gfx10_s_cbranch_execz(union amdgcn_gfx10_insn *insn,
+ short off)
+{
+ insn->sopp.simm16 = off;
+ insn->sopp.op = GFX10_S_CBRANCH_EXECZ;
+ insn->sopp.encoding = GFX10_SOPP_ENCODING;
+
+ return 4;
+}
+
+/* s_cbranch_execnz off - branch if EXEC != 0 */
+inline u32 emit_gfx10_s_cbranch_execnz(union amdgcn_gfx10_insn *insn,
+ short off)
+{
+ insn->sopp.simm16 = off;
+ insn->sopp.op = GFX10_S_CBRANCH_EXECNZ;
+ insn->sopp.encoding = GFX10_SOPP_ENCODING;
+
+ return 4;
+}
+
+/* s_sub_u32 sdst, ssrc0, ssrc1 - sdst = ssrc0 - ssrc1; SCC = borrow */
+inline u32 emit_gfx10_s_sub_u32(union amdgcn_gfx10_insn *insn,
+ u8 sdst, u8 ssrc0, u8 ssrc1)
+{
+ insn->sop2.ssrc0 = ssrc0;
+ insn->sop2.ssrc1 = ssrc1;
+ insn->sop2.sdst = sdst;
+ insn->sop2.op = GFX10_S_SUB_U32;
+ insn->sop2.encoding = GFX10_SOP2_ENCODING;
+
+ return 4;
+}
+
+/* s_cbranch_scc0 off - branch if SCC == 0 (no borrow) */
+inline u32 emit_gfx10_s_cbranch_scc0(union amdgcn_gfx10_insn *insn,
+ short off)
+{
+ insn->sopp.simm16 = off;
+ insn->sopp.op = GFX10_S_CBRANCH_SCC0;
+ insn->sopp.encoding = GFX10_SOPP_ENCODING;
+
+ return 4;
+}
+
+static inline void __emit_gfx10_sop2(union amdgcn_gfx10_insn *insn,
+ int op, int sdst, int ssrc0, int ssrc1)
+{
+ insn->sop2.encoding = GFX10_SOP2_ENCODING;
+ insn->sop2.op = op;
+ insn->sop2.sdst = sdst;
+ insn->sop2.ssrc0 = ssrc0;
+ insn->sop2.ssrc1 = ssrc1;
+}
+
+static inline void __emit_gfx10_sop1(union amdgcn_gfx10_insn *insn,
+ int op, int sdst, int ssrc0)
+{
+ insn->sop1.encoding = GFX10_SOP1_ENCODING;
+ insn->sop1.op = op;
+ insn->sop1.sdst = sdst;
+ insn->sop1.ssrc0 = ssrc0;
+}
+
+static inline void __emit_gfx10_sopp(union amdgcn_gfx10_insn *insn,
+ int op, u16 simm16)
+{
+ insn->sopp.encoding = GFX10_SOPP_ENCODING;
+ insn->sopp.op = op;
+ insn->sopp.simm16 = simm16;
+}
+
+static inline void __emit_gfx10_vop2(union amdgcn_gfx10_insn *insn,
+ int op, int vdst, int vsrc1, int src0)
+{
+ insn->vop2.encoding = GFX10_VOP2_ENCODING;
+ insn->vop2.op = op;
+ insn->vop2.vdst = vdst;
+ insn->vop2.vsrc1 = vsrc1;
+ insn->vop2.src0 = src0;
+}
+
+static inline void __emit_gfx10_smem(union amdgcn_gfx10_insn *insn,
+ int op, int sdata, int sbase_pair,
+ u32 offset)
+{
+ insn->smem.encoding = GFX10_SMEM_ENCODING;
+ insn->smem.op = op;
+ insn->smem.sdata = sdata;
+ insn->smem.sbase = sbase_pair;
+ insn->smem.offset = offset;
+ insn->smem.soffset = GFX10_SRC_NULL;
+}
+
+static inline void __emit_gfx10_ds(union amdgcn_gfx10_insn *insn,
+ int op, int addr, int data0, int vdst,
+ int off0, int off1)
+{
+ insn->ds.encoding = GFX10_DS_ENCODING;
+ insn->ds.op = op;
+ insn->ds.gds = GFX10_DS_LDS;
+ insn->ds.addr = addr;
+ insn->ds.data0 = data0;
+ insn->ds.vdst = vdst;
+ insn->ds.offset0 = off0;
+ insn->ds.offset1 = off1;
+}
+
+static inline void __emit_gfx10_global(union amdgcn_gfx10_insn *insn,
+ int op, int vdst, int vaddr,
+ int vdata, int saddr, int offset)
+{
+ insn->flat.encoding = GFX10_FLAT_ENCODING;
+ insn->flat.seg = GFX10_FLAT_SEG_GLOBAL;
+ insn->flat.op = op;
+ insn->flat.vdst = vdst;
+ insn->flat.addr = vaddr;
+ insn->flat.data = vdata;
+ insn->flat.saddr = saddr;
+ insn->flat.offset = offset;
+}
+
+/* ======================================================================
+ * GFX10 Param-Aware Emit Functions
+ *
+ * Paired with GFX9 equivalents in knod_gfx9_insn.h. Used by shader
+ * emitters (aesgcm_shader.h, ipsec_fused_gfx10.h, ...) that construct
+ * operands via P_S/P_V/P_I/P_L helpers.
+ * ======================================================================
+ */
+
+static inline int __p2e10(struct amdgcn_param32 p)
+{
+ if (p.type == AMDGCN_PARAM_TYPE_LITERAL_CONST)
+ return gfx10_get_param_base(p);
+ return gfx10_get_param_base(p) + p.v;
+}
+
+/* --- GFX10 SOP2 (param32) --- */
+
+#define DEFINE_GFX10_SOP2_P(name, opcode) \
+inline u32 emit_gfx10_##name(union amdgcn_gfx10_insn *insn, \
+ struct amdgcn_param32 dst, \
+ struct amdgcn_param32 src0, \
+ struct amdgcn_param32 src1) \
+{ \
+ insn->sop2.sdst = __p2e10(dst); \
+ insn->sop2.ssrc0 = __p2e10(src0); \
+ insn->sop2.ssrc1 = __p2e10(src1); \
+ insn->sop2.op = opcode; \
+ insn->sop2.encoding = GFX10_SOP2_ENCODING; \
+ if (knod_param_is_literal(src0)) { \
+ insn->sop2.literal = src0.v; \
+ return 8; \
+ } \
+ if (knod_param_is_literal(src1)) { \
+ insn->sop2.literal = src1.v; \
+ return 8; \
+ } \
+ return 4; \
+}
+
+DEFINE_GFX10_SOP2_P(s_add_u32, GFX10_S_ADD_U32)
+DEFINE_GFX10_SOP2_P(s_sub_u32_p, GFX10_S_SUB_U32)
+DEFINE_GFX10_SOP2_P(s_addc_u32, GFX10_S_ADDC_U32)
+DEFINE_GFX10_SOP2_P(s_subb_u32, GFX10_S_SUBB_U32)
+DEFINE_GFX10_SOP2_P(s_and_b32_p, GFX10_S_AND_B32)
+DEFINE_GFX10_SOP2_P(s_lshl_b32, GFX10_S_LSHL_B32)
+DEFINE_GFX10_SOP2_P(s_lshr_b32, GFX10_S_LSHR_B32)
+DEFINE_GFX10_SOP2_P(s_mul_i32, GFX10_S_MUL_I32)
+DEFINE_GFX10_SOP2_P(s_xor_b32, GFX10_S_XOR_B32)
+
+#undef DEFINE_GFX10_SOP2_P
+
+/* --- GFX10 SOPC (param32) --- */
+
+#define DEFINE_GFX10_SOPC_P(name, opcode) \
+inline u32 emit_gfx10_##name(union amdgcn_gfx10_insn *insn, \
+ struct amdgcn_param32 src0, \
+ struct amdgcn_param32 src1) \
+{ \
+ insn->sopc.ssrc0 = __p2e10(src0); \
+ insn->sopc.ssrc1 = __p2e10(src1); \
+ insn->sopc.op = opcode; \
+ insn->sopc.encoding = GFX10_SOPC_ENCODING; \
+ if (knod_param_is_literal(src0)) { \
+ insn->sopc.literal = src0.v; \
+ return 8; \
+ } \
+ if (knod_param_is_literal(src1)) { \
+ insn->sopc.literal = src1.v; \
+ return 8; \
+ } \
+ return 4; \
+}
+
+DEFINE_GFX10_SOPC_P(s_cmp_eq_u32, GFX10_S_CMP_EQ_U32)
+DEFINE_GFX10_SOPC_P(s_cmp_lg_u32, GFX10_S_CMP_LG_U32)
+DEFINE_GFX10_SOPC_P(s_cmp_ge_u32, GFX10_S_CMP_GE_U32)
+DEFINE_GFX10_SOPC_P(s_cmp_lt_u32, GFX10_S_CMP_LT_U32)
+
+#undef DEFINE_GFX10_SOPC_P
+
+/* --- GFX10 SOPP --- */
+
+inline u32 emit_gfx10_s_barrier(union amdgcn_gfx10_insn *insn)
+{
+ insn->sopp.simm16 = 0;
+ insn->sopp.op = GFX10_S_BARRIER;
+ insn->sopp.encoding = GFX10_SOPP_ENCODING;
+ return 4;
+}
+
+inline u32 emit_gfx10_s_cbranch_scc1(union amdgcn_gfx10_insn *insn,
+ short off)
+{
+ insn->sopp.simm16 = off;
+ insn->sopp.op = GFX10_S_CBRANCH_SCC1;
+ insn->sopp.encoding = GFX10_SOPP_ENCODING;
+ return 4;
+}
+
+/*
+ * GFX10 s_waitcnt encoding:
+ * simm16[3:0] = vmcnt[3:0]
+ * simm16[7:4] = expcnt (set to 7 = unused)
+ * simm16[13:8] = lgkmcnt[5:0]
+ * simm16[15:14] = vmcnt[5:4]
+ */
+#define GFX10_WAITCNT(vm, lgkm) \
+ (((((vm) >> 4) & 0x3) << 14) | (((lgkm) & 0x3F) << 8) | \
+ (7 << 4) | ((vm) & 0xF))
+
+inline u32 emit_gfx10_s_waitcnt(union amdgcn_gfx10_insn *insn,
+ int vm, int lgkm)
+{
+ insn->sopp.simm16 = GFX10_WAITCNT(vm, lgkm);
+ insn->sopp.op = GFX10_S_WAITCNT;
+ insn->sopp.encoding = GFX10_SOPP_ENCODING;
+ return 4;
+}
+
+/* --- GFX10 SOP1 --- */
+
+inline u32 emit_gfx10_s_not_b64(union amdgcn_gfx10_insn *insn,
+ u8 sdst, u8 ssrc)
+{
+ insn->sop1.ssrc0 = ssrc;
+ insn->sop1.op = GFX10_S_NOT_B64;
+ insn->sop1.sdst = sdst;
+ insn->sop1.encoding = GFX10_SOP1_ENCODING;
+ return 4;
+}
+
+/* --- GFX10 VOPC (v_cmp_ne_u32 param variant) --- */
+
+inline u32 emit_gfx10_v_cmp_ne_u32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ WARN_ON(src1.type != AMDGCN_PARAM_TYPE_VGPR);
+ insn->vopc.vsrc1 = src1.v;
+ insn->vopc.op = GFX10_V_CMP_NE_U32;
+ insn->vopc.encoding = GFX10_VOPC_ENCODING;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vopc.src0 = gfx10_get_param_base(src0);
+ insn->vopc.literal = src0.v;
+ return 8;
+ }
+ insn->vopc.src0 = gfx10_get_param_base(src0) + src0.v;
+ return 4;
+}
+
+/* --- GFX10 SMEM --- */
+
+#define DEFINE_GFX10_SMEM_P(name, opcode) \
+inline u32 emit_gfx10_##name(union amdgcn_gfx10_insn *insn, \
+ struct amdgcn_param32 dst, \
+ struct amdgcn_param32 src, int offset) \
+{ \
+ insn->smem.sdata = dst.v; \
+ insn->smem.sbase = src.v / 2; \
+ insn->smem.op = opcode; \
+ insn->smem.offset = offset; \
+ insn->smem.soffset = GFX10_SRC_NULL; \
+ insn->smem.encoding = GFX10_SMEM_ENCODING; \
+ return 8; \
+}
+
+DEFINE_GFX10_SMEM_P(s_load_dword, GFX10_S_LOAD_DWORD)
+DEFINE_GFX10_SMEM_P(s_load_dwordx4, GFX10_S_LOAD_DWORDX4)
+
+#undef DEFINE_GFX10_SMEM_P
+
+/* s_dcache_inv - no operands */
+inline u32 emit_gfx10_s_dcache_inv(union amdgcn_gfx10_insn *insn)
+{
+ insn->smem.sdata = 0;
+ insn->smem.sbase = 0;
+ insn->smem.op = GFX10_S_DCACHE_INV;
+ insn->smem.offset = 0;
+ insn->smem.soffset = GFX10_SRC_NULL;
+ insn->smem.encoding = GFX10_SMEM_ENCODING;
+ return 8;
+}
+
+/* --- GFX10 VOP1: v_readfirstlane_b32 --- */
+
+inline u32 emit_gfx10_v_readfirstlane_b32(union amdgcn_gfx10_insn *insn,
+ u8 sdst, u8 vsrc)
+{
+ insn->vop1.encoding = GFX10_VOP1_ENCODING;
+ insn->vop1.vdst = sdst;
+ insn->vop1.op = GFX10_V_READFIRSTLANE_B32;
+ insn->vop1.src0 = GFX10_SRC_VGPR_BASE + vsrc;
+ return 4;
+}
+
+/* --- GFX10 VOP2: v_max_i32 --- */
+
+inline u32 emit_gfx10_v_max_i32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ WARN_ON(dst.type != AMDGCN_PARAM_TYPE_VGPR);
+ WARN_ON(src1.type != AMDGCN_PARAM_TYPE_VGPR);
+
+ insn->vop2.vsrc1 = src1.v;
+ insn->vop2.vdst = dst.v;
+ insn->vop2.op = GFX10_V_MAX_I32;
+ insn->vop2.encoding = GFX10_VOP2_ENCODING;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop2.src0 = gfx10_get_param_base(src0);
+ insn->vop2.literal = src0.v;
+ return 8;
+ }
+ insn->vop2.src0 = gfx10_get_param_base(src0) + src0.v;
+
+ return 4;
+}
+
+/* --- GFX10 VOP3A: v_perm_b32 --- */
+
+inline u32 emit_gfx10_v_perm_b32(union amdgcn_gfx10_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1,
+ struct amdgcn_param32 src2)
+{
+ WARN_ON(dst.type != AMDGCN_PARAM_TYPE_VGPR);
+ /* VOP3 does not support literal constants on GFX10 */
+ WARN_ON(src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST ||
+ src1.type == AMDGCN_PARAM_TYPE_LITERAL_CONST ||
+ src2.type == AMDGCN_PARAM_TYPE_LITERAL_CONST);
+ insn->vop3a.vdst = dst.v;
+ insn->vop3a.abs = 0;
+ insn->vop3a.op_sel = 0;
+ insn->vop3a.clmp = 0;
+ insn->vop3a.op = GFX10_V_PERM_B32;
+ insn->vop3a.encoding = GFX10_VOP3A_ENCODING;
+ insn->vop3a.src0 = gfx10_get_param_base(src0) + src0.v;
+ insn->vop3a.src1 = gfx10_get_param_base(src1) + src1.v;
+ insn->vop3a.src2 = gfx10_get_param_base(src2) + src2.v;
+ insn->vop3a.omod = 0;
+ insn->vop3a.neg = 0;
+
+ return 8;
+}
+
+/* --- GFX10 DS --- */
+
+inline u32 emit_gfx10_ds_write_b32(union amdgcn_gfx10_insn *insn,
+ int addr, int data0)
+{
+ __emit_gfx10_ds(insn, GFX10_DS_WRITE_B32, addr, data0, 0, 0, 0);
+ return 8;
+}
+
+inline u32 emit_gfx10_ds_read_b32(union amdgcn_gfx10_insn *insn,
+ int vdst, int addr)
+{
+ __emit_gfx10_ds(insn, GFX10_DS_READ_B32, addr, 0, vdst, 0, 0);
+ return 8;
+}
+
+inline u32 emit_gfx10_ds_write_b32_off(union amdgcn_gfx10_insn *insn,
+ int addr, int data0, int offset)
+{
+ __emit_gfx10_ds(insn, GFX10_DS_WRITE_B32, addr, data0, 0, offset, 0);
+ return 8;
+}
+
+inline u32 emit_gfx10_ds_read_b32_off(union amdgcn_gfx10_insn *insn,
+ int vdst, int addr, int offset)
+{
+ __emit_gfx10_ds(insn, GFX10_DS_READ_B32, addr, 0, vdst, offset, 0);
+ return 8;
+}
+
+inline u32 emit_gfx10_ds_write_b128(union amdgcn_gfx10_insn *insn,
+ int addr, int data0)
+{
+ __emit_gfx10_ds(insn, GFX10_DS_WRITE_B128, addr, data0, 0, 0, 0);
+ return 8;
+}
+
+inline u32 emit_gfx10_ds_read_b128(union amdgcn_gfx10_insn *insn,
+ int vdst, int addr)
+{
+ __emit_gfx10_ds(insn, GFX10_DS_READ_B128, addr, 0, vdst, 0, 0);
+ return 8;
+}
+
+/* --- GFX10 GLOBAL with saddr mode (scalar base + VGPR offset) --- */
+
+inline u32 emit_gfx10_global_load_dword_saddr(union amdgcn_gfx10_insn *insn,
+ int vdst, int vaddr,
+ int saddr, short off)
+{
+ insn->flat.offset = off;
+ insn->flat.dlc = 0;
+ insn->flat.lds = 0;
+ insn->flat.seg = GFX10_FLAT_SEG_GLOBAL;
+ insn->flat.glc = 0;
+ insn->flat.slc = 0;
+ insn->flat.op = GFX10_GLOBAL_LOAD_DWORD;
+ insn->flat.encoding = GFX10_FLAT_ENCODING;
+ insn->flat.addr = vaddr;
+ insn->flat.data = 0;
+ insn->flat.saddr = saddr / 2;
+ insn->flat.dummy1 = 0;
+ insn->flat.dummy2 = 0;
+ insn->flat.vdst = vdst;
+ return 8;
+}
+
+inline u32 emit_gfx10_global_load_dwordx4_saddr(union amdgcn_gfx10_insn *insn,
+ int vdst, int vaddr,
+ int saddr, short off)
+{
+ insn->flat.offset = off;
+ insn->flat.dlc = 0;
+ insn->flat.lds = 0;
+ insn->flat.seg = GFX10_FLAT_SEG_GLOBAL;
+ insn->flat.glc = 0;
+ insn->flat.slc = 0;
+ insn->flat.op = GFX10_GLOBAL_LOAD_DWORDX4;
+ insn->flat.encoding = GFX10_FLAT_ENCODING;
+ insn->flat.addr = vaddr;
+ insn->flat.data = 0;
+ insn->flat.saddr = saddr / 2;
+ insn->flat.dummy1 = 0;
+ insn->flat.dummy2 = 0;
+ insn->flat.vdst = vdst;
+ return 8;
+}
+
+inline u32 emit_gfx10_global_store_dwordx4_saddr(union amdgcn_gfx10_insn *insn,
+ int vaddr, int vdata,
+ int saddr, short off)
+{
+ insn->flat.offset = off;
+ insn->flat.dlc = 0;
+ insn->flat.lds = 0;
+ insn->flat.seg = GFX10_FLAT_SEG_GLOBAL;
+ insn->flat.glc = 1;
+ insn->flat.slc = 1;
+ insn->flat.op = GFX10_GLOBAL_STORE_DWORDX4;
+ insn->flat.encoding = GFX10_FLAT_ENCODING;
+ insn->flat.addr = vaddr;
+ insn->flat.data = vdata;
+ insn->flat.saddr = saddr / 2;
+ insn->flat.dummy1 = 0;
+ insn->flat.dummy2 = 0;
+ insn->flat.vdst = 0;
+ return 8;
+}
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdkfd/knod/knod_gfx9_insn.h b/drivers/gpu/drm/amd/amdkfd/knod/knod_gfx9_insn.h
new file mode 100644
index 000000000000..ea987188f982
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdkfd/knod/knod_gfx9_insn.h
@@ -0,0 +1,4068 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/* Copyright (c) 2021 Taehee Yoo <ap420073@gmail.com>
+ * Copyright (c) 2021 Hoyeon Lee <hoyeon.rhee@gmail.com>
+ */
+
+#ifndef KFD_AMDGPU_GFX9_INSN_H_INCLUDED
+#define KFD_AMDGPU_GFX9_INSN_H_INCLUDED
+
+#include "knod_amdgpu.h"
+
+/*
+ * knod_amdgpu_insn.h is intentionally NOT included here.
+ * This file provides ISA definitions (structs, enums, encoding constants)
+ * and per-version emit functions that are self-contained.
+ *
+ * The version-dispatch layer (knod_amdgpu_insn.h) includes this file
+ * and adds struct amdgcn_insn + wrapper functions on top.
+ */
+
+#define GFX9_SRC_SGPR_BASE 0
+#define GFX9_SRC_VCC_LO 106
+#define GFX9_SRC_VCC_HI 107
+#define GFX9_SRC_TTPM_BASE 108
+#define GFX9_SRC_M0 124
+#define GFX9_SRC_NULL 125
+#define GFX9_SRC_EXEC_LO 126
+#define GFX9_SRC_EXEC_HI 127
+#define GFX9_SRC_INTEGER_0 128
+#define GFX9_SRC_INTEGER_MINUS_1 193
+#define GFX9_SRC_SHARED_BASE 235
+#define GFX9_SRC_SHARED_LIMIT 236
+#define GFX9_SRC_PRIVATE_BASE 237
+#define GFX9_SRC_PRIVATE_LIMIT 238
+#define GFX9_SRC_POPS_EXITING_WAVE_ID 239
+#define GFX9_SRC_SDWA 249
+#define GFX9_SRC_DDP16 250
+#define GFX9_SRC_VCCZ 251
+#define GFX9_SRC_EXECZ 252
+#define GFX9_SRC_SCC 253
+#define GFX9_SRC_LITERAL_CONST 255
+#define GFX9_SRC_VGPR_BASE 256
+
+static int gfx9_param_base[__AMDGCN_PARAM_TYPE_MAX] = {
+ GFX9_SRC_SGPR_BASE,
+ GFX9_SRC_VCC_LO,
+ GFX9_SRC_VCC_HI,
+ GFX9_SRC_TTPM_BASE,
+ GFX9_SRC_M0,
+ GFX9_SRC_NULL,
+ GFX9_SRC_EXEC_LO,
+ GFX9_SRC_EXEC_HI,
+ GFX9_SRC_INTEGER_0,
+ GFX9_SRC_INTEGER_MINUS_1,
+ GFX9_SRC_SHARED_BASE,
+ GFX9_SRC_SHARED_LIMIT,
+ GFX9_SRC_PRIVATE_BASE,
+ GFX9_SRC_PRIVATE_LIMIT,
+ GFX9_SRC_POPS_EXITING_WAVE_ID,
+ GFX9_SRC_SDWA,
+ GFX9_SRC_DDP16,
+ GFX9_SRC_VCCZ,
+ GFX9_SRC_EXECZ,
+ GFX9_SRC_SCC,
+ GFX9_SRC_LITERAL_CONST,
+ GFX9_SRC_VGPR_BASE,
+};
+
+/* Scalar ALU and Control Format */
+struct amdgcn_gfx9_sop2 {
+ u32 ssrc0:8;
+ u32 ssrc1:8;
+ u32 sdst:7;
+ u32 op:7;
+ u32 encoding:2;
+ u32 literal;
+};
+
+enum amdgcn_gfx9_sop2_opcode {
+ GFX9_S_ADD_U32,
+ GFX9_S_SUB_U32,
+ GFX9_S_ADD_I32,
+ GFX9_S_SUB_I32,
+ GFX9_S_ADDC_U32,
+ GFX9_S_SUBB_U32,
+ GFX9_S_MIN_I32,
+ GFX9_S_MIN_U32,
+ GFX9_S_MAX_I32,
+ GFX9_S_MAX_U32,
+ GFX9_S_CSELECT_B32,
+ GFX9_S_CSELECT_B64,
+ GFX9_S_AND_B32,
+ GFX9_S_AND_B64,
+ GFX9_S_OR_B32,
+ GFX9_S_OR_B64,
+ GFX9_S_XOR_B32,
+ GFX9_S_XOR_B64,
+ GFX9_S_ANDN2_B32,
+ GFX9_S_ANDN2_B64,
+ GFX9_S_ORN2_B32,
+ GFX9_S_ORN2_B64,
+ GFX9_S_NAND_B32,
+ GFX9_S_NAND_B64,
+ GFX9_S_NOR_B32,
+ GFX9_S_NOR_B64,
+ GFX9_S_XNOR_B32,
+ GFX9_S_XNOR_B64,
+ GFX9_S_LSHL_B32,
+ GFX9_S_LSHL_B64,
+ GFX9_S_LSHR_B32,
+ GFX9_S_LSHR_B64,
+ GFX9_S_ASHR_I32,
+ GFX9_S_ASHR_I64,
+ GFX9_S_BFM_B32,
+ GFX9_S_BFM_B64,
+ GFX9_S_MUL_I32,
+ GFX9_S_BFE_U32,
+ GFX9_S_BFE_I32,
+ GFX9_S_BFE_U64,
+ GFX9_S_BFE_I64,
+ GFX9_S_CBRANCH_G_FORK,
+ GFX9_S_ABSDIFF_I32,
+ GFX9_S_RFE_RESTORE_B64,
+ GFX9_S_MUL_HI_U32,
+ GFX9_S_MUL_HI_I32,
+ GFX9_S_LSHL1_ADD_U32,
+ GFX9_S_LSHL2_ADD_U32,
+ GFX9_S_LSHL3_ADD_U32,
+ GFX9_S_LSHL4_ADD_U32,
+ GFX9_S_PACK_LL_B32_B16,
+ GFX9_S_PACK_LH_B32_B16,
+ GFX9_S_PACK_HH_B32_B16,
+};
+
+#define GFX9_SOP2_ENCODING 0x2
+#define GFX9_SOP2_SSRC_SGPR_BASE 0
+#define GFX9_SOP2_SSRC_VCC_LO 106
+#define GFX9_SOP2_SSRC_VCC_HI 107
+#define GFX9_SOP2_SSRC_TTPM_BASE 108
+#define GFX9_SOP2_SSRC_M0 124
+#define GFX9_SOP2_SSRC_NULL 125
+#define GFX9_SOP2_SSRC_EXEC_LO 126
+#define GFX9_SOP2_SSRC_EXEC_HI 127
+#define GFX9_SOP2_SSRC_INTEGER_0 128
+#define GFX9_SOP2_SSRC_INTEGER_MINUS_1 193
+#define GFX9_SOP2_SSRC_SHARED_BASE 235
+#define GFX9_SOP2_SSRC_SHARED_LIMIT 236
+#define GFX9_SOP2_SSRC_PRIVATE_BASE 237
+#define GFX9_SOP2_SSRC_PRIVATE_LIMIT 238
+#define GFX9_SOP2_SSRC_POPS_EXITING_WAVE_ID 239
+#define GFX9_SOP2_SSRC_VCCZ 251
+#define GFX9_SOP2_SSRC_EXECZ 252
+#define GFX9_SOP2_SSRC_SCC 253
+/* SDST
+ * Same codes as SSRC0, above except only codes 0-127 are valid.
+ */
+
+struct amdgcn_gfx9_sopk {
+ u32 simm16:16;
+ u32 sdst:7;
+ u32 op:5;
+ u32 encoding:4;
+};
+
+enum amdgcn_gfx9_sopk_opcode {
+ GFX9_S_MOVK_I32,
+ GFX9_S_CMOVK_I32,
+ GFX9_S_CMPK_EQ_I32,
+ GFX9_S_CMPK_LG_I32,
+ GFX9_S_CMPK_GT_I32,
+ GFX9_S_CMPK_GE_I32,
+ GFX9_S_CMPK_LT_I32,
+ GFX9_S_CMPK_LE_I32,
+ GFX9_S_CMPK_EQ_U32,
+ GFX9_S_CMPK_LG_U32,
+ GFX9_S_CMPK_GT_U32,
+ GFX9_S_CMPK_GE_U32,
+ GFX9_S_CMPK_LT_U32,
+ GFX9_S_CMPK_LE_U32,
+ GFX9_S_ADDK_I32,
+ GFX9_S_MULK_I32,
+ GFX9_S_CBRANCH_I_FORK,
+ GFX9_S_GETREG_B32,
+ GFX9_S_SETREG_B32 = 18,
+ GFX9_S_SETREG_IMM32_B32 = 20,
+ GFX9_S_CALL_B64,
+};
+
+#define GFX9_SOPK_ENCODING 0xb
+#define GFX9_SOPK_SDST_SGPR0_BASE 0
+#define GFX9_SOPK_SDST_VCC_LO 106
+#define GFX9_SOPK_SDST_VCC_HI 107
+#define GFX9_SOPK_SDST_TTPM_BASE 108
+#define GFX9_SOPK_SDST_M0 124
+#define GFX9_SOPK_SDST_NULL 125
+#define GFX9_SOPK_SDST_EXEC_LO 126
+#define GFX9_SOPK_SDST_EXEC_HI 127
+
+struct amdgcn_gfx9_sop1 {
+ u32 ssrc0:8;
+ u32 op:8;
+ u32 sdst:7;
+ u32 encoding:9;
+ u32 literal;
+};
+
+enum amdgcn_gfx9_sop1_opcode {
+ GFX9_S_MOV_B32 = 0,
+ GFX9_S_MOV_B64,
+ GFX9_S_CMOV_B32,
+ GFX9_S_CMOV_B64,
+ GFX9_S_NOT_B32,
+ GFX9_S_NOT_B64,
+ GFX9_S_WQM_B32,
+ GFX9_S_WQM_B64,
+ GFX9_S_BREV_B32,
+ GFX9_S_BREV_B64,
+ GFX9_S_BCNT0_I32_B32,
+ GFX9_S_BCNT0_I32_B64,
+ GFX9_S_BCNT1_I32_B32,
+ GFX9_S_BCNT1_I32_B64,
+ GFX9_S_FF0_I32_B32,
+ GFX9_S_FF0_I32_B64,
+ GFX9_S_FF1_I32_B32,
+ GFX9_S_FF1_I32_B64,
+ GFX9_S_FLBIT_I32_B32,
+ GFX9_S_FLBIT_I32_B64,
+ GFX9_S_FLBIT_I32,
+ GFX9_S_FLBIT_I32_I64,
+ GFX9_S_SEXT_I32_I8,
+ GFX9_S_SEXT_I32_I16,
+ GFX9_S_BITSET0_B32,
+ GFX9_S_BITSET0_B64,
+ GFX9_S_BITSET1_B32,
+ GFX9_S_BITSET1_B64,
+ GFX9_S_GETPC_B64,
+ GFX9_S_SETPC_B64,
+ GFX9_S_SWAPPC_B64,
+ GFX9_S_RFE_B64 = 31,
+ GFX9_S_AND_SAVEEXEC_B64,
+ GFX9_S_OR_SAVEEXEC_B64,
+ GFX9_S_XOR_SAVEEXEC_B64,
+ GFX9_S_ANDN2_SAVEEXEC_B64,
+ GFX9_S_ORN2_SAVEEXEC_B64,
+ GFX9_S_NAND_SAVEEXEC_B64,
+ GFX9_S_NOR_SAVEEXEC_B64,
+ GFX9_S_XNOR_SAVEEXEC_B64,
+ GFX9_S_QUADMASK_B32,
+ GFX9_S_QUADMASK_B64,
+ GFX9_S_MOVRELS_B32,
+ GFX9_S_MOVRELS_B64,
+ GFX9_S_MOVRELD_B32,
+ GFX9_S_MOVRELD_B64,
+ GFX9_S_ABS_I32 = 48,
+ GFX9_S_SET_GPR_IDX_IDX = 50,
+ GFX9_S_ANDN1_SAVEEXEC_B64,
+ GFX9_S_ORN1_SAVEEXEC_B64,
+ GFX9_S_ANDN1_WREXEC_B64,
+ GFX9_S_ANDN2_WREXEC_B64,
+ GFX9_S_BITREPLICATE_B64_B32,
+};
+
+#define GFX9_SOP1_ENCODING 0x17d
+#define GFX9_SOP1_SSRC_SGPR_BASE 0
+#define GFX9_SOP1_SSRC_VCC_LO 106
+#define GFX9_SOP1_SSRC_VCC_HI 107
+#define GFX9_SOP1_SSRC_TTPM_BASE 108
+#define GFX9_SOP1_SSRC_M0 124
+#define GFX9_SOP1_SSRC_NULL 125
+#define GFX9_SOP1_SSRC_EXEC_LO 126
+#define GFX9_SOP1_SSRC_EXEC_HI 127
+#define GFX9_SOP1_SSRC_INTEGER_0 128
+#define GFX9_SOP1_SSRC_INTEGER_MINUS_1 193
+#define GFX9_SOP1_SSRC_SHARED_BASE 235
+#define GFX9_SOP1_SSRC_SHARED_LIMIT 236
+#define GFX9_SOP1_SSRC_PRIVATE_BASE 237
+#define GFX9_SOP1_SSRC_PRIVATE_LIMIT 238
+#define GFX9_SOP1_SSRC_POPS_EXITING_WAVE_ID 239
+#define GFX9_SOP1_SSRC_VCCZ 251
+#define GFX9_SOP1_SSRC_EXECZ 252
+#define GFX9_SOP1_SSRC_SCC 253
+#define GFX9_SOP1_SSRC_LITERAL_CONST 255
+/* SDST
+ * Same codes as SSRC0, above except only codes 0-127 are valid.
+ */
+
+struct amdgcn_gfx9_sopc {
+ u32 ssrc0:8;
+ u32 ssrc1:8;
+ u32 op:7;
+ u32 encoding:9;
+ u32 literal;
+};
+
+enum amdgcn_gfx9_sopc_opcode {
+ GFX9_S_CMP_EQ_I32,
+ GFX9_S_CMP_LG_I32,
+ GFX9_S_CMP_GT_I32,
+ GFX9_S_CMP_GE_I32,
+ GFX9_S_CMP_LT_I32,
+ GFX9_S_CMP_LE_I32,
+ GFX9_S_CMP_EQ_U32,
+ GFX9_S_CMP_LG_U32,
+ GFX9_S_CMP_GT_U32,
+ GFX9_S_CMP_GE_U32,
+ GFX9_S_CMP_LT_U32,
+ GFX9_S_CMP_LE_U32,
+ GFX9_S_BITCMP0_B32,
+ GFX9_S_BITCMP1_B32,
+ GFX9_S_BITCMP0_B64,
+ GFX9_S_BITCMP1_B64,
+ GFX9_S_SETVSKIP,
+ GFX9_S_SET_GPR_IDX_ON,
+ GFX9_S_CMP_EQ_U64,
+ GFX9_S_CMP_LG_U64,
+};
+
+#define GFX9_SOPC_ENCODING 0x17e
+#define GFX9_SOPC_SSRC_SGPR_BASE 0
+#define GFX9_SOPC_SSRC_VCC_LO 106
+#define GFX9_SOPC_SSRC_VCC_HI 107
+#define GFX9_SOPC_SSRC_TTPM_BASE 108
+#define GFX9_SOPC_SSRC_M0 124
+#define GFX9_SOPC_SSRC_NULL 125
+#define GFX9_SOPC_SSRC_EXEC_LO 126
+#define GFX9_SOPC_SSRC_EXEC_HI 127
+#define GFX9_SOPC_SSRC_INTEGER_0 128
+#define GFX9_SOPC_SSRC_INTEGER_MINUS_1 193
+#define GFX9_SOPC_SSRC_SHARED_BASE 235
+#define GFX9_SOPC_SSRC_SHARED_LIMIT 236
+#define GFX9_SOPC_SSRC_PRIVATE_BASE 237
+#define GFX9_SOPC_SSRC_PRIVATE_LIMIT 238
+#define GFX9_SOPC_SSRC_POPS_EXITING_WAVE_ID 239
+#define GFX9_SOPC_SSRC_VCCZ 251
+#define GFX9_SOPC_SSRC_EXECZ 252
+#define GFX9_SOPC_SSRC_SCC 253
+/* SDST
+ * Same codes as SSRC0, above except only codes 0-127 are valid.
+ */
+
+struct amdgcn_gfx9_sopp_vmcnt {
+ u16 vmcnt1:4;
+ u16 expcnt:3;
+ u16 dummy1:1;
+ u16 lgkmcnt:4; /* GFX9: 4-bit counter, max=15 */
+ u16 dummy2:2;
+ u16 vmcnt2:2;
+};
+
+struct amdgcn_gfx9_sopp {
+ u32 simm16:16;
+ u32 op:7;
+ u32 encoding:9;
+};
+
+enum amdgcn_gfx9_sopp_opcode {
+ GFX9_S_NOP,
+ GFX9_S_ENDPGM,
+ GFX9_S_BRANCH,
+ GFX9_S_WAKEUP,
+ GFX9_S_CBRANCH_SCC0,
+ GFX9_S_CBRANCH_SCC1,
+ GFX9_S_CBRANCH_VCCZ,
+ GFX9_S_CBRANCH_VCCNZ,
+ GFX9_S_CBRANCH_EXECZ,
+ GFX9_S_CBRANCH_EXECNZ,
+ GFX9_S_BARRIER,
+ GFX9_S_SETKILL,
+ GFX9_S_WAITCNT,
+ GFX9_S_SETHALT,
+ GFX9_S_SLEEP,
+ GFX9_S_SETPRIO,
+ GFX9_S_SENDMSG,
+ GFX9_S_SENDMSGHALT,
+ GFX9_S_TRAP,
+ GFX9_S_ICACHE_INV,
+ GFX9_S_INCPERFLEVEL,
+ GFX9_S_DECPERFLEVEL,
+ GFX9_S_TTRACEDATA,
+ GFX9_S_CBRANCH_CDBGSYS,
+ GFX9_S_CBRANCH_CDBGUSER,
+ GFX9_S_CBRANCH_CDBGSYS_OR_USER,
+ GFX9_S_CBRANCH_CDBGSYS_AND_USER,
+ GFX9_S_ENDPGM_SAVED,
+ GFX9_S_SET_GPR_IDX_OFF,
+ GFX9_S_SET_GPR_IDX_MODE,
+ GFX9_S_ENDPGM_ORDERED_PS_DONE,
+};
+
+#define GFX9_SOPP_ENCODING 0x17f
+
+/* Scalar Memory Format */
+struct amdgcn_gfx9_smem {
+ u64 sbase:6;
+ u64 sdata:7;
+ u64 dummy1:1;
+ u64 soe:1; /* Scalar Offset Enable */
+ u64 nv:1; /* Non-Volatile */
+ u64 glc:1; /* Globally Memory Coherent */
+ u64 imm:1; /* Immediate Enable */
+ u64 op:8;
+ u64 encoding:6;
+ u64 offset:21;
+ u64 dummy2:4;
+ u64 soffset:7;
+};
+
+enum amdgcn_gfx9_smem_opcode {
+ GFX9_S_LOAD_DWORD,
+ GFX9_S_LOAD_DWORDX2,
+ GFX9_S_LOAD_DWORDX4,
+ GFX9_S_LOAD_DWORDX8,
+ GFX9_S_LOAD_DWORDX16,
+ GFX9_S_SCRATCH_LOAD_DWORD,
+ GFX9_S_SCRATCH_LOAD_DWORDX2,
+ GFX9_S_SCRATCH_LOAD_DWORDX4,
+ GFX9_S_BUFFER_LOAD_DWORD,
+ GFX9_S_BUFFER_LOAD_DWORDX2,
+ GFX9_S_BUFFER_LOAD_DWORDX4,
+ GFX9_S_BUFFER_LOAD_DWORDX8,
+ GFX9_S_BUFFER_LOAD_DWORDX16 = 12,
+ GFX9_S_STORE_DWORD = 16,
+ GFX9_S_STORE_DWORDX2,
+ GFX9_S_STORE_DWORDX4 = 18,
+ GFX9_S_SCRATCH_STORE_DWORD = 21,
+ GFX9_S_SCRATCH_STORE_DWORDX2,
+ GFX9_S_SCRATCH_STORE_DWORDX4,
+ GFX9_S_BUFFER_STORE_DWORD,
+ GFX9_S_BUFFER_STORE_DWORDX2,
+ GFX9_S_BUFFER_STORE_DWORDX4 = 26,
+ GFX9_S_DCACHE_INV = 32,
+ GFX9_S_DCACHE_WB,
+ GFX9_S_DCACHE_INV_VOL,
+ GFX9_S_DCACHE_WB_VOL,
+ GFX9_S_MEMTIME,
+ GFX9_S_MEMREALTIME,
+ GFX9_S_ATC_PROBE,
+ GFX9_S_ATC_PROBE_BUFFER,
+ GFX9_S_DCACHE_DISCARD,
+ GFX9_S_DCACHE_DISCARD_X2 = 41,
+ GFX9_S_BUFFER_ATOMIC_SWAP = 64,
+ GFX9_S_BUFFER_ATOMIC_CMPSWAP,
+ GFX9_S_BUFFER_ATOMIC_ADD,
+ GFX9_S_BUFFER_ATOMIC_SUB,
+ GFX9_S_BUFFER_ATOMIC_SMIN,
+ GFX9_S_BUFFER_ATOMIC_UMIN,
+ GFX9_S_BUFFER_ATOMIC_SMAX,
+ GFX9_S_BUFFER_ATOMIC_UMAX,
+ GFX9_S_BUFFER_ATOMIC_AND,
+ GFX9_S_BUFFER_ATOMIC_OR,
+ GFX9_S_BUFFER_ATOMIC_XOR,
+ GFX9_S_BUFFER_ATOMIC_INC,
+ GFX9_S_BUFFER_ATOMIC_DEC = 76,
+ GFX9_S_BUFFER_ATOMIC_SWAP_X2 = 96,
+ GFX9_S_BUFFER_ATOMIC_CMPSWAP_X2,
+ GFX9_S_BUFFER_ATOMIC_ADD_X2,
+ GFX9_S_BUFFER_ATOMIC_SUB_X2,
+ GFX9_S_BUFFER_ATOMIC_SMIN_X2,
+ GFX9_S_BUFFER_ATOMIC_UMIN_X2,
+ GFX9_S_BUFFER_ATOMIC_SMAX_X2,
+ GFX9_S_BUFFER_ATOMIC_UMAX_X2,
+ GFX9_S_BUFFER_ATOMIC_AND_X2,
+ GFX9_S_BUFFER_ATOMIC_OR_X2,
+ GFX9_S_BUFFER_ATOMIC_XOR_X2,
+ GFX9_S_BUFFER_ATOMIC_INC_X2,
+ GFX9_S_BUFFER_ATOMIC_DEC_X2 = 108,
+ GFX9_S_ATOMIC_SWAP = 128,
+ GFX9_S_ATOMIC_CMPSWAP,
+ GFX9_S_ATOMIC_ADD,
+ GFX9_S_ATOMIC_SUB,
+ GFX9_S_ATOMIC_SMIN,
+ GFX9_S_ATOMIC_UMIN,
+ GFX9_S_ATOMIC_SMAX,
+ GFX9_S_ATOMIC_UMAX,
+ GFX9_S_ATOMIC_AND,
+ GFX9_S_ATOMIC_OR,
+ GFX9_S_ATOMIC_XOR,
+ GFX9_S_ATOMIC_INC,
+ GFX9_S_ATOMIC_DEC = 140,
+ GFX9_S_ATOMIC_SWAP_X2 = 160,
+ GFX9_S_ATOMIC_CMPSWAP_X2,
+ GFX9_S_ATOMIC_ADD_X2,
+ GFX9_S_ATOMIC_SUB_X2,
+ GFX9_S_ATOMIC_SMIN_X2,
+ GFX9_S_ATOMIC_UMIN_X2,
+ GFX9_S_ATOMIC_SMAX_X2,
+ GFX9_S_ATOMIC_UMAX_X2,
+ GFX9_S_ATOMIC_AND_X2,
+ GFX9_S_ATOMIC_OR_X2,
+ GFX9_S_ATOMIC_XOR_X2,
+ GFX9_S_ATOMIC_INC_X2,
+ GFX9_S_ATOMIC_DEC_X2,
+};
+
+#define GFX9_SMEM_ENCODING 0x30
+#define GFX9_SMEM_SOFFSET_NULL 125
+
+/* Vector ALU Format */
+struct amdgcn_gfx9_vop2 {
+ u32 src0:9;
+ u32 vsrc1:8;
+ u32 vdst:8;
+ u32 op:6;
+ u32 encoding:1;
+ u32 literal;
+};
+
+enum amdgcn_gfx9_vop2_opcode {
+ GFX9_V_CNDMASK_B32,
+ GFX9_V_ADD_F32,
+ GFX9_V_SUB_F32,
+ GFX9_V_SUBREV_F32,
+ GFX9_V_MUL_LEGACY_F32,
+ GFX9_V_MUL_F32,
+ GFX9_V_MUL_I32_I24,
+ GFX9_V_MUL_HI_I32_I24,
+ GFX9_V_MUL_U32_U24,
+ GFX9_V_MUL_HI_U32_U24,
+ GFX9_V_MIN_F32,
+ GFX9_V_MAX_F32,
+ GFX9_V_MIN_I32,
+ GFX9_V_MAX_I32,
+ GFX9_V_MIN_U32,
+ GFX9_V_MAX_U32,
+ GFX9_V_LSHRREV_B32,
+ GFX9_V_ASHRREV_I32,
+ GFX9_V_LSHLREV_B32,
+ GFX9_V_AND_B32,
+ GFX9_V_OR_B32,
+ GFX9_V_XOR_B32,
+ GFX9_V_MAC_F32,
+ GFX9_V_MADMK_F32,
+ GFX9_V_MADAK_F32,
+ GFX9_V_ADD_CO_U32,
+ GFX9_V_SUB_CO_U32,
+ GFX9_V_SUBREV_CO_U32,
+ GFX9_V_ADDC_CO_U32,
+ GFX9_V_SUBB_CO_U32,
+ GFX9_V_SUBBREV_CO_U32,
+ GFX9_V_ADD_F16,
+ GFX9_V_SUB_F16,
+ GFX9_V_SUBREV_F16,
+ GFX9_V_MUL_F16,
+ GFX9_V_MAC_F16,
+ GFX9_V_MADMK_F16,
+ GFX9_V_MADAK_F16,
+ GFX9_V_ADD_U16,
+ GFX9_V_SUB_U16,
+ GFX9_V_SUBREV_U16,
+ GFX9_V_MUL_LO_U16,
+ GFX9_V_LSHLREV_B16,
+ GFX9_V_LSHRREV_B16,
+ GFX9_V_ASHRREV_I16,
+ GFX9_V_MAX_F16,
+ GFX9_V_MIN_F16,
+ GFX9_V_MAX_U16,
+ GFX9_V_MAX_I16,
+ GFX9_V_MIN_U16,
+ GFX9_V_MIN_I16,
+ GFX9_V_LDEXP_F16,
+ GFX9_V_ADD_U32,
+ GFX9_V_SUB_U32,
+ GFX9_V_SUBREV_U32,
+};
+
+#define GFX9_VOP2_ENCODING 0x0
+#define GFX9_VOP2_SRC_SGPR_BASE 0
+#define GFX9_VOP2_SRC_VCC_LO 106
+#define GFX9_VOP2_SRC_VCC_HI 107
+#define GFX9_VOP2_SRC_TTPM_BASE 108
+#define GFX9_VOP2_SRC_M0 124
+#define GFX9_VOP2_SRC_NULL 125
+#define GFX9_VOP2_SRC_EXEC_LO 126
+#define GFX9_VOP2_SRC_EXEC_HI 127
+#define GFX9_VOP2_SRC_INTEGER_0 128
+#define GFX9_VOP2_SRC_INTEGER_MINUS_1 193
+#define GFX9_VOP2_SRC_SHARED_BASE 235
+#define GFX9_VOP2_SRC_SHARED_LIMIT 236
+#define GFX9_VOP2_SRC_PRIVATE_BASE 237
+#define GFX9_VOP2_SRC_PRIVATE_LIMIT 238
+#define GFX9_VOP2_SRC_POPS_EXITING_WAVE_ID 239
+#define GFX9_VOP2_SRC_SDWA 249
+#define GFX9_VOP2_SRC_DDP16 250
+#define GFX9_VOP2_SRC_VCCZ 251
+#define GFX9_VOP2_SRC_EXECZ 252
+#define GFX9_VOP2_SRC_SCC 253
+#define GFX9_VOP2_SRC_LITERAL_CONST 255
+#define GFX9_VOP2_SRC_VGPR_BASE 256
+
+struct amdgcn_gfx9_vop1 {
+ u32 src0:9;
+ u32 op:8;
+ u32 vdst:8;
+ u32 encoding:7;
+ u32 literal;
+};
+
+enum amdgcn_gfx9_vop1_opcode {
+ GFX9_V_NOP,
+ GFX9_V_MOV_B32,
+ GFX9_V_READFIRSTLANE_B32,
+ GFX9_V_CVT_I32_F64,
+ GFX9_V_CVT_F64_I32,
+ GFX9_V_CVT_F32_I32,
+ GFX9_V_CVT_F32_U32,
+ GFX9_V_CVT_U32_F32,
+ GFX9_V_CVT_I32_F32 = 8,
+ GFX9_V_CVT_F16_F32 = 10,
+ GFX9_V_CVT_F32_F16,
+ GFX9_V_CVT_RPI_I32_F32,
+ GFX9_V_CVT_FLR_I32_F32,
+ GFX9_V_CVT_OFF_F32_I4,
+ GFX9_V_CVT_F32_F64,
+ GFX9_V_CVT_F64_F32,
+ GFX9_V_CVT_F32_UBYTE0,
+ GFX9_V_CVT_F32_UBYTE1,
+ GFX9_V_CVT_F32_UBYTE2,
+ GFX9_V_CVT_F32_UBYTE3,
+ GFX9_V_CVT_U32_F64,
+ GFX9_V_CVT_F64_U32,
+ GFX9_V_TRUNC_F64,
+ GFX9_V_CEIL_F64,
+ GFX9_V_RNDNE_F64,
+ GFX9_V_FLOOR_F64,
+ GFX9_V_FRACT_F32,
+ GFX9_V_TRUNC_F32,
+ GFX9_V_CEIL_F32,
+ GFX9_V_RNDNE_F32,
+ GFX9_V_FLOOR_F32,
+ GFX9_V_EXP_F32,
+ GFX9_V_LOG_F32,
+ GFX9_V_RCP_F32,
+ GFX9_V_RCP_IFLAG_F32,
+ GFX9_V_RSQ_F32,
+ GFX9_V_RCP_F64,
+ GFX9_V_RSQ_F64,
+ GFX9_V_SQRT_F32,
+ GFX9_V_SQRT_F64,
+ GFX9_V_SIN_F32,
+ GFX9_V_COS_F32,
+ GFX9_V_NOT_B32,
+ GFX9_V_BFREV_B32,
+ GFX9_V_FFBH_U32,
+ GFX9_V_FFBL_B32,
+ GFX9_V_FFBH_I32,
+ GFX9_V_FREXP_EXP_I32_F64,
+ GFX9_V_FREXP_MANT_F64,
+ GFX9_V_FRACT_F64,
+ GFX9_V_FREXP_EXP_I32_F32,
+ GFX9_V_FREXP_MANT_F32,
+ GFX9_V_CLREXCP,
+ /* ISA opcodes 54 and 56 are not defined (reserved gaps). */
+ GFX9_V_SCREEN_PARTITION_4SE_B32 = 55,
+ GFX9_V_CVT_F16_U16 = 57,
+ GFX9_V_CVT_F16_I16,
+ GFX9_V_CVT_U16_F16,
+ GFX9_V_CVT_I16_F16,
+ GFX9_V_RCP_F16,
+ GFX9_V_SQRT_F16,
+ GFX9_V_RSQ_F16,
+ GFX9_V_LOG_F16,
+ GFX9_V_EXP_F16,
+ GFX9_V_FREXP_MANT_F16,
+ GFX9_V_FREXP_EXP_I16_F16,
+ GFX9_V_FLOOR_F16,
+ GFX9_V_CEIL_F16,
+ GFX9_V_TRUNC_F16,
+ GFX9_V_RNDNE_F16,
+ GFX9_V_FRACT_F16,
+ GFX9_V_SIN_F16,
+ GFX9_V_COS_F16,
+ GFX9_V_EXP_LEGACY_F32,
+ GFX9_V_LOG_LEGACY_F32,
+ GFX9_V_CVT_NORM_I16_F16,
+ GFX9_V_CVT_NORM_U16_F16,
+ GFX9_V_SAT_PK_U8_I16 = 79,
+ GFX9_V_SWAP_B32 = 81,
+};
+
+#define GFX9_VOP1_ENCODING 0x3f
+#define GFX9_VOP1_SRC_SGPR_BASE 0
+#define GFX9_VOP1_SRC_VCC_LO 106
+#define GFX9_VOP1_SRC_VCC_HI 107
+#define GFX9_VOP1_SRC_TTPM_BASE 108
+#define GFX9_VOP1_SRC_M0 124
+#define GFX9_VOP1_SRC_NULL 125
+#define GFX9_VOP1_SRC_EXEC_LO 126
+#define GFX9_VOP1_SRC_EXEC_HI 127
+#define GFX9_VOP1_SRC_INTEGER_0 128
+#define GFX9_VOP1_SRC_INTEGER_MINUS_1 193
+#define GFX9_VOP1_SRC_SHARED_BASE 235
+#define GFX9_VOP1_SRC_SHARED_LIMIT 236
+#define GFX9_VOP1_SRC_PRIVATE_BASE 237
+#define GFX9_VOP1_SRC_PRIVATE_LIMIT 238
+#define GFX9_VOP1_SRC_POPS_EXITING_WAVE_ID 239
+#define GFX9_VOP1_SRC_SDWA 249
+#define GFX9_VOP1_SRC_DDP16 250
+#define GFX9_VOP1_SRC_VCCZ 251
+#define GFX9_VOP1_SRC_EXECZ 252
+#define GFX9_VOP1_SRC_SCC 253
+#define GFX9_VOP1_SRC_LITERAL_CONST 255
+#define GFX9_VOP1_SRC_VGPR_BASE 256
+
+struct amdgcn_gfx9_vopc {
+ u32 src0:9;
+ u32 vsrc1:8;
+ u32 op:8;
+ u32 encoding:7;
+ u32 literal;
+};
+
+enum amdgcn_gfx9_vopc_compare_offset16 {
+ GFX9_VOPC16_F = 0,
+ GFX9_VOPC16_LT,
+ GFX9_VOPC16_EQ,
+ GFX9_VOPC16_LE,
+ GFX9_VOPC16_GT,
+ GFX9_VOPC16_LG,
+ GFX9_VOPC16_GE,
+ GFX9_VOPC16_O,
+ GFX9_VOPC16_U,
+ GFX9_VOPC16_NGE,
+ GFX9_VOPC16_NLG,
+ GFX9_VOPC16_NGT,
+ GFX9_VOPC16_NLE,
+ GFX9_VOPC16_NEQ,
+ GFX9_VOPC16_NLT,
+ GFX9_VOPC16_TRU,
+};
+
+enum amdgcn_gfx9_vopc_compare_offset8 {
+ GFX9_VOPC8_F = 0,
+ GFX9_VOPC8_LT,
+ GFX9_VOPC8_EQ,
+ GFX9_VOPC8_LE,
+ GFX9_VOPC8_GT,
+ GFX9_VOPC8_LG,
+ GFX9_VOPC8_GE,
+ GFX9_VOPC8_TRU,
+};
+
+enum amdgcn_gfx9_vopc_opcode {
+ GFX9_V_CMP_CLASS_F32 = 16,
+ GFX9_V_CMPX_CLASS_F32,
+ GFX9_V_CMP_CLASS_F64,
+ GFX9_V_CMPX_CLASS_F64,
+ GFX9_V_CMP_CLASS_F16,
+ GFX9_V_CMPX_CLASS_F16 = 21,
+ GFX9_V_CMP_F_F16 = 32,
+ GFX9_V_CMP_LT_F16,
+ GFX9_V_CMP_EQ_F16,
+ GFX9_V_CMP_LE_F16,
+ GFX9_V_CMP_GT_F16,
+ GFX9_V_CMP_LG_F16,
+ GFX9_V_CMP_GE_F16,
+ GFX9_V_CMP_O_F16,
+ GFX9_V_CMP_U_F16,
+ GFX9_V_CMP_NGE_F16,
+ GFX9_V_CMP_NLG_F16,
+ GFX9_V_CMP_NGT_F16,
+ GFX9_V_CMP_NLE_F16,
+ GFX9_V_CMP_NEQ_F16,
+ GFX9_V_CMP_NLT_F16,
+ GFX9_V_CMP_TRU_F16,
+ GFX9_V_CMPX_F_F16,
+ GFX9_V_CMPX_LT_F16,
+ GFX9_V_CMPX_EQ_F16,
+ GFX9_V_CMPX_LE_F16,
+ GFX9_V_CMPX_GT_F16,
+ GFX9_V_CMPX_LG_F16,
+ GFX9_V_CMPX_GE_F16,
+ GFX9_V_CMPX_O_F16,
+ GFX9_V_CMPX_U_F16,
+ GFX9_V_CMPX_NGE_F16,
+ GFX9_V_CMPX_NLG_F16,
+ GFX9_V_CMPX_NGT_F16,
+ GFX9_V_CMPX_NLE_F16,
+ GFX9_V_CMPX_NEQ_F16,
+ GFX9_V_CMPX_NLT_F16,
+ GFX9_V_CMPX_TRU_F16,
+ GFX9_V_CMP_F_F32,
+ GFX9_V_CMP_LT_F32,
+ GFX9_V_CMP_EQ_F32,
+ GFX9_V_CMP_LE_F32,
+ GFX9_V_CMP_GT_F32,
+ GFX9_V_CMP_LG_F32,
+ GFX9_V_CMP_GE_F32,
+ GFX9_V_CMP_O_F32,
+ GFX9_V_CMP_U_F32,
+ GFX9_V_CMP_NGE_F32,
+ GFX9_V_CMP_NLG_F32,
+ GFX9_V_CMP_NGT_F32,
+ GFX9_V_CMP_NLE_F32,
+ GFX9_V_CMP_NEQ_F32,
+ GFX9_V_CMP_NLT_F32,
+ GFX9_V_CMP_TRU_F32,
+ GFX9_V_CMPX_F_F32,
+ GFX9_V_CMPX_LT_F32,
+ GFX9_V_CMPX_EQ_F32,
+ GFX9_V_CMPX_LE_F32,
+ GFX9_V_CMPX_GT_F32,
+ GFX9_V_CMPX_LG_F32,
+ GFX9_V_CMPX_GE_F32,
+ GFX9_V_CMPX_O_F32,
+ GFX9_V_CMPX_U_F32,
+ GFX9_V_CMPX_NGE_F32,
+ GFX9_V_CMPX_NLG_F32,
+ GFX9_V_CMPX_NGT_F32,
+ GFX9_V_CMPX_NLE_F32,
+ GFX9_V_CMPX_NEQ_F32,
+ GFX9_V_CMPX_NLT_F32,
+ GFX9_V_CMPX_TRU_F32,
+ GFX9_V_CMP_F_F64,
+ GFX9_V_CMP_LT_F64,
+ GFX9_V_CMP_EQ_F64,
+ GFX9_V_CMP_LE_F64,
+ GFX9_V_CMP_GT_F64,
+ GFX9_V_CMP_LG_F64,
+ GFX9_V_CMP_GE_F64,
+ GFX9_V_CMP_O_F64,
+ GFX9_V_CMP_U_F64,
+ GFX9_V_CMP_NGE_F64,
+ GFX9_V_CMP_NLG_F64,
+ GFX9_V_CMP_NGT_F64,
+ GFX9_V_CMP_NLE_F64,
+ GFX9_V_CMP_NEQ_F64,
+ GFX9_V_CMP_NLT_F64,
+ GFX9_V_CMP_TRU_F64,
+ GFX9_V_CMPX_F_F64,
+ GFX9_V_CMPX_LT_F64,
+ GFX9_V_CMPX_EQ_F64,
+ GFX9_V_CMPX_LE_F64,
+ GFX9_V_CMPX_GT_F64,
+ GFX9_V_CMPX_LG_F64,
+ GFX9_V_CMPX_GE_F64,
+ GFX9_V_CMPX_O_F64,
+ GFX9_V_CMPX_U_F64,
+ GFX9_V_CMPX_NGE_F64,
+ GFX9_V_CMPX_NLG_F64,
+ GFX9_V_CMPX_NGT_F64,
+ GFX9_V_CMPX_NLE_F64,
+ GFX9_V_CMPX_NEQ_F64,
+ GFX9_V_CMPX_NLT_F64,
+ GFX9_V_CMPX_TRU_F64 = 127,
+ GFX9_V_CMP_F_I16 = 160,
+ GFX9_V_CMP_LT_I16,
+ GFX9_V_CMP_EQ_I16,
+ GFX9_V_CMP_LE_I16,
+ GFX9_V_CMP_GT_I16,
+ GFX9_V_CMP_NE_I16,
+ GFX9_V_CMP_GE_I16,
+ GFX9_V_CMP_T_I16,
+ GFX9_V_CMP_F_U16,
+ GFX9_V_CMP_LT_U16,
+ GFX9_V_CMP_EQ_U16,
+ GFX9_V_CMP_LE_U16,
+ GFX9_V_CMP_GT_U16,
+ GFX9_V_CMP_NE_U16,
+ GFX9_V_CMP_GE_U16,
+ GFX9_V_CMP_T_U16,
+ GFX9_V_CMPX_F_I16,
+ GFX9_V_CMPX_LT_I16,
+ GFX9_V_CMPX_EQ_I16,
+ GFX9_V_CMPX_LE_I16,
+ GFX9_V_CMPX_GT_I16,
+ GFX9_V_CMPX_NE_I16,
+ GFX9_V_CMPX_GE_I16,
+ GFX9_V_CMPX_T_I16,
+ GFX9_V_CMPX_F_U16,
+ GFX9_V_CMPX_LT_U16,
+ GFX9_V_CMPX_EQ_U16,
+ GFX9_V_CMPX_LE_U16,
+ GFX9_V_CMPX_GT_U16,
+ GFX9_V_CMPX_NE_U16,
+ GFX9_V_CMPX_GE_U16,
+ GFX9_V_CMPX_T_U16,
+ GFX9_V_CMP_F_I32,
+ GFX9_V_CMP_LT_I32,
+ GFX9_V_CMP_EQ_I32,
+ GFX9_V_CMP_LE_I32,
+ GFX9_V_CMP_GT_I32,
+ GFX9_V_CMP_NE_I32,
+ GFX9_V_CMP_GE_I32,
+ GFX9_V_CMP_T_I32,
+ GFX9_V_CMP_F_U32,
+ GFX9_V_CMP_LT_U32,
+ GFX9_V_CMP_EQ_U32,
+ GFX9_V_CMP_LE_U32,
+ GFX9_V_CMP_GT_U32,
+ GFX9_V_CMP_NE_U32,
+ GFX9_V_CMP_GE_U32,
+ GFX9_V_CMP_T_U32,
+ GFX9_V_CMPX_F_I32,
+ GFX9_V_CMPX_LT_I32,
+ GFX9_V_CMPX_EQ_I32,
+ GFX9_V_CMPX_LE_I32,
+ GFX9_V_CMPX_GT_I32,
+ GFX9_V_CMPX_NE_I32,
+ GFX9_V_CMPX_GE_I32,
+ GFX9_V_CMPX_T_I32,
+ GFX9_V_CMPX_F_U32,
+ GFX9_V_CMPX_LT_U32,
+ GFX9_V_CMPX_EQ_U32,
+ GFX9_V_CMPX_LE_U32,
+ GFX9_V_CMPX_GT_U32,
+ GFX9_V_CMPX_NE_U32,
+ GFX9_V_CMPX_GE_U32,
+ GFX9_V_CMPX_T_U32,
+ GFX9_V_CMP_F_I64,
+ GFX9_V_CMP_LT_I64,
+ GFX9_V_CMP_EQ_I64,
+ GFX9_V_CMP_LE_I64,
+ GFX9_V_CMP_GT_I64,
+ GFX9_V_CMP_NE_I64,
+ GFX9_V_CMP_GE_I64,
+ GFX9_V_CMP_T_I64,
+ GFX9_V_CMP_F_U64,
+ GFX9_V_CMP_LT_U64,
+ GFX9_V_CMP_EQ_U64,
+ GFX9_V_CMP_LE_U64,
+ GFX9_V_CMP_GT_U64,
+ GFX9_V_CMP_NE_U64,
+ GFX9_V_CMP_GE_U64,
+ GFX9_V_CMP_T_U64,
+ GFX9_V_CMPX_F_I64,
+ GFX9_V_CMPX_LT_I64,
+ GFX9_V_CMPX_EQ_I64,
+ GFX9_V_CMPX_LE_I64,
+ GFX9_V_CMPX_GT_I64,
+ GFX9_V_CMPX_NE_I64,
+ GFX9_V_CMPX_GE_I64,
+ GFX9_V_CMPX_T_I64,
+ GFX9_V_CMPX_F_U64,
+ GFX9_V_CMPX_LT_U64,
+ GFX9_V_CMPX_EQ_U64,
+ GFX9_V_CMPX_LE_U64,
+ GFX9_V_CMPX_GT_U64,
+ GFX9_V_CMPX_NE_U64,
+ GFX9_V_CMPX_GE_U64,
+ GFX9_V_CMPX_T_U64,
+};
+
+#define GFX9_VOPC_ENCODING 0x3e
+#define GFX9_VOPC_SRC_SGPR_BASE 0
+#define GFX9_VOPC_SRC_VCC_LO 106
+#define GFX9_VOPC_SRC_VCC_HI 107
+#define GFX9_VOPC_SRC_TTPM_BASE 108
+#define GFX9_VOPC_SRC_M0 124
+#define GFX9_VOPC_SRC_NULL 125
+#define GFX9_VOPC_SRC_EXEC_LO 126
+#define GFX9_VOPC_SRC_EXEC_HI 127
+#define GFX9_VOPC_SRC_INTEGER_0 128
+#define GFX9_VOPC_SRC_INTEGER_MINUS_1 193
+#define GFX9_VOPC_SRC_SHARED_BASE 235
+#define GFX9_VOPC_SRC_SHARED_LIMIT 236
+#define GFX9_VOPC_SRC_PRIVATE_BASE 237
+#define GFX9_VOPC_SRC_PRIVATE_LIMIT 238
+#define GFX9_VOPC_SRC_POPS_EXITING_WAVE_ID 239
+#define GFX9_VOPC_SRC_SDWA 249
+#define GFX9_VOPC_SRC_DDP16 250
+#define GFX9_VOPC_SRC_VCCZ 251
+#define GFX9_VOPC_SRC_EXECZ 252
+#define GFX9_VOPC_SRC_SCC 253
+#define GFX9_VOPC_SRC_LITERAL_CONST 255
+#define GFX9_VOPC_SRC_VGPR_BASE 256
+
+struct amdgcn_gfx9_vop3a {
+ u64 vdst:8;
+ u64 abs:3;
+ u64 op_sel:4;
+ u64 clmp:1;
+ u64 op:10;
+ u64 encoding:6;
+ u64 src0:9;
+ u64 src1:9;
+ u64 src2:9;
+ u64 omod:2;
+ u64 neg:3;
+ u32 literal;
+};
+
+enum amdgcn_gfx9_vop3a_opcode {
+ GFX9_V_MAD_LEGACY_F32 = 448,
+ GFX9_V_MAD_F32,
+ GFX9_V_MAD_I32_I24,
+ GFX9_V_MAD_U32_U24,
+ GFX9_V_CUBEID_F32,
+ GFX9_V_CUBESC_F32,
+ GFX9_V_CUBETC_F32,
+ GFX9_V_CUBEMA_F32,
+ GFX9_V_BFE_U32,
+ GFX9_V_BFE_I32,
+ GFX9_V_BFI_B32,
+ GFX9_V_FMA_F32,
+ GFX9_V_FMA_F64,
+ GFX9_V_LERP_U8,
+ GFX9_V_ALIGNBIT_B32,
+ GFX9_V_ALIGNBYTE_B32,
+ GFX9_V_MIN3_F32,
+ GFX9_V_MIN3_I32,
+ GFX9_V_MIN3_U32,
+ GFX9_V_MAX3_F32,
+ GFX9_V_MAX3_I32,
+ GFX9_V_MAX3_U32,
+ GFX9_V_MED3_F32,
+ GFX9_V_MED3_I32,
+ GFX9_V_MED3_U32,
+ GFX9_V_SAD_U8,
+ GFX9_V_SAD_HI_U8,
+ GFX9_V_SAD_U16,
+ GFX9_V_SAD_U32,
+ GFX9_V_CVT_PK_U8_F32,
+ GFX9_V_DIV_FIXUP_F32,
+ GFX9_V_DIV_FIXUP_F64 = 479,
+ GFX9_V_DIV_FMAS_F32 = 482,
+ GFX9_V_DIV_FMAS_F64,
+ GFX9_V_MSAD_U8,
+ GFX9_V_QSAD_PK_U16_U8,
+ GFX9_V_MQSAD_PK_U16_U8,
+ GFX9_V_MQSAD_U32_U8 = 487,
+ GFX9_V_MAD_LEGACY_F16 = 490,
+ GFX9_V_MAD_LEGACY_U16,
+ GFX9_V_MAD_LEGACY_I16,
+ GFX9_V_PERM_B32,
+ GFX9_V_FMA_LEGACY_F16,
+ GFX9_V_DIV_FIXUP_LEGACY_F16,
+ GFX9_V_CVT_PKACCUM_U8_F32,
+ GFX9_V_MAD_U32_U16,
+ GFX9_V_MAD_I32_I16,
+ GFX9_V_XAD_U32,
+ GFX9_V_MIN3_F16,
+ GFX9_V_MIN3_I16,
+ GFX9_V_MIN3_U16,
+ GFX9_V_MAX3_F16,
+ GFX9_V_MAX3_I16,
+ GFX9_V_MAX3_U16,
+ GFX9_V_MED3_F16,
+ GFX9_V_MED3_I16,
+ GFX9_V_MED3_U16,
+ GFX9_V_LSHL_ADD_U32,
+ GFX9_V_ADD_LSHL_U32,
+ GFX9_V_ADD3_U32,
+ GFX9_V_LSHL_OR_B32,
+ GFX9_V_AND_OR_B32,
+ GFX9_V_OR3_B32,
+ GFX9_V_MAD_F16,
+ GFX9_V_MAD_U16,
+ GFX9_V_MAD_I16,
+ GFX9_V_FMA_F16,
+ GFX9_V_DIV_FIXUP_F16 = 519,
+ GFX9_V_INTERP_P1LL_F16 = 628,
+ GFX9_V_INTERP_P1LV_F16,
+ GFX9_V_INTERP_P2_LEGACY_F16,
+ GFX9_V_INTERP_P2_F16 = 631,
+ GFX9_V_ADD_F64 = 640,
+ GFX9_V_MUL_F64,
+ GFX9_V_MIN_F64,
+ GFX9_V_MAX_F64,
+ GFX9_V_LDEXP_F64,
+ GFX9_V_MUL_LO_U32,
+ GFX9_V_MUL_HI_U32,
+ GFX9_V_MUL_HI_I32,
+ GFX9_V_LDEXP_F32,
+ GFX9_V_READLANE_B32,
+ GFX9_V_WRITELANE_B32,
+ GFX9_V_BCNT_U32_B32,
+ GFX9_V_MBCNT_LO_U32_B32,
+ GFX9_V_MBCNT_HI_U32_B32 = 653,
+ GFX9_V_LSHLREV_B64 = 655,
+ GFX9_V_LSHRREV_B64,
+ GFX9_V_ASHRREV_I64,
+ GFX9_V_TRIG_PREOP_F64,
+ GFX9_V_BFM_B32,
+ GFX9_V_CVT_PKNORM_I16_F32,
+ GFX9_V_CVT_PKNORM_U16_F32,
+ GFX9_V_CVT_PKRTZ_F16_F32,
+ GFX9_V_CVT_PK_U16_U32,
+ GFX9_V_CVT_PK_I16_I32,
+ GFX9_V_CVT_PKNORM_I16_F16,
+ GFX9_V_CVT_PKNORM_U16_F16 = 666,
+ GFX9_V_ADD_I32 = 668,
+ GFX9_V_SUB_I32,
+ GFX9_V_ADD_I16,
+ GFX9_V_SUB_I16,
+ GFX9_V_PACK_B32_F16,
+};
+
+enum amdgcn_gfx9_vop3a_abs {
+ GFX9_VOP3A_ABS_SRC0,
+ GFX9_VOP3A_ABS_SRC1,
+ GFX9_VOP3A_ABS_SRC2,
+};
+
+#define GFX9_VOP3A_ENCODING 0x34
+#define GFX9_VOP3A_SRC_SGPR_BASE 0
+#define GFX9_VOP3A_SRC_VCC_LO 106
+#define GFX9_VOP3A_SRC_VCC_HI 107
+#define GFX9_VOP3A_SRC_TTPM_BASE 108
+#define GFX9_VOP3A_SRC_M0 124
+#define GFX9_VOP3A_SRC_NULL 125
+#define GFX9_VOP3A_SRC_EXEC_LO 126
+#define GFX9_VOP3A_SRC_EXEC_HI 127
+#define GFX9_VOP3A_SRC_INTEGER_0 128
+#define GFX9_VOP3A_SRC_INTEGER_MINUS_1 193
+#define GFX9_VOP3A_SRC_SHARED_BASE 235
+#define GFX9_VOP3A_SRC_SHARED_LIMIT 236
+#define GFX9_VOP3A_SRC_PRIVATE_BASE 237
+#define GFX9_VOP3A_SRC_PRIVATE_LIMIT 238
+#define GFX9_VOP3A_SRC_POPS_EXITING_WAVE_ID 239
+#define GFX9_VOP3A_SRC_SDWA 249
+#define GFX9_VOP3A_SRC_DDP16 250
+#define GFX9_VOP3A_SRC_VCCZ 251
+#define GFX9_VOP3A_SRC_EXECZ 252
+#define GFX9_VOP3A_SRC_SCC 253
+#define GFX9_VOP3A_SRC_LITERAL_CONST 255
+#define GFX9_VOP3A_SRC_VGPR_BASE 256
+
+struct amdgcn_gfx9_vop3b {
+ u64 vdst:8;
+ u64 sdst:7;
+ u64 clmp:1;
+ u64 op:10;
+ u64 encoding:6;
+ u64 src0:9;
+ u64 src1:9;
+ u64 src2:9;
+ u64 omod:2;
+ u64 neg:3;
+ u32 literal;
+};
+
+enum amdgcn_gfx9_vop3b_opcode {
+ GFX9_V_DIV_SCALE_F64 = 481,
+ GFX9_V_MAD_U64_U32 = 488,
+ GFX9_V_MAD_I64_I32 = 489,
+};
+
+#define GFX9_VOP3B_ENCODING 0x34
+#define GFX9_VOP3B_SRC_SGPR_BASE 0
+#define GFX9_VOP3B_SRC_VCC_LO 106
+#define GFX9_VOP3B_SRC_VCC_HI 107
+#define GFX9_VOP3B_SRC_TTPM_BASE 108
+#define GFX9_VOP3B_SRC_M0 124
+#define GFX9_VOP3B_SRC_NULL 125
+#define GFX9_VOP3B_SRC_EXEC_LO 126
+#define GFX9_VOP3B_SRC_EXEC_HI 127
+#define GFX9_VOP3B_SRC_INTEGER_0 128
+#define GFX9_VOP3B_SRC_INTEGER_MINUS_1 193
+#define GFX9_VOP3B_SRC_SHARED_BASE 235
+#define GFX9_VOP3B_SRC_SHARED_LIMIT 236
+#define GFX9_VOP3B_SRC_PRIVATE_BASE 237
+#define GFX9_VOP3B_SRC_PRIVATE_LIMIT 238
+#define GFX9_VOP3B_SRC_POPS_EXITING_WAVE_ID 239
+#define GFX9_VOP3B_SRC_SDWA 249
+#define GFX9_VOP3B_SRC_DDP16 250
+#define GFX9_VOP3B_SRC_VCCZ 251
+#define GFX9_VOP3B_SRC_EXECZ 252
+#define GFX9_VOP3B_SRC_SCC 253
+#define GFX9_VOP3B_SRC_LITERAL_CONST 255
+#define GFX9_VOP3B_SRC_VGPR_BASE 256
+
+struct amdgcn_gfx9_vop3p {
+ u64 vdst:8;
+ u64 neg_hi:3;
+ u64 op_sel:3;
+ u64 op_sel_hi2:1;
+ u64 clmp:1;
+ u64 op:7;
+ u64 encoding:9;
+ u64 src0:9;
+ u64 src1:9;
+ u64 src2:9;
+ u64 op_sel_hi:2;
+ u64 neg:3;
+ u32 literal;
+};
+
+enum amdgcn_gfx9_vop3p_opcode {
+ GFX9_V_PK_MAD_I16,
+ GFX9_V_PK_MUL_LO_U16,
+ GFX9_V_PK_ADD_I16,
+ GFX9_V_PK_SUB_I16,
+ GFX9_V_PK_LSHLREV_B16,
+ GFX9_V_PK_LSHRREV_B16,
+ GFX9_V_PK_ASHRREV_I16,
+ GFX9_V_PK_MAX_I16,
+ GFX9_V_PK_MIN_I16,
+ GFX9_V_PK_MAD_U16,
+ GFX9_V_PK_ADD_U16,
+ GFX9_V_PK_SUB_U16,
+ GFX9_V_PK_MAX_U16,
+ GFX9_V_PK_MIN_U16,
+ GFX9_V_PK_FMA_F16,
+ GFX9_V_PK_ADD_F16,
+ GFX9_V_PK_MUL_F16,
+ GFX9_V_PK_MIN_F16,
+ GFX9_V_PK_MAX_F16 = 18,
+ GFX9_V_MAD_MIX_F32 = 32,
+ GFX9_V_MAD_MIXLO_F16,
+ GFX9_V_MAD_MIXHI_F16,
+};
+
+#define GFX9_VOP3P_ENCODING 0x1a7
+#define GFX9_VOP3P_SRC_SGPR_BASE 0
+#define GFX9_VOP3P_SRC_VCC_LO 106
+#define GFX9_VOP3P_SRC_VCC_HI 107
+#define GFX9_VOP3P_SRC_TTPM_BASE 108
+#define GFX9_VOP3P_SRC_M0 124
+#define GFX9_VOP3P_SRC_NULL 125
+#define GFX9_VOP3P_SRC_EXEC_LO 126
+#define GFX9_VOP3P_SRC_EXEC_HI 127
+#define GFX9_VOP3P_SRC_INTEGER_0 128
+#define GFX9_VOP3P_SRC_INTEGER_MINUS_1 193
+#define GFX9_VOP3P_SRC_SHARED_BASE 235
+#define GFX9_VOP3P_SRC_SHARED_LIMIT 236
+#define GFX9_VOP3P_SRC_PRIVATE_BASE 237
+#define GFX9_VOP3P_SRC_PRIVATE_LIMIT 238
+#define GFX9_VOP3P_SRC_POPS_EXITING_WAVE_ID 239
+#define GFX9_VOP3P_SRC_SDWA 249
+#define GFX9_VOP3P_SRC_DDP16 250
+#define GFX9_VOP3P_SRC_VCCZ 251
+#define GFX9_VOP3P_SRC_EXECZ 252
+#define GFX9_VOP3P_SRC_SCC 253
+#define GFX9_VOP3P_SRC_VGPR_BASE 256
+
+struct amdgcn_gfx9_sdwa {
+ u32 src0:8;
+ u32 dst_sel:3;
+ u32 dst_u:2;
+ u32 clmp:1;
+ u32 omod:2;
+ u32 src0_sel:3;
+ u32 src0_sext:1;
+ u32 src0_neg:1;
+ u32 src0_abs:1;
+ u32 dummy1:1;
+ u32 s0:1;
+ u32 src1_sel:3;
+ u32 src1_sext:1;
+ u32 src1_neg:1;
+ u32 src1_abs:1;
+ u32 dummy2:1;
+ u32 s1:1;
+};
+
+struct amdgcn_gfx9_sdwab {
+ u32 src0:8;
+ u32 sdst:7;
+ u32 sd:1;
+ u32 src0_sel:3;
+ u32 src0_sext:1;
+ u32 src0_neg:1;
+ u32 src0_abs:1;
+ u32 dummy1:1;
+ u32 s0:1;
+ u32 src1_sel:3;
+ u32 src1_sext:1;
+ u32 src1_neg:1;
+ u32 src1_abs:1;
+ u32 dummy2:1;
+ u32 s1:1;
+};
+
+struct amdgcn_gfx9_dpp16 {
+};
+
+struct amdgcn_gfx9_dpp8 {
+};
+
+/* Vector Parameter Interpolation Format */
+struct amdgcn_gfx9_vintrp {
+};
+
+/* LDS and GDS Format */
+struct amdgcn_gfx9_ds {
+ u64 offset0:8;
+ u64 offset1:8;
+ u64 gds:1;
+ u64 op:9; /* bits [25:17] */
+ u64 encoding:6; /* bits [31:26] = 0x36 */
+ u64 addr:8;
+ u64 data0:8;
+ u64 data1:8;
+ u64 vdst:8;
+};
+
+enum amdgcn_gfx9_ds_opcode {
+ GFX9_DS_ADD_U32,
+ GFX9_DS_SUB_U32,
+ GFX9_DS_RSUB_U32,
+ GFX9_DS_INC_U32,
+ GFX9_DS_DEC_U32,
+ GFX9_DS_MIN_I32,
+ GFX9_DS_MAX_I32,
+ GFX9_DS_MIN_U32,
+ GFX9_DS_MAX_U32,
+ GFX9_DS_AND_B32,
+ GFX9_DS_OR_B32,
+ GFX9_DS_XOR_B32,
+ GFX9_DS_MSKOR_B32,
+ GFX9_DS_WRITE_B32,
+ GFX9_DS_WRITE2_B32,
+ GFX9_DS_WRITE2ST64_B32,
+ GFX9_DS_CMPST_B32,
+ GFX9_DS_CMPST_F32,
+ GFX9_DS_MIN_F32,
+ GFX9_DS_MAX_F32,
+ GFX9_DS_NOP,
+ GFX9_DS_ADD_F32 = 21,
+ GFX9_DS_WRITE_ADDTID_B32 = 29,
+ GFX9_DS_WRITE_B8,
+ GFX9_DS_WRITE_B16,
+ GFX9_DS_ADD_RTN_U32,
+ GFX9_DS_SUB_RTN_U32,
+ GFX9_DS_RSUB_RTN_U32,
+ GFX9_DS_INC_RTN_U32,
+ GFX9_DS_DEC_RTN_U32,
+ GFX9_DS_MIN_RTN_I32,
+ GFX9_DS_MAX_RTN_I32,
+ GFX9_DS_MIN_RTN_U32,
+ GFX9_DS_MAX_RTN_U32,
+ GFX9_DS_AND_RTN_B32,
+ GFX9_DS_OR_RTN_B32,
+ GFX9_DS_XOR_RTN_B32,
+ GFX9_DS_MSKOR_RTN_B32,
+ GFX9_DS_WRXCHG_RTN_B32,
+ GFX9_DS_WRXCHG2_RTN_B32,
+ GFX9_DS_WRXCHG2ST64_RTN_B32,
+ GFX9_DS_CMPST_RTN_B32,
+ GFX9_DS_CMPST_RTN_F32,
+ GFX9_DS_MIN_RTN_F32,
+ GFX9_DS_MAX_RTN_F32,
+ GFX9_DS_WRAP_RTN_B32,
+ GFX9_DS_ADD_RTN_F32,
+ GFX9_DS_READ_B32,
+ GFX9_DS_READ2_B32,
+ GFX9_DS_READ2ST64_B32,
+ GFX9_DS_READ_I8,
+ GFX9_DS_READ_U8,
+ GFX9_DS_READ_I16,
+ GFX9_DS_READ_U16,
+ GFX9_DS_SWIZZLE_B32,
+ GFX9_DS_PERMUTE_B32,
+ GFX9_DS_BPERMUTE_B32,
+ GFX9_DS_ADD_U64,
+ GFX9_DS_SUB_U64,
+ GFX9_DS_RSUB_U64,
+ GFX9_DS_INC_U64,
+ GFX9_DS_DEC_U64,
+ GFX9_DS_MIN_I64,
+ GFX9_DS_MAX_I64,
+ GFX9_DS_MIN_U64,
+ GFX9_DS_MAX_U64,
+ GFX9_DS_AND_B64,
+ GFX9_DS_OR_B64,
+ GFX9_DS_XOR_B64,
+ GFX9_DS_MSKOR_B64,
+ GFX9_DS_WRITE_B64,
+ GFX9_DS_WRITE2_B64,
+ GFX9_DS_WRITE2ST64_B64,
+ GFX9_DS_CMPST_B64,
+ GFX9_DS_CMPST_F64,
+ GFX9_DS_MIN_F64,
+ GFX9_DS_MAX_F64,
+ GFX9_DS_WRITE_B8_D16_HI,
+ GFX9_DS_WRITE_B16_D16_HI,
+ GFX9_DS_READ_U8_D16,
+ GFX9_DS_READ_U8_D16_HI,
+ GFX9_DS_READ_I8_D16,
+ GFX9_DS_READ_I8_D16_HI,
+ GFX9_DS_READ_U16_D16,
+ GFX9_DS_READ_U16_D16_HI = 91,
+ GFX9_DS_ADD_RTN_U64 = 96,
+ GFX9_DS_SUB_RTN_U64,
+ GFX9_DS_RSUB_RTN_U64,
+ GFX9_DS_INC_RTN_U64,
+ GFX9_DS_DEC_RTN_U64,
+ GFX9_DS_MIN_RTN_I64,
+ GFX9_DS_MAX_RTN_I64,
+ GFX9_DS_MIN_RTN_U64,
+ GFX9_DS_MAX_RTN_U64,
+ GFX9_DS_AND_RTN_B64,
+ GFX9_DS_OR_RTN_B64,
+ GFX9_DS_XOR_RTN_B64,
+ GFX9_DS_MSKOR_RTN_B64,
+ GFX9_DS_WRXCHG_RTN_B64,
+ GFX9_DS_WRXCHG2_RTN_B64,
+ GFX9_DS_WRXCHG2ST64_RTN_B64,
+ GFX9_DS_CMPST_RTN_B64,
+ GFX9_DS_CMPST_RTN_F64,
+ GFX9_DS_MIN_RTN_F64,
+ GFX9_DS_MAX_RTN_F64 = 115,
+ GFX9_DS_READ_B64 = 118,
+ GFX9_DS_READ2_B64 = 119,
+ GFX9_DS_READ2ST64_B64 = 120,
+ /* ISA opcodes 121-125 are reserved gaps. */
+ GFX9_DS_CONDXCHG32_RTN_B64 = 126,
+ /* ISA opcode 127 is a reserved gap. */
+ GFX9_DS_ADD_SRC2_U32 = 128,
+ GFX9_DS_SUB_SRC2_U32,
+ GFX9_DS_RSUB_SRC2_U32,
+ GFX9_DS_INC_SRC2_U32,
+ GFX9_DS_DEC_SRC2_U32,
+ GFX9_DS_MIN_SRC2_I32,
+ GFX9_DS_MAX_SRC2_I32,
+ GFX9_DS_MIN_SRC2_U32,
+ GFX9_DS_MAX_SRC2_U32,
+ GFX9_DS_AND_SRC2_B32,
+ GFX9_DS_OR_SRC2_B32,
+ GFX9_DS_XOR_SRC2_B32 = 139,
+ GFX9_DS_WRITE_SRC2_B32 = 141,
+ GFX9_DS_MIN_SRC2_F32 = 146,
+ GFX9_DS_MAX_SRC2_F32 = 147,
+ GFX9_DS_ADD_SRC2_F32 = 149,
+ GFX9_DS_GWS_SEMA_RELEASE_ALL = 152,
+ GFX9_DS_GWS_INIT,
+ GFX9_DS_GWS_SEMA_V,
+ GFX9_DS_GWS_SEMA_BR,
+ GFX9_DS_GWS_SEMA_P,
+ GFX9_DS_GWS_BARRIER = 157,
+ GFX9_DS_READ_ADDTID_B32 = 182,
+ GFX9_DS_CONSUME = 189,
+ GFX9_DS_APPEND,
+ GFX9_DS_ORDERED_COUNT,
+ GFX9_DS_ADD_SRC2_U64,
+ GFX9_DS_SUB_SRC2_U64,
+ GFX9_DS_RSUB_SRC2_U64,
+ GFX9_DS_INC_SRC2_U64,
+ GFX9_DS_DEC_SRC2_U64,
+ GFX9_DS_MIN_SRC2_I64,
+ GFX9_DS_MAX_SRC2_I64,
+ GFX9_DS_MIN_SRC2_U64,
+ GFX9_DS_MAX_SRC2_U64,
+ GFX9_DS_AND_SRC2_B64,
+ GFX9_DS_OR_SRC2_B64,
+ GFX9_DS_XOR_SRC2_B64 = 203,
+ GFX9_DS_WRITE_SRC2_B64 = 205,
+ GFX9_DS_MIN_SRC2_F64 = 210,
+ GFX9_DS_MAX_SRC2_F64 = 211,
+ GFX9_DS_WRITE_B96 = 222,
+ GFX9_DS_WRITE_B128 = 223,
+ GFX9_DS_READ_B96 = 254,
+ GFX9_DS_READ_B128 = 255,
+};
+
+#define GFX9_DS_ENCODING 0x36
+#define GFX9_DS_GDS 1
+#define GFX9_DS_LDS 0
+
+////////////////////////////////////////////////////
+/* Vector Memory Buffer Formats */
+struct amdgcn_gfx9_mtbuf {
+ u64 offset:12;
+ u64 offen:1;
+ u64 idxen:1;
+ u64 glc:1;
+ u64 op:4;
+ u64 dfmt:4;
+ u64 nfmt:3;
+ u64 encoding:6;
+ u64 vaddr:8;
+ u64 vdata:8;
+ u64 srsrc:5;
+ u64 dummy:1;
+ u64 slc:1;
+ u64 tfe:1;
+ u64 soffset:8;
+};
+
+enum amdgcn_gfx9_mtbuf_opcode {
+ GFX9_TBUFFER_LOAD_FORMAT_X,
+ GFX9_TBUFFER_LOAD_FORMAT_XY,
+ GFX9_TBUFFER_LOAD_FORMAT_XYZ,
+ GFX9_TBUFFER_LOAD_FORMAT_XYZW,
+ GFX9_TBUFFER_STORE_FORMAT_X,
+ GFX9_TBUFFER_STORE_FORMAT_XY,
+ GFX9_TBUFFER_STORE_FORMAT_XYZ,
+ GFX9_TBUFFER_STORE_FORMAT_XYZW,
+ GFX9_TBUFFER_LOAD_FORMAT_D16_X,
+ GFX9_TBUFFER_LOAD_FORMAT_D16_XY,
+ GFX9_TBUFFER_LOAD_FORMAT_D16_XYZ,
+ GFX9_TBUFFER_LOAD_FORMAT_D16_XYZW,
+ GFX9_TBUFFER_STORE_FORMAT_D16_X,
+ GFX9_TBUFFER_STORE_FORMAT_D16_XY,
+ GFX9_TBUFFER_STORE_FORMAT_D16_XYZ,
+ GFX9_TBUFFER_STORE_FORMAT_D16_XYZW,
+};
+
+#define GFX9_MUBUF_ENCODING 0x38
+#define GFX9_MUBUF_SOFFSET_SGPR_BASE 0
+#define GFX9_MUBUF_SOFFSET_VCC_LO 106
+#define GFX9_MUBUF_SOFFSET_VCC_HI 107
+#define GFX9_MUBUF_SOFFSET_TTPM_BASE 108
+#define GFX9_MUBUF_SOFFSET_M0 124
+#define GFX9_MUBUF_SOFFSET_NULL 125
+#define GFX9_MUBUF_SOFFSET_EXEC_LO 126
+#define GFX9_MUBUF_SOFFSET_EXEC_HI 127
+#define GFX9_MUBUF_SOFFSET_INTEGER_0 128
+#define GFX9_MUBUF_SOFFSET_INTEGER_MINUS_1 193
+#define GFX9_MUBUF_SOFFSET_SHARED_BASE 235
+#define GFX9_MUBUF_SOFFSET_SHARED_LIMIT 236
+#define GFX9_MUBUF_SOFFSET_PRIVATE_BASE 237
+#define GFX9_MUBUF_SOFFSET_PRIVATE_LIMIT 238
+#define GFX9_MUBUF_SOFFSET_POPS_EXITING_WAVE_ID 239
+#define GFX9_MUBUF_SOFFSET_VCCZ 251
+#define GFX9_MUBUF_SOFFSET_EXECZ 252
+#define GFX9_MUBUF_SOFFSET_SCC 253
+
+struct amdgcn_gfx9_mubuf {
+ u64 offset:12;
+ u64 offen:1;
+ u64 idxen:1;
+ u64 glc:1;
+ u64 dummy1:1;
+ u64 lds:1;
+ u64 slc:1;
+ u64 op:7;
+ u64 dummy2:1;
+ u64 encoding:6;
+ u64 vaddr:8;
+ u64 vdata:8;
+ u64 srsrc:5;
+ u64 dummy3:2;
+ u64 tfe:1;
+ u64 soffset:8;
+};
+
+enum amdgcn_gfx9_mubuf_opcode {
+ GFX9_BUFFER_LOAD_FORMAT_X,
+ GFX9_BUFFER_LOAD_FORMAT_XY,
+ GFX9_BUFFER_LOAD_FORMAT_XYZ,
+ GFX9_BUFFER_LOAD_FORMAT_XYZW,
+ GFX9_BUFFER_STORE_FORMAT_X,
+ GFX9_BUFFER_STORE_FORMAT_XY,
+ GFX9_BUFFER_STORE_FORMAT_XYZ,
+ GFX9_BUFFER_STORE_FORMAT_XYZW,
+ GFX9_BUFFER_LOAD_FORMAT_D16_X,
+ GFX9_BUFFER_LOAD_FORMAT_D16_XY,
+ GFX9_BUFFER_LOAD_FORMAT_D16_XYZ,
+ GFX9_BUFFER_LOAD_FORMAT_D16_XYZW,
+ GFX9_BUFFER_STORE_FORMAT_D16_X,
+ GFX9_BUFFER_STORE_FORMAT_D16_XY,
+ GFX9_BUFFER_STORE_FORMAT_D16_XYZ,
+ GFX9_BUFFER_STORE_FORMAT_D16_XYZW,
+ GFX9_BUFFER_LOAD_UBYTE,
+ GFX9_BUFFER_LOAD_SBYTE,
+ GFX9_BUFFER_LOAD_USHORT,
+ GFX9_BUFFER_LOAD_SSHORT,
+ GFX9_BUFFER_LOAD_DWORD,
+ GFX9_BUFFER_LOAD_DWORDX2,
+ GFX9_BUFFER_LOAD_DWORDX3,
+ GFX9_BUFFER_LOAD_DWORDX4,
+ GFX9_BUFFER_STORE_BYTE,
+ GFX9_BUFFER_STORE_BYTE_D16_HI,
+ GFX9_BUFFER_STORE_SHORT,
+ GFX9_BUFFER_STORE_SHORT_D16_HI,
+ GFX9_BUFFER_STORE_DWORD,
+ GFX9_BUFFER_STORE_DWORDX2,
+ GFX9_BUFFER_STORE_DWORDX3,
+ GFX9_BUFFER_STORE_DWORDX4,
+ GFX9_BUFFER_LOAD_UBYTE_D16,
+ GFX9_BUFFER_LOAD_UBYTE_D16_HI,
+ GFX9_BUFFER_LOAD_SBYTE_D16,
+ GFX9_BUFFER_LOAD_SBYTE_D16_HI,
+ GFX9_BUFFER_LOAD_SHORT_D16,
+ GFX9_BUFFER_LOAD_SHORT_D16_HI,
+ GFX9_BUFFER_LOAD_FORMAT_D16_HI_X,
+ GFX9_BUFFER_STORE_FORMAT_D16_HI_X = 39,
+ GFX9_BUFFER_STORE_LDS_DWORD = 61,
+ GFX9_BUFFER_WBINVL1,
+ GFX9_BUFFER_WBINVL1_VOL,
+ GFX9_BUFFER_ATOMIC_SWAP,
+ GFX9_BUFFER_ATOMIC_CMPSWAP,
+ GFX9_BUFFER_ATOMIC_ADD,
+ GFX9_BUFFER_ATOMIC_SUB,
+ GFX9_BUFFER_ATOMIC_SMIN,
+ GFX9_BUFFER_ATOMIC_UMIN,
+ GFX9_BUFFER_ATOMIC_SMAX,
+ GFX9_BUFFER_ATOMIC_UMAX,
+ GFX9_BUFFER_ATOMIC_AND,
+ GFX9_BUFFER_ATOMIC_OR,
+ GFX9_BUFFER_ATOMIC_XOR,
+ GFX9_BUFFER_ATOMIC_INC,
+ GFX9_BUFFER_ATOMIC_DEC = 76,
+ GFX9_BUFFER_ATOMIC_SWAP_X2 = 96,
+ GFX9_BUFFER_ATOMIC_CMPSWAP_X2,
+ GFX9_BUFFER_ATOMIC_ADD_X2,
+ GFX9_BUFFER_ATOMIC_SUB_X2,
+ GFX9_BUFFER_ATOMIC_SMIN_X2,
+ GFX9_BUFFER_ATOMIC_UMIN_X2,
+ GFX9_BUFFER_ATOMIC_SMAX_X2,
+ GFX9_BUFFER_ATOMIC_UMAX_X2,
+ GFX9_BUFFER_ATOMIC_AND_X2,
+ GFX9_BUFFER_ATOMIC_OR_X2,
+ GFX9_BUFFER_ATOMIC_XOR_X2,
+ GFX9_BUFFER_ATOMIC_INC_X2,
+ GFX9_BUFFER_ATOMIC_DEC_X2,
+};
+
+/* Vector Memory Image Format */
+struct amdgcn_gfx9_mimg {
+};
+
+#define GFX9_FLAT_ENCODING 0x37
+#define GFX9_FLAT_SADDR_SGPR_BASE 0
+#define GFX9_FLAT_SADDR_VCC_LO 106
+#define GFX9_FLAT_SADDR_VCC_HI 107
+#define GFX9_FLAT_SADDR_TTPM_BASE 108
+#define GFX9_FLAT_SADDR_M0 124
+#define GFX9_FLAT_SADDR_NULL 125
+#define GFX9_FLAT_SADDR_EXEC_LO 126
+#define GFX9_FLAT_SADDR_EXEC_HI 127
+#define GFX9_FLAT_SADDR_INTEGER_0 128
+#define GFX9_FLAT_SADDR_INTEGER_MINUS_1 193
+#define GFX9_FLAT_SADDR_SHARED_BASE 235
+#define GFX9_FLAT_SADDR_SHARED_LIMIT 236
+#define GFX9_FLAT_SADDR_PRIVATE_BASE 237
+#define GFX9_FLAT_SADDR_PRIVATE_LIMIT 238
+#define GFX9_FLAT_SADDR_POPS_EXITING_WAVE_ID 239
+#define GFX9_FLAT_SADDR_VCCZ 251
+#define GFX9_FLAT_SADDR_EXECZ 252
+#define GFX9_FLAT_SADDR_SCC 253
+
+/*
+ * Scalar SGPR that provides an offset address. To disable, set this field to
+ * 0x7F. Meaning of this field is different for Scratch and Global:
+ * Flat: Unused.
+ * Scratch: Use an SGPR (instead of VGPR) for the address.
+ * Global: Use the SGPR to provide a base address; the VGPR provides a
+ * 32-bit offset.
+ */
+#define GFX9_FLAT_SADDR_DISABLE 0x7f
+#define GFX9_FLAT_SEG_FLAT 0
+#define GFX9_FLAT_SEG_SCRATCH 1
+#define GFX9_FLAT_SEG_GLOBAL 2
+
+/* Flat Formats */
+struct amdgcn_gfx9_flat {
+ u64 offset:13;
+ u64 lds:1;
+ u64 seg:2;
+ u64 glc:1;
+ u64 slc:1;
+ u64 op:7;
+ u64 dummy:1;
+ u64 encoding:6;
+ u64 addr:8;
+ u64 data:8;
+ u64 saddr:7;
+ u64 nv:1;
+ u64 vdst:8;
+};
+
+enum amdgcn_gfx9_global_opcode {
+ GFX9_GLOBAL_LOAD_UBYTE = 16,
+ GFX9_GLOBAL_LOAD_SBYTE,
+ GFX9_GLOBAL_LOAD_USHORT,
+ GFX9_GLOBAL_LOAD_SSHORT,
+ GFX9_GLOBAL_LOAD_DWORD,
+ GFX9_GLOBAL_LOAD_DWORDX2,
+ GFX9_GLOBAL_LOAD_DWORDX3,
+ GFX9_GLOBAL_LOAD_DWORDX4,
+ GFX9_GLOBAL_STORE_BYTE,
+ GFX9_GLOBAL_STORE_BYTE_D16_HI,
+ GFX9_GLOBAL_STORE_SHORT,
+ GFX9_GLOBAL_STORE_SHORT_D16_HI,
+ GFX9_GLOBAL_STORE_DWORD,
+ GFX9_GLOBAL_STORE_DWORDX2,
+ GFX9_GLOBAL_STORE_DWORDX3,
+ GFX9_GLOBAL_STORE_DWORDX4,
+ GFX9_GLOBAL_LOAD_UBYTE_D16,
+ GFX9_GLOBAL_LOAD_UBYTE_D16_HI,
+ GFX9_GLOBAL_LOAD_SBYTE_D16,
+ GFX9_GLOBAL_LOAD_SBYTE_D16_HI,
+ GFX9_GLOBAL_LOAD_SHORT_D16,
+ GFX9_GLOBAL_LOAD_SHORT_D16_HI = 37,
+ GFX9_GLOBAL_ATOMIC_SWAP = 64,
+ GFX9_GLOBAL_ATOMIC_CMPSWAP,
+ GFX9_GLOBAL_ATOMIC_ADD,
+ GFX9_GLOBAL_ATOMIC_SUB,
+ GFX9_GLOBAL_ATOMIC_SMIN,
+ GFX9_GLOBAL_ATOMIC_UMIN,
+ GFX9_GLOBAL_ATOMIC_SMAX,
+ GFX9_GLOBAL_ATOMIC_UMAX,
+ GFX9_GLOBAL_ATOMIC_AND,
+ GFX9_GLOBAL_ATOMIC_OR,
+ GFX9_GLOBAL_ATOMIC_XOR,
+ GFX9_GLOBAL_ATOMIC_INC,
+ GFX9_GLOBAL_ATOMIC_DEC = 76,
+ GFX9_GLOBAL_ATOMIC_SWAP_X2 = 96,
+ GFX9_GLOBAL_ATOMIC_CMPSWAP_X2,
+ GFX9_GLOBAL_ATOMIC_ADD_X2,
+ GFX9_GLOBAL_ATOMIC_SUB_X2,
+ GFX9_GLOBAL_ATOMIC_SMIN_X2,
+ GFX9_GLOBAL_ATOMIC_UMIN_X2,
+ GFX9_GLOBAL_ATOMIC_SMAX_X2,
+ GFX9_GLOBAL_ATOMIC_UMAX_X2,
+ GFX9_GLOBAL_ATOMIC_AND_X2,
+ GFX9_GLOBAL_ATOMIC_OR_X2,
+ GFX9_GLOBAL_ATOMIC_XOR_X2,
+ GFX9_GLOBAL_ATOMIC_INC_X2,
+ GFX9_GLOBAL_ATOMIC_DEC_X2,
+};
+
+/* Export Format */
+struct amdgcn_gfx9_exp {
+};
+
+union amdgcn_gfx9_insn {
+ struct amdgcn_gfx9_sop2 sop2;
+ struct amdgcn_gfx9_sopk sopk;
+ struct amdgcn_gfx9_sop1 sop1;
+ struct amdgcn_gfx9_sopc sopc;
+ struct amdgcn_gfx9_sopp sopp;
+ struct amdgcn_gfx9_smem smem;
+ struct amdgcn_gfx9_vop2 vop2;
+ struct amdgcn_gfx9_vop1 vop1;
+ struct amdgcn_gfx9_vopc vopc;
+ struct amdgcn_gfx9_vop3a vop3a;
+ struct amdgcn_gfx9_vop3b vop3b;
+ struct amdgcn_gfx9_vop3p vop3p;
+ struct amdgcn_gfx9_sdwa sdwa;
+ struct amdgcn_gfx9_sdwab sdwab;
+ struct amdgcn_gfx9_dpp16 dpp16;
+ struct amdgcn_gfx9_dpp8 dpp8;
+ struct amdgcn_gfx9_vintrp vintrp;
+ struct amdgcn_gfx9_ds ds;
+ struct amdgcn_gfx9_mtbuf mtbuf;
+ struct amdgcn_gfx9_mubuf mubuf;
+ struct amdgcn_gfx9_mimg mimg;
+ struct amdgcn_gfx9_flat flat;
+ struct amdgcn_gfx9_exp exp;
+};
+
+/* Cast macro - convert u32 *buf position to GFX9 insn union pointer. */
+#define I9(buf, n) ((union amdgcn_gfx9_insn *)&(buf)[(n)])
+
+inline u32 gfx9_get_param_base(struct amdgcn_param32 param)
+{
+ return gfx9_param_base[param.type];
+}
+
+inline u32 emit_gfx9_s_load_dwordx2(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src, int offset)
+{
+ /* s_load_dwordx2 <dst/sdata>, <src/sbase>, <offset>
+ * sdata: register, load to.
+ * sbase: sgpr-pair, load from.
+ * offset: offset of kernarg_address in the hsa_kernel_dispatch_packet.
+ *
+ * sdata = *(sbase + offset);
+ * param = (__global struct _knod_bpf_param *)pkt.kernarg_address;
+ */
+
+ insn->smem.sbase = KNOD_AMDGPU_REG_PAIR(src.v);
+ insn->smem.sdata = dst.v;
+ insn->smem.dummy1 = 0;
+ insn->smem.soe = 0;
+ insn->smem.nv = 0;
+ insn->smem.glc = 0;
+ insn->smem.imm = 1;
+ insn->smem.op = GFX9_S_LOAD_DWORDX2;
+ insn->smem.encoding = GFX9_SMEM_ENCODING;
+ insn->smem.offset = offset;
+ insn->smem.dummy2 = 0;
+ insn->smem.soffset = 0;
+
+ return 8;
+}
+
+inline u32 emit_gfx9_v_bfe_i32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1,
+ struct amdgcn_param32 src2)
+{
+ WARN_ON(dst.type != AMDGCN_PARAM_TYPE_VGPR);
+ WARN_ON(src1.type == AMDGCN_PARAM_TYPE_LITERAL_CONST ||
+ src2.type == AMDGCN_PARAM_TYPE_LITERAL_CONST);
+ insn->vop3a.vdst = dst.v;
+ insn->vop3a.abs = 0;
+ insn->vop3a.op_sel = 0;
+ insn->vop3a.clmp = 0;
+ insn->vop3a.op = GFX9_V_BFE_I32;
+ insn->vop3a.encoding = GFX9_VOP3A_ENCODING;
+ insn->vop3a.src1 = gfx9_get_param_base(src1) + src1.v;
+ insn->vop3a.src2 = gfx9_get_param_base(src2) + src2.v;
+ insn->vop3a.omod = 0;
+ insn->vop3a.neg = 0;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop3a.src0 = gfx9_get_param_base(src0);
+ insn->vop3a.literal = src0.v;
+ return 12;
+ }
+ insn->vop3a.src0 = gfx9_get_param_base(src0) + src0.v;
+
+ return 8;
+}
+
+inline u32 emit_gfx9_v_bfe_u32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1,
+ struct amdgcn_param32 src2)
+{
+ /* v_bfe_u32 v0, v0, 8
+ * <dst>: destination vgpr.
+ * <src>: initialized to workitem in first bitfields.
+ * 8: bitfield size to extract from.
+ *
+ * extract workitem ID from <src>
+ */
+ WARN_ON(dst.type != AMDGCN_PARAM_TYPE_VGPR);
+ WARN_ON(src1.type == AMDGCN_PARAM_TYPE_LITERAL_CONST ||
+ src2.type == AMDGCN_PARAM_TYPE_LITERAL_CONST);
+ insn->vop3a.vdst = dst.v;
+ insn->vop3a.abs = 0;
+ insn->vop3a.op_sel = 0;
+ insn->vop3a.clmp = 0;
+ insn->vop3a.op = GFX9_V_BFE_U32;
+ insn->vop3a.encoding = GFX9_VOP3A_ENCODING;
+ insn->vop3a.src1 = gfx9_get_param_base(src1) + src1.v;
+ insn->vop3a.src2 = gfx9_get_param_base(src2) + src2.v;
+ insn->vop3a.omod = 0;
+ insn->vop3a.neg = 0;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop3a.src0 = gfx9_get_param_base(src0);
+ insn->vop3a.literal = src0.v;
+ return 12;
+ }
+ insn->vop3a.src0 = gfx9_get_param_base(src0) + src0.v;
+
+ return 8;
+}
+
+inline u32 emit_gfx9_v_bfi_b32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1,
+ struct amdgcn_param32 src2)
+{
+ WARN_ON(dst.type != AMDGCN_PARAM_TYPE_VGPR);
+ WARN_ON(src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST ||
+ src1.type == AMDGCN_PARAM_TYPE_LITERAL_CONST ||
+ src2.type == AMDGCN_PARAM_TYPE_LITERAL_CONST);
+ insn->vop3a.vdst = dst.v;
+ insn->vop3a.abs = 0;
+ insn->vop3a.op_sel = 0;
+ insn->vop3a.clmp = 0;
+ insn->vop3a.op = GFX9_V_BFI_B32;
+ insn->vop3a.encoding = GFX9_VOP3A_ENCODING;
+ insn->vop3a.src1 = gfx9_get_param_base(src1) + src1.v;
+ insn->vop3a.src2 = gfx9_get_param_base(src2) + src2.v;
+ insn->vop3a.omod = 0;
+ insn->vop3a.neg = 0;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop3a.src0 = gfx9_get_param_base(src0);
+ insn->vop3a.literal = src0.v;
+ return 12;
+ }
+ insn->vop3a.src0 = gfx9_get_param_base(src0) + src0.v;
+
+ return 8;
+}
+
+inline u32 emit_gfx9_v_lshl_add_u32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1,
+ struct amdgcn_param32 src2)
+{
+ /* v_lshl_add_u32 <dst>, <src0>, <src1>, <src2>
+ *
+ * <dst> = (<src0> << <src1>) + <src2>;
+ */
+
+ insn->vop3a.vdst = dst.v;
+ insn->vop3a.abs = 0;
+ insn->vop3a.op_sel = 0;
+ insn->vop3a.clmp = 0;
+ insn->vop3a.op = GFX9_V_LSHL_ADD_U32;
+ insn->vop3a.encoding = GFX9_VOP3A_ENCODING;
+ insn->vop3a.src1 = gfx9_get_param_base(src1) + src1.v;
+ insn->vop3a.src2 = gfx9_get_param_base(src2) + src2.v;
+ insn->vop3a.omod = 0;
+ insn->vop3a.neg = 0;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop3a.src0 = gfx9_get_param_base(src0);
+ insn->vop3a.literal = src0.v;
+ return 12;
+ }
+ insn->vop3a.src0 = gfx9_get_param_base(src0) + src0.v;
+
+ return 8;
+}
+
+inline u32 emit_gfx9_v_lshl_or_b32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1,
+ struct amdgcn_param32 src2)
+{
+ /* v_lshl_or_b32 <dst>, <src0>, <src1>, <src2>
+ *
+ * D.u = (S0.u << S1.u[4:0]) | S2.u
+ */
+
+ WARN_ON(src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST ||
+ src1.type == AMDGCN_PARAM_TYPE_LITERAL_CONST ||
+ src2.type == AMDGCN_PARAM_TYPE_LITERAL_CONST ||
+ src2.type == AMDGCN_PARAM_TYPE_LITERAL_CONST);
+
+ insn->vop3a.vdst = dst.v;
+ insn->vop3a.abs = 0;
+ insn->vop3a.op_sel = 0;
+ insn->vop3a.clmp = 0;
+ insn->vop3a.op = GFX9_V_LSHL_OR_B32;
+ insn->vop3a.encoding = GFX9_VOP3A_ENCODING;
+ insn->vop3a.src1 = gfx9_get_param_base(src1) + src1.v;
+ insn->vop3a.src2 = gfx9_get_param_base(src2) + src2.v;
+ insn->vop3a.omod = 0;
+ insn->vop3a.neg = 0;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop3a.src0 = gfx9_get_param_base(src0);
+ insn->vop3a.literal = src0.v;
+ return 12;
+ }
+ insn->vop3a.src0 = gfx9_get_param_base(src0) + src0.v;
+
+ return 8;
+}
+
+inline u32 emit_gfx9_v_mad_u64_u32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param64 vdst,
+ struct amdgcn_param32 sdst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1,
+ struct amdgcn_param64 src2)
+{
+ /* v_mad_u64_u32 <dst>, <dst2>, <src0_imm>, <src1>, <src2>
+ * <vdst>: destination vgpr.
+ * <sdst>: destination sgpr.
+ * <src0>: source vgpr.
+ * <src1>: source vgpr.
+ * <src2>: source vgpr.
+ *
+ * {vcc_out, D.u64} = S0.u32 * S1.u32 + S2.u64.
+ * ctx = ¶m->sub[idx].ctx;
+ */
+
+ WARN_ON(src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST ||
+ src1.type == AMDGCN_PARAM_TYPE_LITERAL_CONST ||
+ src2.lo.type == AMDGCN_PARAM_TYPE_LITERAL_CONST ||
+ src2.hi.type == AMDGCN_PARAM_TYPE_LITERAL_CONST);
+ insn->vop3b.vdst = vdst.lo.v;
+ insn->vop3b.sdst = sdst.v;
+ insn->vop3b.clmp = 0;
+ insn->vop3b.op = GFX9_V_MAD_U64_U32;
+ insn->vop3b.encoding = GFX9_VOP3B_ENCODING;
+ insn->vop3b.src1 = gfx9_get_param_base(src1) + src1.v;
+ insn->vop3b.src2 = gfx9_get_param_base(src2.lo) + src2.lo.v;
+ insn->vop3b.omod = 0;
+ insn->vop3b.neg = 0;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop3b.src0 = gfx9_get_param_base(src0);
+ insn->vop3b.literal = src0.v;
+ return 12;
+ }
+ insn->vop3b.src0 = gfx9_get_param_base(src0) + src0.v;
+
+ return 8;
+}
+
+inline u32 emit_gfx9_s_mov_b32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src)
+{
+ if (src.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->sop1.ssrc0 = gfx9_get_param_base(src);
+ insn->sop1.literal = src.v;
+ } else {
+ insn->sop1.ssrc0 = gfx9_get_param_base(src) + src.v;
+ }
+ insn->sop1.op = GFX9_S_MOV_B32;
+ insn->sop1.sdst = gfx9_get_param_base(dst) + dst.v;
+ insn->sop1.encoding = GFX9_SOP1_ENCODING;
+
+ if (src.type == AMDGCN_PARAM_TYPE_LITERAL_CONST)
+ return 8;
+ return 4;
+}
+
+/*
+ * v_readfirstlane_b32 sdst, vsrc
+ * Copies VGPR[vsrc] from the first active lane to SGPR[sdst].
+ * VOP1 encoding, dst = SGPR (not VGPR).
+ */
+inline u32 emit_gfx9_v_readfirstlane_b32(union amdgcn_gfx9_insn *insn,
+ u8 sdst, u8 vsrc)
+{
+ insn->vop1.encoding = GFX9_VOP1_ENCODING;
+ insn->vop1.vdst = sdst;
+ insn->vop1.op = GFX9_V_READFIRSTLANE_B32;
+ insn->vop1.src0 = GFX9_SRC_VGPR_BASE + vsrc;
+ return 4;
+}
+
+inline u32 emit_gfx9_v_mov_b32_e32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src)
+{
+ if (dst.type != AMDGCN_PARAM_TYPE_VGPR)
+ WARN_ON_ONCE(1);
+ insn->vop1.encoding = GFX9_VOP1_ENCODING;
+ insn->vop1.vdst = dst.v;
+ insn->vop1.op = GFX9_V_MOV_B32;
+ if (src.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop1.src0 = GFX9_VOP1_SRC_LITERAL_CONST;
+ insn->vop1.literal = src.v;
+
+ return 8;
+ }
+ insn->vop1.src0 = gfx9_get_param_base(src) + src.v;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_v_add_co_u32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ WARN_ON(dst.type != AMDGCN_PARAM_TYPE_VGPR);
+ WARN_ON(src1.type != AMDGCN_PARAM_TYPE_VGPR);
+
+ insn->vop2.vsrc1 = src1.v;
+ insn->vop2.vdst = dst.v;
+ insn->vop2.op = GFX9_V_ADD_CO_U32;
+ insn->vop2.encoding = GFX9_VOP2_ENCODING;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop2.src0 = gfx9_get_param_base(src0);
+ insn->vop2.literal = src0.v;
+ return 8;
+ }
+ insn->vop2.src0 = gfx9_get_param_base(src0) + src0.v;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_v_add_u32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ WARN_ON(dst.type != AMDGCN_PARAM_TYPE_VGPR);
+ WARN_ON(src1.type != AMDGCN_PARAM_TYPE_VGPR);
+
+ insn->vop2.vsrc1 = src1.v;
+ insn->vop2.vdst = dst.v;
+ insn->vop2.op = GFX9_V_ADD_U32;
+ insn->vop2.encoding = GFX9_VOP2_ENCODING;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop2.src0 = gfx9_get_param_base(src0);
+ insn->vop2.literal = src0.v;
+ return 8;
+ }
+ insn->vop2.src0 = gfx9_get_param_base(src0) + src0.v;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_v_sub_u32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ WARN_ON(dst.type != AMDGCN_PARAM_TYPE_VGPR);
+ WARN_ON(src1.type != AMDGCN_PARAM_TYPE_VGPR);
+
+ insn->vop2.vsrc1 = src1.v;
+ insn->vop2.vdst = dst.v;
+ insn->vop2.op = GFX9_V_SUB_U32;
+ insn->vop2.encoding = GFX9_VOP2_ENCODING;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop2.src0 = gfx9_get_param_base(src0);
+ insn->vop2.literal = src0.v;
+ return 8;
+ }
+ insn->vop2.src0 = gfx9_get_param_base(src0) + src0.v;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_v_addc_co_u32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ insn->vop2.vsrc1 = src1.v;
+ insn->vop2.vdst = dst.v;
+ insn->vop2.op = GFX9_V_ADDC_CO_U32;
+ insn->vop2.encoding = GFX9_VOP2_ENCODING;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ WARN_ON_ONCE(1);
+ insn->vop2.src0 = gfx9_get_param_base(src0);
+ insn->vop2.literal = src0.v;
+ return 8;
+ }
+ insn->vop2.src0 = gfx9_get_param_base(src0) + src0.v;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_v_xor_b32_e32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ insn->vop2.vsrc1 = src1.v;
+ insn->vop2.vdst = dst.v;
+ insn->vop2.op = GFX9_V_XOR_B32;
+ insn->vop2.encoding = GFX9_VOP2_ENCODING;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop2.src0 = gfx9_get_param_base(src0);
+ insn->vop2.literal = src0.v;
+ return 8;
+ }
+ insn->vop2.src0 = gfx9_get_param_base(src0) + src0.v;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_v_or_b32_e32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ insn->vop2.vsrc1 = src1.v;
+ insn->vop2.vdst = dst.v;
+ insn->vop2.op = GFX9_V_OR_B32;
+ insn->vop2.encoding = GFX9_VOP2_ENCODING;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop2.src0 = gfx9_get_param_base(src0);
+ insn->vop2.literal = src0.v;
+ return 8;
+ }
+ insn->vop2.src0 = gfx9_get_param_base(src0) + src0.v;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_v_cndmask_b32_e32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ insn->vop2.vsrc1 = src1.v;
+ insn->vop2.vdst = dst.v;
+ insn->vop2.op = GFX9_V_CNDMASK_B32;
+ insn->vop2.encoding = GFX9_VOP2_ENCODING;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop2.src0 = gfx9_get_param_base(src0);
+ insn->vop2.literal = src0.v;
+ return 8;
+ }
+ insn->vop2.src0 = gfx9_get_param_base(src0) + src0.v;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_v_and_b32_e32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ insn->vop2.vsrc1 = src1.v;
+ insn->vop2.vdst = dst.v;
+ insn->vop2.op = GFX9_V_AND_B32;
+ insn->vop2.encoding = GFX9_VOP2_ENCODING;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop2.src0 = gfx9_get_param_base(src0);
+ insn->vop2.literal = src0.v;
+ return 8;
+ }
+ insn->vop2.src0 = gfx9_get_param_base(src0) + src0.v;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_v_sub_co_u32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ insn->vop2.vsrc1 = src1.v;
+ insn->vop2.vdst = dst.v;
+ insn->vop2.op = GFX9_V_SUB_CO_U32;
+ insn->vop2.encoding = GFX9_VOP2_ENCODING;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop2.src0 = gfx9_get_param_base(src0);
+ insn->vop2.literal = src0.v;
+ return 8;
+ }
+ insn->vop2.src0 = gfx9_get_param_base(src0) + src0.v;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_v_subrev_co_u32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ /* D.u = S1.u - S0.u; */
+ insn->vop2.vsrc1 = src1.v;
+ insn->vop2.vdst = dst.v;
+ insn->vop2.op = GFX9_V_SUBREV_CO_U32;
+ insn->vop2.encoding = GFX9_VOP2_ENCODING;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop2.src0 = gfx9_get_param_base(src0);
+ insn->vop2.literal = src0.v;
+ return 8;
+ }
+ insn->vop2.src0 = gfx9_get_param_base(src0) + src0.v;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_v_subbrev_co_u32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ /* D.u = S1.u - S0.u; */
+ insn->vop2.vsrc1 = src1.v;
+ insn->vop2.vdst = dst.v;
+ insn->vop2.op = GFX9_V_SUBBREV_CO_U32;
+ insn->vop2.encoding = GFX9_VOP2_ENCODING;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop2.src0 = gfx9_get_param_base(src0);
+ insn->vop2.literal = src0.v;
+ return 8;
+ }
+ insn->vop2.src0 = gfx9_get_param_base(src0) + src0.v;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_v_subb_co_u32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+
+{
+ /* dst = src0 - src1 */
+ insn->vop2.vsrc1 = src1.v;
+ insn->vop2.vdst = dst.v;
+ insn->vop2.op = GFX9_V_SUBB_CO_U32;
+ insn->vop2.encoding = GFX9_VOP2_ENCODING;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop2.src0 = gfx9_get_param_base(src0);
+ insn->vop2.literal = src0.v;
+ return 8;
+ }
+ insn->vop2.src0 = gfx9_get_param_base(src0) + src0.v;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_v_mul_lo_u32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ insn->vop3a.vdst = dst.v;
+ insn->vop3a.abs = 0;
+ insn->vop3a.op_sel = 0;
+ insn->vop3a.clmp = 0;
+ insn->vop3a.op = GFX9_V_MUL_LO_U32;
+ insn->vop3a.encoding = GFX9_VOP3A_ENCODING;
+ insn->vop3a.src1 = gfx9_get_param_base(src1) + src1.v;
+ /* 128: src2=0 reads s0 on gfx9 */
+ insn->vop3a.src2 = GFX9_SRC_INTEGER_0;
+ insn->vop3a.omod = 0;
+ insn->vop3a.neg = 0;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop3a.src0 = gfx9_get_param_base(src0);
+ insn->vop3a.literal = src0.v;
+ return 12;
+ }
+ insn->vop3a.src0 = gfx9_get_param_base(src0) + src0.v;
+
+ return 8;
+}
+
+/* v_mbcnt_lo_u32_b32 vdst, src0, vsrc1
+ * vdst = popcount(src0 & ((1 << lane_id[4:0]) - 1)) + vsrc1
+ */
+inline u32 emit_gfx9_v_mbcnt_lo_u32_b32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ insn->vop3a.vdst = dst.v;
+ insn->vop3a.abs = 0;
+ insn->vop3a.op_sel = 0;
+ insn->vop3a.clmp = 0;
+ insn->vop3a.op = GFX9_V_MBCNT_LO_U32_B32;
+ insn->vop3a.encoding = GFX9_VOP3A_ENCODING;
+ insn->vop3a.src0 = gfx9_get_param_base(src0) + src0.v;
+ insn->vop3a.src1 = gfx9_get_param_base(src1) + src1.v;
+ /* 128: src2=0 reads s0 on gfx9 */
+ insn->vop3a.src2 = GFX9_SRC_INTEGER_0;
+ insn->vop3a.omod = 0;
+ insn->vop3a.neg = 0;
+
+ return 8;
+}
+
+/* v_mbcnt_hi_u32_b32 vdst, src0, vsrc1
+ * vdst = popcount(src0 & ((1 << lane_id[5]) - 1)) + vsrc1
+ */
+inline u32 emit_gfx9_v_mbcnt_hi_u32_b32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ insn->vop3a.vdst = dst.v;
+ insn->vop3a.abs = 0;
+ insn->vop3a.op_sel = 0;
+ insn->vop3a.clmp = 0;
+ insn->vop3a.op = GFX9_V_MBCNT_HI_U32_B32;
+ insn->vop3a.encoding = GFX9_VOP3A_ENCODING;
+ insn->vop3a.src0 = gfx9_get_param_base(src0) + src0.v;
+ insn->vop3a.src1 = gfx9_get_param_base(src1) + src1.v;
+ /* 128: src2=0 reads s0 on gfx9 */
+ insn->vop3a.src2 = GFX9_SRC_INTEGER_0;
+ insn->vop3a.omod = 0;
+ insn->vop3a.neg = 0;
+
+ return 8;
+}
+
+inline u32 emit_gfx9_v_mul_hi_u32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ insn->vop3a.vdst = dst.v;
+ insn->vop3a.abs = 0;
+ insn->vop3a.op_sel = 0;
+ insn->vop3a.clmp = 0;
+ insn->vop3a.op = GFX9_V_MUL_HI_U32;
+ insn->vop3a.encoding = GFX9_VOP3A_ENCODING;
+ insn->vop3a.src1 = gfx9_get_param_base(src1) + src1.v;
+ /* 128: src2=0 reads s0 on gfx9 */
+ insn->vop3a.src2 = GFX9_SRC_INTEGER_0;
+ insn->vop3a.omod = 0;
+ insn->vop3a.neg = 0;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop3a.src0 = gfx9_get_param_base(src0);
+ insn->vop3a.literal = src0.v;
+ return 12;
+ }
+ insn->vop3a.src0 = gfx9_get_param_base(src0) + src0.v;
+
+ return 8;
+}
+
+inline u32 emit_gfx9_v_lshlrev_b64(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src0,
+ struct amdgcn_param64 src1)
+{
+ /* dst = s1 << s0 */
+ insn->vop3a.vdst = dst.lo.v;
+ insn->vop3a.abs = 0;
+ insn->vop3a.op_sel = 0;
+ insn->vop3a.clmp = 0;
+ insn->vop3a.op = GFX9_V_LSHLREV_B64;
+ insn->vop3a.encoding = GFX9_VOP3A_ENCODING;
+ insn->vop3a.src1 = gfx9_get_param_base(src1.lo) + src1.lo.v;
+ /* 128: src2=0 reads s0 on gfx9 */
+ insn->vop3a.src2 = GFX9_SRC_INTEGER_0;
+ insn->vop3a.omod = 0;
+ insn->vop3a.neg = 0;
+ if (src0.lo.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop3a.src0 = gfx9_get_param_base(src0.lo);
+ insn->vop3a.literal = src0.lo.v;
+ return 12;
+ }
+ insn->vop3a.src0 = gfx9_get_param_base(src0.lo) + src0.lo.v;
+
+ return 8;
+}
+
+inline u32 emit_gfx9_v_lshlrev_b32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ /* dst = s1 << s0 */
+ WARN_ON(dst.type != AMDGCN_PARAM_TYPE_VGPR);
+ insn->vop2.vsrc1 = src1.v;
+ insn->vop2.vdst = dst.v;
+ insn->vop2.op = GFX9_V_LSHLREV_B32;
+ insn->vop2.encoding = GFX9_VOP2_ENCODING;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop2.src0 = gfx9_get_param_base(src0);
+ insn->vop2.literal = src0.v;
+ return 8;
+ }
+ insn->vop2.src0 = gfx9_get_param_base(src0) + src0.v;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_v_lshrrev_b32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ /* dst = s1 << s0 */
+ WARN_ON(dst.type != AMDGCN_PARAM_TYPE_VGPR);
+ insn->vop2.vsrc1 = src1.v;
+ insn->vop2.vdst = dst.v;
+ insn->vop2.op = GFX9_V_LSHRREV_B32;
+ insn->vop2.encoding = GFX9_VOP2_ENCODING;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop2.src0 = gfx9_get_param_base(src0);
+ insn->vop2.literal = src0.v;
+ return 8;
+ }
+ insn->vop2.src0 = gfx9_get_param_base(src0) + src0.v;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_v_lshrrev_b64(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src0,
+ struct amdgcn_param64 src1)
+{
+ /* dst = s1 >> s0 */
+ insn->vop3a.vdst = dst.lo.v;
+ insn->vop3a.abs = 0;
+ insn->vop3a.op_sel = 0;
+ insn->vop3a.clmp = 0;
+ insn->vop3a.op = GFX9_V_LSHRREV_B64;
+ insn->vop3a.encoding = GFX9_VOP3A_ENCODING;
+ insn->vop3a.src1 = gfx9_get_param_base(src1.lo) + src1.lo.v;
+ /* 128: src2=0 reads s0 on gfx9 */
+ insn->vop3a.src2 = GFX9_SRC_INTEGER_0;
+ insn->vop3a.omod = 0;
+ insn->vop3a.neg = 0;
+ if (src0.lo.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop3a.src0 = gfx9_get_param_base(src0.lo);
+ insn->vop3a.literal = src0.lo.v;
+ return 12;
+ }
+ insn->vop3a.src0 = gfx9_get_param_base(src0.lo) + src0.lo.v;
+
+ return 8;
+}
+
+inline u32 emit_gfx9_v_ashrrev_i64(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src0,
+ struct amdgcn_param64 src1)
+{
+ /* dst = s1 >> s0 */
+ insn->vop3a.vdst = dst.lo.v;
+ insn->vop3a.abs = 0;
+ insn->vop3a.op_sel = 0;
+ insn->vop3a.clmp = 0;
+ insn->vop3a.op = GFX9_V_ASHRREV_I64;
+ insn->vop3a.encoding = GFX9_VOP3A_ENCODING;
+ insn->vop3a.src1 = gfx9_get_param_base(src1.lo) + src1.lo.v;
+ /* 128: src2=0 reads s0 on gfx9 */
+ insn->vop3a.src2 = GFX9_SRC_INTEGER_0;
+ insn->vop3a.omod = 0;
+ insn->vop3a.neg = 0;
+ if (src0.lo.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop3a.src0 = gfx9_get_param_base(src0.lo);
+ insn->vop3a.literal = src0.lo.v;
+ return 12;
+ }
+ insn->vop3a.src0 = gfx9_get_param_base(src0.lo) + src0.lo.v;
+
+ return 8;
+}
+
+inline u32 emit_gfx9_v_ashrrev_i32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ /* dst = s1 >> s0 */
+ insn->vop3a.vdst = dst.v;
+ insn->vop3a.abs = 0;
+ insn->vop3a.op_sel = 0;
+ insn->vop3a.clmp = 0;
+ /*
+ * VOP2 ops encoded in VOP3A use opcode 0x100 + vop2_op (V_ASHRREV_I32
+ * is a VOP2 instruction). Without the +0x100 the op field decodes as a
+ * different VOP3A instruction, so the arithmetic shift (used for 64-bit
+ * sign-extension, e.g. bpf_xdp_adjust_head's delta) produced garbage.
+ */
+ insn->vop3a.op = GFX9_V_ASHRREV_I32 + 0x100;
+ insn->vop3a.encoding = GFX9_VOP3A_ENCODING;
+ insn->vop3a.src1 = gfx9_get_param_base(src1) + src1.v;
+ /* 128: src2=0 reads s0 on gfx9 */
+ insn->vop3a.src2 = GFX9_SRC_INTEGER_0;
+ insn->vop3a.omod = 0;
+ insn->vop3a.neg = 0;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop3a.src0 = gfx9_get_param_base(src0);
+ insn->vop3a.literal = src0.v;
+ return 12;
+ }
+ insn->vop3a.src0 = gfx9_get_param_base(src0) + src0.v;
+
+ return 8;
+}
+
+inline u32 emit_gfx9_v_alignbit_b32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1,
+ struct amdgcn_param32 src2)
+{
+ /* D.u = ({S0,S1} >> S2.u[4:0]) & 0xffffffff. */
+ WARN_ON(dst.type != AMDGCN_PARAM_TYPE_VGPR);
+ insn->vop3a.vdst = dst.v;
+ insn->vop3a.abs = 0;
+ insn->vop3a.op_sel = 0;
+ insn->vop3a.clmp = 0;
+ insn->vop3a.op = GFX9_V_ALIGNBIT_B32;
+ insn->vop3a.encoding = GFX9_VOP3A_ENCODING;
+ insn->vop3a.src1 = gfx9_get_param_base(src1) + src1.v;
+ insn->vop3a.src2 = gfx9_get_param_base(src2) + src2.v;
+ insn->vop3a.omod = 0;
+ insn->vop3a.neg = 0;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop3a.src0 = gfx9_get_param_base(src0);
+ insn->vop3a.literal = src0.v;
+ return 12;
+ }
+ insn->vop3a.src0 = gfx9_get_param_base(src0) + src0.v;
+
+ return 8;
+}
+
+inline u32 emit_gfx9_v_perm_b32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1,
+ struct amdgcn_param32 src2)
+{
+ /*
+ * v_perm_b32: D.u[31:24] = src[sel[14:12]],
+ * D.u[23:16] = src[sel[10:8]], ...
+ * For bswap32: sel = 0x00010203 reverses bytes.
+ */
+ WARN_ON(dst.type != AMDGCN_PARAM_TYPE_VGPR);
+ /* VOP3 does not support literal constants on GFX9 */
+ WARN_ON(src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST ||
+ src1.type == AMDGCN_PARAM_TYPE_LITERAL_CONST ||
+ src2.type == AMDGCN_PARAM_TYPE_LITERAL_CONST);
+ insn->vop3a.vdst = dst.v;
+ insn->vop3a.abs = 0;
+ insn->vop3a.op_sel = 0;
+ insn->vop3a.clmp = 0;
+ insn->vop3a.op = GFX9_V_PERM_B32;
+ insn->vop3a.encoding = GFX9_VOP3A_ENCODING;
+ insn->vop3a.src0 = gfx9_get_param_base(src0) + src0.v;
+ insn->vop3a.src1 = gfx9_get_param_base(src1) + src1.v;
+ insn->vop3a.src2 = gfx9_get_param_base(src2) + src2.v;
+ insn->vop3a.omod = 0;
+ insn->vop3a.neg = 0;
+
+ return 8;
+}
+
+inline u32 emit_gfx9_v_cmp_eq_u64(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src)
+{
+ /* VCC = S0 == S1 */
+ insn->vopc.src0 = gfx9_get_param_base(dst) + dst.v;
+ insn->vopc.vsrc1 = src.v;
+ insn->vopc.op = GFX9_V_CMP_EQ_U64;
+ insn->vopc.encoding = GFX9_VOPC_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_v_cmp_eq_u32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src)
+{
+ /*
+ * VOPC src0 supports SGPR/inline/VGPR but NOT literal
+ * (no 8-byte path)
+ */
+ WARN_ON(dst.type == AMDGCN_PARAM_TYPE_LITERAL_CONST);
+ WARN_ON(src.type != AMDGCN_PARAM_TYPE_VGPR);
+ /* VCC = S0 == S1 */
+ insn->vopc.src0 = gfx9_get_param_base(dst) + dst.v;
+ insn->vopc.vsrc1 = src.v;
+ insn->vopc.op = GFX9_V_CMP_EQ_U32;
+ insn->vopc.encoding = GFX9_VOPC_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_v_cmp_gt_u64(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src)
+{
+ /* VCC = S0 > S1 */
+ insn->vopc.src0 = gfx9_get_param_base(dst.lo) + dst.lo.v;
+ insn->vopc.vsrc1 = src.lo.v;
+ insn->vopc.op = GFX9_V_CMP_GT_U64;
+ insn->vopc.encoding = GFX9_VOPC_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_v_cmp_gt_i64(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src)
+{
+ /* VCC = S0 > S1 (signed) */
+ insn->vopc.src0 = gfx9_get_param_base(dst.lo) + dst.lo.v;
+ insn->vopc.vsrc1 = src.lo.v;
+ insn->vopc.op = GFX9_V_CMP_GT_I64;
+ insn->vopc.encoding = GFX9_VOPC_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_v_cmp_ge_u32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src)
+{
+ /* VCC = S0 >= S1 */
+ insn->vopc.src0 = gfx9_get_param_base(dst) + dst.v;
+ insn->vopc.vsrc1 = src.v;
+ insn->vopc.op = GFX9_V_CMP_GE_U32;
+ insn->vopc.encoding = GFX9_VOPC_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_v_cmp_gt_u32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src)
+{
+ /* VCC = S0 > S1 */
+ insn->vopc.src0 = gfx9_get_param_base(dst) + dst.v;
+ insn->vopc.vsrc1 = src.v;
+ insn->vopc.op = GFX9_V_CMP_GT_U32;
+ insn->vopc.encoding = GFX9_VOPC_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_v_cmp_lt_u32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src)
+{
+ /* VCC = S0 < S1 */
+ insn->vopc.src0 = gfx9_get_param_base(dst) + dst.v;
+ insn->vopc.vsrc1 = src.v;
+ insn->vopc.op = GFX9_V_CMP_LT_U32;
+ insn->vopc.encoding = GFX9_VOPC_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_v_cmp_le_u32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src)
+{
+ /* VCC = S0 <= S1 */
+ insn->vopc.src0 = gfx9_get_param_base(dst) + dst.v;
+ insn->vopc.vsrc1 = src.v;
+ insn->vopc.op = GFX9_V_CMP_LE_U32;
+ insn->vopc.encoding = GFX9_VOPC_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_v_cmp_gt_i32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src)
+{
+ /* VCC = S0 > S1 (signed) */
+ insn->vopc.src0 = gfx9_get_param_base(dst) + dst.v;
+ insn->vopc.vsrc1 = src.v;
+ insn->vopc.op = GFX9_V_CMP_GT_I32;
+ insn->vopc.encoding = GFX9_VOPC_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_v_cmp_ge_i32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src)
+{
+ /* VCC = S0 >= S1 (signed) */
+ insn->vopc.src0 = gfx9_get_param_base(dst) + dst.v;
+ insn->vopc.vsrc1 = src.v;
+ insn->vopc.op = GFX9_V_CMP_GE_I32;
+ insn->vopc.encoding = GFX9_VOPC_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_v_cmp_lt_i32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src)
+{
+ /* VCC = S0 < S1 (signed) */
+ insn->vopc.src0 = gfx9_get_param_base(dst) + dst.v;
+ insn->vopc.vsrc1 = src.v;
+ insn->vopc.op = GFX9_V_CMP_LT_I32;
+ insn->vopc.encoding = GFX9_VOPC_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_v_cmp_le_i32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src)
+{
+ /* VCC = S0 <= S1 (signed) */
+ insn->vopc.src0 = gfx9_get_param_base(dst) + dst.v;
+ insn->vopc.vsrc1 = src.v;
+ insn->vopc.op = GFX9_V_CMP_LE_I32;
+ insn->vopc.encoding = GFX9_VOPC_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_v_cmpx_lt_u32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src)
+{
+ /* EXEC &= (S0 < S1) */
+ insn->vopc.src0 = gfx9_get_param_base(dst) + dst.v;
+ insn->vopc.vsrc1 = src.v;
+ insn->vopc.op = GFX9_V_CMPX_LT_U32;
+ insn->vopc.encoding = GFX9_VOPC_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_v_cmp_ge_u64(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src)
+{
+ /* VCC = S0 >= S1 */
+ insn->vopc.src0 = gfx9_get_param_base(dst.lo) + dst.lo.v;
+ insn->vopc.vsrc1 = src.lo.v;
+ insn->vopc.op = GFX9_V_CMP_GE_U64;
+ insn->vopc.encoding = GFX9_VOPC_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_v_cmp_ge_i64(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src)
+{
+ /* VCC = S0 >= S1 (signed) */
+ insn->vopc.src0 = gfx9_get_param_base(dst.lo) + dst.lo.v;
+ insn->vopc.vsrc1 = src.lo.v;
+ insn->vopc.op = GFX9_V_CMP_GE_I64;
+ insn->vopc.encoding = GFX9_VOPC_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_v_cmp_lt_u64(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src)
+{
+ /* VCC = S0 < S1 */
+ insn->vopc.src0 = gfx9_get_param_base(dst.lo) + dst.lo.v;
+ insn->vopc.vsrc1 = src.lo.v;
+ insn->vopc.op = GFX9_V_CMP_LT_U64;
+ insn->vopc.encoding = GFX9_VOPC_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_v_cmp_lt_i64(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src)
+{
+ /* VCC = S0 < S1 (signed) */
+ insn->vopc.src0 = gfx9_get_param_base(dst.lo) + dst.lo.v;
+ insn->vopc.vsrc1 = src.lo.v;
+ insn->vopc.op = GFX9_V_CMP_LT_I64;
+ insn->vopc.encoding = GFX9_VOPC_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_v_cmp_le_u64(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src)
+{
+ /* VCC = S0 <= S1 */
+ insn->vopc.src0 = gfx9_get_param_base(dst.lo) + dst.lo.v;
+ insn->vopc.vsrc1 = src.lo.v;
+ insn->vopc.op = GFX9_V_CMP_LE_U64;
+ insn->vopc.encoding = GFX9_VOPC_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_v_cmp_le_i64(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src)
+{
+ /* VCC = S0 <= S1 (signed) */
+ insn->vopc.src0 = gfx9_get_param_base(dst.lo) + dst.lo.v;
+ insn->vopc.vsrc1 = src.lo.v;
+ insn->vopc.op = GFX9_V_CMP_LE_I64;
+ insn->vopc.encoding = GFX9_VOPC_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_buffer_load_ubyte(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src,
+ short off)
+{
+ /* buffer_load_ubyte <dst>, <src>, s[0:3], 0 offen offset:<off> */
+ insn->mubuf.offset = off;
+ insn->mubuf.offen = 1;
+ insn->mubuf.idxen = 0;
+ insn->mubuf.glc = 0;
+ insn->mubuf.dummy1 = 0;
+ insn->mubuf.lds = 0;
+ insn->mubuf.op = GFX9_BUFFER_LOAD_UBYTE;
+ insn->mubuf.dummy2 = 0;
+ insn->mubuf.encoding = GFX9_MUBUF_ENCODING;
+ insn->mubuf.vaddr = src.v;
+ insn->mubuf.vdata = dst.v;
+ insn->mubuf.srsrc = 0; /* s[0:3] */
+ insn->mubuf.dummy3 = 0;
+ insn->mubuf.tfe = 0;
+ insn->mubuf.soffset = GFX9_MUBUF_SOFFSET_INTEGER_0;
+
+ return 8;
+}
+
+inline u32 emit_gfx9_buffer_load_ushort(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src,
+ short off)
+{
+ /* buffer_load_ushort <dst>, <src>, s[0:3], 0 offen offset:<off> */
+ insn->mubuf.offset = off;
+ insn->mubuf.offen = 1;
+ insn->mubuf.idxen = 0;
+ insn->mubuf.glc = 0;
+ insn->mubuf.dummy1 = 0;
+ insn->mubuf.lds = 0;
+ insn->mubuf.op = GFX9_BUFFER_LOAD_USHORT;
+ insn->mubuf.dummy2 = 0;
+ insn->mubuf.encoding = GFX9_MUBUF_ENCODING;
+ insn->mubuf.vaddr = src.v;
+ insn->mubuf.vdata = dst.v;
+ insn->mubuf.srsrc = 0; /* s[0:3] */
+ insn->mubuf.dummy3 = 0;
+ insn->mubuf.tfe = 0;
+ insn->mubuf.soffset = GFX9_MUBUF_SOFFSET_INTEGER_0;
+
+ return 8;
+}
+
+inline u32 emit_gfx9_buffer_load_dword(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src,
+ short off)
+{
+ /* buffer_load_dword <dst>, <src>, s[0:3], 0 offen offset:<off> */
+ insn->mubuf.offset = off;
+ insn->mubuf.offen = 1;
+ insn->mubuf.idxen = 0;
+ insn->mubuf.glc = 0;
+ insn->mubuf.dummy1 = 0;
+ insn->mubuf.lds = 0;
+ insn->mubuf.op = GFX9_BUFFER_LOAD_DWORD;
+ insn->mubuf.dummy2 = 0;
+ insn->mubuf.encoding = GFX9_MUBUF_ENCODING;
+ insn->mubuf.vaddr = src.v;
+ insn->mubuf.vdata = dst.v;
+ insn->mubuf.srsrc = 0; /* s[0:3] */
+ insn->mubuf.dummy3 = 0;
+ insn->mubuf.tfe = 0;
+ insn->mubuf.soffset = GFX9_MUBUF_SOFFSET_INTEGER_0;
+
+ return 8;
+}
+
+inline u32 emit_gfx9_buffer_load_dwordx2(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src,
+ short off)
+{
+ /* buffer_load_dwordx2 <dst>, <src>, s[0:3], 0 offen offset:<off> */
+ insn->mubuf.offset = off;
+ insn->mubuf.offen = 1;
+ insn->mubuf.idxen = 0;
+ insn->mubuf.glc = 0;
+ insn->mubuf.dummy1 = 0;
+ insn->mubuf.lds = 0;
+ insn->mubuf.op = GFX9_BUFFER_LOAD_DWORDX2;
+ insn->mubuf.dummy2 = 0;
+ insn->mubuf.encoding = GFX9_MUBUF_ENCODING;
+ insn->mubuf.vaddr = src.v;
+ insn->mubuf.vdata = dst.v;
+ insn->mubuf.srsrc = 0; /* s[0:3] */
+ insn->mubuf.dummy3 = 0;
+ insn->mubuf.tfe = 0;
+ insn->mubuf.soffset = GFX9_MUBUF_SOFFSET_INTEGER_0;
+
+ return 8;
+}
+
+inline u32 emit_gfx9_buffer_load_dwordx4(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src,
+ short off)
+{
+ /* buffer_load_dwordx4 <dst>, <src>, s[0:3], 0 offen offset:<off> */
+ insn->mubuf.offset = off;
+ insn->mubuf.offen = 1;
+ insn->mubuf.idxen = 0;
+ insn->mubuf.glc = 0;
+ insn->mubuf.dummy1 = 0;
+ insn->mubuf.lds = 0;
+ insn->mubuf.op = GFX9_BUFFER_LOAD_DWORDX4;
+ insn->mubuf.dummy2 = 0;
+ insn->mubuf.encoding = GFX9_MUBUF_ENCODING;
+ insn->mubuf.vaddr = src.v;
+ insn->mubuf.vdata = dst.v;
+ insn->mubuf.srsrc = 0; /* s[0:3] */
+ insn->mubuf.dummy3 = 0;
+ insn->mubuf.tfe = 0;
+ insn->mubuf.soffset = GFX9_MUBUF_SOFFSET_INTEGER_0;
+
+ return 8;
+}
+
+inline u32 emit_gfx9_global_load_ubyte(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src,
+ short off)
+{
+ /* global_load_ubyte <dst>, <srcs>, off offset:<off> */
+ insn->flat.offset = off;
+ insn->flat.lds = 0;
+ insn->flat.seg = GFX9_FLAT_SEG_GLOBAL;
+ insn->flat.glc = 0;
+ insn->flat.slc = 0;
+ insn->flat.op = GFX9_GLOBAL_LOAD_UBYTE;
+ insn->flat.dummy = 0;
+ insn->flat.encoding = GFX9_FLAT_ENCODING;
+ insn->flat.addr = src.v;
+ insn->flat.data = 0; /* ignored? */
+ insn->flat.saddr = GFX9_FLAT_SADDR_DISABLE;
+ insn->flat.nv = 0;
+ insn->flat.vdst = dst.v;
+
+ return 8;
+}
+
+inline u32 emit_gfx9_global_load_ushort(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src,
+ short off)
+{
+ /* global_load_ushort <dst>, <srcs>, off offset:<off> */
+ insn->flat.offset = off;
+ insn->flat.lds = 0;
+ insn->flat.seg = GFX9_FLAT_SEG_GLOBAL;
+ insn->flat.glc = 0;
+ insn->flat.slc = 0;
+ insn->flat.op = GFX9_GLOBAL_LOAD_USHORT;
+ insn->flat.dummy = 0;
+ insn->flat.encoding = GFX9_FLAT_ENCODING;
+ insn->flat.addr = src.v;
+ insn->flat.data = 0; /* ignored? */
+ insn->flat.saddr = GFX9_FLAT_SADDR_DISABLE;
+ insn->flat.nv = 0;
+ insn->flat.vdst = dst.v;
+
+ return 8;
+}
+
+inline u32 emit_gfx9_global_load_dword(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src,
+ short off)
+{
+ /* global_load_dword <dst>, <srcs>, off offset:<off> */
+ insn->flat.offset = off;
+ insn->flat.lds = 0;
+ insn->flat.seg = GFX9_FLAT_SEG_GLOBAL;
+ insn->flat.glc = 0;
+ insn->flat.slc = 0;
+ insn->flat.op = GFX9_GLOBAL_LOAD_DWORD;
+ insn->flat.dummy = 0;
+ insn->flat.encoding = GFX9_FLAT_ENCODING;
+ insn->flat.addr = src.v;
+ insn->flat.data = 0; /* ignored? */
+ insn->flat.saddr = GFX9_FLAT_SADDR_DISABLE;
+ insn->flat.nv = 0;
+ insn->flat.vdst = dst.v;
+
+ return 8;
+}
+
+inline u32 emit_gfx9_global_load_dwordx2(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src,
+ short off)
+{
+ /* global_load_dwordx2 <dst>, <srcs>, off offset:<off> */
+ insn->flat.offset = off;
+ insn->flat.lds = 0;
+ insn->flat.seg = GFX9_FLAT_SEG_GLOBAL;
+ insn->flat.glc = 0;
+ insn->flat.slc = 0;
+ insn->flat.op = GFX9_GLOBAL_LOAD_DWORDX2;
+ insn->flat.dummy = 0;
+ insn->flat.encoding = GFX9_FLAT_ENCODING;
+ insn->flat.addr = src.v;
+ insn->flat.data = 0; /* ignored? */
+ insn->flat.saddr = GFX9_FLAT_SADDR_DISABLE;
+ insn->flat.nv = 0;
+ insn->flat.vdst = dst.v;
+
+ return 8;
+}
+
+inline u32 emit_gfx9_global_load_dwordx4(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src,
+ short off)
+{
+ /* global_load_dwordx4 <dst>, <srcs>, off offset:<off> */
+ insn->flat.offset = off;
+ insn->flat.lds = 0;
+ insn->flat.seg = GFX9_FLAT_SEG_GLOBAL;
+ insn->flat.glc = 0;
+ insn->flat.slc = 0;
+ insn->flat.op = GFX9_GLOBAL_LOAD_DWORDX4;
+ insn->flat.dummy = 0;
+ insn->flat.encoding = GFX9_FLAT_ENCODING;
+ insn->flat.addr = src.v;
+ insn->flat.data = 0; /* ignored? */
+ insn->flat.saddr = GFX9_FLAT_SADDR_DISABLE;
+ insn->flat.nv = 0;
+ insn->flat.vdst = dst.v;
+
+ return 8;
+}
+
+inline u32 emit_gfx9_global_load_dwordx4_glc(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src,
+ short off)
+{
+ /* global_load_dwordx4 <dst>, <srcs>, off offset:<off> glc slc
+ * GLC=1: bypass L1 (TCP). SLC=1: bypass L2 (TCC).
+ * Both needed for VRAM written by external DMA (NIC via PCIe)
+ * since L2 is not invalidated on external writes.
+ */
+ insn->flat.offset = off;
+ insn->flat.lds = 0;
+ insn->flat.seg = GFX9_FLAT_SEG_GLOBAL;
+ insn->flat.glc = 1;
+ insn->flat.slc = 1;
+ insn->flat.op = GFX9_GLOBAL_LOAD_DWORDX4;
+ insn->flat.dummy = 0;
+ insn->flat.encoding = GFX9_FLAT_ENCODING;
+ insn->flat.addr = src.v;
+ insn->flat.data = 0;
+ insn->flat.saddr = GFX9_FLAT_SADDR_DISABLE;
+ insn->flat.nv = 0;
+ insn->flat.vdst = dst.v;
+
+ return 8;
+}
+
+inline u32 emit_gfx9_global_store_byte(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src, int off)
+{
+ /* global_store_byte <src>, <dst>, off offset:<off>
+ * *(char *)(dst + off) = src;
+ */
+ insn->flat.offset = off;
+ insn->flat.lds = 0;
+ insn->flat.seg = GFX9_FLAT_SEG_GLOBAL;
+ insn->flat.glc = 0;
+ insn->flat.slc = 0;
+ insn->flat.op = GFX9_GLOBAL_STORE_BYTE;
+ insn->flat.dummy = 0;
+ insn->flat.encoding = GFX9_FLAT_ENCODING;
+ insn->flat.addr = dst.v;
+ insn->flat.data = src.v;
+ insn->flat.saddr = GFX9_FLAT_SADDR_DISABLE;
+ insn->flat.nv = 0;
+ insn->flat.vdst = 0;
+
+ return 8;
+}
+
+inline u32 emit_gfx9_global_store_short(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src, int off)
+{
+ /* global_store_short <src>, <dst>, off offset:<off>
+ * *(char *)(dst + off) = src;
+ */
+ insn->flat.offset = off;
+ insn->flat.lds = 0;
+ insn->flat.seg = GFX9_FLAT_SEG_GLOBAL;
+ insn->flat.glc = 0;
+ insn->flat.slc = 0;
+ insn->flat.op = GFX9_GLOBAL_STORE_SHORT;
+ insn->flat.dummy = 0;
+ insn->flat.encoding = GFX9_FLAT_ENCODING;
+ insn->flat.addr = dst.v;
+ insn->flat.data = src.v;
+ insn->flat.saddr = GFX9_FLAT_SADDR_DISABLE;
+ insn->flat.nv = 0;
+ insn->flat.vdst = 0;
+
+ return 8;
+}
+
+inline u32 emit_gfx9_global_store_dword(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src, int off)
+{
+ /* global_store_dword <src>, <dst>, off offset:<off>
+ * *(char *)(dst + off) = src;
+ */
+ insn->flat.offset = off;
+ insn->flat.lds = 0;
+ insn->flat.seg = GFX9_FLAT_SEG_GLOBAL;
+ insn->flat.glc = 0;
+ insn->flat.slc = 0;
+ insn->flat.op = GFX9_GLOBAL_STORE_DWORD;
+ insn->flat.dummy = 0;
+ insn->flat.encoding = GFX9_FLAT_ENCODING;
+ insn->flat.addr = dst.v;
+ insn->flat.data = src.v;
+ insn->flat.saddr = GFX9_FLAT_SADDR_DISABLE;
+ insn->flat.nv = 0;
+ insn->flat.vdst = 0;
+
+ return 8;
+}
+
+/* global_atomic_add vdst, addr, data, off (GLC=1: return old value)
+ * old_val = *(u32 *)(addr + off); *(u32 *)(addr + off) += data; vdst = old_val
+ */
+/* GFX9 global atomic: opcode only differs, all else identical */
+#define DEFINE_GFX9_GLOBAL_ATOMIC(name, opcode) \
+inline u32 emit_gfx9_global_atomic_##name(union amdgcn_gfx9_insn *insn,\
+ struct amdgcn_param32 vdst, \
+ struct amdgcn_param32 addr, \
+ struct amdgcn_param32 data, \
+ int off, int glc) \
+{ \
+ insn->flat.offset = off; \
+ insn->flat.lds = 0; \
+ insn->flat.seg = GFX9_FLAT_SEG_GLOBAL; \
+ insn->flat.glc = glc; \
+ insn->flat.slc = 0; \
+ insn->flat.op = (opcode); \
+ insn->flat.dummy = 0; \
+ insn->flat.encoding = GFX9_FLAT_ENCODING; \
+ insn->flat.addr = addr.v; \
+ insn->flat.data = data.v; \
+ insn->flat.saddr = GFX9_FLAT_SADDR_DISABLE; \
+ insn->flat.nv = 0; \
+ insn->flat.vdst = vdst.v; \
+ return 8; \
+}
+
+DEFINE_GFX9_GLOBAL_ATOMIC(add, GFX9_GLOBAL_ATOMIC_ADD)
+DEFINE_GFX9_GLOBAL_ATOMIC(and, GFX9_GLOBAL_ATOMIC_AND)
+DEFINE_GFX9_GLOBAL_ATOMIC(or, GFX9_GLOBAL_ATOMIC_OR)
+DEFINE_GFX9_GLOBAL_ATOMIC(xor, GFX9_GLOBAL_ATOMIC_XOR)
+DEFINE_GFX9_GLOBAL_ATOMIC(swap, GFX9_GLOBAL_ATOMIC_SWAP)
+DEFINE_GFX9_GLOBAL_ATOMIC(cmpswap, GFX9_GLOBAL_ATOMIC_CMPSWAP)
+DEFINE_GFX9_GLOBAL_ATOMIC(add_x2, GFX9_GLOBAL_ATOMIC_ADD_X2)
+DEFINE_GFX9_GLOBAL_ATOMIC(and_x2, GFX9_GLOBAL_ATOMIC_AND_X2)
+DEFINE_GFX9_GLOBAL_ATOMIC(or_x2, GFX9_GLOBAL_ATOMIC_OR_X2)
+DEFINE_GFX9_GLOBAL_ATOMIC(xor_x2, GFX9_GLOBAL_ATOMIC_XOR_X2)
+DEFINE_GFX9_GLOBAL_ATOMIC(swap_x2, GFX9_GLOBAL_ATOMIC_SWAP_X2)
+DEFINE_GFX9_GLOBAL_ATOMIC(cmpswap_x2, GFX9_GLOBAL_ATOMIC_CMPSWAP_X2)
+
+inline u32 emit_gfx9_global_store_dwordx2(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src, int off)
+{
+ /* global_store_dwordx2 <src>, <dst>, off offset:<off>
+ * *(char *)(dst + off) = src;
+ */
+ insn->flat.offset = off;
+ insn->flat.lds = 0;
+ insn->flat.seg = GFX9_FLAT_SEG_GLOBAL;
+ insn->flat.glc = 0;
+ insn->flat.slc = 0;
+ insn->flat.op = GFX9_GLOBAL_STORE_DWORDX2;
+ insn->flat.dummy = 0;
+ insn->flat.encoding = GFX9_FLAT_ENCODING;
+ insn->flat.addr = dst.v;
+ insn->flat.data = src.v;
+ insn->flat.saddr = GFX9_FLAT_SADDR_DISABLE;
+ insn->flat.nv = 0;
+ insn->flat.vdst = 0;
+
+ return 8;
+}
+
+inline u32 emit_gfx9_global_store_dwordx4(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src, int off)
+{
+ /* global_store_dwordx4 <src>, <dst>, off offset:<off>
+ * *(char *)(dst + off) = src;
+ */
+ insn->flat.offset = off;
+ insn->flat.lds = 0;
+ insn->flat.seg = GFX9_FLAT_SEG_GLOBAL;
+ insn->flat.glc = 0;
+ insn->flat.slc = 0;
+ insn->flat.op = GFX9_GLOBAL_STORE_DWORDX4;
+ insn->flat.dummy = 0;
+ insn->flat.encoding = GFX9_FLAT_ENCODING;
+ insn->flat.addr = dst.v;
+ insn->flat.data = src.v;
+ insn->flat.saddr = GFX9_FLAT_SADDR_DISABLE;
+ insn->flat.nv = 0;
+ insn->flat.vdst = 0;
+
+ return 8;
+}
+
+inline u32 emit_gfx9_ds_read2_b64(union amdgcn_gfx9_insn *insn,
+ int dst, int src, short off0, short off1)
+{
+ /*
+ * ds_read2_b64 v[<dst>+3:<dst>], v<src> off
+ * offset0:<off0> offset1:<off1>
+ */
+
+ insn->ds.offset0 = off0;
+ insn->ds.offset1 = off1;
+ insn->ds.gds = GFX9_DS_LDS;
+ insn->ds.op = GFX9_DS_READ2_B64;
+ insn->ds.encoding = GFX9_DS_ENCODING;
+ insn->ds.addr = src;
+ insn->ds.data0 = 0;
+ insn->ds.data1 = 0;
+ insn->ds.vdst = dst;
+
+ return 8;
+}
+
+inline u32 emit_gfx9_ds_read_b64(union amdgcn_gfx9_insn *insn,
+ int dst, int src, short off)
+{
+ /* ds_read_b64 v[<dst>+1:<dst>], v<src> offset:<off> */
+
+ insn->ds.offset0 = off & 0xff;
+ insn->ds.offset1 = off >> 8;
+ insn->ds.gds = GFX9_DS_LDS;
+ insn->ds.op = GFX9_DS_READ_B64;
+ insn->ds.encoding = GFX9_DS_ENCODING;
+ insn->ds.addr = src;
+ insn->ds.data0 = 0;
+ insn->ds.data1 = 0;
+ insn->ds.vdst = dst;
+
+ return 8;
+}
+
+inline u32 emit_gfx9_ds_read_b32(union amdgcn_gfx9_insn *insn,
+ int dst, int src, short off)
+{
+ /* ds_read_b32 v<dst>, v<src> offset:<off> */
+
+ insn->ds.offset0 = off & 0xff;
+ insn->ds.offset1 = off >> 8;
+ insn->ds.gds = GFX9_DS_LDS;
+ insn->ds.op = GFX9_DS_READ_B32;
+ insn->ds.encoding = GFX9_DS_ENCODING;
+ insn->ds.addr = src;
+ insn->ds.data0 = 0;
+ insn->ds.data1 = 0;
+ insn->ds.vdst = dst;
+
+ return 8;
+}
+
+inline u32 emit_gfx9_ds_read_u16(union amdgcn_gfx9_insn *insn,
+ int dst, int src, short off)
+{
+ /* ds_read_u16 v<dst>, v<src> offset:<off> */
+
+ insn->ds.offset0 = off & 0xff;
+ insn->ds.offset1 = off >> 8;
+ insn->ds.gds = GFX9_DS_LDS;
+ insn->ds.op = GFX9_DS_READ_U16;
+ insn->ds.encoding = GFX9_DS_ENCODING;
+ insn->ds.addr = src;
+ insn->ds.data0 = 0;
+ insn->ds.data1 = 0;
+ insn->ds.vdst = dst;
+
+ return 8;
+}
+
+inline u32 emit_gfx9_ds_read_u8(union amdgcn_gfx9_insn *insn,
+ int dst, int src, short off)
+{
+ /* ds_read_u8 v<dst>, v<src> offset:<off> */
+
+ insn->ds.offset0 = off & 0xff;
+ insn->ds.offset1 = off >> 8;
+ insn->ds.gds = GFX9_DS_LDS;
+ insn->ds.op = GFX9_DS_READ_U8;
+ insn->ds.encoding = GFX9_DS_ENCODING;
+ insn->ds.addr = src;
+ insn->ds.data0 = 0;
+ insn->ds.data1 = 0;
+ insn->ds.vdst = dst;
+
+ return 8;
+}
+
+inline u32 emit_gfx9_ds_write2_b64(union amdgcn_gfx9_insn *insn,
+ int dst, int src0, int src1,
+ short off0, short off1)
+{
+ /*
+ * ds_write2_b64 v[<dst>+3:<dst>], v<src> off
+ * offset0:<off0> offset1:<off1>
+ */
+
+ insn->ds.offset0 = off0;
+ insn->ds.offset1 = off1;
+ insn->ds.gds = GFX9_DS_LDS;
+ insn->ds.op = GFX9_DS_WRITE2_B64;
+ insn->ds.encoding = GFX9_DS_ENCODING;
+ insn->ds.addr = dst;
+ insn->ds.data0 = src0;
+ insn->ds.data1 = src1;
+ insn->ds.vdst = 0;
+
+ return 8;
+}
+
+inline u32 emit_gfx9_s_branch(union amdgcn_gfx9_insn *insn,
+ short off)
+{
+ insn->sopp.simm16 = off;
+ insn->sopp.op = GFX9_S_BRANCH;
+ insn->sopp.encoding = GFX9_SOPP_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_s_cbranch_vccz(union amdgcn_gfx9_insn *insn,
+ short off)
+{
+ insn->sopp.simm16 = off;
+ insn->sopp.op = GFX9_S_CBRANCH_VCCZ;
+ insn->sopp.encoding = GFX9_SOPP_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_s_cbranch_vccnz(union amdgcn_gfx9_insn *insn,
+ short off)
+{
+ insn->sopp.simm16 = off;
+ insn->sopp.op = GFX9_S_CBRANCH_VCCNZ;
+ insn->sopp.encoding = GFX9_SOPP_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_branch_fixup(union amdgcn_gfx9_insn *insn,
+ short off)
+{
+ WARN_ON(insn->sopp.op != GFX9_S_BRANCH &&
+ insn->sopp.op != GFX9_S_CBRANCH_SCC0 &&
+ insn->sopp.op != GFX9_S_CBRANCH_VCCZ &&
+ insn->sopp.op != GFX9_S_CBRANCH_VCCNZ &&
+ insn->sopp.op != GFX9_S_CBRANCH_EXECZ &&
+ insn->sopp.op != GFX9_S_CBRANCH_EXECNZ);
+ insn->sopp.simm16 = off;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_s_waitcnt_lgkmcnt(union amdgcn_gfx9_insn *insn)
+{
+ struct amdgcn_gfx9_sopp_vmcnt vmcnt;
+ u16 simm16;
+ /* s_waitcnt lgkmcnt (0)
+ * wait for memory
+ */
+ vmcnt.vmcnt1 = -1;
+ vmcnt.vmcnt2 = -1;
+ vmcnt.expcnt = -1;
+ vmcnt.dummy1 = 0;
+ vmcnt.dummy2 = 0;
+ vmcnt.lgkmcnt = 0;
+ memcpy(&simm16, &vmcnt, sizeof(u16));
+ insn->sopp.simm16 = simm16;
+ insn->sopp.op = GFX9_S_WAITCNT;
+ insn->sopp.encoding = GFX9_SOPP_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_s_waitcnt_vmcnt(union amdgcn_gfx9_insn *insn)
+{
+ struct amdgcn_gfx9_sopp_vmcnt vmcnt;
+ u16 simm16;
+ /* s_waitcnt lgkmcnt (0)
+ * wait for memory
+ */
+ vmcnt.vmcnt1 = 0;
+ vmcnt.vmcnt2 = 0;
+ vmcnt.expcnt = -1;
+ vmcnt.dummy1 = 0;
+ vmcnt.dummy2 = 0;
+ vmcnt.lgkmcnt = -1;
+ memcpy(&simm16, &vmcnt, sizeof(u16));
+ insn->sopp.simm16 = simm16;
+ insn->sopp.op = GFX9_S_WAITCNT;
+ insn->sopp.encoding = GFX9_SOPP_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_s_waitcnt_vmcnt_lgkmcnt(union amdgcn_gfx9_insn *insn)
+{
+ struct amdgcn_gfx9_sopp_vmcnt vmcnt;
+ u16 simm16;
+ /* s_waitcnt lgkmcnt (0)
+ * wait for memory
+ */
+ vmcnt.vmcnt1 = 0;
+ vmcnt.vmcnt2 = 0;
+ vmcnt.expcnt = -1;
+ vmcnt.dummy1 = 0;
+ vmcnt.dummy2 = 0;
+ vmcnt.lgkmcnt = 0;
+ memcpy(&simm16, &vmcnt, sizeof(u16));
+ insn->sopp.simm16 = simm16;
+ insn->sopp.op = GFX9_S_WAITCNT;
+ insn->sopp.encoding = GFX9_SOPP_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_s_nop(union amdgcn_gfx9_insn *insn)
+{
+ insn->sopp.simm16 = 0;
+ insn->sopp.op = GFX9_S_NOP;
+ insn->sopp.encoding = GFX9_SOPP_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_s_endpgm(union amdgcn_gfx9_insn *insn)
+{
+ insn->sopp.simm16 = 0;
+ insn->sopp.op = GFX9_S_ENDPGM;
+ insn->sopp.encoding = GFX9_SOPP_ENCODING;
+
+ return 4;
+}
+
+inline u32 emit_gfx9_s_icache_inv(union amdgcn_gfx9_insn *insn)
+{
+ insn->sopp.simm16 = 0;
+ insn->sopp.op = GFX9_S_ICACHE_INV;
+ insn->sopp.encoding = GFX9_SOPP_ENCODING;
+
+ return 4;
+}
+
+/* Structurized CFG instructions.
+ * These use raw SGPR indices for 64-bit pair operations involving
+ * EXEC (126) and VCC (106) special registers.
+ */
+
+/* s_and_saveexec_b64 s[sdst:sdst+1], s[ssrc:ssrc+1]
+ * sdst = EXEC; EXEC &= ssrc; SCC = (EXEC != 0)
+ */
+inline u32 emit_gfx9_s_and_saveexec_b64(union amdgcn_gfx9_insn *insn,
+ u8 sdst, u8 ssrc)
+{
+ insn->sop1.ssrc0 = ssrc;
+ insn->sop1.op = GFX9_S_AND_SAVEEXEC_B64;
+ insn->sop1.sdst = sdst;
+ insn->sop1.encoding = GFX9_SOP1_ENCODING;
+
+ return 4;
+}
+
+/* s_bcnt1_i32_b64 sdst, s[ssrc:ssrc+1]
+ * sdst = popcount(ssrc). SCC = (sdst != 0)
+ */
+inline u32 emit_gfx9_s_bcnt1_i32_b64(union amdgcn_gfx9_insn *insn,
+ u8 sdst, u8 ssrc)
+{
+ insn->sop1.ssrc0 = ssrc;
+ insn->sop1.op = GFX9_S_BCNT1_I32_B64;
+ insn->sop1.sdst = sdst;
+ insn->sop1.encoding = GFX9_SOP1_ENCODING;
+
+ return 4;
+}
+
+/* s_mov_b64 s[sdst:sdst+1], s[ssrc:ssrc+1] (or special src like 0) */
+inline u32 emit_gfx9_s_mov_b64(union amdgcn_gfx9_insn *insn,
+ u8 sdst, u8 ssrc)
+{
+ insn->sop1.ssrc0 = ssrc;
+ insn->sop1.op = GFX9_S_MOV_B64;
+ insn->sop1.sdst = sdst;
+ insn->sop1.encoding = GFX9_SOP1_ENCODING;
+
+ return 4;
+}
+
+/* s_and_b64 s[sdst:sdst+1], s[ssrc0:ssrc0+1], s[ssrc1:ssrc1+1] */
+inline u32 emit_gfx9_s_and_b64(union amdgcn_gfx9_insn *insn,
+ u8 sdst, u8 ssrc0, u8 ssrc1)
+{
+ insn->sop2.ssrc0 = ssrc0;
+ insn->sop2.ssrc1 = ssrc1;
+ insn->sop2.sdst = sdst;
+ insn->sop2.op = GFX9_S_AND_B64;
+ insn->sop2.encoding = GFX9_SOP2_ENCODING;
+
+ return 4;
+}
+
+/* s_or_b64 s[sdst:sdst+1], s[ssrc0:ssrc0+1], s[ssrc1:ssrc1+1] */
+inline u32 emit_gfx9_s_or_b64(union amdgcn_gfx9_insn *insn,
+ u8 sdst, u8 ssrc0, u8 ssrc1)
+{
+ insn->sop2.ssrc0 = ssrc0;
+ insn->sop2.ssrc1 = ssrc1;
+ insn->sop2.sdst = sdst;
+ insn->sop2.op = GFX9_S_OR_B64;
+ insn->sop2.encoding = GFX9_SOP2_ENCODING;
+
+ return 4;
+}
+
+/* s_andn2_b64 s[sdst:sdst+1], s[ssrc0:ssrc0+1], s[ssrc1:ssrc1+1]
+ * sdst = ssrc0 & ~ssrc1
+ */
+inline u32 emit_gfx9_s_andn2_b64(union amdgcn_gfx9_insn *insn,
+ u8 sdst, u8 ssrc0, u8 ssrc1)
+{
+ insn->sop2.ssrc0 = ssrc0;
+ insn->sop2.ssrc1 = ssrc1;
+ insn->sop2.sdst = sdst;
+ insn->sop2.op = GFX9_S_ANDN2_B64;
+ insn->sop2.encoding = GFX9_SOP2_ENCODING;
+
+ return 4;
+}
+
+/* s_and_b32 s[sdst], s[ssrc0], s[ssrc1] */
+inline u32 emit_gfx9_s_and_b32(union amdgcn_gfx9_insn *insn,
+ u8 sdst, u8 ssrc0, u8 ssrc1)
+{
+ insn->sop2.ssrc0 = ssrc0;
+ insn->sop2.ssrc1 = ssrc1;
+ insn->sop2.sdst = sdst;
+ insn->sop2.op = GFX9_S_AND_B32;
+ insn->sop2.encoding = GFX9_SOP2_ENCODING;
+
+ return 4;
+}
+
+/* s_or_b32 s[sdst], s[ssrc0], s[ssrc1] */
+inline u32 emit_gfx9_s_or_b32(union amdgcn_gfx9_insn *insn,
+ u8 sdst, u8 ssrc0, u8 ssrc1)
+{
+ insn->sop2.ssrc0 = ssrc0;
+ insn->sop2.ssrc1 = ssrc1;
+ insn->sop2.sdst = sdst;
+ insn->sop2.op = GFX9_S_OR_B32;
+ insn->sop2.encoding = GFX9_SOP2_ENCODING;
+
+ return 4;
+}
+
+/* s_andn2_b32 s[sdst], s[ssrc0], s[ssrc1]
+ * sdst = ssrc0 & ~ssrc1
+ */
+inline u32 emit_gfx9_s_andn2_b32(union amdgcn_gfx9_insn *insn,
+ u8 sdst, u8 ssrc0, u8 ssrc1)
+{
+ insn->sop2.ssrc0 = ssrc0;
+ insn->sop2.ssrc1 = ssrc1;
+ insn->sop2.sdst = sdst;
+ insn->sop2.op = GFX9_S_ANDN2_B32;
+ insn->sop2.encoding = GFX9_SOP2_ENCODING;
+
+ return 4;
+}
+
+/* s_cbranch_execz off - branch if EXEC == 0 */
+inline u32 emit_gfx9_s_cbranch_execz(union amdgcn_gfx9_insn *insn,
+ short off)
+{
+ insn->sopp.simm16 = off;
+ insn->sopp.op = GFX9_S_CBRANCH_EXECZ;
+ insn->sopp.encoding = GFX9_SOPP_ENCODING;
+
+ return 4;
+}
+
+/* s_cbranch_execnz off - branch if EXEC != 0 */
+inline u32 emit_gfx9_s_cbranch_execnz(union amdgcn_gfx9_insn *insn,
+ short off)
+{
+ insn->sopp.simm16 = off;
+ insn->sopp.op = GFX9_S_CBRANCH_EXECNZ;
+ insn->sopp.encoding = GFX9_SOPP_ENCODING;
+
+ return 4;
+}
+
+/* s_sub_u32 sdst, ssrc0, ssrc1 - sdst = ssrc0 - ssrc1; SCC = borrow */
+inline u32 emit_gfx9_s_sub_u32(union amdgcn_gfx9_insn *insn,
+ u8 sdst, u8 ssrc0, u8 ssrc1)
+{
+ insn->sop2.ssrc0 = ssrc0;
+ insn->sop2.ssrc1 = ssrc1;
+ insn->sop2.sdst = sdst;
+ insn->sop2.op = GFX9_S_SUB_U32;
+ insn->sop2.encoding = GFX9_SOP2_ENCODING;
+
+ return 4;
+}
+
+/* s_cbranch_scc0 off - branch if SCC == 0 (no borrow) */
+inline u32 emit_gfx9_s_cbranch_scc0(union amdgcn_gfx9_insn *insn,
+ short off)
+{
+ insn->sopp.simm16 = off;
+ insn->sopp.op = GFX9_S_CBRANCH_SCC0;
+ insn->sopp.encoding = GFX9_SOPP_ENCODING;
+
+ return 4;
+}
+
+static inline void __emit_gfx9_sop2(union amdgcn_gfx9_insn *insn,
+ int op, int sdst, int ssrc0, int ssrc1)
+{
+ insn->sop2.encoding = GFX9_SOP2_ENCODING;
+ insn->sop2.op = op;
+ insn->sop2.sdst = sdst;
+ insn->sop2.ssrc0 = ssrc0;
+ insn->sop2.ssrc1 = ssrc1;
+}
+
+static inline void __emit_gfx9_sop1(union amdgcn_gfx9_insn *insn,
+ int op, int sdst, int ssrc0)
+{
+ insn->sop1.encoding = GFX9_SOP1_ENCODING;
+ insn->sop1.op = op;
+ insn->sop1.sdst = sdst;
+ insn->sop1.ssrc0 = ssrc0;
+}
+
+static inline void __emit_gfx9_sopp(union amdgcn_gfx9_insn *insn,
+ int op, u16 simm16)
+{
+ insn->sopp.encoding = GFX9_SOPP_ENCODING;
+ insn->sopp.op = op;
+ insn->sopp.simm16 = simm16;
+}
+
+static inline void __emit_gfx9_sopc(union amdgcn_gfx9_insn *insn,
+ int op, int ssrc0, int ssrc1)
+{
+ insn->sopc.encoding = GFX9_SOPC_ENCODING;
+ insn->sopc.op = op;
+ insn->sopc.ssrc0 = ssrc0;
+ insn->sopc.ssrc1 = ssrc1;
+}
+
+static inline void __emit_gfx9_vop1(union amdgcn_gfx9_insn *insn,
+ int op, int vdst, int src0)
+{
+ insn->vop1.encoding = GFX9_VOP1_ENCODING;
+ insn->vop1.op = op;
+ insn->vop1.vdst = vdst;
+ insn->vop1.src0 = src0;
+}
+
+static inline void __emit_gfx9_vop2(union amdgcn_gfx9_insn *insn,
+ int op, int vdst, int vsrc1, int src0)
+{
+ insn->vop2.encoding = GFX9_VOP2_ENCODING;
+ insn->vop2.op = op;
+ insn->vop2.vdst = vdst;
+ insn->vop2.vsrc1 = vsrc1;
+ insn->vop2.src0 = src0;
+}
+
+static inline void __emit_gfx9_vopc(union amdgcn_gfx9_insn *insn,
+ int op, int vsrc1, int src0)
+{
+ insn->vopc.encoding = GFX9_VOPC_ENCODING;
+ insn->vopc.op = op;
+ insn->vopc.vsrc1 = vsrc1;
+ insn->vopc.src0 = src0;
+}
+
+static inline void __emit_gfx9_smem(union amdgcn_gfx9_insn *insn,
+ int op, int sdata, int sbase_pair,
+ u32 offset)
+{
+ insn->smem.encoding = GFX9_SMEM_ENCODING;
+ insn->smem.op = op;
+ insn->smem.sdata = sdata;
+ insn->smem.sbase = sbase_pair;
+ insn->smem.imm = 1;
+ insn->smem.offset = offset;
+ insn->smem.soffset = GFX9_SRC_NULL;
+}
+
+static inline void __emit_gfx9_ds(union amdgcn_gfx9_insn *insn,
+ int op, int addr, int data0, int vdst,
+ int off0, int off1)
+{
+ insn->ds.encoding = GFX9_DS_ENCODING;
+ insn->ds.op = op;
+ insn->ds.gds = GFX9_DS_LDS;
+ insn->ds.addr = addr;
+ insn->ds.data0 = data0;
+ insn->ds.vdst = vdst;
+ insn->ds.offset0 = off0;
+ insn->ds.offset1 = off1;
+}
+
+static inline void __emit_gfx9_global(union amdgcn_gfx9_insn *insn,
+ int op, int vdst, int vaddr,
+ int vdata, int saddr, int offset)
+{
+ insn->flat.encoding = GFX9_FLAT_ENCODING;
+ insn->flat.seg = GFX9_FLAT_SEG_GLOBAL;
+ insn->flat.op = op;
+ insn->flat.vdst = vdst;
+ insn->flat.addr = vaddr;
+ insn->flat.data = vdata;
+ insn->flat.saddr = saddr;
+ insn->flat.offset = offset;
+}
+
+/* ======================================================================
+ * GFX9 Param-Aware Emit Functions
+ *
+ * Used by shader-emitters (aesgcm_shader.h, ipsec_fused_gfx9.h, ...) that
+ * construct param32 operands via P_S/P_V/P_I/P_L helpers. Paired with
+ * the GFX10 equivalents in knod_gfx10_insn.h.
+ * ======================================================================
+ */
+
+static inline int __p2e9(struct amdgcn_param32 p)
+{
+ if (p.type == AMDGCN_PARAM_TYPE_LITERAL_CONST)
+ return gfx9_get_param_base(p);
+ return gfx9_get_param_base(p) + p.v;
+}
+
+/* --- SOP2 family (param32 version) --- */
+
+#define DEFINE_GFX9_SOP2_P(name, opcode) \
+inline u32 emit_gfx9_##name(union amdgcn_gfx9_insn *insn, \
+ struct amdgcn_param32 dst, \
+ struct amdgcn_param32 src0, \
+ struct amdgcn_param32 src1) \
+{ \
+ insn->sop2.sdst = __p2e9(dst); \
+ insn->sop2.ssrc0 = __p2e9(src0); \
+ insn->sop2.ssrc1 = __p2e9(src1); \
+ insn->sop2.op = opcode; \
+ insn->sop2.encoding = GFX9_SOP2_ENCODING; \
+ if (knod_param_is_literal(src0)) { \
+ insn->sop2.literal = src0.v; \
+ return 8; \
+ } \
+ if (knod_param_is_literal(src1)) { \
+ insn->sop2.literal = src1.v; \
+ return 8; \
+ } \
+ return 4; \
+}
+
+DEFINE_GFX9_SOP2_P(s_add_u32, GFX9_S_ADD_U32)
+DEFINE_GFX9_SOP2_P(s_sub_u32_p, GFX9_S_SUB_U32)
+DEFINE_GFX9_SOP2_P(s_addc_u32, GFX9_S_ADDC_U32)
+DEFINE_GFX9_SOP2_P(s_subb_u32, GFX9_S_SUBB_U32)
+DEFINE_GFX9_SOP2_P(s_and_b32_p, GFX9_S_AND_B32)
+DEFINE_GFX9_SOP2_P(s_lshl_b32, GFX9_S_LSHL_B32)
+DEFINE_GFX9_SOP2_P(s_lshr_b32, GFX9_S_LSHR_B32)
+DEFINE_GFX9_SOP2_P(s_mul_i32, GFX9_S_MUL_I32)
+DEFINE_GFX9_SOP2_P(s_xor_b32, GFX9_S_XOR_B32)
+
+#undef DEFINE_GFX9_SOP2_P
+
+/* --- SOPC family (param32 version) --- */
+
+#define DEFINE_GFX9_SOPC_P(name, opcode) \
+inline u32 emit_gfx9_##name(union amdgcn_gfx9_insn *insn, \
+ struct amdgcn_param32 src0, \
+ struct amdgcn_param32 src1) \
+{ \
+ insn->sopc.ssrc0 = __p2e9(src0); \
+ insn->sopc.ssrc1 = __p2e9(src1); \
+ insn->sopc.op = opcode; \
+ insn->sopc.encoding = GFX9_SOPC_ENCODING; \
+ if (knod_param_is_literal(src0)) { \
+ insn->sopc.literal = src0.v; \
+ return 8; \
+ } \
+ if (knod_param_is_literal(src1)) { \
+ insn->sopc.literal = src1.v; \
+ return 8; \
+ } \
+ return 4; \
+}
+
+DEFINE_GFX9_SOPC_P(s_cmp_eq_u32, GFX9_S_CMP_EQ_U32)
+DEFINE_GFX9_SOPC_P(s_cmp_lg_u32, GFX9_S_CMP_LG_U32)
+DEFINE_GFX9_SOPC_P(s_cmp_ge_u32, GFX9_S_CMP_GE_U32)
+DEFINE_GFX9_SOPC_P(s_cmp_lt_u32, GFX9_S_CMP_LT_U32)
+
+#undef DEFINE_GFX9_SOPC_P
+
+/* --- SOPP family --- */
+
+inline u32 emit_gfx9_s_barrier(union amdgcn_gfx9_insn *insn)
+{
+ insn->sopp.simm16 = 0;
+ insn->sopp.op = GFX9_S_BARRIER;
+ insn->sopp.encoding = GFX9_SOPP_ENCODING;
+ return 4;
+}
+
+/* waitcnt with arbitrary vm/lgkm */
+#define GFX9_WAITCNT(vm, lgkm) (((lgkm) << 8) | (7 << 4) | (vm))
+
+inline u32 emit_gfx9_s_waitcnt(union amdgcn_gfx9_insn *insn,
+ int vm, int lgkm)
+{
+ insn->sopp.simm16 = GFX9_WAITCNT(vm, lgkm);
+ insn->sopp.op = GFX9_S_WAITCNT;
+ insn->sopp.encoding = GFX9_SOPP_ENCODING;
+ return 4;
+}
+
+inline u32 emit_gfx9_s_cbranch_scc1(union amdgcn_gfx9_insn *insn,
+ short off)
+{
+ insn->sopp.simm16 = off;
+ insn->sopp.op = GFX9_S_CBRANCH_SCC1;
+ insn->sopp.encoding = GFX9_SOPP_ENCODING;
+ return 4;
+}
+
+/* --- SOP1 family --- */
+
+inline u32 emit_gfx9_s_not_b64(union amdgcn_gfx9_insn *insn,
+ u8 sdst, u8 ssrc)
+{
+ insn->sop1.ssrc0 = ssrc;
+ insn->sop1.op = GFX9_S_NOT_B64;
+ insn->sop1.sdst = sdst;
+ insn->sop1.encoding = GFX9_SOP1_ENCODING;
+ return 4;
+}
+
+/* --- VOPC (v_cmp_ne_u32 param variant) --- */
+
+inline u32 emit_gfx9_v_cmp_ne_u32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ WARN_ON(src1.type != AMDGCN_PARAM_TYPE_VGPR);
+ insn->vopc.vsrc1 = src1.v;
+ insn->vopc.op = GFX9_V_CMP_NE_U32;
+ insn->vopc.encoding = GFX9_VOPC_ENCODING;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vopc.src0 = gfx9_get_param_base(src0);
+ insn->vopc.literal = src0.v;
+ return 8;
+ }
+ insn->vopc.src0 = gfx9_get_param_base(src0) + src0.v;
+ return 4;
+}
+
+/* --- SMEM family (param32 version) --- */
+
+#define DEFINE_GFX9_SMEM_P(name, opcode) \
+inline u32 emit_gfx9_##name(union amdgcn_gfx9_insn *insn, \
+ struct amdgcn_param32 dst, \
+ struct amdgcn_param32 src, int offset) \
+{ \
+ insn->smem.sdata = dst.v; \
+ insn->smem.sbase = src.v / 2; \
+ insn->smem.op = opcode; \
+ insn->smem.imm = 1; \
+ insn->smem.offset = offset; \
+ insn->smem.encoding = GFX9_SMEM_ENCODING; \
+ return 8; \
+}
+
+DEFINE_GFX9_SMEM_P(s_load_dword, GFX9_S_LOAD_DWORD)
+DEFINE_GFX9_SMEM_P(s_load_dwordx4, GFX9_S_LOAD_DWORDX4)
+DEFINE_GFX9_SMEM_P(s_load_dwordx8, GFX9_S_LOAD_DWORDX8)
+
+#undef DEFINE_GFX9_SMEM_P
+
+/* s_dcache_inv - no operands, just opcode + encoding */
+inline u32 emit_gfx9_s_dcache_inv(union amdgcn_gfx9_insn *insn)
+{
+ insn->smem.sdata = 0;
+ insn->smem.sbase = 0;
+ insn->smem.op = GFX9_S_DCACHE_INV;
+ insn->smem.imm = 0;
+ insn->smem.offset = 0;
+ insn->smem.encoding = GFX9_SMEM_ENCODING;
+ return 8;
+}
+
+/* --- SOPK: s_getreg_b32 --- */
+
+/*
+ * HWREG encoding for s_getreg_b32 simm16:
+ * bits[5:0] = hwreg_id
+ * bits[10:6] = offset (bit position)
+ * bits[15:11] = size - 1 (in bits)
+ */
+#define GFX9_HWREG(id, off, sz) (((sz) - 1) << 11 | (off) << 6 | (id))
+#define GFX9_HW_REG_LDS_ALLOC 6
+
+inline u32 emit_gfx9_s_getreg_b32(union amdgcn_gfx9_insn *insn,
+ int sdst, u16 hwreg)
+{
+ insn->sopk.encoding = GFX9_SOPK_ENCODING;
+ insn->sopk.op = GFX9_S_GETREG_B32;
+ insn->sopk.sdst = sdst;
+ insn->sopk.simm16 = hwreg;
+ return 4;
+}
+
+/* --- DS family --- */
+
+inline u32 emit_gfx9_gds_write_b32(union amdgcn_gfx9_insn *insn,
+ int addr, int data0)
+{
+ __emit_gfx9_ds(insn, GFX9_DS_WRITE_B32, addr, data0, 0, 0, 0);
+ insn->ds.gds = GFX9_DS_GDS;
+ return 8;
+}
+
+inline u32 emit_gfx9_gds_read_b32(union amdgcn_gfx9_insn *insn,
+ int vdst, int addr, int offset)
+{
+ __emit_gfx9_ds(insn, GFX9_DS_READ_B32, addr, 0, vdst, offset, 0);
+ insn->ds.gds = GFX9_DS_GDS;
+ return 8;
+}
+
+inline u32 emit_gfx9_ds_write_b32(union amdgcn_gfx9_insn *insn,
+ int addr, int data0)
+{
+ __emit_gfx9_ds(insn, GFX9_DS_WRITE_B32, addr, data0, 0, 0, 0);
+ return 8;
+}
+
+inline u32 emit_gfx9_ds_write_b32_off(union amdgcn_gfx9_insn *insn,
+ int addr, int data0, int offset)
+{
+ __emit_gfx9_ds(insn, GFX9_DS_WRITE_B32, addr, data0, 0, offset, 0);
+ return 8;
+}
+
+inline u32 emit_gfx9_ds_read_b32_off(union amdgcn_gfx9_insn *insn,
+ int vdst, int addr, int offset)
+{
+ __emit_gfx9_ds(insn, GFX9_DS_READ_B32, addr, 0, vdst, offset, 0);
+ return 8;
+}
+
+/* ds_write_b128 v<addr>, v[<data>:<data>+3] - write 128 bits to LDS */
+inline u32 emit_gfx9_ds_write_b128(union amdgcn_gfx9_insn *insn,
+ int addr, int data0)
+{
+ __emit_gfx9_ds(insn, GFX9_DS_WRITE_B128, addr, data0, 0, 0, 0);
+ return 8;
+}
+
+/* ds_read_b128 v[<vdst>:<vdst>+3], v<addr> - read 128 bits from LDS */
+inline u32 emit_gfx9_ds_read_b128(union amdgcn_gfx9_insn *insn,
+ int vdst, int addr)
+{
+ __emit_gfx9_ds(insn, GFX9_DS_READ_B128, addr, 0, vdst, 0, 0);
+ return 8;
+}
+
+/* v_max_i32 vdst, src0, vsrc1 - VOP2 */
+inline u32 emit_gfx9_v_max_i32(union amdgcn_gfx9_insn *insn,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ WARN_ON(dst.type != AMDGCN_PARAM_TYPE_VGPR);
+ WARN_ON(src1.type != AMDGCN_PARAM_TYPE_VGPR);
+
+ insn->vop2.vsrc1 = src1.v;
+ insn->vop2.vdst = dst.v;
+ insn->vop2.op = GFX9_V_MAX_I32;
+ insn->vop2.encoding = GFX9_VOP2_ENCODING;
+ if (src0.type == AMDGCN_PARAM_TYPE_LITERAL_CONST) {
+ insn->vop2.src0 = gfx9_get_param_base(src0);
+ insn->vop2.literal = src0.v;
+ return 8;
+ }
+ insn->vop2.src0 = gfx9_get_param_base(src0) + src0.v;
+
+ return 4;
+}
+
+#endif
--
2.43.0
^ permalink raw reply related
* [RFC PATCH net-next 09/13] drm/amdkfd: add BPF-to-GPU JIT offload
From: Taehee Yoo @ 2026-07-19 17:58 UTC (permalink / raw)
To: Alex Deucher, Alexei Starovoitov, amd-gfx, Andrew Lunn,
Andrii Nakryiko, Bill Wendling, bpf, Christian König,
Daniel Borkmann, David Airlie, David S. Miller, Donald Hunter,
dri-devel, Eduard Zingerman, Emil Tsalapatis, Eric Dumazet,
Felix Kuehling, Hoyeon Lee, Ilias Apalodimas, Jakub Kicinski,
Jesper Dangaard Brouer, Jiri Olsa, John Fastabend, Justin Stitt,
Kees Cook, Kumar Kartikeya Dwivedi, Leon Romanovsky,
linaro-mm-sig, linux-hardening, linux-kernel, linux-kselftest,
linux-media, linux-rdma, llvm, Mark Bloch, Martin KaFai Lau,
Michael Chan, Nathan Chancellor, netdev, Nick Desaulniers,
Paolo Abeni, Pavan Chebbi, Saeed Mahameed, Shuah Khan,
Simona Vetter, Simon Horman, Song Liu, Stanislav Fomichev,
Sumit Semwal, Taehee Yoo, Tariq Toukan, Yonghong Song
In-Reply-To: <20260719175857.4071636-1-ap420073@gmail.com>
Add the knod BPF feature: an XDP program attached in offload mode is
JIT-compiled from eBPF to an AMD GCN shader and dispatched on the GPU
against packets DMA'd into GPU memory, keeping the PASS/DROP/TX verdict
path off the host CPU. Built as a separate module (knod_bpf).
Signed-off-by: Taehee Yoo <ap420073@gmail.com>
(cherry picked from commit 132d51819ffced59b5890e14bd39cab2e9c12eb4)
---
drivers/gpu/drm/amd/amdkfd/Kconfig | 11 +
drivers/gpu/drm/amd/amdkfd/Makefile | 2 +
drivers/gpu/drm/amd/amdkfd/knod/knod_bpf.c | 11554 +++++++++++++++++++
drivers/gpu/drm/amd/amdkfd/knod/knod_bpf.h | 597 +
4 files changed, 12164 insertions(+)
create mode 100644 drivers/gpu/drm/amd/amdkfd/knod/knod_bpf.c
create mode 100644 drivers/gpu/drm/amd/amdkfd/knod/knod_bpf.h
diff --git a/drivers/gpu/drm/amd/amdkfd/Kconfig b/drivers/gpu/drm/amd/amdkfd/Kconfig
index d93f1af749ff..708aa5fc051b 100644
--- a/drivers/gpu/drm/amd/amdkfd/Kconfig
+++ b/drivers/gpu/drm/amd/amdkfd/Kconfig
@@ -49,3 +49,14 @@ config HSA_AMD_KNOD
Say N to drop the KNOD core along with the BPF and IPsec offloads
layered on top of it. If unsure, say Y.
+
+config HSA_AMD_KNOD_BPF
+ tristate "KNOD BPF"
+ depends on HSA_AMD_KNOD
+ help
+ GPU-accelerated XDP/BPF packet processing via KNOD. BPF programs
+ attached in XDP mode are JIT-compiled to AMD GCN shaders and run
+ on the GPU against packets DMA'd directly into GPU memory, keeping
+ the verdict path (PASS/DROP/TX) off the host CPU.
+
+ If unsure, say N.
diff --git a/drivers/gpu/drm/amd/amdkfd/Makefile b/drivers/gpu/drm/amd/amdkfd/Makefile
index 1834faa54863..4df3850e1466 100644
--- a/drivers/gpu/drm/amd/amdkfd/Makefile
+++ b/drivers/gpu/drm/amd/amdkfd/Makefile
@@ -75,3 +75,5 @@ endif
ifneq ($(CONFIG_HSA_AMD_KNOD),)
AMDKFD_FILES += $(AMDKFD_PATH)/kfd_knod.o
endif
+
+obj-$(CONFIG_HSA_AMD_KNOD_BPF) += $(AMDKFD_PATH)/knod/knod_bpf.o
diff --git a/drivers/gpu/drm/amd/amdkfd/knod/knod_bpf.c b/drivers/gpu/drm/amd/amdkfd/knod/knod_bpf.c
new file mode 100644
index 000000000000..f4f48e1b9f1c
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdkfd/knod/knod_bpf.c
@@ -0,0 +1,11554 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/* Copyright (c) 2021 Taehee Yoo <ap420073@gmail.com>
+ * Copyright (c) 2021 Hoyeon Lee <hoyeon.rhee@gmail.com>
+ */
+
+#include <linux/cpumask.h>
+#include <linux/types.h>
+#include <linux/mutex.h>
+#include <linux/slab.h>
+#include <linux/delay.h>
+#include <linux/sched.h>
+#include <linux/workqueue.h>
+#include <linux/file.h>
+#include <linux/jhash.h>
+#include <drm/ttm/ttm_tt.h>
+#include <net/page_pool/helpers.h>
+#include "kfd_priv.h"
+#include "kfd_hsa.h"
+#include "knod_bpf.h"
+#include "kfd_migrate.h"
+#include "kfd_events.h"
+#include "kfd_device_queue_manager.h"
+#include <linux/reciprocal_div.h>
+#include <linux/jhash.h>
+#include <net/knod.h>
+#include <net/netdev_rx_queue.h>
+
+/*+--------+---------+-------+------+--+-----+------+------+--------+
+ *| v0-v21 | v22-v59 |v60-v61| v62 |63|64-65|66-67 |68-69 | v70-127|
+ *+--------+---------+-------+------+--+-----+------+------+--------+
+ *|BPF REGS|TMP REGS |CTX REG| WIDX |R |DATA |D_END |PGBASE|PKTCACHE|
+ *+--------+---------+-------+------+--+-----+------+------+--------+
+ *+-----------------+
+ *| v128-v255 |
+ *+-----------------+
+ *| BPF STACK(512B) |
+ *+-----------------+
+ */
+
+/* Temp register map
+ *+-------------+-------------+---------------+---------------+
+ *|TREG0 - TREG2|TREG3 - TREG9|TREG10 - TREG16|TREG17 - TREG18|
+ *+-------------+-------------+---------------+---------------+
+ *| General Use | Key cache A | Key in MAP | JHASH Temp Reg|
+ *+-------------+-------------+---------------+---------------+
+ * Available Key cache size is 56.
+ * So, key size of map can't be exceed 56B.
+ */
+
+#define KNOD_AMDGPU_VREG0_LO 0 /* v0 */
+#define KNOD_AMDGPU_VREG0_HI 1
+#define KNOD_AMDGPU_VREG1_LO 2
+#define KNOD_AMDGPU_VREG1_HI 3
+#define KNOD_AMDGPU_VREG2_LO 4
+#define KNOD_AMDGPU_VREG2_HI 5
+#define KNOD_AMDGPU_VREG3_LO 6
+#define KNOD_AMDGPU_VREG3_HI 7
+#define KNOD_AMDGPU_VREG4_LO 8
+#define KNOD_AMDGPU_VREG4_HI 9
+#define KNOD_AMDGPU_VREG5_LO 10
+#define KNOD_AMDGPU_VREG5_HI 11
+#define KNOD_AMDGPU_VREG6_LO 12
+#define KNOD_AMDGPU_VREG6_HI 13
+#define KNOD_AMDGPU_VREG7_LO 14
+#define KNOD_AMDGPU_VREG7_HI 15
+#define KNOD_AMDGPU_VREG8_LO 16
+#define KNOD_AMDGPU_VREG8_HI 17
+#define KNOD_AMDGPU_VREG9_LO 18
+#define KNOD_AMDGPU_VREG9_HI 19
+#define KNOD_AMDGPU_FRAME_POINTER_VREG_LO 20 /* v20 */
+#define KNOD_AMDGPU_FRAME_POINTER_VREG_HI 21 /* v20 */
+
+#define KNOD_AMDGPU_TMP_VREG0_LO 22
+#define KNOD_AMDGPU_TMP_VREG0_HI 23
+#define KNOD_AMDGPU_TMP_VREG1_LO 24
+#define KNOD_AMDGPU_TMP_VREG1_HI 25
+#define KNOD_AMDGPU_TMP_VREG2_LO 26
+#define KNOD_AMDGPU_TMP_VREG2_HI 27
+#define KNOD_AMDGPU_TMP_VREG3_LO 28
+#define KNOD_AMDGPU_TMP_VREG3_HI 29
+#define KNOD_AMDGPU_TMP_VREG4_LO 30
+#define KNOD_AMDGPU_TMP_VREG4_HI 31
+#define KNOD_AMDGPU_TMP_VREG5_LO 32
+#define KNOD_AMDGPU_TMP_VREG5_HI 33
+#define KNOD_AMDGPU_TMP_VREG6_LO 34
+#define KNOD_AMDGPU_TMP_VREG6_HI 35
+#define KNOD_AMDGPU_TMP_VREG7_LO 36
+#define KNOD_AMDGPU_TMP_VREG7_HI 37
+#define KNOD_AMDGPU_TMP_VREG8_LO 38
+#define KNOD_AMDGPU_TMP_VREG8_HI 39
+#define KNOD_AMDGPU_TMP_VREG9_LO 40
+#define KNOD_AMDGPU_TMP_VREG9_HI 41
+#define KNOD_AMDGPU_TMP_VREG10_LO 42
+#define KNOD_AMDGPU_TMP_VREG10_HI 43
+#define KNOD_AMDGPU_TMP_VREG11_LO 44
+#define KNOD_AMDGPU_TMP_VREG11_HI 45
+#define KNOD_AMDGPU_TMP_VREG12_LO 46
+#define KNOD_AMDGPU_TMP_VREG12_HI 47
+#define KNOD_AMDGPU_TMP_VREG13_LO 48
+#define KNOD_AMDGPU_TMP_VREG13_HI 49
+#define KNOD_AMDGPU_TMP_VREG14_LO 50
+#define KNOD_AMDGPU_TMP_VREG14_HI 51
+#define KNOD_AMDGPU_TMP_VREG15_LO 52
+#define KNOD_AMDGPU_TMP_VREG15_HI 53
+#define KNOD_AMDGPU_TMP_VREG16_LO 54
+#define KNOD_AMDGPU_TMP_VREG16_HI 55
+#define KNOD_AMDGPU_TMP_VREG17_LO 56
+#define KNOD_AMDGPU_TMP_VREG17_HI 57
+#define KNOD_AMDGPU_TMP_VREG18_LO 58
+#define KNOD_AMDGPU_TMP_VREG18_HI 59
+#define KNOD_AMDGPU_TMP_VREG_MAX KNOD_AMDGPU_TMP_VREG18_HI
+#define KNOD_AMDGPU_CTX_VREG_LO 60
+#define KNOD_AMDGPU_CTX_VREG_HI 61
+#define KNOD_AMDGPU_IDX_VREG 62
+#define KNOD_AMDGPU_RESERVED 63
+/*
+ * After prologue step 4, IDX_VREG is no longer needed.
+ * v62:v63 are repurposed to hold slot_addr (spsc_bd GTT address)
+ * through BPF execution and into the epilogue.
+ *
+ * BACKLOG_IDX_VREG (v58) saves the backlog index from IDX_VREG
+ * before step 6 overwrites it. Used in epilogue for XDP_PASS.
+ */
+#define KNOD_AMDGPU_BACKLOG_IDX_VREG KNOD_AMDGPU_TMP_VREG18_LO /* v58 */
+#define KNOD_AMDGPU_SLOT_VREG_LO KNOD_AMDGPU_IDX_VREG /* v62 */
+#define KNOD_AMDGPU_SLOT_VREG_HI KNOD_AMDGPU_RESERVED /* v63 */
+/*
+ * DATA/DATA_END VGPRs: hold packet gaddr and end address.
+ * Set in prologue, read by BPF ctx->data / ctx->data_end accesses.
+ * Replaces GTT round-trip (prologue store -> BPF load).
+ */
+#define KNOD_AMDGPU_DATA_VREG_LO 64
+#define KNOD_AMDGPU_DATA_VREG_HI 65
+#define KNOD_AMDGPU_DATA_END_VREG_LO 66
+#define KNOD_AMDGPU_DATA_END_VREG_HI 67
+#define KNOD_AMDGPU_PAGE_BASE_VREG_LO 68
+#define KNOD_AMDGPU_PAGE_BASE_VREG_HI 69
+#define KNOD_AMDGPU_PKT_CACHE_VREG0 70
+#define KNOD_AMDGPU_PKT_CACHE_VREG_MAX 127 /* 0 ~ 127 vgprs are available */
+#define KNOD_AMDGPU_STACK_VREG0 128
+#define KNOD_AMDGPU_STACK_VREG_MAX 255 /* 128 ~ 255 vgprs are available */
+
+#define KNOD_BPF_PROG_BUF_SIZE 32768
+
+/* Index for r64.
+ * r64[TREG64_0]
+ */
+#define TREG64_0 0
+#define TREG64_1 1
+#define TREG64_2 2
+#define TREG64_3 3
+#define KEY_IN_PKT_64 TREG64_3
+#define TREG64_4 4
+#define TREG64_5 5
+#define TREG64_6 6
+#define TREG64_7 7
+#define TREG64_8 8
+#define TREG64_9 9
+#define TREG64_10 10
+#define KEY_IN_MAP_64 TREG64_10
+#define TREG64_11 11
+#define TREG64_12 12
+#define TREG64_13 13
+#define TREG64_14 14
+#define TREG64_15 15
+#define TREG64_16 16
+#define TREG64_17 17
+#define TREG64_18 18
+
+#define MAX_MAP_KEY_SIZE 56
+
+/* Index for r32.
+ * r32[TREG32_0_LO]
+ */
+#define TREG32_0_LO 0
+#define TREG32_0_HI 1
+#define TREG32_1_LO 2
+#define TREG32_1_HI 3
+#define TREG32_2_LO 4
+#define TREG32_2_HI 5
+#define TREG32_3_LO 6
+#define KEY_IN_PKT_32 TREG32_3_LO
+#define TREG32_3_HI 7
+#define TREG32_4_LO 8
+#define TREG32_4_HI 9
+#define TREG32_5_LO 10
+#define TREG32_5_HI 11
+#define TREG32_6_LO 12
+#define TREG32_6_HI 13
+#define TREG32_7_LO 14
+#define TREG32_7_HI 15
+#define TREG32_8_LO 16
+#define TREG32_8_HI 17
+#define TREG32_9_LO 18
+#define TREG32_9_HI 19
+#define TREG32_10_LO 20
+#define KEY_IN_MAP_32 TREG32_10_LO
+#define TREG32_10_HI 21
+#define TREG32_11_LO 22
+#define TREG32_11_HI 23
+#define TREG32_12_LO 24
+#define TREG32_12_HI 25
+#define TREG32_13_LO 26
+#define TREG32_13_HI 27
+#define TREG32_14_LO 28
+#define TREG32_14_HI 29
+#define TREG32_15_LO 30
+#define TREG32_15_HI 31
+#define TREG32_16_LO 32
+#define TREG32_16_HI 33
+#define TREG32_17_LO 34
+#define TREG32_17_HI 35
+#define TREG32_18_LO 36
+#define TREG32_18_HI 37
+#define TREG32_MAX TREG32_18_HI
+
+/*+--------+--------------------------------------+---+---+
+ *| s[0:3] |s[4:5] s[6:7] s[8:9] s[10:11] s[12:13]|s14|s15|
+ *+--------+--------------------------------------+---+---+
+ *| PSB | USER SGPRs (disp/queue/karg/id/flat) |WGX|QID|
+ *+--------+--------------------------------------+---+---+
+ *+----------------+------+---+---+------+-------+-------------------+
+ *| s[16:27] |s28:29|s30|s31|s32:33|s34:35 | s[36:105] |
+ *+----------------+------+---+---+------+-------+-------------------+
+ *|TMP_SREG 0-5 |PARAM |FP | - | GFX9 | DONE | EXEC_SAVE PAIRS |
+ *|(6 x 64-bit) |SREG | | |BROKE!| MASK | (max 35, GFX10) |
+ *+----------------+------+---+---+------+-------+-------------------+
+ * Implicit: VCC = s[106:107] EXEC = s[126:127]
+ *
+ * user_sgpr_count=14, same on GFX9 and GFX10.
+ * enable_sgpr_private_segment_size is disabled so that workgroup_id_y
+ * lands at s15 and TMP_SREG0_LO stays at s16 (keeps 64-bit SGPR pair
+ * alignment; avoids shifting the entire TMP/PARAM/FRAME layout).
+ */
+#define KNOD_AMDGPU_PSB_SREG 0 /* s[0:3] private_segment_buffer */
+#define KNOD_AMDGPU_DISPATCH_PTR_SREG 4 /* s[4:5] dispatch_ptr */
+#define KNOD_AMDGPU_ARG_SREG 4 /* alias for dispatch_ptr */
+#define KNOD_AMDGPU_QUEUE_PTR_SREG 6 /* s[6:7] queue_ptr */
+#define KNOD_AMDGPU_KERNARG_PTR_SREG 8 /* s[8:9] kernarg_segment_ptr */
+#define KNOD_AMDGPU_DISPATCH_ID_SREG 10 /* s[10:11] dispatch_id */
+#define KNOD_AMDGPU_FLAT_SCR_INIT_SREG 12 /* s[12:13] flat_scratch_init */
+#define KNOD_AMDGPU_WORKGROUP_ID_X_SREG 14 /* s14 workgroup_id_x */
+#define KNOD_AMDGPU_WORKGROUP_ID_Y_SREG 15 /* s15 workgroup_id_y = queue_id */
+#define KNOD_AMDGPU_TMP_SREG0_LO 16
+#define KNOD_AMDGPU_TMP_SREG0_HI 17
+#define KNOD_AMDGPU_TMP_SREG1_LO 18
+#define KNOD_AMDGPU_TMP_SREG1_HI 19
+#define KNOD_AMDGPU_TMP_SREG2_LO 20
+#define KNOD_AMDGPU_TMP_SREG2_HI 21
+#define KNOD_AMDGPU_TMP_SREG3_LO 22
+#define KNOD_AMDGPU_TMP_SREG3_HI 23
+#define KNOD_AMDGPU_TMP_SREG4_LO 24
+#define KNOD_AMDGPU_TMP_SREG4_HI 25
+#define KNOD_AMDGPU_TMP_SREG5_LO 26
+#define KNOD_AMDGPU_TMP_SREG5_HI 27
+#define KNOD_AMDGPU_PARAM_SREG_LO 28 /* s28 */
+#define KNOD_AMDGPU_PARAM_SREG_HI 29 /* s29 */
+#define KNOD_AMDGPU_FRAME_POINTER_SREG 30 /* s30 */
+
+/* Structurized CFG: EXEC mask save/restore SGPRs.
+ * done_mask tracks lanes that have reached BPF_EXIT.
+ * exec_save pairs store EXEC at branch points for restore at merge points.
+ * GFX9: s[0:101] addressable (102 SGPRs), GFX10: s[0:105] (106 SGPRs).
+ * NOTE: s[32:33] is corrupted by GFX9 hardware - do NOT use on GFX9.
+ * GFX10 uses s[32:33] for done_mask and starts exec_save at s[34].
+ */
+/* Common SGPR special register indices (same on GFX9 and GFX10) */
+#define AMDGCN_SREG_VCC_LO 106
+#define AMDGCN_SREG_EXEC_LO 126
+#define AMDGCN_SREG_INTEGER_0 128
+#define AMDGCN_SREG_INTEGER_1 129
+
+/* s[34:35] - must not overlap TMP_SREGs */
+#define KNOD_AMDGPU_DONE_MASK_SREG 34
+#define KNOD_AMDGPU_EXEC_SAVE_SREG_BASE 36 /* s[36:37], s[38:39], ... */
+#define KNOD_AMDGPU_INITIAL_EXEC_SREG_GFX9 100
+#define KNOD_AMDGPU_INITIAL_EXEC_SREG_GFX10 104
+/* exec_save can fill up to each ISA's top usable SGPR pair. GFX9 lays the
+ * initial in-bounds EXEC snapshot immediately after the pairs a program
+ * actually uses, so small programs keep the old 64-SGPR occupancy window.
+ * GFX10 keeps the original high fixed snapshot pair.
+ */
+#define KNOD_AMDGPU_EXEC_SAVE_SREG_MAX_GFX9 99
+#define KNOD_AMDGPU_EXEC_SAVE_SREG_MAX_GFX10 103
+#define KNOD_AMDGPU_MAX_EXEC_SAVE_PAIRS_GFX9 \
+ ((KNOD_AMDGPU_EXEC_SAVE_SREG_MAX_GFX9 - KNOD_AMDGPU_EXEC_SAVE_SREG_BASE + 1) / 2)
+#define KNOD_AMDGPU_MAX_EXEC_SAVE_PAIRS_GFX10 \
+ ((KNOD_AMDGPU_EXEC_SAVE_SREG_MAX_GFX10 - KNOD_AMDGPU_EXEC_SAVE_SREG_BASE + 1) / 2)
+
+static u8 knod_bpf_gfx9_sgpr_granule(unsigned int sgprs_used)
+{
+ if (sgprs_used <= 16)
+ return 0;
+
+ return 2 * (DIV_ROUND_UP(sgprs_used, 16) - 1);
+}
+
+unsigned int knod_bpf_workgroups = KNOD_BPF_WORKGROUPS_DEFAULT;
+MODULE_PARM_DESC(workgroups, "Workgroup size, multiple of 64, Min(64) Default/Max(256)");
+module_param_named(workgroups, knod_bpf_workgroups, int, 0600);
+
+unsigned int knod_bpf_expire = KNOD_BPF_EXPIRE_DEFAULT;
+MODULE_PARM_DESC(queue_expire, "Queue expire time(ms), Min(1), Default(10), Max(1000)");
+module_param_named(queue_expire, knod_bpf_expire, int, 0600);
+
+unsigned int knod_bpf_pkt_cache;
+MODULE_PARM_DESC(packet_cache, "Use packet cache, 0=Off(Default), 1=On");
+module_param_named(packet_cache, knod_bpf_pkt_cache, int, 0600);
+
+unsigned int knod_bpf_wave32;
+MODULE_PARM_DESC(wave32, "Use wave32 0=Off(Default), 1=On");
+module_param_named(wave32, knod_bpf_wave32, int, 0600);
+
+#define KNOD_EA(extack, msg) NL_SET_ERR_MSG_MOD((extack), msg)
+
+DEFINE_STATIC_KEY_FALSE(knod_stats_key);
+
+static const u32 bl_bounds[KNOD_BL_BUCKETS - 1] = {
+ 16, 64, 256, 1024, 4096, 8192, 16384
+};
+
+static const char * const lat_labels[] = {
+ "< 1us", "1-2us", "2-4us", "4-8us", "8-16us",
+ "16-32us", "32-64us", "64-128us", "128-256us", ">= 256us",
+};
+
+static const char * const bl_labels[] = {
+ "1-16", "17-64", "65-256", "257-1K",
+ "1K-4K", "4K-8K", "8K-16K", ">= 16K",
+};
+
+static LIST_HEAD(priv_list);
+struct amdgcn_param64 r64[20], sr64[6], p64[4], bpf_reg64[11];
+struct amdgcn_param32 r32[40]; /* last two is CTX */
+struct amdgcn_param32 stack[128];
+struct amdgcn_param32 pkt_cache[64];
+
+struct amdgcn_label {
+ struct knod_insn_meta *meta;
+ int insn_idx;
+};
+
+struct amdgcn_branch_fixup {
+ struct amdgcn_label *target_label;
+ struct knod_insn_meta *meta;
+ int insn_idx;
+};
+
+struct knod_accel_xdp_ops accel_xdp_ops;
+
+static int knod_prog_prepare_insns(struct knod_bpf_priv *priv,
+ struct knod_prog *knod_prog);
+static int knod_bpf_worker(void *arg);
+static void knod_bpf_drain_worker(struct knod_bpf_priv *priv);
+static void knod_prog_free(struct knod_prog *knod_prog);
+static void knod_emit_pass_addr_store(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta);
+static void knod_setup_bpf_prog(struct bpf_prog *prog);
+
+static void knod_bpf_gpu_mem_fence(struct knod_bpf_priv *priv)
+{
+ if (!priv)
+ return;
+
+ /* drain the WC store buffer before the GPU reads the map */
+ wmb();
+}
+
+static unsigned int knod_bpf_active_rxq_count(struct net_device *netdev)
+{
+ unsigned int nr_rxq;
+
+ if (!netdev)
+ return 0;
+
+ nr_rxq = READ_ONCE(netdev->real_num_rx_queues);
+ if (!nr_rxq)
+ nr_rxq = netdev->num_rx_queues;
+
+ return min_t(unsigned int, nr_rxq, KNOD_SPSC_MAX);
+}
+
+static void knod_bpf_fill_dispatch(struct knod_bpf_priv *priv,
+ struct knod_bpf_work_sq *sqw,
+ struct knod_dispatch_params *p)
+{
+ struct knod_bpf_param *param = sqw->param->kaddr;
+
+ p->workgroup_size_x = knod_bpf_workgroups;
+ p->grid_size_x = priv->batch_size;
+ p->grid_size_y = param->nr_queues;
+ p->private_segment_size = 8192;
+ p->group_segment_size = 8192;
+ p->kernel_object =
+ (u64)priv->knod->kernels[READ_ONCE(priv->active_idx)]->gaddr;
+ p->kernarg_address = sqw->param->gaddr;
+}
+
+static void debug_kernel_descriptor(struct kernel_descriptor *kernel_code)
+{
+ knod_jit_dbg(" kernel_code->group_segment_fixed_size = %d\n",
+ kernel_code->group_segment_fixed_size);
+ knod_jit_dbg(" kernel_code->private_segment_fixed_size = %d\n",
+ kernel_code->private_segment_fixed_size);
+ knod_jit_dbg(" kernel_code->kernarg_size = %d\n",
+ kernel_code->kernarg_size);
+ knod_jit_dbg(" kernel_code->kernel_code_entry_byte_offset = %lld\n",
+ kernel_code->kernel_code_entry_byte_offset);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc3.accum_offset = %d\n",
+ kernel_code->compute_pgm_rsrc3.accum_offset);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc3.reserved0 = %d\n",
+ kernel_code->compute_pgm_rsrc3.reserved0);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc3.tg_split = %d\n",
+ kernel_code->compute_pgm_rsrc3.tg_split);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc3.reserved1 = %d\n",
+ kernel_code->compute_pgm_rsrc3.reserved1);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc1.granulated_workitem_vgpr_count = %d\n",
+ kernel_code->compute_pgm_rsrc1.granulated_workitem_vgpr_count);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc1.granulated_wavefront_sgpr_count = %d\n",
+ kernel_code->compute_pgm_rsrc1.granulated_wavefront_sgpr_count);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc1.priority = %d\n",
+ kernel_code->compute_pgm_rsrc1.priority);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc1.float_round_mode_32 = %d\n",
+ kernel_code->compute_pgm_rsrc1.float_round_mode_32);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc1.float_round_mode_16_64 = %d\n",
+ kernel_code->compute_pgm_rsrc1.float_round_mode_16_64);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc1.float_denorm_mode_32 = %d\n",
+ kernel_code->compute_pgm_rsrc1.float_denorm_mode_32);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc1.float_denorm_mode_16_64 = %d\n",
+ kernel_code->compute_pgm_rsrc1.float_denorm_mode_16_64);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc1.priv = %d\n",
+ kernel_code->compute_pgm_rsrc1.priv);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc1.enable_dx10_clamp = %d\n",
+ kernel_code->compute_pgm_rsrc1.enable_dx10_clamp);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc1.debug_mode = %d\n",
+ kernel_code->compute_pgm_rsrc1.debug_mode);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc1.enable_ieee_mode = %d\n",
+ kernel_code->compute_pgm_rsrc1.enable_ieee_mode);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc1.bulky = %d\n",
+ kernel_code->compute_pgm_rsrc1.bulky);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc1.cdbg_user = %d\n",
+ kernel_code->compute_pgm_rsrc1.cdbg_user);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc1.fp16_ovfl = %d\n",
+ kernel_code->compute_pgm_rsrc1.fp16_ovfl);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc1.reserved0 = %d\n",
+ kernel_code->compute_pgm_rsrc1.reserved0);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc1.wgp_mode = %d\n",
+ kernel_code->compute_pgm_rsrc1.wgp_mode);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc1.mem_ordered = %d\n",
+ kernel_code->compute_pgm_rsrc1.mem_ordered);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc1.fwd_progress = %d\n",
+ kernel_code->compute_pgm_rsrc1.fwd_progress);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc2.enable_private_segment = %d\n",
+ kernel_code->compute_pgm_rsrc2.enable_private_segment);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc2.user_sgpr_count = %d\n",
+ kernel_code->compute_pgm_rsrc2.user_sgpr_count);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc2.enable_trap_handler = %d\n",
+ kernel_code->compute_pgm_rsrc2.enable_trap_handler);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc2.enable_sgpr_workgroup_id_x = %d\n",
+ kernel_code->compute_pgm_rsrc2.enable_sgpr_workgroup_id_x);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc2.enable_sgpr_workgroup_id_y = %d\n",
+ kernel_code->compute_pgm_rsrc2.enable_sgpr_workgroup_id_y);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc2.enable_sgpr_workgroup_id_z = %d\n",
+ kernel_code->compute_pgm_rsrc2.enable_sgpr_workgroup_id_z);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc2.enable_sgpr_workgroup_info = %d\n",
+ kernel_code->compute_pgm_rsrc2.enable_sgpr_workgroup_info);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc2.enable_vgpr_workitem_id = %d\n",
+ kernel_code->compute_pgm_rsrc2.enable_vgpr_workitem_id);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc2.enable_exception_address_watch = %d\n",
+ kernel_code->compute_pgm_rsrc2.enable_exception_address_watch);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc2.enable_exception_memory = %d\n",
+ kernel_code->compute_pgm_rsrc2.enable_exception_memory);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc2.granulated_lds_size = %d\n",
+ kernel_code->compute_pgm_rsrc2.granulated_lds_size);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc2.enable_exception_ieee_754_fp_invalid_operation = %d\n",
+ kernel_code->compute_pgm_rsrc2
+ .enable_exception_ieee_754_fp_invalid_operation);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc2.enable_exception_fp_denormal_source = %d\n",
+ kernel_code->compute_pgm_rsrc2
+ .enable_exception_fp_denormal_source);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc2.enable_exception_ieee_754_fp_division_by_zero = %d\n",
+ kernel_code->compute_pgm_rsrc2
+ .enable_exception_ieee_754_fp_division_by_zero);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc2.enable_exception_ieee_754_fp_overflow = %d\n",
+ kernel_code->compute_pgm_rsrc2
+ .enable_exception_ieee_754_fp_overflow);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc2.enable_exception_ieee_754_fp_underflow = %d\n",
+ kernel_code->compute_pgm_rsrc2
+ .enable_exception_ieee_754_fp_underflow);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc2.enable_exception_ieee_754_fp_inexact = %d\n",
+ kernel_code->compute_pgm_rsrc2
+ .enable_exception_ieee_754_fp_inexact);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc2.enable_exception_int_divide_by_zero = %d\n",
+ kernel_code->compute_pgm_rsrc2
+ .enable_exception_int_divide_by_zero);
+ knod_jit_dbg(" kernel_code->compute_pgm_rsrc2.reserved0 = %d\n",
+ kernel_code->compute_pgm_rsrc2.reserved0);
+ knod_jit_dbg(" kernel_code->code_properties.enable_sgpr_private_segment_buffer = %d\n",
+ kernel_code->code_properties
+ .enable_sgpr_private_segment_buffer);
+ knod_jit_dbg(" kernel_code->code_properties.enable_sgpr_dispatch_ptr = %d\n",
+ kernel_code->code_properties.enable_sgpr_dispatch_ptr);
+ knod_jit_dbg(" kernel_code->code_properties.enable_sgpr_queue_ptr = %d\n",
+ kernel_code->code_properties.enable_sgpr_queue_ptr);
+ knod_jit_dbg(" kernel_code->code_properties.enable_sgpr_kernarg_segment_ptr = %d\n",
+ kernel_code->code_properties.enable_sgpr_kernarg_segment_ptr);
+ knod_jit_dbg(" kernel_code->code_properties.enable_sgpr_dispatch_id = %d\n",
+ kernel_code->code_properties.enable_sgpr_dispatch_id);
+ knod_jit_dbg(" kernel_code->code_properties.enable_sgpr_flat_scratch_init = %d\n",
+ kernel_code->code_properties.enable_sgpr_flat_scratch_init);
+ knod_jit_dbg(" kernel_code->code_properties.enable_sgpr_private_segment_size = %d\n",
+ kernel_code->code_properties.enable_sgpr_private_segment_size);
+ knod_jit_dbg(" kernel_code->code_properties.reserved0 = %d\n",
+ kernel_code->code_properties.reserved0);
+ knod_jit_dbg(" kernel_code->code_properties.enable_wavefront_size32 = %d\n",
+ kernel_code->code_properties.enable_wavefront_size32);
+ knod_jit_dbg(" kernel_code->code_properties.uses_dynamic_stack = %d\n",
+ kernel_code->code_properties.uses_dynamic_stack);
+ knod_jit_dbg(" kernel_code->code_properties.reserved1 = %d\n",
+ kernel_code->code_properties.reserved1);
+}
+
+static void kfd_kernel_gfx9_init(struct knod *knod)
+{
+ struct kernel_descriptor *kernel_code = knod->kernels[0]->kaddr;
+
+ kernel_code->group_segment_fixed_size = 0;
+ kernel_code->private_segment_fixed_size = 8192;
+ kernel_code->kernarg_size = 64;
+ kernel_code->kernel_code_entry_byte_offset = 1024;
+
+ /* GFX10+ or GFX90A+ */
+ kernel_code->compute_pgm_rsrc3.accum_offset = 0;
+ kernel_code->compute_pgm_rsrc3.reserved0 = 0;
+ kernel_code->compute_pgm_rsrc3.tg_split = 0;
+ kernel_code->compute_pgm_rsrc3.reserved1 = 0;
+
+ kernel_code->compute_pgm_rsrc1.granulated_workitem_vgpr_count =
+ (256 / 4) - 1;
+ /*
+ * Start with the small GFX9 window. BPF install updates each slot
+ * descriptor when a program needs a larger exec_save/initial_exec
+ * range.
+ */
+ kernel_code->compute_pgm_rsrc1.granulated_wavefront_sgpr_count =
+ knod_bpf_gfx9_sgpr_granule(52);
+ kernel_code->compute_pgm_rsrc1.priority = 0;
+ kernel_code->compute_pgm_rsrc1.float_round_mode_32 = 0;
+ kernel_code->compute_pgm_rsrc1.float_round_mode_16_64 = 0;
+ kernel_code->compute_pgm_rsrc1.float_denorm_mode_32 = 3;
+ kernel_code->compute_pgm_rsrc1.float_denorm_mode_16_64 = 3;
+ kernel_code->compute_pgm_rsrc1.priv = 0;
+ kernel_code->compute_pgm_rsrc1.enable_dx10_clamp = 1;
+ kernel_code->compute_pgm_rsrc1.debug_mode = 0;
+ kernel_code->compute_pgm_rsrc1.enable_ieee_mode = 1;
+ kernel_code->compute_pgm_rsrc1.bulky = 0;
+ kernel_code->compute_pgm_rsrc1.cdbg_user = 0;
+ kernel_code->compute_pgm_rsrc1.fp16_ovfl = 0;
+ kernel_code->compute_pgm_rsrc1.reserved0 = 0;
+ kernel_code->compute_pgm_rsrc1.wgp_mode = 0;
+ kernel_code->compute_pgm_rsrc1.mem_ordered = 0;
+ kernel_code->compute_pgm_rsrc1.fwd_progress = 0;
+
+ kernel_code->compute_pgm_rsrc2.enable_private_segment = 0;
+ kernel_code->compute_pgm_rsrc2.user_sgpr_count = 14; /* 4+2+2+2+2+2 */
+ kernel_code->compute_pgm_rsrc2.enable_trap_handler = 0;
+ kernel_code->compute_pgm_rsrc2.enable_sgpr_workgroup_id_x = 1;
+ kernel_code->compute_pgm_rsrc2.enable_sgpr_workgroup_id_y = 1;
+ kernel_code->compute_pgm_rsrc2.enable_sgpr_workgroup_id_z = 0;
+ kernel_code->compute_pgm_rsrc2.enable_sgpr_workgroup_info = 0;
+ kernel_code->compute_pgm_rsrc2.enable_vgpr_workitem_id = 0;
+ kernel_code->compute_pgm_rsrc2.enable_exception_address_watch = 0;
+ kernel_code->compute_pgm_rsrc2.enable_exception_memory = 0;
+ kernel_code->compute_pgm_rsrc2.granulated_lds_size = 0;
+ kernel_code->compute_pgm_rsrc2
+ .enable_exception_ieee_754_fp_invalid_operation = 0;
+ kernel_code->compute_pgm_rsrc2.enable_exception_fp_denormal_source = 0;
+ kernel_code->compute_pgm_rsrc2
+ .enable_exception_ieee_754_fp_division_by_zero = 0;
+ kernel_code->compute_pgm_rsrc2
+ .enable_exception_ieee_754_fp_overflow = 0;
+ kernel_code->compute_pgm_rsrc2
+ .enable_exception_ieee_754_fp_underflow = 0;
+ kernel_code->compute_pgm_rsrc2.enable_exception_ieee_754_fp_inexact = 0;
+ kernel_code->compute_pgm_rsrc2.enable_exception_int_divide_by_zero = 0;
+ kernel_code->compute_pgm_rsrc2.reserved0 = 0;
+
+ /*
+ * User SGPR layout - loaded in fixed order, disabled entries are
+ * skipped (not reserved). The resulting SGPR map depends on which
+ * flags are enabled:
+ *
+ * enable_sgpr_private_segment_buffer -> 4 SGPRs (s[0:3])
+ * enable_sgpr_dispatch_ptr -> 2 SGPRs (s[4:5])
+ * enable_sgpr_queue_ptr -> 2 SGPRs
+ * enable_sgpr_kernarg_segment_ptr -> 2 SGPRs
+ * enable_sgpr_dispatch_id -> 2 SGPRs
+ * enable_sgpr_flat_scratch_init -> 2 SGPRs
+ * enable_sgpr_private_segment_size -> 1 SGPR
+ *
+ * System SGPRs (WorkgroupId etc.) follow immediately after the
+ * last user SGPR. user_sgpr_count must match the total above.
+ */
+ /* 4 SGPRs */
+ kernel_code->code_properties.enable_sgpr_private_segment_buffer = 1;
+ /* 2 SGPRs */
+ kernel_code->code_properties.enable_sgpr_dispatch_ptr = 1;
+ /* 2 SGPRs */
+ kernel_code->code_properties.enable_sgpr_queue_ptr = 1;
+ /* 2 SGPRs */
+ kernel_code->code_properties.enable_sgpr_kernarg_segment_ptr = 1;
+ /* 2 SGPRs */
+ kernel_code->code_properties.enable_sgpr_dispatch_id = 1;
+ /* 2 SGPRs */
+ kernel_code->code_properties.enable_sgpr_flat_scratch_init = 1;
+ /* disabled -> s14/s15 free for workgroup_id */
+ kernel_code->code_properties.enable_sgpr_private_segment_size = 0;
+ /* total = 14 SGPRs */
+ kernel_code->code_properties.reserved0 = 0;
+ /* GFX10+ */
+ kernel_code->code_properties.enable_wavefront_size32 = 0;
+ kernel_code->code_properties.uses_dynamic_stack = 0;
+ kernel_code->code_properties.reserved1 = 0;
+
+ debug_kernel_descriptor(kernel_code);
+}
+
+static void kfd_kernel_gfx10_init(struct knod *knod)
+{
+ struct kernel_descriptor *kernel_code = knod->kernels[0]->kaddr;
+
+ kernel_code->group_segment_fixed_size = 0;
+ kernel_code->private_segment_fixed_size = 8192;
+ kernel_code->kernarg_size = 64;
+ kernel_code->kernel_code_entry_byte_offset = 1024;
+
+ /*
+ * User SGPR layout - loaded in fixed order, disabled entries are
+ * skipped (not reserved). The resulting SGPR map depends on which
+ * flags are enabled:
+ *
+ * enable_sgpr_private_segment_buffer -> 4 SGPRs (s[0:3])
+ * enable_sgpr_dispatch_ptr -> 2 SGPRs (s[4:5])
+ * enable_sgpr_queue_ptr -> 2 SGPRs
+ * enable_sgpr_kernarg_segment_ptr -> 2 SGPRs
+ * enable_sgpr_dispatch_id -> 2 SGPRs
+ * enable_sgpr_flat_scratch_init -> 2 SGPRs
+ * enable_sgpr_private_segment_size -> 1 SGPR
+ *
+ * System SGPRs (WorkgroupId etc.) follow immediately after the
+ * last user SGPR. user_sgpr_count must match the total above.
+ */
+ /* 4 SGPRs */
+ kernel_code->code_properties.enable_sgpr_private_segment_buffer = 1;
+ /* 2 SGPRs */
+ kernel_code->code_properties.enable_sgpr_dispatch_ptr = 1;
+ /* 2 SGPRs */
+ kernel_code->code_properties.enable_sgpr_queue_ptr = 1;
+ /* 2 SGPRs */
+ kernel_code->code_properties.enable_sgpr_kernarg_segment_ptr = 1;
+ /* 2 SGPRs */
+ kernel_code->code_properties.enable_sgpr_dispatch_id = 1;
+ /* 2 SGPRs */
+ kernel_code->code_properties.enable_sgpr_flat_scratch_init = 1;
+ /* disabled -> s14/s15 free for workgroup_id */
+ kernel_code->code_properties.enable_sgpr_private_segment_size = 0;
+ /* total = 14 SGPRs */
+ kernel_code->code_properties.reserved0 = 0;
+ if (knod_bpf_wave32)
+ kernel_code->code_properties.enable_wavefront_size32 = 1;
+ else
+ kernel_code->code_properties.enable_wavefront_size32 = 0;
+ kernel_code->code_properties.uses_dynamic_stack = 0;
+ kernel_code->code_properties.reserved1 = 0;
+
+ kernel_code->compute_pgm_rsrc3.accum_offset = 0;
+ kernel_code->compute_pgm_rsrc3.reserved0 = 0;
+ kernel_code->compute_pgm_rsrc3.tg_split = 0;
+ kernel_code->compute_pgm_rsrc3.reserved1 = 0;
+
+ if (kernel_code->code_properties.enable_wavefront_size32 == 1)
+ kernel_code->compute_pgm_rsrc1.granulated_workitem_vgpr_count =
+ (256 / 8) - 1;
+ else
+ kernel_code->compute_pgm_rsrc1.granulated_workitem_vgpr_count =
+ (256 / 4) - 1;
+ kernel_code->compute_pgm_rsrc1.granulated_wavefront_sgpr_count = 0;
+ kernel_code->compute_pgm_rsrc1.priority = 0;
+ kernel_code->compute_pgm_rsrc1.float_round_mode_32 = 0;
+ kernel_code->compute_pgm_rsrc1.float_round_mode_16_64 = 0;
+ kernel_code->compute_pgm_rsrc1.float_denorm_mode_32 = 3;
+ kernel_code->compute_pgm_rsrc1.float_denorm_mode_16_64 = 3;
+ kernel_code->compute_pgm_rsrc1.priv = 0;
+ kernel_code->compute_pgm_rsrc1.enable_dx10_clamp = 1;
+ kernel_code->compute_pgm_rsrc1.debug_mode = 0;
+ kernel_code->compute_pgm_rsrc1.enable_ieee_mode = 1;
+ kernel_code->compute_pgm_rsrc1.bulky = 0;
+ kernel_code->compute_pgm_rsrc1.cdbg_user = 0;
+ kernel_code->compute_pgm_rsrc1.fp16_ovfl = 0;
+ kernel_code->compute_pgm_rsrc1.reserved0 = 0;
+ kernel_code->compute_pgm_rsrc1.wgp_mode = 0;
+ kernel_code->compute_pgm_rsrc1.mem_ordered = 1;
+ kernel_code->compute_pgm_rsrc1.fwd_progress = 0;
+
+ kernel_code->compute_pgm_rsrc2.enable_private_segment = 0;
+ kernel_code->compute_pgm_rsrc2.user_sgpr_count = 14; /* 4+2+2+2+2+2 */
+ kernel_code->compute_pgm_rsrc2.enable_trap_handler = 0;
+ kernel_code->compute_pgm_rsrc2.enable_sgpr_workgroup_id_x = 1;
+ kernel_code->compute_pgm_rsrc2.enable_sgpr_workgroup_id_y = 1;
+ kernel_code->compute_pgm_rsrc2.enable_sgpr_workgroup_id_z = 0;
+ kernel_code->compute_pgm_rsrc2.enable_sgpr_workgroup_info = 0;
+ kernel_code->compute_pgm_rsrc2.enable_vgpr_workitem_id = 1;
+ kernel_code->compute_pgm_rsrc2.enable_exception_address_watch = 0;
+ kernel_code->compute_pgm_rsrc2.enable_exception_memory = 0;
+ kernel_code->compute_pgm_rsrc2.granulated_lds_size = 0;
+ kernel_code->compute_pgm_rsrc2
+ .enable_exception_ieee_754_fp_invalid_operation = 0;
+ kernel_code->compute_pgm_rsrc2.enable_exception_fp_denormal_source = 0;
+ kernel_code->compute_pgm_rsrc2
+ .enable_exception_ieee_754_fp_division_by_zero = 0;
+ kernel_code->compute_pgm_rsrc2
+ .enable_exception_ieee_754_fp_overflow = 0;
+ kernel_code->compute_pgm_rsrc2
+ .enable_exception_ieee_754_fp_underflow = 0;
+ kernel_code->compute_pgm_rsrc2.enable_exception_ieee_754_fp_inexact = 0;
+ kernel_code->compute_pgm_rsrc2.enable_exception_int_divide_by_zero = 0;
+ kernel_code->compute_pgm_rsrc2.reserved0 = 0;
+
+ debug_kernel_descriptor(kernel_code);
+}
+
+static int kfd_kernel_init(struct knod *knod, struct knod_bpf_priv *priv)
+{
+ struct kernel_descriptor *kd;
+
+ if (!knod->kernels[1])
+ return -ENOMEM;
+
+ /*
+ * Pass-through starts on slot 0; the first XDP prog attach stages into
+ * slot 1 and flips the active index there, ping-ponging on each
+ * install.
+ */
+ priv->active_idx = 0;
+
+ if (priv->isa_version == 9)
+ kfd_kernel_gfx9_init(knod);
+ else if (priv->isa_version == 10)
+ kfd_kernel_gfx10_init(knod);
+
+ /*
+ * Slot 1 must carry the same kernel-descriptor as slot 0 -- gfx init
+ * only touches slot 0, and slot 1's BO is otherwise uninitialised,
+ * which stalls the compute queue. Copy the kd + pre-code region.
+ */
+ kd = knod->kernels[0]->kaddr;
+ memcpy(knod->kernels[1]->kaddr, knod->kernels[0]->kaddr,
+ kd->kernel_code_entry_byte_offset);
+ knod_bpf_gpu_mem_fence(priv);
+
+ return 0;
+}
+
+static struct knod_bpf_work_sq *
+__knod_get_free_work_sq(struct knod_bpf_priv *priv)
+{
+ return list_first_entry_or_null(&priv->free_list_sqw,
+ struct knod_bpf_work_sq, list);
+}
+
+/* Prepare a dispatch: peek SPSC rings and fill params, but do not submit.
+ * Returns the prepared sqw (with backlogs > 0), or NULL if nothing to do.
+ *
+ * A single in-flight AQL queue means the worker never has to reserve SPSC
+ * ranges ahead of the current dispatch. The SPSC acquired pointer is advanced
+ * only after the GPU finishes the dispatch that consumed those entries.
+ */
+static struct knod_bpf_work_sq *knod_prepare_bpf(struct knod_bpf_priv *priv)
+{
+ int i, cnt, backlogs = 0;
+ struct knod_dev *knodev = priv->knodev;
+ struct knod_bpf_work_sq *sqw;
+ struct knod_bpf_param *param;
+
+ if (READ_ONCE(priv->installing_kernel))
+ return NULL;
+
+ if (!priv->pass_prog_buf && !READ_ONCE(priv->prog))
+ return NULL;
+
+ sqw = __knod_get_free_work_sq(priv);
+ if (!sqw)
+ return NULL;
+
+ param = (struct knod_bpf_param *)sqw->param->kaddr;
+ memset(sqw->queue_idx, 0, sizeof(sqw->queue_idx));
+
+ /* 2D dispatch: queue_id = workgroup_id_y, tid = workitem within WG.
+ * Per-queue bds live in sqw->bds[i * batch_size + tid] and shader
+ * indexes sub[] / sqw->bds[] using (queue_id * batch_size + tid).
+ * No cumulative start_idx -- each queue's slot range is fixed by i.
+ */
+ for (i = 0; i < priv->nr_works; i++) {
+ int slot = i * priv->batch_size;
+ unsigned int skip = 0, j;
+
+ /* Stage past every in-flight dispatch's claim on this queue so
+ * the new sqw reads disjoint SPSC slots. Peek self-limits: if
+ * the ring holds fewer entries past @skip, cnt shrinks (or 0).
+ */
+ for (j = 0; j < priv->inflight_cnt; j++)
+ skip += priv->inflight[j]->queue_idx[i];
+
+ param->queues[i].count = 0;
+ spsc_peek_at(&knodev->wpriv[i].spsc_bds, skip,
+ (void **)&sqw->bds[slot],
+ priv->batch_size, &cnt);
+ if (!cnt) {
+ sqw->queue_idx[i] = 0;
+ param->queues[i].count = 0;
+ continue;
+ }
+
+ /* Fill queue descriptor for GPU direct SPSC read.
+ * ring_start is the absolute ring position where this sqw
+ * begins - shader reads slots[(ring_start + tid) & mask].
+ * Offset by skip to keep staged sqws disjoint.
+ */
+ param->queues[i].pool_gaddr = knodev->wpriv[i].spsc_pool_gaddr;
+ param->queues[i].base_gaddr = priv->queue_base_gaddr[i];
+ param->queues[i].count = cnt;
+ param->queues[i].ring_start =
+ knodev->wpriv[i].spsc_bds.acquired + skip;
+ param->queues[i].ring_mask =
+ knodev->wpriv[i].spsc_bds.mask;
+
+ backlogs += cnt;
+ sqw->queue_idx[i] = cnt;
+ }
+ sqw->backlogs = backlogs;
+ param->nr_backlogs = backlogs;
+ param->nr_queues = priv->nr_works;
+ param->spsc_stride = ALIGN(sizeof(struct spsc_bd), SMP_CACHE_BYTES);
+ for (i = 0; i < priv->nr_works; i++) {
+ param->pass_count[i] = 0;
+ param->pass_meta_buf_gaddr[i] = priv->pass_meta_buf ?
+ priv->pass_meta_buf->gaddr +
+ (u64)i * priv->pass_pkts_per_queue *
+ KNOD_PASS_SLOT_SIZE :
+ 0;
+ }
+ param->ktime_ns = ktime_get_ns();
+
+ if (!sqw->backlogs)
+ return NULL;
+
+ list_del_init(&sqw->list);
+ return sqw;
+}
+
+/* Submit a prepared sqw: write AQL packet, ring doorbell, record stats. */
+static void knod_submit_bpf(struct knod_bpf_priv *priv,
+ struct knod_bpf_work_sq *sqw)
+{
+ struct amd_signal *signal =
+ (struct amd_signal *)priv->knod->kaql[0].queue_signal->kaddr;
+ struct knod_bpf_stats *stats = &priv->stats;
+ struct knod_dispatch_params p;
+ int i, bucket = KNOD_BL_BUCKETS - 1;
+
+ /* The @inflight_cnt dispatches already in flight decrement the signal
+ * before this one, so this sqw completes when the signal drops below
+ * (current value - inflight_cnt).
+ */
+ sqw->sigval = signal->value - priv->inflight_cnt;
+ sqw->expire = jiffies + msecs_to_jiffies(knod_bpf_expire);
+ if (static_branch_unlikely(&knod_stats_key)) {
+ sqw->dispatch_time = ktime_get();
+
+ stats->backlogs_total += sqw->backlogs;
+ for (i = 0; i < KNOD_BL_BUCKETS - 1; i++) {
+ if (sqw->backlogs <= bl_bounds[i]) {
+ bucket = i;
+ break;
+ }
+ }
+ stats->backlogs_hist[bucket]++;
+ }
+
+ knod_bpf_fill_dispatch(priv, sqw, &p);
+ /* publish dispatch params before the AQL packet becomes visible */
+ wmb();
+ knod_setup_header(priv->knod, &p, 0);
+}
+
+/* Phase 1: advance SPSC consumer pointers so next dispatch can peek
+ * new entries.
+ */
+static void knod_complete_acquire(struct knod_bpf_priv *priv,
+ struct knod_bpf_work_sq *sqw)
+{
+ struct knod_dev *knodev = priv->knodev;
+ int i;
+
+ for (i = 0; i < priv->nr_works; i++) {
+ if (sqw->queue_idx[i] >= 1) {
+ spsc_acquire(&knodev->wpriv[i].spsc_bds, NULL,
+ sqw->queue_idx[i], NULL);
+ }
+ }
+}
+
+/* Phase 2: schedule NAPI and free sqw. Can run after the next dispatch
+ * has been submitted - napi_schedule overlaps with GPU execution.
+ */
+static void knod_complete_napi(struct knod_bpf_priv *priv,
+ struct knod_bpf_work_sq *sqw)
+{
+ struct knod_dev *knodev = priv->knodev;
+ struct knod_bpf_stats *stats = &priv->stats;
+ ktime_t start;
+ int i;
+
+ if (static_branch_unlikely(&knod_stats_key))
+ start = ktime_get();
+
+ for (i = 0; i < priv->nr_works; i++) {
+ if (sqw->queue_idx[i] >= 1)
+ knod_napi_kick(&knodev->wpriv[i]);
+ }
+
+ sqw->backlogs = 0;
+ sqw->expire = 0;
+ list_add_tail_rcu(&sqw->list, &priv->free_list_sqw);
+
+ if (static_branch_unlikely(&knod_stats_key)) {
+ u64 ns = ktime_to_ns(ktime_sub(ktime_get(), start));
+
+ stats->decode_act_total_ns += ns;
+ stats->decode_act_count++;
+ if (ns > stats->decode_act_max_ns)
+ stats->decode_act_max_ns = ns;
+ }
+}
+
+static void knod_bpf_update_kernel_descriptor(struct knod_bpf_priv *priv,
+ struct kernel_descriptor *kd,
+ const struct knod_prog *knod_prog)
+{
+ unsigned int sgprs_used;
+
+ if (priv->isa_version != 9 || !knod_prog)
+ return;
+
+ sgprs_used = knod_prog->initial_exec_sreg + 2;
+ kd->compute_pgm_rsrc1.granulated_wavefront_sgpr_count =
+ knod_bpf_gfx9_sgpr_granule(sgprs_used);
+}
+
+/*
+ * Install kernel code into the inactive slot and atomically flip the active
+ * index. The active slot is never modified while the GPU dispatches it, so
+ * the swap never races the live pipeline, and the worker is not touched: new
+ * dispatches pick up the new slot, the in-flight one finishes on the old slot.
+ */
+static void knod_bpf_install_kernel(struct knod_bpf_priv *priv,
+ const struct knod_prog *knod_prog,
+ const void *code, u32 size)
+{
+ struct kernel_descriptor *kd;
+ struct knod *knod = priv->knod;
+ struct knod_mem *slot;
+ u32 entry_off;
+ u32 image_len;
+ int idx;
+
+ if (!code || !size || !knod->kernels[1])
+ return;
+
+ /*
+ * Before the worker runs, install in place; once it is dispatching,
+ * stage into the inactive slot and flip the active index so the live
+ * pipeline never reads a half-written slot.
+ */
+ if (!priv->start || !knod->worker)
+ idx = priv->active_idx;
+ else
+ idx = priv->active_idx ^ 1;
+
+ slot = knod->kernels[idx];
+ kd = slot->kaddr;
+ knod_bpf_update_kernel_descriptor(priv, kd, knod_prog);
+ entry_off = kd->kernel_code_entry_byte_offset;
+ if (WARN_ON(entry_off >= slot->size))
+ return;
+ if (WARN_ON(size > slot->size - entry_off))
+ size = slot->size - entry_off;
+ image_len = entry_off + size;
+
+ memcpy(slot->kaddr + entry_off, code, size);
+ if (image_len < slot->size) {
+ u32 clear_end = min_t(u32, slot->size,
+ entry_off + KNOD_BPF_PROG_BUF_SIZE);
+
+ if (image_len < clear_end)
+ memset(slot->kaddr + image_len, 0,
+ clear_end - image_len);
+ }
+ /*
+ * kernels[] is write-combining VRAM. smp_wmb() is only a compiler
+ * barrier on x86 and does NOT drain the WC buffers, so the GPU could
+ * fetch half-written code and spin. wmb() (sfence) flushes WC to VRAM
+ * before we publish the new slot; the dispatch doorbell is ordered
+ * behind it.
+ */
+ wmb();
+ knod_bpf_gpu_mem_fence(priv);
+ WRITE_ONCE(priv->kernel_image_len[idx], image_len);
+
+ if (idx != priv->active_idx)
+ WRITE_ONCE(priv->active_idx, idx);
+}
+
+/*
+ * Keep the just-built pass-kernel IR for the debugfs "insn" dump, so it can
+ * show the pass-through kernel when no XDP prog is attached. The machine code
+ * already lives in the kernel slot; this only retains the meta list. Rebuilt
+ * on every start (old metas freed first), released in knod_priv_exit().
+ */
+static void knod_bpf_retain_pass_ir(struct knod_bpf_priv *priv,
+ struct knod_prog *src)
+{
+ struct knod_insn_meta *meta, *tmp;
+ struct knod_prog *dst = priv->pass_knod_prog;
+
+ if (!dst) {
+ dst = kzalloc_obj(*dst, GFP_KERNEL);
+ if (!dst)
+ return;
+ INIT_LIST_HEAD(&dst->pre_insns);
+ INIT_LIST_HEAD(&dst->insns);
+ INIT_LIST_HEAD(&dst->post_insns);
+ priv->pass_knod_prog = dst;
+ } else {
+ list_for_each_entry_safe(meta, tmp, &dst->pre_insns, l) {
+ list_del(&meta->l);
+ kfree(meta);
+ }
+ list_for_each_entry_safe(meta, tmp, &dst->insns, l) {
+ list_del(&meta->l);
+ kfree(meta);
+ }
+ list_for_each_entry_safe(meta, tmp, &dst->post_insns, l) {
+ list_del(&meta->l);
+ kfree(meta);
+ }
+ }
+ list_splice_init(&src->pre_insns, &dst->pre_insns);
+ list_splice_init(&src->insns, &dst->insns);
+ list_splice_init(&src->post_insns, &dst->post_insns);
+}
+
+static void knod_bpf_layout_sregs(struct knod_bpf_priv *priv,
+ struct knod_prog *knod_prog)
+{
+ if (priv->isa_version != 9)
+ return;
+
+ knod_prog->initial_exec_sreg =
+ knod_prog->exec_save_base + knod_prog->exec_save_pairs_used * 2;
+}
+
+static int knod_bpf_jit_pass_kernel(struct knod_bpf_priv *priv)
+{
+ struct list_head *lists[2];
+ struct knod_insn_meta *meta, *tmp, *epi;
+ struct amdgcn_param32 p[3];
+ struct knod *knod = priv->knod;
+ struct knod_prog pass_prog;
+ int pass_branch_idx;
+ u32 pass_dwords;
+ u8 *buf, *ptr;
+ u32 total = 0;
+ int i, j, li, err;
+
+ memset(&pass_prog, 0, sizeof(pass_prog));
+ INIT_LIST_HEAD(&pass_prog.pre_insns);
+ INIT_LIST_HEAD(&pass_prog.insns);
+ INIT_LIST_HEAD(&pass_prog.post_insns);
+ pass_prog.knod = knod;
+ pass_prog.knodev = priv->knodev;
+ if (priv->isa_version == 10) {
+ pass_prog.done_mask_sreg = 32;
+ pass_prog.exec_save_base = 34;
+ pass_prog.initial_exec_sreg =
+ KNOD_AMDGPU_INITIAL_EXEC_SREG_GFX10;
+ } else {
+ pass_prog.done_mask_sreg = KNOD_AMDGPU_DONE_MASK_SREG;
+ pass_prog.exec_save_base = KNOD_AMDGPU_EXEC_SAVE_SREG_BASE;
+ pass_prog.initial_exec_sreg =
+ KNOD_AMDGPU_INITIAL_EXEC_SREG_GFX9;
+ }
+
+ knod_bpf_layout_sregs(priv, &pass_prog);
+ err = knod_prog_prepare_insns(priv, &pass_prog);
+ if (err)
+ return err;
+
+ epi = kzalloc_obj(*epi, GFP_KERNEL);
+ if (!epi) {
+ err = -ENOMEM;
+ goto free_pro;
+ }
+
+ /* BPF/XDP actions are 32-bit values; mlx5 consumes bd->act as low32. */
+ knod_vset32(&p[0], KNOD_AMDGPU_VREG0_LO);
+ knod_iset32(&p[1], XDP_PASS);
+ knod_emit(priv, epi, v_mov_b32_e32, p[0], p[1]);
+
+ knod_vset32(&p[0], KNOD_AMDGPU_VREG0_LO);
+ knod_vset32(&p[1], KNOD_AMDGPU_SLOT_VREG_LO);
+ knod_emit(priv, epi, global_store_dword, p[0], p[1],
+ offsetof(struct spsc_bd, act));
+
+ /* XDP_PASS detection: v_cmp_eq_u32 XDP_PASS, R0 -> VCC */
+ knod_iset32(&p[0], XDP_PASS);
+ knod_vset32(&p[1], KNOD_AMDGPU_VREG0_LO);
+ knod_emit(priv, epi, v_cmp_eq_u32, p[0], p[1]);
+
+ pass_branch_idx = epi->amdgpu_insns;
+ knod_emit(priv, epi, s_cbranch_vccz, 0);
+
+ /* EXEC &= VCC - only PASS lanes proceed */
+ knod_emit(priv, epi, s_and_b64, AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_EXEC_LO, AMDGCN_SREG_VCC_LO);
+
+ /* v_mov param addr to VGPR pair for pass_count atomic */
+ knod_vset32(&p[0], KNOD_AMDGPU_TMP_VREG9_LO);
+ knod_sset32(&p[1], KNOD_AMDGPU_PARAM_SREG_LO);
+ knod_emit(priv, epi, v_mov_b32_e32, p[0], p[1]);
+
+ knod_vset32(&p[0], KNOD_AMDGPU_TMP_VREG9_HI);
+ knod_sset32(&p[1], KNOD_AMDGPU_PARAM_SREG_HI);
+ knod_emit(priv, epi, v_mov_b32_e32, p[0], p[1]);
+
+ /* v_mov v2, s15 (queue_idx -> VGPR) */
+ knod_vset32(&p[0], KNOD_AMDGPU_VREG1_LO);
+ knod_sset32(&p[1], KNOD_AMDGPU_WORKGROUP_ID_Y_SREG);
+ knod_emit(priv, epi, v_mov_b32_e32, p[0], p[1]);
+
+ /* v_lshlrev_b32 v2, 2, v2 (queue_idx * 4) */
+ knod_vset32(&p[0], KNOD_AMDGPU_VREG1_LO);
+ knod_iset32(&p[1], 2);
+ knod_vset32(&p[2], KNOD_AMDGPU_VREG1_LO);
+ knod_emit(priv, epi, v_lshlrev_b32, p[0], p[1], p[2]);
+
+ /* v_add_u32 TMP9_LO, v2, TMP9_LO (param_addr += queue_idx * 4) */
+ knod_vset32(&p[0], KNOD_AMDGPU_TMP_VREG9_LO);
+ knod_vset32(&p[1], KNOD_AMDGPU_VREG1_LO);
+ knod_vset32(&p[2], KNOD_AMDGPU_TMP_VREG9_LO);
+ knod_emit(priv, epi, v_add_u32, p[0], p[1], p[2]);
+
+ /* v_mov TMP10_LO, 1 */
+ knod_vset32(&p[0], KNOD_AMDGPU_TMP_VREG10_LO);
+ knod_iset32(&p[1], 1);
+ knod_emit(priv, epi, v_mov_b32_e32, p[0], p[1]);
+
+ /* global_atomic_add pass_count[q]++, GLC=1 -> old_val in TMP10_LO */
+ knod_vset32(&p[0], KNOD_AMDGPU_TMP_VREG10_LO);
+ knod_vset32(&p[1], KNOD_AMDGPU_TMP_VREG9_LO);
+ knod_vset32(&p[2], KNOD_AMDGPU_TMP_VREG10_LO);
+ knod_emit(priv, epi, global_atomic_add, p[0], p[1], p[2],
+ offsetof(struct knod_bpf_param, pass_count), 1);
+
+ /* s_waitcnt vmcnt(0) */
+ knod_emit(priv, epi, s_waitcnt_vmcnt);
+
+ /* v_sub_u32 TMP9_LO, TMP9_LO, v2 (restore param_addr_lo) */
+ knod_vset32(&p[0], KNOD_AMDGPU_TMP_VREG9_LO);
+ knod_vset32(&p[1], KNOD_AMDGPU_TMP_VREG9_LO);
+ knod_vset32(&p[2], KNOD_AMDGPU_VREG1_LO);
+ knod_emit(priv, epi, v_sub_u32, p[0], p[1], p[2]);
+
+ /* old_val * 2 for pass_indices u16 stride */
+ knod_vset32(&p[0], KNOD_AMDGPU_TMP_VREG10_LO);
+ knod_iset32(&p[1], 1);
+ knod_vset32(&p[2], KNOD_AMDGPU_TMP_VREG10_LO);
+ knod_emit(priv, epi, v_lshlrev_b32, p[0], p[1], p[2]);
+
+ /* addr_lo += old_val * 2 */
+ knod_vset32(&p[0], KNOD_AMDGPU_TMP_VREG9_LO);
+ knod_vset32(&p[1], KNOD_AMDGPU_TMP_VREG10_LO);
+ knod_vset32(&p[2], KNOD_AMDGPU_TMP_VREG9_LO);
+ knod_emit(priv, epi, v_add_u32, p[0], p[1], p[2]);
+
+ /* global_store_short pass_indices[old_val], BACKLOG_IDX_VREG */
+ knod_vset32(&p[0], KNOD_AMDGPU_BACKLOG_IDX_VREG);
+ knod_vset32(&p[1], KNOD_AMDGPU_TMP_VREG9_LO);
+ knod_emit(priv, epi, global_store_short, p[0], p[1],
+ offsetof(struct knod_bpf_param, pass_indices));
+
+ /* Store len + src_addr to pass_meta_buf slot header */
+ knod_emit_pass_addr_store(priv, epi);
+
+ /* Patch s_cbranch_vccz offset (skip pass handling) */
+ pass_dwords = 0;
+ for (j = pass_branch_idx + 1; j < epi->amdgpu_insns; j++)
+ pass_dwords += epi->amdgpu_insn[j].size / 4;
+ emit_s_cbranch_vccz(priv->isa_version,
+ &epi->amdgpu_insn[pass_branch_idx], pass_dwords);
+
+ knod_emit(priv, epi, s_endpgm);
+
+ if (priv->isa_version >= 10) {
+ for (j = 0; j < 16 && epi->amdgpu_insns < KNOD_META_INSNS; j++)
+ knod_emit(priv, epi, s_code_end);
+ }
+ list_add_tail(&epi->l, &pass_prog.post_insns);
+
+ /* Linearize prologue + epilogue into pass_prog_buf */
+ lists[0] = &pass_prog.pre_insns;
+ lists[1] = &pass_prog.post_insns;
+
+ for (li = 0; li < 2; li++) {
+ list_for_each_entry(meta, lists[li], l)
+ for (i = 0; i < meta->amdgpu_insns; i++)
+ total += meta->amdgpu_insn[i].size;
+ }
+
+ kfree(priv->pass_prog_buf);
+ buf = kzalloc(total, GFP_KERNEL);
+ if (!buf) {
+ err = -ENOMEM;
+ goto free_all;
+ }
+
+ ptr = buf;
+ for (li = 0; li < 2; li++) {
+ list_for_each_entry(meta, lists[li], l)
+ for (i = 0; i < meta->amdgpu_insns; i++) {
+ memcpy(ptr, &meta->amdgpu_insn[i],
+ meta->amdgpu_insn[i].size);
+ ptr += meta->amdgpu_insn[i].size;
+ }
+ }
+
+ priv->pass_prog_buf = buf;
+ priv->pass_prog_size = total;
+
+ knod_bpf_install_kernel(priv, &pass_prog, priv->pass_prog_buf,
+ priv->pass_prog_size);
+ /* Remember which slot now holds pass so detach can flip back to it. */
+ priv->pass_idx = priv->active_idx;
+
+ pr_info("knod_bpf: pass kernel JIT'd %u bytes\n", priv->pass_prog_size);
+ err = 0;
+ /* Retain the IR (moves the lists out) before the cleanup below
+ * frees.
+ */
+ knod_bpf_retain_pass_ir(priv, &pass_prog);
+
+free_all:
+ list_for_each_entry_safe(meta, tmp, &pass_prog.post_insns, l) {
+ list_del_init(&meta->l);
+ kfree(meta);
+ }
+free_pro:
+ list_for_each_entry_safe(meta, tmp, &pass_prog.pre_insns, l) {
+ list_del_init(&meta->l);
+ kfree(meta);
+ }
+ return err;
+}
+
+static void knod_bpf_reset_sqw(struct knod_bpf_work_sq *sqw)
+{
+ if (!sqw)
+ return;
+
+ sqw->backlogs = 0;
+ sqw->expire = 0;
+}
+
+static void knod_bpf_wait_sqw(struct knod_bpf_priv *priv,
+ struct knod_bpf_work_sq *sqw)
+{
+ struct amd_signal *signal;
+ unsigned long deadline;
+
+ if (!sqw)
+ return;
+
+ signal = (struct amd_signal *)
+ priv->knod->kaql[0].queue_signal->kaddr;
+ deadline = jiffies + msecs_to_jiffies(1000);
+
+ while (sqw->sigval <= READ_ONCE(signal->value) &&
+ time_before(jiffies, deadline))
+ usleep_range(100, 200);
+
+ if (sqw->sigval <= READ_ONCE(signal->value))
+ pr_warn("knod: timed out waiting for GPU dispatch completion\n");
+}
+
+static void knod_bpf_drain_worker(struct knod_bpf_priv *priv)
+{
+ struct knod_bpf_work_sq *sqw;
+
+ /* stop() runs on interface-down AND on every feature switch, both
+ * with mlx5 RX possibly still producing into knodev->wpriv[].spsc_bds.
+ * So we only quiesce the GPU here; the NIC-owned RX SPSC rings are
+ * drained on interface-down by mlx5e_rx_offload_stop().
+ */
+ while (priv->inflight_cnt) {
+ sqw = priv->inflight[--priv->inflight_cnt];
+ priv->inflight[priv->inflight_cnt] = NULL;
+ knod_bpf_wait_sqw(priv, sqw);
+ knod_bpf_reset_sqw(sqw);
+ list_add_tail_rcu(&sqw->list, &priv->free_list_sqw);
+ }
+}
+
+static void knod_bpf_drain(struct knod_bpf_priv *priv)
+{
+ knod_bpf_drain_worker(priv);
+}
+
+static void knod_bpf_stop_worker(struct knod_bpf_priv *priv)
+{
+ priv->start = 0;
+ if (priv->worker_task) {
+ kthread_stop(priv->worker_task);
+ put_task_struct(priv->worker_task);
+ priv->worker_task = NULL;
+ }
+ synchronize_net();
+}
+
+static void knod_bpf_configure_worker(struct knod_bpf_priv *priv)
+{
+ knod_bpf_stop_worker(priv);
+ knod_bpf_drain(priv);
+
+ priv->inflight_cnt = 0;
+}
+
+static int knod_bpf_start_worker(struct knod_bpf_priv *priv)
+{
+ struct task_struct *p;
+
+ p = kthread_run(knod_bpf_worker, priv, "knod_%d_0",
+ priv->knodev->accel->id);
+ if (IS_ERR(p))
+ return PTR_ERR(p);
+
+ get_task_struct(p);
+ priv->worker_task = p;
+ return 0;
+}
+
+static bool knod_bpf_uses_percpu(struct knod_bpf_priv *priv)
+{
+ struct knod_bpf_map *knod_map;
+
+ list_for_each_entry(knod_map, &priv->knodev->accel->xdp.bound_maps,
+ list)
+ if (knod_map->knod_map_obj->map_type ==
+ BPF_MAP_TYPE_PERCPU_ARRAY)
+ return true;
+ return false;
+}
+
+/* Fan out one workgroup per CU (rounded down to a power of two). PERCPU maps
+ * keep one instance per RX queue, so a queue must stay on a single workgroup;
+ * force xgroups=1 when the program uses them. Non-PERCPU maps are globally
+ * shared with atomics, so fan-out is safe there. This replaces the old manual
+ * xgroups knob.
+ */
+static unsigned int knod_bpf_auto_xgroups(struct knod_bpf_priv *priv)
+{
+ struct amdgpu_device *adev = NULL;
+ unsigned int cus, xgroups;
+
+ if (knod_bpf_uses_percpu(priv))
+ return 1;
+
+ if (priv->knod->dev)
+ adev = priv->knod->dev->adev;
+ else if (priv->knod->process && priv->knod->process->pdds[0])
+ adev = priv->knod->process->pdds[0]->dev->adev;
+ if (!adev || !priv->nr_works)
+ return 1;
+
+ cus = adev->gfx.cu_info.number;
+ xgroups = cus / priv->nr_works;
+ if (!xgroups)
+ xgroups = 1;
+ return rounddown_pow_of_two(xgroups);
+}
+
+/* Per-queue dispatch batch = workgroups * xgroups packets, capped by the
+ * static descriptor array and rounded down to a power of two (the shader
+ * derives the flat slot as queue_id << ilog2(batch_size) + local_idx).
+ */
+static unsigned int knod_bpf_batch_size(struct knod_bpf_priv *priv)
+{
+ unsigned int xgroups = knod_bpf_auto_xgroups(priv);
+ unsigned int max_flat = KNOD_BPF_BACKLOGS_MAX / priv->nr_works;
+ unsigned int batch = min_t(unsigned int,
+ knod_bpf_workgroups * xgroups, max_flat);
+
+ if (!batch)
+ batch = knod_bpf_workgroups;
+ return rounddown_pow_of_two(batch);
+}
+
+static void knod_bpf_start(struct knod_dev *knodev)
+{
+ struct knod_bpf_priv *priv =
+ (struct knod_bpf_priv *)knodev->accel->xdp.priv;
+ struct bpf_prog *prog;
+ unsigned int active_rxq;
+ int err;
+
+ priv->start = 1;
+ active_rxq = knod_bpf_active_rxq_count(knodev->netdev);
+ if (active_rxq && active_rxq != priv->nr_works)
+ pr_warn("knod_bpf: active rx queues changed from %d to %u; using initialized count\n",
+ priv->nr_works, active_rxq);
+
+ priv->batch_size = knod_bpf_batch_size(priv);
+
+ knod_jit_dbg(" batch_size = %d\n", priv->batch_size);
+ knod_bpf_configure_worker(priv);
+ pr_info("knod_bpf: using single AQL queue, rx_works=%d active_rxq=%u batch_size=%d xgroups=%u\n",
+ priv->nr_works, active_rxq, priv->batch_size,
+ priv->batch_size / knod_bpf_workgroups);
+
+ if (knod_bpf_jit_pass_kernel(priv))
+ pr_warn("knod_bpf: pass kernel JIT failed\n");
+
+ prog = READ_ONCE(priv->prog);
+ if (prog)
+ knod_setup_bpf_prog(prog);
+
+ priv->start = 1;
+ err = knod_bpf_start_worker(priv);
+ if (err) {
+ pr_err("knod_bpf: start_worker failed: %d\n", err);
+ priv->start = 0;
+ return;
+ }
+}
+
+static void knod_bpf_stop(struct knod_dev *knodev)
+{
+ struct knod_bpf_priv *priv =
+ (struct knod_bpf_priv *)knodev->accel->xdp.priv;
+
+ knod_bpf_stop_worker(priv);
+ knod_bpf_drain(priv);
+
+ kfree(priv->pass_prog_buf);
+ priv->pass_prog_buf = NULL;
+ priv->pass_prog_size = 0;
+}
+
+/*
+ * Flip the dispatched kernel back to pass-through when the XDP prog is
+ * detached. The pass slot already holds the pass code, so this is just an
+ * atomic index flip -- no re-copy.
+ */
+static void knod_bpf_reload_pass(struct knod_dev *knodev)
+{
+ struct knod_bpf_priv *priv = knodev->accel->xdp.priv;
+
+ if (priv)
+ WRITE_ONCE(priv->active_idx, priv->pass_idx);
+}
+
+static void knod_setup_bpf_prog(struct bpf_prog *prog)
+{
+ struct knod_prog *knod_prog = prog->aux->offload->dev_priv;
+ struct knod_dev *knodev = knod_prog->knodev;
+ struct knod_insn_meta *meta, *tmp;
+ struct knod_bpf_priv *priv;
+ u8 *kernel_ptr, *ptr;
+ u32 total_bytes;
+ u32 *debug_ptr;
+ int i;
+
+ priv = (struct knod_bpf_priv *)knodev->accel->xdp.priv;
+ WRITE_ONCE(priv->installing_kernel, true);
+
+ if (prog) {
+ WRITE_ONCE(priv->prog, NULL);
+ kernel_ptr = priv->prog_buf;
+ memset(priv->prog_buf, 0, KNOD_BPF_PROG_BUF_SIZE);
+
+ list_for_each_entry(meta, &priv->knod_prog->pre_insns, l) {
+ for (i = 0; i < meta->amdgpu_insns; i++) {
+ ptr = (u8 *)&meta->amdgpu_insn[i];
+ debug_ptr = (u32 *)ptr;
+
+ memcpy(kernel_ptr, ptr,
+ meta->amdgpu_insn[i].size);
+ kernel_ptr += meta->amdgpu_insn[i].size;
+
+ if (meta->amdgpu_insn[i].size == 4) {
+ knod_jit_dbg(" 0x%.8X\t%.8X\n",
+ meta->amdgpu_insn_idx,
+ debug_ptr[0]);
+ } else if (meta->amdgpu_insn[i].size == 8) {
+ knod_jit_dbg(" 0x%.8X\t%.8X %.8X\n",
+ meta->amdgpu_insn_idx,
+ debug_ptr[0], debug_ptr[1]);
+ } else if (meta->amdgpu_insn[i].size == 12) {
+ knod_jit_dbg(" 0x%.8X\t%.8X %.8X %.8X\n",
+ meta->amdgpu_insn_idx,
+ debug_ptr[0],
+ debug_ptr[1], debug_ptr[2]);
+ } else {
+ WARN_ON_ONCE(1);
+ }
+ }
+ }
+
+ list_for_each_entry(meta, &priv->knod_prog->insns, l) {
+ for (i = 0; i < meta->amdgpu_insns; i++) {
+ ptr = (u8 *)&meta->amdgpu_insn[i];
+ debug_ptr = (u32 *)ptr;
+
+ memcpy(kernel_ptr, ptr,
+ meta->amdgpu_insn[i].size);
+ kernel_ptr += meta->amdgpu_insn[i].size;
+ if (meta->amdgpu_insn[i].size == 4) {
+ knod_jit_dbg(" 0x%.8X\t%.8X\n",
+ meta->amdgpu_insn_idx,
+ debug_ptr[0]);
+ } else if (meta->amdgpu_insn[i].size == 8) {
+ knod_jit_dbg(" 0x%.8X\t%.8X %.8X\n",
+ meta->amdgpu_insn_idx,
+ debug_ptr[0], debug_ptr[1]);
+ } else if (meta->amdgpu_insn[i].size == 12) {
+ knod_jit_dbg(" 0x%.8X\t%.8X %.8X %.8X\n",
+ meta->amdgpu_insn_idx,
+ debug_ptr[0],
+ debug_ptr[1], debug_ptr[2]);
+ } else {
+ WARN_ON_ONCE(1);
+ }
+ }
+ }
+
+ list_for_each_entry(meta, &priv->knod_prog->post_insns, l) {
+ for (i = 0; i < meta->amdgpu_insns; i++) {
+ ptr = (u8 *)&meta->amdgpu_insn[i];
+ debug_ptr = (u32 *)ptr;
+
+ memcpy(kernel_ptr, ptr,
+ meta->amdgpu_insn[i].size);
+ kernel_ptr += meta->amdgpu_insn[i].size;
+ if (meta->amdgpu_insn[i].size == 4) {
+ knod_jit_dbg(" %.8X\n", debug_ptr[0]);
+ } else if (meta->amdgpu_insn[i].size == 8) {
+ knod_jit_dbg(" %.8X %.8X\n",
+ debug_ptr[0], debug_ptr[1]);
+ } else if (meta->amdgpu_insn[i].size == 12) {
+ knod_jit_dbg(" %.8X %.8X %.8X\n",
+ debug_ptr[0],
+ debug_ptr[1], debug_ptr[2]);
+ } else {
+ WARN_ON_ONCE(1);
+ }
+ }
+ }
+ total_bytes = kernel_ptr - (u8 *)priv->prog_buf;
+
+ pr_debug("KNOD JIT: total binary size = %u bytes (limit %u)\n",
+ total_bytes, KNOD_BPF_PROG_BUF_SIZE);
+ if (WARN_ON(total_bytes > KNOD_BPF_PROG_BUF_SIZE))
+ total_bytes = KNOD_BPF_PROG_BUF_SIZE;
+ knod_bpf_install_kernel(priv, knod_prog, priv->prog_buf,
+ total_bytes);
+ WRITE_ONCE(priv->prog, prog);
+ } else {
+ WRITE_ONCE(priv->prog, NULL);
+ list_for_each_entry_safe(meta, tmp, &priv->knod_prog->pre_insns,
+ l) {
+ list_del_init(&meta->l);
+ kfree(meta);
+ }
+
+ list_for_each_entry_safe(meta, tmp, &priv->knod_prog->insns,
+ l) {
+ list_del_init(&meta->l);
+ kfree(meta);
+ }
+
+ list_for_each_entry_safe(meta, tmp,
+ &priv->knod_prog->post_insns, l) {
+ list_del_init(&meta->l);
+ kfree(meta);
+ }
+
+ /* bbs points into the metas just freed */
+ kfree(priv->knod_prog->bbs);
+ priv->knod_prog->bbs = NULL;
+ priv->knod_prog->n_bbs = 0;
+
+ if (priv->pass_prog_buf)
+ knod_bpf_install_kernel(priv, priv->pass_knod_prog,
+ priv->pass_prog_buf,
+ priv->pass_prog_size);
+ }
+ WRITE_ONCE(priv->installing_kernel, false);
+}
+
+static int knod_bpf_map_hash_init_elem(struct knod_bpf_map *knod_map,
+ struct knod_bpf_map_obj *knod_map_obj)
+{
+ unsigned int *queue = (unsigned int *)knod_map->queue_mem->kaddr;
+ unsigned int *bucket = (unsigned int *)&knod_map_obj->bucket[0];
+ void *elems = knod_map->hash_elems_mem->kaddr;
+ struct knod_bpf_hash_elem_obj *e;
+ int i, elem_size;
+
+ elem_size = sizeof(struct knod_bpf_hash_elem_obj) +
+ roundup(knod_map_obj->key_size, 4) +
+ roundup(knod_map_obj->value_size, 4);
+ knod_map_obj->meta.hmeta.elem_size = elem_size;
+
+ for (i = 0; i < knod_map_obj->meta.hmeta.n_buckets; i++)
+ bucket[i] = KNOD_BPF_HASH_NEXT_END;
+
+ for (i = 0; i < knod_map_obj->max_entries; i++) {
+ e = elems + (i * elem_size);
+ e->next = KNOD_BPF_HASH_NEXT_END;
+ queue[i] = i;
+ }
+ knod_map_obj->meta.hmeta.cur = knod_map_obj->max_entries - 1;
+
+ return 0;
+}
+
+static inline unsigned char *
+knod_bpf_hash_elem_kv(struct knod_bpf_hash_elem_obj *e)
+{
+ return (unsigned char *)e + offsetof(struct knod_bpf_hash_elem_obj, kv);
+}
+
+static inline void *
+knod_bpf_array_value_ptr(struct knod_bpf_map_obj *knod_map_obj,
+ unsigned int idx)
+{
+ return (unsigned char *)knod_map_obj +
+ offsetof(struct knod_bpf_map_obj, bucket) +
+ (size_t)idx * knod_map_obj->value_size;
+}
+
+static int __knod_bpf_map_alloc(struct knod_dev *knodev,
+ struct bpf_offloaded_map *offmap)
+{
+ struct knod_bpf_priv *priv =
+ (struct knod_bpf_priv *)knodev->accel->xdp.priv;
+ struct knod_mem *mem, *queue_mem, *hash_elems_mem, *gc_mem;
+ int order, size, queue_size, i, value_size, nents, err;
+ int n_instances = 1;
+ int flags = KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE |
+ KFD_IOC_ALLOC_MEM_FLAGS_COHERENT |
+ KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC |
+ KFD_IOC_ALLOC_MEM_FLAGS_VRAM;
+ struct knod_bpf_map_obj *knod_map_obj;
+ struct knod *knod = priv->knod;
+ struct knod_bpf_map *knod_map;
+ unsigned int gc_size;
+ unsigned int *q;
+
+ if (offmap->map.map_type == BPF_MAP_TYPE_HASH) {
+ value_size = sizeof(unsigned int);
+ nents = roundup_pow_of_two(offmap->map.max_entries);
+ } else {
+ value_size = offmap->map.value_size;
+ nents = offmap->map.max_entries;
+ }
+
+ /* PERCPU_ARRAY keeps one value array per GPU workgroup (percpu
+ * instance) so each CU updates its own copy - no cross-CU atomic
+ * contention. Instances map 1:1 to the per-cpu value buffer, so
+ * allocate num_possible_cpus of them (workgroup_id_y indexes into it).
+ */
+ if (offmap->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY)
+ n_instances = num_possible_cpus();
+
+ size = sizeof(struct knod_bpf_map_obj) +
+ (value_size * nents * n_instances);
+ if (offmap->map.map_type == BPF_MAP_TYPE_HASH)
+ size += sizeof(unsigned int) * nents;
+ order = get_order(size);
+
+ mem = knod_alloc_mem(knod, PAGE_SIZE << order, flags);
+ if (IS_ERR(mem))
+ return -ENOMEM;
+
+ memset(mem->kaddr, 0, size);
+ knod_map = kzalloc_obj(struct knod_bpf_map, GFP_KERNEL);
+ if (!knod_map) {
+ knod_free_mem(knod, mem);
+ return -ENOMEM;
+ }
+
+ knod_map->mem = mem;
+ knod_map->queue_mem = NULL;
+ knod_map->hash_elems_mem = NULL;
+ knod_map->offmap = offmap;
+ knod_map->priv = priv;
+ if (offmap->dev_priv)
+ WARN_ON_ONCE(1);
+ offmap->dev_priv = knod_map;
+
+ knod_map_obj = (struct knod_bpf_map_obj *)mem->kaddr;
+ knod_map_obj->key_size = offmap->map.key_size;
+ if (knod_map_obj->key_size > MAX_MAP_KEY_SIZE) {
+ pr_warn("request key size is %d, but max key size is %d\n",
+ knod_map_obj->key_size, MAX_MAP_KEY_SIZE);
+ return -ENOMEM;
+ }
+ knod_map_obj->value_size = offmap->map.value_size;
+ knod_map_obj->max_entries = nents;
+ knod_map_obj->id = offmap->map.id;
+ knod_map_obj->map_type = offmap->map.map_type;
+ if (knod_map_obj->map_type == BPF_MAP_TYPE_HASH) {
+ knod_map_obj->meta.hmeta.n_buckets = nents;
+ if (offmap->map.map_flags & BPF_F_ZERO_SEED)
+ knod_map_obj->meta.hmeta.hashrnd = 0;
+ else
+ knod_map_obj->meta.hmeta.hashrnd = get_random_u32();
+ } else {
+ knod_map_obj->meta.ameta.per_instance_size = value_size * nents;
+ knod_map_obj->meta.ameta.n_instances = n_instances;
+ }
+ knod_map->knod_map_obj = knod_map_obj;
+ /* map->flags = ? */
+ knod_jit_dbg(" map_id = %d\n", knod_map_obj->id);
+
+ if (knod_map_obj->map_type == BPF_MAP_TYPE_HASH) {
+ queue_size = sizeof(unsigned int) * nents;
+ queue_size = PAGE_SIZE << get_order(queue_size);
+ queue_mem = knod_alloc_mem(knod, queue_size, flags);
+ if (IS_ERR(queue_mem)) {
+ knod_free_mem(knod, mem);
+ kfree(knod_map);
+ return -ENOMEM;
+ }
+
+ memset(queue_mem->kaddr, 0, queue_mem->size);
+ q = queue_mem->kaddr;
+ for (i = 0; i < knod_map_obj->meta.hmeta.n_buckets; i++)
+ q[i] = i;
+ knod_map->queue_mem = queue_mem;
+ knod_map_obj->meta.hmeta.q = (struct _queue *)queue_mem->gaddr;
+
+ queue_size = (sizeof(struct knod_bpf_hash_elem_obj) +
+ roundup(knod_map_obj->key_size, 4) +
+ roundup(knod_map_obj->value_size, 4)) *
+ knod_map_obj->max_entries;
+ queue_size = PAGE_SIZE << get_order(queue_size);
+
+ hash_elems_mem = knod_alloc_mem(knod, queue_size, flags);
+ if (IS_ERR(hash_elems_mem)) {
+ knod_free_mem(knod, queue_mem);
+ knod_free_mem(knod, mem);
+ kfree(knod_map);
+ return -ENOMEM;
+ }
+
+ memset(hash_elems_mem->kaddr, 0, queue_size);
+ knod_map->hash_elems_mem = hash_elems_mem;
+ knod_map_obj->meta.hmeta.elems = (void *)hash_elems_mem->gaddr;
+ knod_bpf_map_hash_init_elem(knod_map, knod_map_obj);
+
+ /* GC list for GPU-side delete: elem_ids pending unlink */
+ gc_size = sizeof(unsigned int) * nents;
+ gc_size = PAGE_SIZE << get_order(gc_size);
+ gc_mem = knod_alloc_mem(knod, gc_size, flags);
+ if (IS_ERR(gc_mem)) {
+ knod_free_mem(knod, hash_elems_mem);
+ knod_free_mem(knod, queue_mem);
+ knod_free_mem(knod, mem);
+ kfree(knod_map);
+ return -ENOMEM;
+ }
+ memset(gc_mem->kaddr, 0, gc_size);
+ knod_map->gc_mem = gc_mem;
+ knod_map_obj->meta.hmeta.gc_count = 0;
+ knod_map_obj->meta.hmeta.gc_list = (void *)gc_mem->gaddr;
+ }
+
+ err = __knod_map_mem(knod, mem);
+ if (err) {
+ pr_err("knod_bpf: failed to GPU-map map BO\n");
+ goto err_map;
+ }
+ if (knod_map_obj->map_type == BPF_MAP_TYPE_HASH) {
+ err = __knod_map_mem(knod, queue_mem);
+ if (err) {
+ pr_err("knod_bpf: failed to GPU-map queue BO\n");
+ goto err_map;
+ }
+ err = __knod_map_mem(knod, hash_elems_mem);
+ if (err) {
+ pr_err("knod_bpf: failed to GPU-map hash_elems BO\n");
+ goto err_map;
+ }
+ err = __knod_map_mem(knod, knod_map->gc_mem);
+ if (err) {
+ pr_err("knod_bpf: failed to GPU-map gc BO\n");
+ goto err_map;
+ }
+ }
+ knod_bpf_gpu_mem_fence(priv);
+
+ mutex_lock(&knodev->lock);
+ list_add(&knod_map->list, &knodev->accel->xdp.bound_maps);
+ mutex_unlock(&knodev->lock);
+ return 0;
+
+err_map:
+ if (knod_map_obj->map_type == BPF_MAP_TYPE_HASH) {
+ knod_free_mem(knod, knod_map->gc_mem);
+ knod_free_mem(knod, hash_elems_mem);
+ knod_free_mem(knod, queue_mem);
+ }
+ knod_free_mem(knod, mem);
+ kfree(knod_map);
+ return err;
+}
+
+static void knod_bpf_map_setup(struct bpf_prog *prog)
+{
+ struct knod_prog *knod_prog = prog->aux->offload->dev_priv;
+ struct knod_dev *knodev = knod_prog->knodev;
+ struct knod_bpf_map *knod_map;
+ struct knod_bpf_map_obj *map;
+ struct knod_mem *mem;
+
+ mutex_lock(&knodev->lock);
+ list_for_each_entry(knod_map, &knodev->accel->xdp.bound_maps, list) {
+ mem = knod_map->mem;
+ map = mem->kaddr;
+ map->id = knod_map->offmap->map.id;
+ map->map_type = knod_map->offmap->map.map_type;
+ knod_jit_dbg(" id = %d type = %d\n", knod_map->offmap->map.id,
+ knod_map->offmap->map.map_type);
+ }
+ mutex_unlock(&knodev->lock);
+}
+
+static struct knod_bpf_hash_elem_obj *
+knod_bpf_map_hash_pop(struct knod_bpf_map *knod_map,
+ struct knod_bpf_map_obj *knod_map_obj)
+{
+ void *elems = knod_map->hash_elems_mem->kaddr;
+ unsigned int *queue = (unsigned int *)knod_map->queue_mem->kaddr;
+ struct knod_bpf_hash_elem_obj *e;
+ int elem_id;
+
+ if (knod_map_obj->meta.hmeta.cur < 1)
+ return NULL;
+
+ elem_id = queue[knod_map_obj->meta.hmeta.cur];
+ knod_jit_dbg(" elem_id = 0x%x\n", elem_id);
+ e = elems + (elem_id * knod_map_obj->meta.hmeta.elem_size);
+ knod_map_obj->meta.hmeta.cur--;
+ e->next = KNOD_BPF_HASH_NEXT_END;
+
+ return e;
+}
+
+static struct knod_bpf_hash_elem_obj *
+knod_bpf_map_hash_alloc_elem(struct knod_bpf_map *knod_map,
+ struct knod_bpf_map_obj *knod_map_obj,
+ void *key, void *value)
+{
+ struct knod_bpf_hash_elem_obj *e;
+
+ e = knod_bpf_map_hash_pop(knod_map, knod_map_obj);
+ if (!e)
+ return NULL;
+
+ unsafe_memcpy(knod_bpf_hash_elem_kv(e), key, knod_map_obj->key_size,
+ "knod hash elems are variable-sized GPU map records");
+ unsafe_memcpy(knod_bpf_hash_elem_kv(e) + knod_map_obj->key_size,
+ value, knod_map_obj->value_size,
+ "knod hash elems are variable-sized GPU map records");
+ /* VRAM is ioremap_wc - drain new elem's next and kv stores before
+ * the caller publishes a pointer to this elem.
+ */
+ wmb();
+ return e;
+}
+
+static int knod_bpf_map_hash_lookup_elem(struct knod_bpf_map *knod_map,
+ struct knod_bpf_map_obj *knod_map_obj,
+ void *key,
+ void *value)
+{
+ void *elems = knod_map->hash_elems_mem->kaddr;
+ unsigned int hash, elem_id, elem_size;
+ struct knod_bpf_hash_elem_obj *e;
+ unsigned int *bucket;
+
+ hash = jhash((const void *)key, knod_map_obj->key_size,
+ knod_map_obj->meta.hmeta.hashrnd);
+ knod_jit_dbg(" hash = %x\n", hash);
+ hash = hash & (knod_map_obj->meta.hmeta.n_buckets - 1);
+ knod_jit_dbg(" hash = %x\n", hash);
+ bucket = (unsigned int *)&knod_map_obj->bucket[0];
+
+ elem_id = bucket[hash];
+ if (elem_id == KNOD_BPF_HASH_NEXT_END)
+ return -ENOENT;
+
+ elem_size = knod_map_obj->meta.hmeta.elem_size;
+
+ e = elems + (elem_id * elem_size);
+ while (1) {
+ if (!(e->next & KNOD_BPF_HASH_NEXT_DELETED) &&
+ !memcmp(&e->kv[0], (const unsigned char *)key,
+ knod_map_obj->key_size)) {
+ memcpy(value,
+ (unsigned char *)&e->kv[0] +
+ knod_map_obj->key_size,
+ knod_map_obj->value_size);
+ return 0;
+ }
+ unsigned int real_next = e->next & KNOD_BPF_HASH_NEXT_MASK;
+
+ if (real_next == KNOD_BPF_HASH_NEXT_END)
+ return -ENOENT;
+ e = elems + (real_next * elem_size);
+ }
+
+ return -ENOENT;
+}
+
+static int knod_bpf_map_hash_update_elem(struct knod_bpf_map *knod_map,
+ struct knod_bpf_map_obj *knod_map_obj,
+ void *key,
+ void *value)
+{
+ void *elems = knod_map->hash_elems_mem->kaddr;
+ unsigned int hash, elem_id, elem_size;
+ struct knod_bpf_hash_elem_obj *e, *ne;
+ unsigned int *bucket;
+
+ hash = jhash((const void *)key, knod_map_obj->key_size,
+ knod_map_obj->meta.hmeta.hashrnd);
+ hash = hash & (knod_map_obj->meta.hmeta.n_buckets - 1);
+ bucket = (unsigned int *)&knod_map_obj->bucket[0];
+
+ elem_size = knod_map_obj->meta.hmeta.elem_size;
+ elem_id = bucket[hash];
+ if (elem_id == KNOD_BPF_HASH_NEXT_END) {
+ ne = knod_bpf_map_hash_alloc_elem(knod_map, knod_map_obj, key,
+ value);
+ if (!ne)
+ return -ENOMEM;
+ bucket[hash] = ((void *)ne - (void *)elems) / elem_size;
+ return 0;
+ }
+
+ e = elems + (elem_id * elem_size);
+ while (1) {
+ if (!(e->next & KNOD_BPF_HASH_NEXT_DELETED) &&
+ !memcmp(&e->kv[0], (const unsigned char *)key,
+ knod_map_obj->key_size)) {
+ unsafe_memcpy(knod_bpf_hash_elem_kv(e) +
+ knod_map_obj->key_size,
+ value, knod_map_obj->value_size,
+ "knod hash elems are variable-sized GPU map records");
+ return 0;
+ }
+ unsigned int real_next = e->next & KNOD_BPF_HASH_NEXT_MASK;
+
+ if (real_next == KNOD_BPF_HASH_NEXT_END) {
+ ne = knod_bpf_map_hash_alloc_elem(knod_map,
+ knod_map_obj,
+ key, value);
+ if (!ne)
+ return -ENOMEM;
+ e->next = (e->next & KNOD_BPF_HASH_NEXT_DELETED) |
+ (((void *)ne - (void *)elems) / elem_size);
+ return 0;
+ }
+ e = elems + (real_next * elem_size);
+ }
+
+ return -ENOENT;
+}
+
+static int knod_bpf_map_hash_delete_elem(struct knod_bpf_map *knod_map,
+ struct knod_bpf_map_obj *knod_map_obj,
+ void *key)
+{
+ void *elems = knod_map->hash_elems_mem->kaddr;
+ unsigned int *queue = knod_map->queue_mem->kaddr;
+ unsigned int hash, elem_id, elem_size, cur;
+ struct knod_bpf_hash_elem_obj *e, *pe;
+ unsigned int *bucket;
+
+ hash = jhash((const void *)key, knod_map_obj->key_size,
+ knod_map_obj->meta.hmeta.hashrnd);
+ hash = hash & (knod_map_obj->meta.hmeta.n_buckets - 1);
+ bucket = (unsigned int *)&knod_map_obj->bucket[0];
+
+ elem_id = bucket[hash];
+ if (elem_id == KNOD_BPF_HASH_NEXT_END)
+ return -ENOENT;
+
+ elem_size = knod_map_obj->meta.hmeta.elem_size;
+
+ e = elems + (elem_id * elem_size);
+ pe = e;
+ while (1) {
+ if (!(e->next & KNOD_BPF_HASH_NEXT_DELETED) &&
+ !memcmp(&e->kv[0], (const unsigned char *)key,
+ knod_map_obj->key_size)) {
+ unsigned int e_next = e->next & KNOD_BPF_HASH_NEXT_MASK;
+ unsigned int del_id = ((void *)e - elems) / elem_size;
+
+ /* Unlink (GPU is paused - safe) */
+ if (pe != e)
+ pe->next = (pe->next &
+ KNOD_BPF_HASH_NEXT_DELETED) |
+ e_next;
+ else
+ bucket[hash] = e_next;
+
+ e->next = KNOD_BPF_HASH_NEXT_END;
+
+ /* Return elem to queue */
+ cur = knod_map_obj->meta.hmeta.cur;
+ queue[cur] = del_id;
+ knod_map_obj->meta.hmeta.cur = cur + 1;
+ return 0;
+ }
+ unsigned int real_next = e->next & KNOD_BPF_HASH_NEXT_MASK;
+
+ if (real_next == KNOD_BPF_HASH_NEXT_END)
+ return -ENOENT;
+ pe = e;
+ e = elems + (real_next * elem_size);
+ }
+
+ return -ENOENT;
+}
+
+static int knod_bpf_map_hash_get_first_key(struct bpf_offloaded_map *offmap,
+ void *nkey)
+{
+ struct knod_bpf_map *knod_map = (struct knod_bpf_map *)offmap->dev_priv;
+ struct knod_bpf_map_obj *knod_map_obj;
+ unsigned int *bucket, elem_size, i;
+ struct knod_bpf_hash_elem_obj *e;
+ void *elems;
+
+ knod_map_obj = knod_map->knod_map_obj;
+ bucket = (unsigned int *)&knod_map_obj->bucket[0];
+ elems = knod_map->hash_elems_mem->kaddr;
+
+ elem_size = knod_map_obj->meta.hmeta.elem_size;
+
+ for (i = 0; i < knod_map_obj->meta.hmeta.n_buckets; i++) {
+ unsigned int eid;
+
+ if (bucket[i] == KNOD_BPF_HASH_NEXT_END)
+ continue;
+ eid = bucket[i];
+ while (eid != KNOD_BPF_HASH_NEXT_END) {
+ e = elems + (eid * elem_size);
+ if (!(e->next & KNOD_BPF_HASH_NEXT_DELETED)) {
+ unsafe_memcpy(nkey, knod_bpf_hash_elem_kv(e),
+ knod_map_obj->key_size,
+ "knod hash elems are variable-sized GPU map records");
+ return 0;
+ }
+ eid = e->next & KNOD_BPF_HASH_NEXT_MASK;
+ }
+ }
+
+ return -ENOENT;
+}
+
+static int knod_bpf_map_hash_get_next_key(struct bpf_offloaded_map *offmap,
+ void *key, void *nkey)
+{
+ struct knod_bpf_map *knod_map = (struct knod_bpf_map *)offmap->dev_priv;
+ struct knod_bpf_map_obj *knod_map_obj;
+ unsigned int *bucket, elem_size, i;
+ struct knod_bpf_hash_elem_obj *e;
+ bool found = false;
+ unsigned int hash;
+ void *elems;
+
+ knod_map_obj = knod_map->knod_map_obj;
+
+ bucket = (unsigned int *)&knod_map_obj->bucket[0];
+ elems = knod_map->hash_elems_mem->kaddr;
+
+ hash = jhash((const void *)key, knod_map_obj->key_size,
+ knod_map_obj->meta.hmeta.hashrnd);
+ hash = hash & (knod_map_obj->meta.hmeta.n_buckets - 1);
+ elem_size = knod_map_obj->meta.hmeta.elem_size;
+
+ for (i = hash; i < knod_map_obj->meta.hmeta.n_buckets; i++) {
+ unsigned int eid;
+
+ if (bucket[i] == KNOD_BPF_HASH_NEXT_END)
+ continue;
+
+ eid = bucket[i];
+ while (eid != KNOD_BPF_HASH_NEXT_END) {
+ e = elems + (eid * elem_size);
+ if (!(e->next & KNOD_BPF_HASH_NEXT_DELETED)) {
+ if (found &&
+ memcmp(&e->kv[0],
+ (const unsigned char *)key,
+ knod_map_obj->key_size)) {
+ unsafe_memcpy(nkey,
+ knod_bpf_hash_elem_kv(e),
+ knod_map_obj->key_size,
+ "knod hash elems are variable-sized GPU map records");
+ return 0;
+ }
+ if (!memcmp(&e->kv[0],
+ (const unsigned char *)key,
+ knod_map_obj->key_size))
+ found = true;
+ }
+ eid = e->next & KNOD_BPF_HASH_NEXT_MASK;
+ }
+ }
+
+ return -ENOENT;
+}
+
+static void knod_bpf_map_free(struct knod_dev *knodev,
+ struct bpf_offloaded_map *offmap)
+{
+ struct knod_bpf_map *knod_map = offmap->dev_priv;
+ struct knod_bpf_priv *priv = knodev->accel->xdp.priv;
+
+ if (!knod_map)
+ return;
+ /*
+ * Defer the BO free: an in-flight prog dispatch may still reference
+ * this map's VRAM. Move it from bound_maps onto dead_maps under
+ * knodev->lock (the lock that guards the add); the worker reaps it
+ * from there after its next completion, by which point the in-flight
+ * dispatch on the old slot has retired (clean atomic flip).
+ */
+ mutex_lock(&knodev->lock);
+ list_del(&knod_map->list);
+ list_add(&knod_map->list, &priv->dead_maps);
+ mutex_unlock(&knodev->lock);
+ offmap->dev_priv = NULL;
+}
+
+static int __knod_bpf_map_lookup_elem(struct bpf_offloaded_map *offmap,
+ void *key, void *value)
+{
+ unsigned int idx = *(unsigned int *)key;
+ struct knod_bpf_map_obj *knod_map_obj;
+ struct knod_bpf_map *knod_map;
+ void *bucket;
+ u32 stride;
+ int i;
+
+ knod_map = (struct knod_bpf_map *)offmap->dev_priv;
+ if (!knod_map || !knod_map->mem || !knod_map->mem->kaddr ||
+ (knod_map->hash_elems_mem && !knod_map->hash_elems_mem->kaddr)) {
+ pr_err("knod_bpf: lookup on freed/invalid map (dev_priv=%p)\n",
+ offmap->dev_priv);
+ return -ENODEV;
+ }
+ knod_map_obj = knod_map->knod_map_obj;
+
+ if (knod_map_obj->map_type == BPF_MAP_TYPE_ARRAY) {
+ if (*(unsigned int *)key >= knod_map_obj->max_entries)
+ return -ENOENT;
+ bucket = knod_bpf_array_value_ptr(knod_map_obj, idx);
+
+ unsafe_memcpy(value, bucket, knod_map_obj->value_size,
+ "knod array values live in a variable-sized GPU map tail");
+ } else if (knod_map_obj->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
+ if (idx >= knod_map_obj->max_entries)
+ return -ENOENT;
+ stride = round_up(knod_map_obj->value_size, 8);
+ bucket = &knod_map_obj->bucket[0];
+ bucket += (idx * knod_map_obj->value_size);
+ for (i = 0; i < knod_map_obj->meta.ameta.n_instances; i++)
+ unsafe_memcpy(value + i * stride,
+ bucket + i *
+ knod_map_obj->meta.ameta
+ .per_instance_size,
+ knod_map_obj->value_size,
+ "knod percpu array values live in a variable-sized GPU map tail");
+ } else if (knod_map_obj->map_type == BPF_MAP_TYPE_HASH) {
+ return knod_bpf_map_hash_lookup_elem(knod_map, knod_map_obj,
+ key, value);
+ }
+
+ return 0;
+}
+
+static void knod_bpf_map_op_begin(struct knod_bpf_priv *priv)
+{
+ /*
+ * Serialize concurrent map ops but do NOT park the worker: stopping it
+ * mid-flight strands the in-flight dispatch and stalls the GPU compute
+ * queue. A map value updated while the GPU reads it may be seen torn,
+ * which is a transient inconsistency the BPF prog tolerates.
+ */
+ mutex_lock(&priv->map_op_lock);
+}
+
+static void knod_bpf_map_op_end(struct knod_bpf_priv *priv)
+{
+ knod_bpf_gpu_mem_fence(priv);
+ mutex_unlock(&priv->map_op_lock);
+}
+
+static int __knod_bpf_map_update_elem(struct bpf_offloaded_map *offmap,
+ void *key, void *value, u64 flags)
+{
+ struct knod_bpf_map *knod_map = (struct knod_bpf_map *)offmap->dev_priv;
+ struct knod_bpf_map_obj *knod_map_obj;
+ struct knod_bpf_priv *priv;
+ unsigned int idx = *(unsigned int *)key;
+ struct knod_dev *knodev;
+ void *bucket;
+ u32 stride;
+ int i;
+
+ if (!knod_map || !knod_map->mem || !knod_map->mem->kaddr)
+ return -ENODEV;
+ knod_map_obj = knod_map->knod_map_obj;
+ priv = knod_map->priv;
+ knodev = priv->knodev;
+ if (knod_map_obj->map_type == BPF_MAP_TYPE_ARRAY) {
+ if (idx >= knod_map_obj->max_entries)
+ return -ENOENT;
+
+ bucket = knod_bpf_array_value_ptr(knod_map_obj, idx);
+ unsafe_memcpy(bucket, value, knod_map_obj->value_size,
+ "knod array values live in a variable-sized GPU map tail");
+ knod_bpf_gpu_mem_fence(priv);
+ return 0;
+ } else if (knod_map_obj->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
+ if (idx >= knod_map_obj->max_entries)
+ return -ENOENT;
+ stride = round_up(knod_map_obj->value_size, 8);
+ bucket = &knod_map_obj->bucket[0];
+ bucket += (idx * knod_map_obj->value_size);
+ for (i = 0; i < knod_map_obj->meta.ameta.n_instances; i++)
+ unsafe_memcpy(bucket + i *
+ knod_map_obj->meta.ameta
+ .per_instance_size,
+ value + i * stride,
+ knod_map_obj->value_size,
+ "knod percpu array values live in a variable-sized GPU map tail");
+ knod_bpf_gpu_mem_fence(priv);
+ return 0;
+ } else if (knod_map_obj->map_type == BPF_MAP_TYPE_HASH) {
+ int ret = -ENOENT;
+
+ mutex_lock(&knodev->lock);
+ list_for_each_entry(knod_map, &knodev->accel->xdp.bound_maps,
+ list) {
+ if (knod_map->knod_map_obj == knod_map_obj) {
+ mutex_unlock(&knodev->lock);
+ knod_bpf_map_op_begin(priv);
+ ret = knod_bpf_map_hash_update_elem(knod_map,
+ knod_map_obj,
+ key, value);
+ knod_bpf_map_op_end(priv);
+ return ret;
+ }
+ }
+ mutex_unlock(&knodev->lock);
+ }
+
+ return -ENOENT;
+}
+
+static int __knod_bpf_map_delete_elem(struct bpf_offloaded_map *offmap,
+ void *key)
+{
+ struct knod_bpf_map *knod_map = (struct knod_bpf_map *)offmap->dev_priv;
+ struct knod_bpf_map_obj *knod_map_obj;
+ struct knod_bpf_priv *priv;
+ int ret;
+
+ if (!knod_map || !knod_map->mem || !knod_map->mem->kaddr)
+ return -ENODEV;
+ knod_map_obj = knod_map->knod_map_obj;
+ priv = knod_map->priv;
+
+ if (knod_map_obj->map_type == BPF_MAP_TYPE_ARRAY ||
+ knod_map_obj->map_type == BPF_MAP_TYPE_PERCPU_ARRAY)
+ return 0;
+ else if (knod_map_obj->map_type == BPF_MAP_TYPE_HASH) {
+ knod_bpf_map_op_begin(priv);
+ ret = knod_bpf_map_hash_delete_elem(knod_map, knod_map_obj,
+ key);
+ knod_bpf_map_op_end(priv);
+ return ret;
+ }
+
+ return -ENOENT;
+}
+
+static void knod_bpf_map_gc_process(struct knod_bpf_map *knod_map)
+{
+ struct knod_bpf_map_obj *knod_map_obj = knod_map->knod_map_obj;
+ unsigned int *gc_list = knod_map->gc_mem->kaddr;
+ unsigned int *queue = knod_map->queue_mem->kaddr;
+ void *elems = knod_map->hash_elems_mem->kaddr;
+ unsigned int *bucket = (unsigned int *)&knod_map_obj->bucket[0];
+ unsigned int elem_size = knod_map_obj->meta.hmeta.elem_size;
+ unsigned int gc_count, cur, i;
+
+ gc_count = READ_ONCE(knod_map_obj->meta.hmeta.gc_count);
+ if (!gc_count)
+ return;
+
+ for (i = 0; i < gc_count; i++) {
+ unsigned int del_id = gc_list[i];
+ struct knod_bpf_hash_elem_obj *del_elem =
+ elems + (del_id * elem_size);
+ unsigned int hash, eid;
+ struct knod_bpf_hash_elem_obj *e, *pe;
+
+ hash = jhash(&del_elem->kv[0], knod_map_obj->key_size,
+ knod_map_obj->meta.hmeta.hashrnd);
+ hash = hash & (knod_map_obj->meta.hmeta.n_buckets - 1);
+
+ eid = bucket[hash];
+ pe = NULL;
+ while (eid != KNOD_BPF_HASH_NEXT_END) {
+ e = elems + (eid * elem_size);
+ if (e == del_elem) {
+ unsigned int next = e->next &
+ KNOD_BPF_HASH_NEXT_MASK;
+ if (pe)
+ pe->next =
+ (pe->next &
+ KNOD_BPF_HASH_NEXT_DELETED) |
+ next;
+ else
+ bucket[hash] = next;
+
+ e->next = KNOD_BPF_HASH_NEXT_END;
+
+ cur = knod_map_obj->meta.hmeta.cur;
+ queue[cur] = del_id;
+ knod_map_obj->meta.hmeta.cur = cur + 1;
+ break;
+ }
+ pe = e;
+ eid = e->next & KNOD_BPF_HASH_NEXT_MASK;
+ }
+ }
+
+ WRITE_ONCE(knod_map_obj->meta.hmeta.gc_count, 0);
+}
+
+/*
+ * Per-loop map maintenance, run from the worker loop head (outside any
+ * rcu_read_lock_bh, since knod_free_mem() may sleep). All bound_maps access
+ * is serialized under knodev->lock -- the same lock map_alloc/map_free use:
+ * GC live HASH maps, then reap maps that detach moved onto dead_maps. The
+ * worker only reaches here after completing the previous dispatch, so the
+ * clean atomic flip guarantees the GPU no longer reads a reaped map's BOs.
+ */
+#define KNOD_BPF_MAPS_TICK_INTERVAL 65536
+
+static void knod_bpf_maps_tick(struct knod_bpf_priv *priv)
+{
+ struct knod_dev *knodev = priv->knodev;
+ struct knod_bpf_map *knod_map, *tmp;
+ LIST_HEAD(reap);
+
+ if (list_empty(&knodev->accel->xdp.bound_maps) &&
+ list_empty(&priv->dead_maps))
+ return;
+
+ if (list_empty(&priv->dead_maps) &&
+ (++priv->maps_tick_skip & (KNOD_BPF_MAPS_TICK_INTERVAL - 1)))
+ return;
+
+ mutex_lock(&knodev->lock);
+ list_for_each_entry(knod_map, &knodev->accel->xdp.bound_maps, list) {
+ if (knod_map->knod_map_obj->map_type == BPF_MAP_TYPE_HASH)
+ knod_bpf_map_gc_process(knod_map);
+ }
+ list_splice_init(&priv->dead_maps, &reap);
+ mutex_unlock(&knodev->lock);
+
+ list_for_each_entry_safe(knod_map, tmp, &reap, list) {
+ if (knod_map->gc_mem)
+ knod_free_mem(priv->knod, knod_map->gc_mem);
+ if (knod_map->queue_mem)
+ knod_free_mem(priv->knod, knod_map->queue_mem);
+ if (knod_map->hash_elems_mem)
+ knod_free_mem(priv->knod, knod_map->hash_elems_mem);
+ if (knod_map->mem)
+ knod_free_mem(priv->knod, knod_map->mem);
+ kfree(knod_map);
+ }
+}
+
+/* Completion mode: 0 = event (default, sleep on the AQL signal interrupt),
+ * 1 = poll (busy-spin the signal value). Selectable via debugfs.
+ */
+static bool knod_bpf_poll_mode;
+
+/* Max spacing (microseconds) between dispatch-ahead submissions. Once a
+ * dispatch is in flight the worker waits up to this long before submitting
+ * the next so it batches the packets arriving meanwhile, letting inflight
+ * grow >= 2 without degenerating into one-packet dispatches. This is a
+ * ceiling only: an empty pipe submits at once to keep the GPU fed, and a
+ * completed dispatch is always retired without waiting. To actually build
+ * depth the value must be below the GPU execution time of a dispatch.
+ * 0 disables spacing (submit as soon as the ring has anything).
+ */
+static u32 knod_bpf_dispatch_delay_us = 20;
+
+static void knod_bpf_wait_event(struct knod_bpf_priv *priv)
+{
+ struct kfd_event_data events = {
+ .event_id = priv->knod->aql_event[0].id,
+ };
+ u32 timeout_ms = knod_bpf_expire;
+ u32 wait_result;
+
+ knod_wait_on_events(priv->knod->process, 1, &events, true,
+ &timeout_ms, &wait_result);
+}
+
+static bool knod_bpf_submit_work(struct knod_bpf_priv *priv)
+{
+ struct knod_bpf_work_sq *sqw;
+ struct knod_bpf_stats *stats = &priv->stats;
+ ktime_t dispatch_start;
+
+ if (priv->inflight_cnt >= KNOD_BPF_INFLIGHT)
+ return false;
+
+ /* Pace dispatch-ahead so the next dispatch batches the packets that
+ * arrive during this window instead of firing one-packet dispatches.
+ * An empty pipe skips the wait so the GPU is never left idle.
+ */
+ if (priv->inflight_cnt &&
+ ktime_before(ktime_get(), priv->next_dispatch_time))
+ return false;
+
+ if (static_branch_unlikely(&knod_stats_key))
+ dispatch_start = ktime_get();
+
+ sqw = knod_prepare_bpf(priv);
+ if (!sqw)
+ return false;
+
+ if (static_branch_unlikely(&knod_stats_key)) {
+ u64 dns = ktime_to_ns(ktime_sub(ktime_get(),
+ dispatch_start));
+
+ stats->dispatch_total_ns += dns;
+ stats->dispatch_count++;
+ if (dns > stats->dispatch_max_ns)
+ stats->dispatch_max_ns = dns;
+ }
+
+ knod_submit_bpf(priv, sqw);
+ priv->inflight[priv->inflight_cnt++] = sqw;
+ priv->next_dispatch_time =
+ ktime_add_us(ktime_get(),
+ READ_ONCE(knod_bpf_dispatch_delay_us));
+ return true;
+}
+
+static void knod_bpf_record_completion(struct knod_bpf_priv *priv,
+ struct knod_bpf_work_sq *sqw)
+{
+ struct knod_bpf_stats *stats = &priv->stats;
+ u64 ns;
+ int bucket;
+
+ if (!static_branch_unlikely(&knod_stats_key))
+ return;
+
+ ns = ktime_to_ns(ktime_sub(ktime_get(), sqw->dispatch_time));
+ stats->completion_total_ns += ns;
+ stats->completion_count++;
+
+ if (ns > stats->completion_max_ns)
+ stats->completion_max_ns = ns;
+
+ if (ns < 1000)
+ bucket = 0;
+ else
+ bucket = min(ilog2(ns / 1000) + 1,
+ KNOD_LAT_BUCKETS - 1);
+ stats->completion_hist[bucket]++;
+}
+
+static bool knod_bpf_poll_complete(struct knod_bpf_priv *priv,
+ struct knod_bpf_work_sq *sqw)
+{
+ struct amd_signal *signal;
+
+ if (!sqw)
+ return false;
+
+ signal = (struct amd_signal *)
+ priv->knod->kaql[0].queue_signal->kaddr;
+
+ if (sqw->sigval > READ_ONCE(signal->value)) {
+ knod_bpf_record_completion(priv, sqw);
+ return true;
+ }
+
+ if (time_after(jiffies, sqw->expire)) {
+ pr_warn_ratelimited("knod_bpf: poll expire (sigval=%lld signal=%lld expire_ms=%u)\n",
+ sqw->sigval, READ_ONCE(signal->value), knod_bpf_expire);
+ knod_bpf_record_completion(priv, sqw);
+ return true;
+ }
+
+ return false;
+}
+
+static void knod_bpf_schedule_pending_napi(struct knod_bpf_priv *priv)
+{
+ struct knod_dev *knodev = priv->knodev;
+ int qi;
+
+ for (qi = 0; qi < priv->nr_works; qi++) {
+ if (spsc_pending(&knodev->wpriv[qi].spsc_bds) &&
+ knodev->wpriv[qi].napi)
+ napi_schedule(knodev->wpriv[qi].napi);
+ }
+}
+
+static int knod_bpf_worker(void *arg)
+{
+ struct knod_bpf_priv *priv = arg;
+ struct knod_bpf_work_sq *sqw;
+ bool progressed;
+
+ while (!kthread_should_stop()) {
+ if (kthread_should_park()) {
+ knod_bpf_drain_worker(priv);
+ kthread_parkme();
+ continue;
+ }
+
+ knod_bpf_maps_tick(priv);
+
+ progressed = false;
+
+ rcu_read_lock_bh();
+ /* Retire completed dispatches oldest-first: the signal is
+ * monotonic so inflight[0] finishes before inflight[1..].
+ */
+ while (priv->inflight_cnt &&
+ knod_bpf_poll_complete(priv, priv->inflight[0])) {
+ sqw = priv->inflight[0];
+ if (--priv->inflight_cnt)
+ memmove(priv->inflight, priv->inflight + 1,
+ priv->inflight_cnt *
+ sizeof(priv->inflight[0]));
+ priv->inflight[priv->inflight_cnt] = NULL;
+ knod_complete_acquire(priv, sqw);
+ knod_complete_napi(priv, sqw);
+ progressed = true;
+ }
+
+ /* Keep the pipe full: dispatch ahead up to KNOD_BPF_INFLIGHT.
+ * Staging self-limits, so this stops once the ring is drained.
+ */
+ while (knod_bpf_submit_work(priv))
+ progressed = true;
+ rcu_read_unlock_bh();
+
+ if (!priv->inflight_cnt) {
+ knod_bpf_schedule_pending_napi(priv);
+ usleep_range(100, 200);
+ } else if (!progressed) {
+ /* Room to dispatch ahead but the pacing window has not
+ * opened yet: spin so the next submit fires on time and
+ * a completion is retired the instant it lands. Block
+ * on the event only when the pipe is full (nothing to
+ * submit) or spacing is disabled.
+ */
+ if (priv->inflight_cnt < KNOD_BPF_INFLIGHT &&
+ ktime_before(ktime_get(), priv->next_dispatch_time))
+ cpu_relax();
+ else if (knod_bpf_poll_mode)
+ cpu_relax();
+ else
+ knod_bpf_wait_event(priv);
+ }
+ }
+
+ return 0;
+}
+
+static void knod_bpf_sq_init(struct knod_bpf_priv *priv)
+{
+ struct knod_bpf_work_sq *sqw;
+ int i;
+
+ priv->worker_task = NULL;
+ priv->inflight_cnt = 0;
+ INIT_LIST_HEAD(&priv->free_list_sqw);
+
+ for (i = 0; i < 32; i++) {
+ sqw = kvzalloc_obj(struct knod_bpf_work_sq, GFP_KERNEL);
+ if (!sqw)
+ continue;
+
+ sqw->param = knod_alloc_mem(priv->knod,
+ sizeof(struct knod_bpf_param),
+ KFD_IOC_ALLOC_MEM_FLAGS_GTT |
+ KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE |
+ KFD_IOC_ALLOC_MEM_FLAGS_COHERENT);
+ if (!sqw->param) {
+ kvfree(sqw);
+ continue;
+ }
+ memset(sqw->param->kaddr, 0, sizeof(struct knod_bpf_param));
+ INIT_LIST_HEAD(&sqw->list);
+ list_add(&sqw->list, &priv->free_list_sqw);
+ sqw->backlogs = 0;
+ }
+}
+
+static void knod_bpf_free_sqw(struct knod_bpf_priv *priv,
+ struct knod_bpf_work_sq *sqw)
+{
+ if (!sqw)
+ return;
+
+ knod_free_mem(priv->knod, sqw->param);
+ kfree(sqw);
+}
+
+static void knod_bpf_free_sqw_list(struct knod_bpf_priv *priv,
+ struct list_head *head)
+{
+ struct knod_bpf_work_sq *sqw, *tmp;
+
+ list_for_each_entry_safe(sqw, tmp, head, list) {
+ list_del(&sqw->list);
+ knod_bpf_free_sqw(priv, sqw);
+ }
+}
+
+static void knod_bpf_sq_exit(struct knod_bpf_priv *priv)
+{
+ if (!priv->knod)
+ return;
+
+ knod_bpf_stop_worker(priv);
+ knod_bpf_drain(priv);
+
+ knod_bpf_free_sqw_list(priv, &priv->free_list_sqw);
+ priv->inflight_cnt = 0;
+}
+
+static void knod_priv_exit(struct knod_bpf_priv *priv)
+{
+ struct knod_dev *knodev = priv->knodev;
+ struct knod_bpf_map *knod_map, *tmp;
+ LIST_HEAD(reap);
+
+ knod_bpf_sq_exit(priv);
+
+ /*
+ * The dispatch worker is not stopped until the next feature registers
+ * its own worker, so it may still be running knod_bpf_maps_tick() here.
+ * Serialize under knodev->lock and splice both lists to a local one:
+ * whichever side splices first frees them, the other sees them empty.
+ * Free outside the lock since knod_free_mem() may sleep.
+ */
+ mutex_lock(&knodev->lock);
+ list_splice_init(&knodev->accel->xdp.bound_maps, &reap);
+ list_splice_init(&priv->dead_maps, &reap);
+ mutex_unlock(&knodev->lock);
+
+ list_for_each_entry_safe(knod_map, tmp, &reap, list) {
+ if (knod_map->gc_mem)
+ knod_free_mem(priv->knod, knod_map->gc_mem);
+ if (knod_map->queue_mem)
+ knod_free_mem(priv->knod, knod_map->queue_mem);
+ if (knod_map->hash_elems_mem)
+ knod_free_mem(priv->knod, knod_map->hash_elems_mem);
+ if (knod_map->mem)
+ knod_free_mem(priv->knod, knod_map->mem);
+ kfree(knod_map);
+ }
+
+ kfree(priv->prog_buf);
+ kfree(priv->pass_prog_buf);
+ if (priv->pass_knod_prog) {
+ knod_prog_free(priv->pass_knod_prog);
+ priv->pass_knod_prog = NULL;
+ }
+ /* kernels[] are owned by knod (freed in knod_release_ctx), not here */
+ if (priv->pass_meta_buf)
+ knod_free_mem(priv->knod, priv->pass_meta_buf);
+}
+
+static int knod_priv_init(struct knod_bpf_priv *priv)
+{
+ struct knod_dev *knodev = priv->knodev;
+ int pass_meta_buf_size;
+ int index;
+
+ priv->prog = NULL;
+ mutex_init(&priv->map_op_lock);
+ INIT_LIST_HEAD(&priv->dead_maps);
+ priv->maps_tick_skip = 0;
+
+ priv->nr_works = knod_bpf_active_rxq_count(knodev->netdev);
+ if (!priv->nr_works) {
+ pr_warn("knod_bpf: no active RX queues for %s\n",
+ knodev->netdev ? knodev->netdev->name : "<null>");
+ return -EINVAL;
+ }
+
+ priv->prog_buf = kzalloc(KNOD_BPF_PROG_BUF_SIZE, GFP_KERNEL);
+ if (!priv->prog_buf)
+ return -ENOMEM;
+
+ for (index = 0; index < priv->nr_works; index++)
+ priv->queue_base_gaddr[index] = priv->knod->buf[index]->gaddr;
+
+ /* Per-queue PASS slot count; sizes the shader pass_meta_buf below.
+ * At most one PASS packet per dispatched slot, i.e. batch_size.
+ */
+ priv->pass_pkts_per_queue = knod_bpf_batch_size(priv);
+
+ /* Allocate GTT buffer for per-queue shader PASS copy */
+ pass_meta_buf_size = priv->nr_works * priv->pass_pkts_per_queue *
+ KNOD_PASS_SLOT_SIZE;
+ priv->pass_meta_buf = knod_alloc_mem(priv->knod, pass_meta_buf_size,
+ KFD_IOC_ALLOC_MEM_FLAGS_GTT |
+ KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE |
+ KFD_IOC_ALLOC_MEM_FLAGS_COHERENT);
+ if (IS_ERR(priv->pass_meta_buf)) {
+ pr_warn("KNOD: failed to allocate pass_meta_buf\n");
+ priv->pass_meta_buf = NULL;
+ knod_priv_exit(priv);
+ return -ENOMEM;
+ }
+ pr_debug("KNOD: pass_meta_buf gaddr=0x%llx..0x%llx size=%d nr_q=%d pass_pkts_per_queue=%u\n",
+ priv->pass_meta_buf->gaddr,
+ priv->pass_meta_buf->gaddr + pass_meta_buf_size,
+ priv->pass_meta_buf->size, priv->nr_works,
+ priv->pass_pkts_per_queue);
+
+ /* GPU->host delivery pages come from the framework per-queue page_pool
+ * (knodev->wpriv[q].pass_pool): the producer allocs from it and the
+ * NAPI drain recycles, so no per-feature delivery BO is allocated here.
+ */
+
+ knod_bpf_sq_init(priv);
+
+ return 0;
+}
+
+static struct knod_bpf_priv *__knod_accel_xdp_init(struct knod_accel *accel,
+ struct knod_dev *knodev)
+{
+ struct knod *knod = (struct knod *)knodev->accel->priv;
+ struct knod_bpf_priv *priv;
+
+ priv = kzalloc_obj(struct knod_bpf_priv, GFP_KERNEL);
+ if (!priv)
+ return ERR_PTR(-ENOMEM);
+
+ INIT_LIST_HEAD(&priv->list);
+ if (knod_bpf_workgroups % 64) {
+ knod_bpf_workgroups /= 64;
+ knod_bpf_workgroups++;
+ knod_bpf_workgroups *= 64;
+ }
+
+ if (knod_bpf_workgroups < KNOD_BPF_WORKGROUPS_MIN ||
+ knod_bpf_workgroups > KNOD_BPF_WORKGROUPS_MAX)
+ knod_bpf_workgroups = KNOD_BPF_WORKGROUPS_DEFAULT;
+
+ if (knod_bpf_expire < KNOD_BPF_EXPIRE_MIN ||
+ knod_bpf_expire > KNOD_BPF_EXPIRE_MAX)
+ knod_bpf_expire = KNOD_BPF_EXPIRE_DEFAULT;
+ pr_debug("workgroup size %d\n", knod_bpf_workgroups);
+ pr_debug("expire time = %dms\n", knod_bpf_expire);
+ pr_debug("packet cache = %d", knod_bpf_pkt_cache);
+
+ INIT_LIST_HEAD(&accel->xdp.bound_maps);
+ accel->flags |= KNOD_FLAGS_XDP;
+ accel->xdp.priv = priv;
+ list_add(&priv->list, &priv_list);
+
+ priv->knod = knod;
+ priv->accel = accel;
+ priv->knodev = knodev;
+ priv->dev = knodev->netdev;
+
+ priv->isa_version = knod->isa_version;
+
+ /*
+ * Only permanent per-attach state is set up here; the GPU compute
+ * buffers (knod_priv_init/kfd_kernel_init) are allocated by
+ * ->activate() when the BPF feature is selected.
+ */
+
+ return priv;
+}
+
+/* Feature select: allocate the BPF GPU compute resources. */
+static int knod_bpf_activate(struct knod_dev *knodev)
+{
+ struct knod_accel *accel = knodev->accel;
+ struct knod_bpf_priv *priv = accel->xdp.priv;
+ struct knod *knod = accel->priv;
+
+ /*
+ * Pin the module while BPF is the selected feature: the core calls
+ * into these ops, so it must not be unloaded until feature->none.
+ * (No-op when built in - THIS_MODULE is NULL.)
+ */
+ if (!try_module_get(THIS_MODULE))
+ return -ENODEV;
+
+ if (knod_priv_init(priv)) {
+ WARN_ON_ONCE(1);
+ module_put(THIS_MODULE);
+ return -EINVAL;
+ }
+ if (kfd_kernel_init(knod, priv)) {
+ knod_priv_exit(priv);
+ module_put(THIS_MODULE);
+ return -ENOMEM;
+ }
+
+ priv->start = 0;
+ return 0;
+}
+
+/* Feature deselect: free the BPF GPU compute resources. */
+static void knod_bpf_deactivate(struct knod_dev *knodev)
+{
+ struct knod_bpf_priv *priv = knodev->accel->xdp.priv;
+
+ knod_priv_exit(priv);
+ module_put(THIS_MODULE);
+}
+
+/* True while a user XDP prog or offloaded map is still bound to this accel. */
+static bool knod_bpf_busy(struct knod_dev *knodev)
+{
+ struct knod_accel *accel = knodev->accel;
+ struct knod_bpf_priv *priv = accel->xdp.priv;
+
+ if (!priv)
+ return false;
+ return READ_ONCE(priv->prog) || !list_empty(&accel->xdp.bound_maps);
+}
+
+static void __knod_accel_xdp_exit(struct knod_accel *accel,
+ struct knod_bpf_priv *priv)
+{
+ /* GPU compute buffers are freed by ->deactivate(); free the rest. */
+ memset(&accel->xdp, 0, sizeof(struct knod_accel_xdp));
+ accel->flags &= ~KNOD_FLAGS_XDP;
+ list_del(&priv->list);
+ kfree(priv);
+}
+
+static struct knod_insn_meta *knod_bpf_goto_meta(struct knod_prog *knod_prog,
+ struct knod_insn_meta *meta,
+ unsigned int insn_idx)
+{
+ unsigned int forward, backward, i;
+
+ backward = meta->bpf_insn_idx - insn_idx;
+ forward = insn_idx - meta->bpf_insn_idx;
+
+ if (min(forward, backward) > knod_prog->n_insns - insn_idx - 1) {
+ backward = knod_prog->n_insns - insn_idx - 1;
+ meta = knod_prog_last_meta(knod_prog);
+ }
+ if (min(forward, backward) > insn_idx && backward > insn_idx) {
+ forward = insn_idx;
+ meta = knod_prog_first_meta(knod_prog);
+ }
+
+ if (forward < backward)
+ for (i = 0; i < forward; i++)
+ meta = knod_meta_next(meta);
+ else
+ for (i = 0; i < backward; i++)
+ meta = knod_meta_prev(meta);
+
+ return meta;
+}
+
+static int knod_bpf_check_stack_access(struct knod_prog *knod_prog,
+ struct knod_insn_meta *meta,
+ const struct bpf_reg_state *reg,
+ struct bpf_verifier_env *env)
+{
+ s32 old_off, new_off;
+
+ if (reg->frameno != env->cur_state->curframe)
+ meta->flags |= FLAG_INSN_PTR_CALLER_STACK_FRAME;
+
+ if (!tnum_is_const(reg->var_off)) {
+ knod_jit_dbg(" variable ptr stack access\n");
+ return -EINVAL;
+ }
+
+ if (meta->ptr.type == NOT_INIT)
+ return 0;
+
+ old_off = meta->ptr.var_off.value;
+ new_off = reg->var_off.value;
+
+ meta->ptr_not_const |= old_off != new_off;
+
+ if (!meta->ptr_not_const)
+ return 0;
+
+ if (old_off % 4 == new_off % 4)
+ return 0;
+
+ knod_jit_dbg(" stack access changed location was:%d is:%d\n",
+ old_off, new_off);
+ return -EINVAL;
+}
+
+static struct knod_insn_meta *
+knod_bpf_lookup_prev_meta_by_dreg(struct knod_prog *knod_prog,
+ struct knod_insn_meta *meta,
+ int dreg_id)
+{
+ list_for_each_entry_continue_reverse(meta, &knod_prog->insns, l) {
+ if (!is_mbpf_alu(meta) &&
+ !is_mbpf_load(meta) &&
+ !is_mbpf_store(meta))
+ continue;
+ if (meta->insn.dst_reg == dreg_id)
+ return meta;
+ }
+
+ return NULL;
+}
+
+static int knod_bpf_check_ptr(struct knod_prog *knod_prog,
+ struct knod_insn_meta *meta,
+ struct bpf_verifier_env *env, u8 reg_no)
+{
+ const struct bpf_reg_state *reg = cur_regs(env) + reg_no;
+ int err;
+
+ if (reg->type != PTR_TO_CTX &&
+ reg->type != PTR_TO_STACK &&
+ reg->type != PTR_TO_MAP_VALUE &&
+ reg->type != PTR_TO_PACKET) {
+ knod_jit_dbg(" unsupported ptr type: %d\n", reg->type);
+ return -EINVAL;
+ }
+
+ if (reg->type == PTR_TO_STACK) {
+ err = knod_bpf_check_stack_access(knod_prog, meta, reg, env);
+ if (err)
+ return err;
+ }
+
+ if (meta->ptr.type != NOT_INIT && meta->ptr.type != reg->type) {
+ knod_jit_dbg(" ptr type changed for instruction %d -> %d\n",
+ meta->ptr.type,
+ reg->type);
+ return -EINVAL;
+ }
+
+ meta->ptr = *reg;
+
+ return 0;
+}
+
+static int knod_bpf_update_ptr_off(struct knod_prog *knod_prog,
+ struct knod_insn_meta *meta,
+ struct bpf_verifier_env *env)
+{
+ struct knod_bpf_reg_state *sreg = &meta->sreg;
+ struct knod_bpf_reg_state *dreg = &meta->dreg;
+ struct knod_insn_meta *prev_meta;
+
+ if (is_mbpf_load(meta)) {
+ if (sreg->reg.type == PTR_TO_PACKET ||
+ sreg->reg.type == PTR_TO_STACK) {
+ prev_meta = knod_bpf_lookup_prev_meta_by_dreg(
+ knod_prog, meta, meta->insn.src_reg);
+ if (!prev_meta) {
+ knod_jit_dbg(" Invalid\n");
+ return -EINVAL;
+ }
+ if (sreg->reg.type == PTR_TO_PACKET)
+ sreg->packet_off = prev_meta->dreg.packet_off;
+ else
+ sreg->stack_off = prev_meta->dreg.stack_off;
+ }
+ } else if (is_mbpf_store(meta)) {
+ if (dreg->reg.type == PTR_TO_PACKET ||
+ dreg->reg.type == PTR_TO_PACKET) {
+ prev_meta = knod_bpf_lookup_prev_meta_by_dreg(
+ knod_prog, meta, meta->insn.dst_reg);
+ if (!prev_meta) {
+ knod_jit_dbg(" Invalid\n");
+ return -EINVAL;
+ }
+ if (dreg->reg.type == PTR_TO_PACKET)
+ dreg->packet_off = prev_meta->dreg.packet_off;
+ else
+ dreg->stack_off = prev_meta->dreg.stack_off;
+ }
+ }
+
+ return 0;
+}
+
+static int knod_bpf_check_store(struct knod_prog *knod_prog,
+ struct knod_insn_meta *meta,
+ struct bpf_verifier_env *env)
+{
+ const struct bpf_reg_state *reg = cur_regs(env) + meta->insn.dst_reg;
+
+ if (reg->type == PTR_TO_CTX) {
+ if (knod_prog->type == BPF_PROG_TYPE_XDP) {
+ /* XDP ctx accesses must be 4B in size */
+ switch (meta->insn.off) {
+ case offsetof(struct xdp_md, rx_queue_index):
+ knod_jit_dbg(" queue selection not supported by FW\n");
+ return -EOPNOTSUPP;
+ }
+ }
+ knod_jit_dbg(" unsupported store to context field\n");
+ return -EOPNOTSUPP;
+ }
+
+ return knod_bpf_check_ptr(knod_prog, meta, env, meta->insn.dst_reg);
+}
+
+/* NOTE:
+ * knod_bpf_lookup_prev_meta_by_dreg(), src_reg vs dst_reg ???????/
+ */
+static int knod_bpf_check_alu(struct knod_prog *knod_prog,
+ struct knod_insn_meta *meta,
+ struct bpf_verifier_env *env)
+{
+ const struct bpf_reg_state *sreg = cur_regs(env) + meta->insn.src_reg;
+ const struct bpf_reg_state *dreg = cur_regs(env) + meta->insn.dst_reg;
+ struct knod_bpf_reg_state *ksreg = &meta->sreg;
+ struct knod_bpf_reg_state *kdreg = &meta->dreg;
+ struct knod_insn_meta *prev_meta;
+ int imm;
+
+ meta->umin_src = min(meta->umin_src, reg_umin(sreg));
+ meta->umax_src = max(meta->umax_src, reg_umax(sreg));
+ meta->umin_dst = min(meta->umin_dst, reg_umin(dreg));
+ meta->umax_dst = max(meta->umax_dst, reg_umax(dreg));
+
+ /* AMDGPU doesn't have divide instructions, we support divide by
+ * constant through reciprocal multiplication. Given NFP support
+ * multiplication no bigger than u32, we'd require divisor and dividend
+ * no bigger than that as well.
+ *
+ * Also eBPF doesn't support signed divide and has enforced this on C
+ * language level by failing compilation. However LLVM assembler hasn't
+ * enforced this, so it is possible for negative constant to leak in as
+ * a BPF_K operand through assembly code, we reject such cases as well.
+ */
+ if (is_mbpf_div(meta)) {
+ if (meta->umax_dst > U32_MAX) {
+ knod_jit_dbg(" dividend is not within u32 value range\n");
+ return -EINVAL;
+ }
+ if (mbpf_src(meta) == BPF_X) {
+ if (meta->umin_src != meta->umax_src) {
+ knod_jit_dbg(" divisor is not constant\n");
+ return -EINVAL;
+ }
+ if (meta->umax_src > U32_MAX) {
+ knod_jit_dbg(" divisor is not within u32 value range\n");
+ return -EINVAL;
+ }
+ }
+ if (mbpf_src(meta) == BPF_K && meta->insn.imm < 0) {
+ knod_jit_dbg(" divide by negative constant is not supported\n");
+ return -EINVAL;
+ }
+ }
+
+ if (dreg->type == PTR_TO_STACK) {
+ imm = meta->insn.imm;
+
+ switch (meta->insn.code) {
+ /* ALU
+ * If a destination register contains a pointer of STACK,
+ * offset should not be minus.
+ */
+ case BPF_ALU | BPF_MOV | BPF_X:
+ case BPF_ALU64 | BPF_MOV | BPF_X:
+ //r[d] = r[s];
+ kdreg->stack_off = ksreg->stack_off;
+ break;
+ case BPF_ALU | BPF_MOV | BPF_K:
+ case BPF_ALU64 | BPF_MOV | BPF_K:
+ //r[d] = imm;
+ kdreg->stack_off = ksreg->stack_off;
+ break;
+ case BPF_ALU | BPF_XOR | BPF_X:
+ case BPF_ALU64 | BPF_XOR | BPF_X:
+ //r[d] ^= r[s];
+ knod_jit_dbg(" PTR_TO_STACK with BPF_X is not supported\n");
+ return -EINVAL;
+ case BPF_ALU | BPF_XOR | BPF_K:
+ case BPF_ALU64 | BPF_XOR | BPF_K:
+ //r[d] ^= imm;
+ prev_meta = knod_bpf_lookup_prev_meta_by_dreg(
+ knod_prog, meta, meta->insn.dst_reg);
+ if (!prev_meta) {
+ knod_jit_dbg(" Invalid\n");
+ return -EINVAL;
+ }
+ kdreg->stack_off = prev_meta->dreg.stack_off ^ imm;
+ break;
+ case BPF_ALU | BPF_MOD | BPF_X:
+ case BPF_ALU64 | BPF_MOD | BPF_X:
+ //r[d] %= r[s];
+ knod_jit_dbg(" PTR_TO_STACK with BPF_X is not supported\n");
+ return -EINVAL;
+ case BPF_ALU | BPF_MOD | BPF_K:
+ case BPF_ALU64 | BPF_MOD | BPF_K:
+ //r[d] %= imm;
+ prev_meta = knod_bpf_lookup_prev_meta_by_dreg(
+ knod_prog, meta, meta->insn.dst_reg);
+ if (!prev_meta) {
+ knod_jit_dbg(" Invalid\n");
+ return -EINVAL;
+ }
+ kdreg->stack_off = prev_meta->dreg.stack_off % imm;
+ break;
+ case BPF_ALU | BPF_AND | BPF_X:
+ case BPF_ALU64 | BPF_AND | BPF_X:
+ //r[d] &= r[s];
+ knod_jit_dbg(" PTR_TO_STACK with BPF_X is not supported\n");
+ return -EINVAL;
+ case BPF_ALU | BPF_AND | BPF_K:
+ case BPF_ALU64 | BPF_AND | BPF_K:
+ //r[d] &= imm;
+ prev_meta = knod_bpf_lookup_prev_meta_by_dreg(
+ knod_prog, meta, meta->insn.dst_reg);
+ if (!prev_meta) {
+ knod_jit_dbg(" Invalid\n");
+ return -EINVAL;
+ }
+ kdreg->stack_off = prev_meta->dreg.stack_off & imm;
+ break;
+ case BPF_ALU | BPF_OR | BPF_X:
+ case BPF_ALU64 | BPF_OR | BPF_X:
+ //r[d] |= r[s];
+ knod_jit_dbg(" PTR_TO_STACK with BPF_X is not supported\n");
+ return -EINVAL;
+ case BPF_ALU | BPF_OR | BPF_K:
+ case BPF_ALU64 | BPF_OR | BPF_K:
+ //r[d] |= imm;
+ prev_meta = knod_bpf_lookup_prev_meta_by_dreg(
+ knod_prog, meta, meta->insn.dst_reg);
+ if (!prev_meta) {
+ knod_jit_dbg(" Invalid\n");
+ return -EINVAL;
+ }
+ kdreg->stack_off = prev_meta->dreg.stack_off | imm;
+ break;
+ case BPF_ALU | BPF_ADD | BPF_X:
+ case BPF_ALU64 | BPF_ADD | BPF_X:
+ //r[d] += r[s];
+ knod_jit_dbg(" PTR_TO_STACK with BPF_X is not supported\n");
+ return -EINVAL;
+ case BPF_ALU | BPF_ADD | BPF_K:
+ case BPF_ALU64 | BPF_ADD | BPF_K:
+ //r[d] += imm;
+ prev_meta = knod_bpf_lookup_prev_meta_by_dreg(
+ knod_prog, meta, meta->insn.dst_reg);
+ if (!prev_meta) {
+ knod_jit_dbg(" Invalid\n");
+ return -EINVAL;
+ }
+ kdreg->stack_off = prev_meta->dreg.stack_off + imm;
+ break;
+ case BPF_ALU | BPF_SUB | BPF_X:
+ case BPF_ALU64 | BPF_SUB | BPF_X:
+ //r[d] -= r[s];
+ knod_jit_dbg(" PTR_TO_STACK with BPF_X is not supported\n");
+ return -EINVAL;
+ case BPF_ALU | BPF_SUB | BPF_K:
+ case BPF_ALU64 | BPF_SUB | BPF_K:
+ //r[d] -= imm;
+ prev_meta = knod_bpf_lookup_prev_meta_by_dreg(
+ knod_prog, meta, meta->insn.dst_reg);
+ if (!prev_meta) {
+ knod_jit_dbg(" Invalid\n");
+ return -EINVAL;
+ }
+ kdreg->stack_off = prev_meta->dreg.stack_off - imm;
+ break;
+ case BPF_ALU | BPF_MUL | BPF_X:
+ case BPF_ALU64 | BPF_MUL | BPF_X:
+ //r[d] *= r[s];
+ knod_jit_dbg(" PTR_TO_STACK with BPF_X is not supported\n");
+ return -EINVAL;
+ case BPF_ALU | BPF_MUL | BPF_K:
+ case BPF_ALU64 | BPF_MUL | BPF_K:
+ //r[d] *= imm;
+ prev_meta = knod_bpf_lookup_prev_meta_by_dreg(
+ knod_prog, meta, meta->insn.dst_reg);
+ if (!prev_meta) {
+ knod_jit_dbg(" Invalid\n");
+ return -EINVAL;
+ }
+ kdreg->stack_off = prev_meta->dreg.stack_off * imm;
+ break;
+ case BPF_ALU | BPF_DIV | BPF_X:
+ case BPF_ALU64 | BPF_DIV | BPF_X:
+ //r[d] /= r[s];
+ knod_jit_dbg(" PTR_TO_STACK with BPF_X is not supported\n");
+ return -EINVAL;
+ case BPF_ALU | BPF_DIV | BPF_K:
+ case BPF_ALU64 | BPF_DIV | BPF_K:
+ //r[d] /= imm;
+ prev_meta = knod_bpf_lookup_prev_meta_by_dreg(
+ knod_prog, meta, meta->insn.dst_reg);
+ if (!prev_meta) {
+ knod_jit_dbg(" Invalid\n");
+ return -EINVAL;
+ }
+ kdreg->stack_off = prev_meta->dreg.stack_off / imm;
+ break;
+ case BPF_ALU | BPF_NEG:
+ case BPF_ALU64 | BPF_NEG:
+ //r[d] = -r[d];
+ break;
+ case BPF_ALU | BPF_LSH | BPF_X:
+ case BPF_ALU64 | BPF_LSH | BPF_X:
+ //r[d] <<= r[s];
+ knod_jit_dbg(" PTR_TO_STACK with BPF_X is not supported\n");
+ return -EINVAL;
+ case BPF_ALU | BPF_LSH | BPF_K:
+ case BPF_ALU64 | BPF_LSH | BPF_K:
+ //r[d] <<= imm;
+ prev_meta = knod_bpf_lookup_prev_meta_by_dreg(
+ knod_prog, meta, meta->insn.dst_reg);
+ if (!prev_meta) {
+ knod_jit_dbg(" Invalid\n");
+ return -EINVAL;
+ }
+ kdreg->stack_off = prev_meta->dreg.stack_off << imm;
+ break;
+ case BPF_ALU | BPF_RSH | BPF_X:
+ case BPF_ALU64 | BPF_RSH | BPF_X:
+ //r[d] >>= r[s];
+ knod_jit_dbg(" PTR_TO_STACK with BPF_X is not supported\n");
+ return -EINVAL;
+ case BPF_ALU | BPF_RSH | BPF_K:
+ case BPF_ALU64 | BPF_RSH | BPF_K:
+ //r[d] >>= imm;
+ prev_meta = knod_bpf_lookup_prev_meta_by_dreg(
+ knod_prog, meta, meta->insn.dst_reg);
+ if (!prev_meta) {
+ knod_jit_dbg(" Invalid\n");
+ return -EINVAL;
+ }
+ kdreg->stack_off = prev_meta->dreg.stack_off >> imm;
+ break;
+ case BPF_ALU | BPF_ARSH | BPF_X:
+ case BPF_ALU64 | BPF_ARSH | BPF_X:
+ //r[d] >>= r[s];
+ knod_jit_dbg(" PTR_TO_STACK with BPF_X is not supported\n");
+ return -EINVAL;
+ case BPF_ALU | BPF_ARSH | BPF_K:
+ case BPF_ALU64 | BPF_ARSH | BPF_K:
+ //r[d] >>= imm;
+ prev_meta = knod_bpf_lookup_prev_meta_by_dreg(
+ knod_prog, meta, meta->insn.dst_reg);
+ if (!prev_meta) {
+ knod_jit_dbg(" Invalid\n");
+ return -EINVAL;
+ }
+ kdreg->stack_off = prev_meta->dreg.stack_off >> imm;
+ break;
+ }
+ knod_jit_dbg(" %d: dreg->stack_off = %d\n", meta->bpf_insn_idx,
+ kdreg->stack_off);
+ }
+
+ if (dreg->type == PTR_TO_PACKET) {
+ imm = meta->insn.imm;
+
+ switch (meta->insn.code) {
+ /* ALU
+ * If a destination register contains a pointer of STACK,
+ * offset should not be minus.
+ */
+ case BPF_ALU | BPF_MOV | BPF_X:
+ case BPF_ALU64 | BPF_MOV | BPF_X:
+ //r[d] = r[s];
+ kdreg->packet_off = ksreg->packet_off;
+ break;
+ case BPF_ALU | BPF_MOV | BPF_K:
+ case BPF_ALU64 | BPF_MOV | BPF_K:
+ //r[d] = imm;
+ kdreg->packet_off = ksreg->packet_off;
+ break;
+ case BPF_ALU | BPF_XOR | BPF_X:
+ case BPF_ALU64 | BPF_XOR | BPF_X:
+ //r[d] ^= r[s];
+ knod_jit_dbg(" PTR_TO_STACK with BPF_X is not supported\n");
+ return -EINVAL;
+ case BPF_ALU | BPF_XOR | BPF_K:
+ case BPF_ALU64 | BPF_XOR | BPF_K:
+ //r[d] ^= imm;
+ prev_meta = knod_bpf_lookup_prev_meta_by_dreg(
+ knod_prog, meta, meta->insn.dst_reg);
+ if (!prev_meta) {
+ knod_jit_dbg(" Invalid\n");
+ return -EINVAL;
+ }
+ kdreg->packet_off = prev_meta->dreg.packet_off ^ imm;
+ break;
+ case BPF_ALU | BPF_MOD | BPF_X:
+ case BPF_ALU64 | BPF_MOD | BPF_X:
+ //r[d] %= r[s];
+ knod_jit_dbg(" PTR_TO_STACK with BPF_X is not supported\n");
+ return -EINVAL;
+ case BPF_ALU | BPF_MOD | BPF_K:
+ case BPF_ALU64 | BPF_MOD | BPF_K:
+ //r[d] %= imm;
+ prev_meta = knod_bpf_lookup_prev_meta_by_dreg(
+ knod_prog, meta, meta->insn.dst_reg);
+ if (!prev_meta) {
+ knod_jit_dbg(" Invalid\n");
+ return -EINVAL;
+ }
+ kdreg->packet_off = prev_meta->dreg.packet_off % imm;
+ break;
+ case BPF_ALU | BPF_AND | BPF_X:
+ case BPF_ALU64 | BPF_AND | BPF_X:
+ //r[d] &= r[s];
+ knod_jit_dbg(" PTR_TO_STACK with BPF_X is not supported\n");
+ return -EINVAL;
+ case BPF_ALU | BPF_AND | BPF_K:
+ case BPF_ALU64 | BPF_AND | BPF_K:
+ //r[d] &= imm;
+ prev_meta = knod_bpf_lookup_prev_meta_by_dreg(
+ knod_prog, meta, meta->insn.dst_reg);
+ if (!prev_meta) {
+ knod_jit_dbg(" Invalid\n");
+ return -EINVAL;
+ }
+ kdreg->packet_off = prev_meta->dreg.packet_off & imm;
+ break;
+ case BPF_ALU | BPF_OR | BPF_X:
+ case BPF_ALU64 | BPF_OR | BPF_X:
+ //r[d] |= r[s];
+ knod_jit_dbg(" PTR_TO_STACK with BPF_X is not supported\n");
+ return -EINVAL;
+ case BPF_ALU | BPF_OR | BPF_K:
+ case BPF_ALU64 | BPF_OR | BPF_K:
+ //r[d] |= imm;
+ prev_meta = knod_bpf_lookup_prev_meta_by_dreg(
+ knod_prog, meta, meta->insn.dst_reg);
+ if (!prev_meta) {
+ knod_jit_dbg(" Invalid\n");
+ return -EINVAL;
+ }
+ kdreg->packet_off = prev_meta->dreg.packet_off | imm;
+ break;
+ case BPF_ALU | BPF_ADD | BPF_X:
+ case BPF_ALU64 | BPF_ADD | BPF_X:
+ //r[d] += r[s];
+ knod_jit_dbg(" PTR_TO_STACK with BPF_X is not supported\n");
+ return -EINVAL;
+ case BPF_ALU | BPF_ADD | BPF_K:
+ case BPF_ALU64 | BPF_ADD | BPF_K:
+ //r[d] += imm;
+ prev_meta = knod_bpf_lookup_prev_meta_by_dreg(
+ knod_prog, meta, meta->insn.dst_reg);
+ if (!prev_meta) {
+ knod_jit_dbg(" Invalid\n");
+ return -EINVAL;
+ }
+ kdreg->packet_off = prev_meta->dreg.packet_off + imm;
+ break;
+ case BPF_ALU | BPF_SUB | BPF_X:
+ case BPF_ALU64 | BPF_SUB | BPF_X:
+ //r[d] -= r[s];
+ knod_jit_dbg(" PTR_TO_STACK with BPF_X is not supported\n");
+ return -EINVAL;
+ case BPF_ALU | BPF_SUB | BPF_K:
+ case BPF_ALU64 | BPF_SUB | BPF_K:
+ //r[d] -= imm;
+ prev_meta = knod_bpf_lookup_prev_meta_by_dreg(
+ knod_prog, meta, meta->insn.dst_reg);
+ if (!prev_meta) {
+ knod_jit_dbg(" Invalid\n");
+ return -EINVAL;
+ }
+ kdreg->packet_off = prev_meta->dreg.packet_off - imm;
+ break;
+ case BPF_ALU | BPF_MUL | BPF_X:
+ case BPF_ALU64 | BPF_MUL | BPF_X:
+ //r[d] *= r[s];
+ knod_jit_dbg(" PTR_TO_STACK with BPF_X is not supported\n");
+ return -EINVAL;
+ case BPF_ALU | BPF_MUL | BPF_K:
+ case BPF_ALU64 | BPF_MUL | BPF_K:
+ //r[d] *= imm;
+ prev_meta = knod_bpf_lookup_prev_meta_by_dreg(
+ knod_prog, meta, meta->insn.dst_reg);
+ if (!prev_meta) {
+ knod_jit_dbg(" Invalid\n");
+ return -EINVAL;
+ }
+ kdreg->packet_off = prev_meta->dreg.packet_off * imm;
+ break;
+ case BPF_ALU | BPF_DIV | BPF_X:
+ case BPF_ALU64 | BPF_DIV | BPF_X:
+ //r[d] /= r[s];
+ knod_jit_dbg(" PTR_TO_STACK with BPF_X is not supported\n");
+ return -EINVAL;
+ case BPF_ALU | BPF_DIV | BPF_K:
+ case BPF_ALU64 | BPF_DIV | BPF_K:
+ //r[d] /= imm;
+ prev_meta = knod_bpf_lookup_prev_meta_by_dreg(
+ knod_prog, meta, meta->insn.dst_reg);
+ if (!prev_meta) {
+ knod_jit_dbg(" Invalid\n");
+ return -EINVAL;
+ }
+ kdreg->packet_off = prev_meta->dreg.packet_off / imm;
+ break;
+ case BPF_ALU | BPF_NEG:
+ case BPF_ALU64 | BPF_NEG:
+ //r[d] = -r[d];
+ break;
+ case BPF_ALU | BPF_LSH | BPF_X:
+ case BPF_ALU64 | BPF_LSH | BPF_X:
+ //r[d] <<= r[s];
+ knod_jit_dbg(" PTR_TO_STACK with BPF_X is not supported\n");
+ return -EINVAL;
+ case BPF_ALU | BPF_LSH | BPF_K:
+ case BPF_ALU64 | BPF_LSH | BPF_K:
+ //r[d] <<= imm;
+ prev_meta = knod_bpf_lookup_prev_meta_by_dreg(
+ knod_prog, meta, meta->insn.dst_reg);
+ if (!prev_meta) {
+ knod_jit_dbg(" Invalid\n");
+ return -EINVAL;
+ }
+ kdreg->packet_off = prev_meta->dreg.packet_off << imm;
+ break;
+ case BPF_ALU | BPF_RSH | BPF_X:
+ case BPF_ALU64 | BPF_RSH | BPF_X:
+ //r[d] >>= r[s];
+ knod_jit_dbg(" PTR_TO_STACK with BPF_X is not supported\n");
+ return -EINVAL;
+ case BPF_ALU | BPF_RSH | BPF_K:
+ case BPF_ALU64 | BPF_RSH | BPF_K:
+ //r[d] >>= imm;
+ prev_meta = knod_bpf_lookup_prev_meta_by_dreg(
+ knod_prog, meta, meta->insn.dst_reg);
+ if (!prev_meta) {
+ knod_jit_dbg(" Invalid\n");
+ return -EINVAL;
+ }
+ kdreg->packet_off = prev_meta->dreg.packet_off >> imm;
+ break;
+ case BPF_ALU | BPF_ARSH | BPF_X:
+ case BPF_ALU64 | BPF_ARSH | BPF_X:
+ //r[d] >>= r[s];
+ knod_jit_dbg(" PTR_TO_STACK with BPF_X is not supported\n");
+ return -EINVAL;
+ case BPF_ALU | BPF_ARSH | BPF_K:
+ case BPF_ALU64 | BPF_ARSH | BPF_K:
+ //r[d] >>= imm;
+ prev_meta = knod_bpf_lookup_prev_meta_by_dreg(
+ knod_prog, meta, meta->insn.dst_reg);
+ if (!prev_meta) {
+ knod_jit_dbg(" Invalid\n");
+ return -EINVAL;
+ }
+ kdreg->packet_off = prev_meta->dreg.packet_off >> imm;
+ break;
+ }
+ knod_jit_dbg(" %d: dreg->packet_off = %d\n", meta->bpf_insn_idx,
+ kdreg->packet_off);
+ }
+ return 0;
+}
+
+static int knod_bpf_verify_insn(struct bpf_verifier_env *env,
+ int insn_idx, int prev_insn)
+{
+ struct knod_prog *knod_prog = env->prog->aux->offload->dev_priv;
+ const struct bpf_reg_state *sreg, *dreg, *kreg, *vreg;
+ struct knod_insn_meta *meta = knod_prog->meta;
+ struct knod_insn_meta *prev_meta;
+ int err = 0;
+
+ meta = knod_bpf_goto_meta(knod_prog, meta, insn_idx);
+ sreg = cur_regs(env) + meta->insn.src_reg;
+ dreg = cur_regs(env) + meta->insn.dst_reg;
+ knod_prog->meta = meta;
+ meta->sreg.reg = *sreg;
+ meta->dreg.reg = *dreg;
+
+ knod_bpf_update_ptr_off(knod_prog, meta, env);
+
+ if (meta->insn.src_reg >= MAX_BPF_REG ||
+ meta->insn.dst_reg >= MAX_BPF_REG) {
+ knod_jit_dbg(" program uses extended registers - jit hardening?\n");
+ err = -EINVAL;
+ goto out;
+ }
+
+ if (is_mbpf_load(meta)) {
+ err = knod_bpf_check_ptr(knod_prog, meta, env,
+ meta->insn.src_reg);
+ goto out;
+ }
+ if (is_mbpf_store(meta)) {
+ err = knod_bpf_check_store(knod_prog, meta, env);
+ goto out;
+ }
+
+ if (is_mbpf_map_call(meta)) {
+ kreg = cur_regs(env) + 2;
+ meta->kreg.reg = *kreg;
+
+ prev_meta = knod_bpf_lookup_prev_meta_by_dreg(knod_prog,
+ meta,
+ 2);
+ if (!prev_meta) {
+ knod_jit_dbg(" Invalid\n");
+ err = -EINVAL;
+ goto out;
+ }
+ if (kreg->type == PTR_TO_PACKET)
+ meta->kreg.packet_off = prev_meta->dreg.packet_off;
+ else
+ meta->kreg.stack_off = prev_meta->dreg.stack_off;
+ if (knod_prog->max_stack_off > meta->kreg.stack_off)
+ knod_prog->max_stack_off = meta->kreg.stack_off;
+ if (knod_prog->max_packet_off < meta->kreg.packet_off)
+ knod_prog->max_packet_off = meta->kreg.packet_off;
+
+ /* bpf_map_update_elem: track r3 (value pointer) */
+ if (meta->insn.imm == 2) {
+ vreg = cur_regs(env) + 3;
+ meta->vreg.reg = *vreg;
+
+ prev_meta = knod_bpf_lookup_prev_meta_by_dreg(knod_prog,
+ meta,
+ 3);
+ if (!prev_meta) {
+ knod_jit_dbg(" Invalid vreg\n");
+ err = -EINVAL;
+ goto out;
+ }
+ if (vreg->type == PTR_TO_PACKET)
+ meta->vreg.packet_off =
+ prev_meta->dreg.packet_off;
+ else
+ meta->vreg.stack_off =
+ prev_meta->dreg.stack_off;
+ if (knod_prog->max_stack_off > meta->vreg.stack_off)
+ knod_prog->max_stack_off = meta->vreg.stack_off;
+ if (knod_prog->max_packet_off < meta->vreg.packet_off)
+ knod_prog->max_packet_off =
+ meta->vreg.packet_off;
+ }
+ }
+
+ if (is_mbpf_alu(meta))
+ err = knod_bpf_check_alu(knod_prog, meta, env);
+
+ /* less stack offset is bigger */
+ if (knod_prog->max_stack_off > meta->sreg.stack_off)
+ knod_prog->max_stack_off = meta->sreg.stack_off;
+ if (knod_prog->max_stack_off > meta->dreg.stack_off)
+ knod_prog->max_stack_off = meta->dreg.stack_off;
+ if (knod_prog->max_packet_off < meta->sreg.packet_off)
+ knod_prog->max_packet_off = meta->sreg.packet_off;
+ if (knod_prog->max_packet_off < meta->dreg.packet_off)
+ knod_prog->max_packet_off = meta->dreg.packet_off;
+
+out:
+ if (err)
+ pr_warn("knod_bpf: verifier rejected bpf insn %d (code 0x%02x off %d imm %d): %d\n",
+ insn_idx, meta->insn.code, meta->insn.off,
+ meta->insn.imm, err);
+ return err;
+}
+
+static int knod_bpf_finalize(struct bpf_verifier_env *env)
+{
+ return 0;
+}
+
+static int knod_bpf_offload(struct knod_dev *knodev,
+ struct bpf_prog *prog, bool oldprog)
+{
+ struct knod_bpf_priv *priv = knodev->accel->xdp.priv;
+
+ WARN(!!knod_dev_offloaded(knodev) != oldprog,
+ "bad offload state, expected offload %sto be active",
+ oldprog ? "" : "not ");
+
+ WRITE_ONCE(priv->prog, prog);
+ knod_dev_offload(knodev, prog);
+
+ /*
+ * Uninstalling the prog: reload the pass kernel now, while the prog's
+ * maps are still valid, so the worker stops dispatching prog code that
+ * is about to reference freed maps.
+ */
+ if (!prog)
+ knod_bpf_reload_pass(knodev);
+
+ return 0;
+}
+
+static int knod_bpf_xdp_offload_prog(struct knod_dev *knodev,
+ struct netdev_bpf *bpf)
+{
+ if (!knod_dev_active(knodev) && !bpf->prog)
+ return 0;
+
+ if (!knod_dev_active(knodev) && bpf->prog &&
+ knodev->accel->xdp.bpf_offloaded) {
+ return -EBUSY;
+ }
+
+ return knod_bpf_offload(knodev, bpf->prog, knod_dev_active(knodev));
+}
+
+static int knod_bpf_xdp_set_prog(struct knod_dev *knodev,
+ struct netdev_bpf *bpf)
+{
+ int err;
+
+ if (bpf->command == XDP_SETUP_PROG_HW) {
+ err = knod_bpf_xdp_offload_prog(knodev, bpf);
+ if (err)
+ return err;
+ }
+
+ xdp_attachment_setup(&knodev->accel->xdp.xdp_hw, bpf);
+
+ return 0;
+}
+
+static void knod_wait_vmcnt(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta)
+{
+ knod_emit(priv, meta, s_waitcnt_vmcnt);
+}
+
+static void knod_global_load_size_cache(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ struct amdgcn_param32 *d,
+ struct amdgcn_param32 s,
+ int dst_idx, int start_off, int length)
+{
+ int off = start_off;
+
+ /* length is 4B aligned */
+ while (length) {
+ if (length >= 16) {
+ knod_emit(priv, meta, global_load_dwordx4, d[dst_idx],
+ s, off);
+ length -= 16;
+ off += 16;
+ dst_idx += 4;
+ } else if (length >= 8) {
+ knod_emit(priv, meta, global_load_dwordx2, d[dst_idx],
+ s, off);
+ length -= 8;
+ off += 8;
+ dst_idx += 2;
+ } else if (length >= 4) {
+ knod_emit(priv, meta, global_load_dword, d[dst_idx],
+ s, off);
+ length -= 4;
+ off += 4;
+ dst_idx += 1;
+ }
+ }
+
+ knod_wait_vmcnt(priv, meta);
+}
+
+static void knod_global_store_size_cache(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ struct amdgcn_param32 *d,
+ struct amdgcn_param32 s,
+ int dst_idx, int start_off, int length)
+{
+ int off = start_off;
+
+ /* length is 4B aligned */
+ while (length) {
+ if (length >= 16) {
+ knod_emit(priv, meta, global_store_dwordx4, d[dst_idx],
+ s, off);
+ length -= 16;
+ off += 16;
+ dst_idx += 4;
+ } else if (length >= 8) {
+ knod_emit(priv, meta, global_store_dwordx2, d[dst_idx],
+ s, off);
+ length -= 8;
+ off += 8;
+ dst_idx += 2;
+ } else if (length >= 4) {
+ knod_emit(priv, meta, global_store_dword, d[dst_idx],
+ s, off);
+ length -= 4;
+ off += 4;
+ dst_idx += 1;
+ }
+ }
+
+ knod_wait_vmcnt(priv, meta);
+}
+
+static int knod_prog_prepare_insns(struct knod_bpf_priv *priv,
+ struct knod_prog *knod_prog)
+{
+ struct amdgcn_param64 param64[3];
+ struct amdgcn_param32 param[10];
+ struct knod_insn_meta *meta;
+ int bs_shift;
+
+ meta = kzalloc_obj(*meta, GFP_KERNEL);
+ if (!meta)
+ return -ENOMEM;
+
+ meta->amdgpu_insn_idx = 0;
+
+ /* Invalidate SQC instruction cache so that a re-uploaded shader
+ * at the same VRAM address is fetched from memory, not from the
+ * stale I-cache. Must be the very first instruction at the entry
+ * point so that every shader version has s_icache_inv at the same
+ * offset - the cached old version executes s_icache_inv too,
+ * which flushes the cache before divergent code is reached.
+ */
+ knod_emit(priv, meta, s_icache_inv);
+ knod_emit(priv, meta, s_waitcnt_vmcnt_lgkmcnt);
+
+ knod_sset32(¶m[0], KNOD_AMDGPU_PARAM_SREG_LO);
+ knod_sset32(¶m[1], KNOD_AMDGPU_ARG_SREG);
+ /* param = (__global struct _knod_bpf_param *)pkt.kernarg_address; */
+ knod_emit(priv, meta, s_load_dwordx2, param[0], param[1],
+ offsetof(struct hsa_kernel_dispatch_packet, kernarg_address));
+
+ knod_vset32(¶m[0], KNOD_AMDGPU_IDX_VREG);
+ knod_vset32(¶m[1], KNOD_AMDGPU_VREG0_LO);
+ knod_iset32(¶m[2], 0);
+ /* 10bits, lidx can up to 1024, Do not edit */
+ knod_iset32(¶m[3], 10);
+ /* extract workitem ID to reserved vgpr register.
+ * In the 2D-dispatch layout, IDX_VREG holds the per-workgroup tid
+ * (0..workgroup_size_x-1). queue_id = workgroup_id_y (s15). The
+ * flat index (queue_id * batch_size + local_idx) is computed later,
+ * after queue_desc has been loaded and the v_cmpx bounds check has
+ * narrowed EXEC to lanes with local_idx < count.
+ */
+ knod_emit(priv, meta, v_bfe_i32, param[0], param[1], param[2],
+ param[3]);
+ if (priv->batch_size > knod_bpf_workgroups) {
+ /* local_idx = workgroup_id_x * workgroup_size_x
+ * + workitem_id.
+ */
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG5_LO);
+ knod_sset32(¶m[1], KNOD_AMDGPU_WORKGROUP_ID_X_SREG);
+ knod_iset32(¶m[2], knod_bpf_workgroups);
+ knod_emit(priv, meta,
+ v_mul_lo_u32, param[0], param[1], param[2]);
+ knod_vset32(¶m[0], KNOD_AMDGPU_IDX_VREG);
+ knod_vset32(¶m[1], KNOD_AMDGPU_TMP_VREG5_LO);
+ knod_vset32(¶m[2], KNOD_AMDGPU_IDX_VREG);
+ knod_emit(priv, meta, v_add_u32, param[0], param[1], param[2]);
+ }
+ /* set frame pointer to 0 */
+ knod_sset32(¶m[0], KNOD_AMDGPU_FRAME_POINTER_SREG);
+ knod_iset32(¶m[1], 0);
+ knod_emit(priv, meta, s_mov_b32, param[0], param[1]);
+ /* wait for s_load_dwordx2 (kernarg_address) */
+ knod_emit(priv, meta, s_waitcnt_vmcnt_lgkmcnt);
+
+ /* load {nr_backlogs, _pad} from param (offset 0, 8-byte aligned) */
+ knod_sset32(¶m[0], KNOD_AMDGPU_TMP_SREG1_LO);
+ knod_sset32(¶m[1], KNOD_AMDGPU_PARAM_SREG_LO);
+ knod_emit(priv, meta, s_load_dwordx2, param[0], param[1], 0);
+ knod_emit(priv, meta, s_waitcnt_vmcnt_lgkmcnt);
+
+ /* NOTE: the bounds check `EXEC &= (tid < count)` is deferred until
+ * after the queue descriptor load (queue<->workgroup binding).
+ * nr_backlogs is no longer the right upper bound because lanes with
+ * tid > this queue's count must be masked, not just the ones past
+ * the aggregate backlog total.
+ */
+
+ /* Initialize done_mask to 0 for structurized CFG */
+ knod_emit(priv, meta, s_mov_b64, knod_prog->done_mask_sreg,
+ AMDGCN_SREG_INTEGER_0);
+
+ /* =========================================================
+ * Queue-descriptor prep (2D dispatch: queue_id = workgroup_id_y)
+ *
+ * Loads this WG's queue descriptor from VRAM, performs the
+ * per-lane bounds check (local_idx < queues[queue_id].count), and
+ * converts IDX_VREG from local_idx to flat_IDX (queue_id *
+ * batch_size + local_idx) which the rest of the prologue/BPF body
+ * expects. TMP_VREG9_LO is repurposed to hold the saved local
+ * tid for use as local_idx in the slot-address step.
+ * =========================================================
+ */
+ bs_shift = ilog2(priv->batch_size);
+
+ /* a. queue_id = workgroup_id_y (broadcast scalar to VGPR LO). */
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_sset32(¶m[1], KNOD_AMDGPU_WORKGROUP_ID_Y_SREG);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0], param[1]);
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG0_HI);
+ knod_iset32(¶m[1], 0);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0], param[1]);
+
+ /* b. Copy queue_id into TMP_VREG5_LO - separate scratch used as
+ * the v_mad src-multiplicand. Avoids dst/src1 overlap on the
+ * following v_mad_u64_u32 (dst=TMP_VREG0_LO:HI).
+ */
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG5_LO);
+ knod_vset32(¶m[1], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0], param[1]);
+
+ /* c. TMP_VREG0 = PARAM + queue_id * sizeof(queue_desc). */
+ knod_vset64(¶m64[0], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_sset32(¶m[0], KNOD_AMDGPU_TMP_SREG0_LO);
+ knod_iset32(¶m[1],
+ sizeof(struct knod_bpf_queue_desc));
+ knod_vset32(¶m[2], KNOD_AMDGPU_TMP_VREG5_LO);
+ knod_sset64(¶m64[1], KNOD_AMDGPU_PARAM_SREG_LO);
+ knod_emit(priv, meta, v_mad_u64_u32, param64[0], param[0],
+ param[1], param[2], param64[1]);
+
+ /* d. TMP_VREG0 += offsetof(queues). */
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_iset32(¶m[1],
+ offsetof(struct knod_bpf_param, queues));
+ knod_vset32(¶m[2], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_emit(priv, meta, v_add_co_u32, param[0], param[1], param[2]);
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG0_HI);
+ knod_iset32(¶m[1], 0);
+ knod_emit(priv, meta, v_add_co_ci_u32_e32, param[0], param[1],
+ param[0]);
+
+ /* e. Load pool_gaddr + base_gaddr (offset 0, 16 bytes) into
+ * TMP_VREG1_LO..TMP_VREG2_HI (v24..v27 - must be consecutive).
+ */
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG1_LO);
+ knod_vset32(¶m[1], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_emit(priv, meta, global_load_dwordx4, param[0], param[1], 0);
+
+ /* f. Load count + _pad + ring_start + ring_mask (offset 16, 16
+ * bytes) into TMP_VREG3_LO..TMP_VREG4_HI.
+ */
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG3_LO);
+ knod_vset32(¶m[1], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_emit(priv, meta, global_load_dwordx4, param[0], param[1],
+ offsetof(struct knod_bpf_queue_desc, count));
+ knod_emit(priv, meta, s_waitcnt_vmcnt);
+
+ /* g. Bounds check: EXEC &= (workitem_id < count). */
+ knod_vset32(¶m[0], KNOD_AMDGPU_IDX_VREG);
+ knod_vset32(¶m[1], KNOD_AMDGPU_TMP_VREG3_LO);
+ knod_emit(priv, meta, v_cmpx_lt_u32, param[0], param[1]);
+
+ /* Snapshot the in-bounds lane mask. The unified epilogue uses this
+ * to publish one verdict for every lane the dispatch claimed, even if
+ * a malformed or newly added CFG path fails to join done_mask.
+ */
+ knod_emit(priv, meta, s_mov_b64, knod_prog->initial_exec_sreg,
+ AMDGCN_SREG_EXEC_LO);
+
+ /* h. Save per-queue local_idx to TMP_VREG9_LO.
+ * The slot-address step consumes this value; keeping it in a
+ * dedicated VGPR lets us overwrite IDX_VREG with flat_IDX for
+ * the CTX address computation that immediately follows.
+ */
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG9_LO);
+ knod_vset32(¶m[1], KNOD_AMDGPU_IDX_VREG);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0], param[1]);
+
+ /* i. IDX_VREG = (queue_id << ilog2(batch_size)) + local_idx
+ * -> flat_IDX into the sub[] / sqw->bds[] arrays, matching the
+ * CPU-side layout `sqw->bds[queue_id * batch_size + local_idx]`.
+ * batch_size is rounded down to a power of two at start so the
+ * shift is exact.
+ */
+ knod_vset32(¶m[0], KNOD_AMDGPU_IDX_VREG);
+ knod_sset32(¶m[1], KNOD_AMDGPU_WORKGROUP_ID_Y_SREG);
+ knod_iset32(¶m[2], bs_shift);
+ knod_vset32(¶m[3], KNOD_AMDGPU_IDX_VREG);
+ knod_emit(priv, meta, v_lshl_add_u32, param[0], param[1],
+ param[2], param[3]);
+
+ /* ctx = ¶m->sub[flat_IDX].ctx;
+ * v_mad: VREG1 = sizeof(sub_obj) * flat_IDX + PARAM_SREG
+ * then add offsetof(sub) = 8 to account for nr_backlogs/_pad
+ */
+ knod_vset64(¶m64[0], KNOD_AMDGPU_VREG1_LO);
+ knod_sset32(¶m[0], KNOD_AMDGPU_TMP_SREG0_LO);
+ knod_iset32(¶m[1], sizeof(struct knod_bpf_subparam_obj));
+ knod_vset32(¶m[2], KNOD_AMDGPU_IDX_VREG);
+ knod_sset64(¶m64[1], KNOD_AMDGPU_PARAM_SREG_LO);
+ knod_emit(priv, meta, v_mad_u64_u32, param64[0], param[0],
+ param[1], param[2], param64[1]);
+ /* + offsetof(struct knod_bpf_param, sub) */
+ knod_vset32(¶m[0], KNOD_AMDGPU_VREG1_LO);
+ knod_iset32(¶m[1], offsetof(struct knod_bpf_param, sub));
+ knod_vset32(¶m[2], KNOD_AMDGPU_VREG1_LO);
+ knod_emit(priv, meta, v_add_co_u32, param[0], param[1], param[2]);
+ knod_vset32(¶m[0], KNOD_AMDGPU_VREG1_HI);
+ knod_iset32(¶m[1], 0);
+ knod_emit(priv, meta, v_add_co_ci_u32_e32, param[0], param[1],
+ param[0]);
+ knod_vset32(¶m[0], KNOD_AMDGPU_CTX_VREG_LO);
+ knod_vset32(¶m[1], KNOD_AMDGPU_VREG1_LO);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0], param[1]);
+ knod_vset32(¶m[0], KNOD_AMDGPU_CTX_VREG_HI);
+ knod_vset32(¶m[1], KNOD_AMDGPU_VREG1_HI);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0], param[1]);
+ knod_vset32(¶m[0], KNOD_AMDGPU_FRAME_POINTER_VREG_LO);
+ knod_iset32(¶m[1], 0x200);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0], param[1]);
+ knod_vset32(¶m[0], KNOD_AMDGPU_FRAME_POINTER_VREG_HI);
+ knod_iset32(¶m[1], 0);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0], param[1]);
+
+ /* 5. slot = (ring_start + local_idx) & ring_mask.
+ * local_idx = saved per-queue local_idx in TMP_VREG9_LO.
+ */
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG5_LO);
+ knod_vset32(¶m[1], KNOD_AMDGPU_TMP_VREG4_LO);
+ knod_vset32(¶m[2], KNOD_AMDGPU_TMP_VREG9_LO);
+ knod_emit(priv, meta, v_add_u32, param[0], param[1], param[2]);
+ knod_vset32(¶m[1], KNOD_AMDGPU_TMP_VREG4_HI);
+ knod_vset32(¶m[2], KNOD_AMDGPU_TMP_VREG5_LO);
+ knod_emit(priv, meta, v_and_b32_e32, param[0], param[1], param[2]);
+
+ /* Save backlog index before step 6 overwrites IDX_VREG -> SLOT_VREG.
+ * v_mov_b32 BACKLOG_IDX_VREG(v58), IDX_VREG(v62)
+ * Used in epilogue for XDP_PASS pass_indices[] write.
+ */
+ knod_vset32(¶m[0], KNOD_AMDGPU_BACKLOG_IDX_VREG);
+ knod_vset32(¶m[1], KNOD_AMDGPU_IDX_VREG);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0], param[1]);
+
+ /* 6. slot_addr = pool_gaddr + slot * spsc_stride, where
+ * spsc_stride = ALIGN(sizeof(spsc_bd), SMP_CACHE_BYTES)
+ * Compute directly into SLOT_VREG (v62:v63).
+ */
+ knod_vset32(¶m[0], KNOD_AMDGPU_SLOT_VREG_LO);
+ knod_iset32(¶m[1],
+ ilog2(ALIGN(sizeof(struct spsc_bd), SMP_CACHE_BYTES)));
+ knod_emit(priv, meta, v_lshlrev_b32, param[0], param[1], param[2]);
+ /* slot_addr = pool_gaddr + slot_offset */
+ knod_vset32(¶m[1], KNOD_AMDGPU_TMP_VREG1_LO);
+ knod_emit(priv, meta, v_add_co_u32, param[0], param[0], param[1]);
+ knod_vset32(¶m[0], KNOD_AMDGPU_SLOT_VREG_HI);
+ knod_iset32(¶m[1], 0);
+ knod_vset32(¶m[2], KNOD_AMDGPU_TMP_VREG1_HI);
+ knod_emit(priv, meta, v_add_co_ci_u32_e32, param[0], param[1],
+ param[2]);
+
+ /* 7. Load spsc_bd: {off(u16)|len(u16), page_idx} via single dwordx2
+ * TMP_VREG6_LO (v34) = off|len, TMP_VREG6_HI (v35) = page_idx
+ */
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG6_LO);
+ knod_vset32(¶m[1], KNOD_AMDGPU_SLOT_VREG_LO);
+ knod_emit(priv, meta, global_load_dwordx2, param[0], param[1],
+ offsetof(struct spsc_bd, off));
+ knod_emit(priv, meta, s_waitcnt_vmcnt);
+
+ /* 8. data = base_gaddr + (page_idx << PAGE_SHIFT) + off
+ * Compute directly into DATA_VREG (v64:v65).
+ */
+ knod_vset32(¶m[0], KNOD_AMDGPU_DATA_VREG_LO);
+ knod_iset32(¶m[1], PAGE_SHIFT);
+ knod_vset32(¶m[2], KNOD_AMDGPU_TMP_VREG6_HI);
+ knod_emit(priv, meta, v_lshlrev_b32, param[0], param[1], param[2]);
+ knod_vset32(¶m[0], KNOD_AMDGPU_DATA_VREG_HI);
+ knod_iset32(¶m[1], 32 - PAGE_SHIFT);
+ knod_emit(priv, meta, v_lshrrev_b32, param[0], param[1], param[2]);
+
+ /* data = base_gaddr + page_gaddr */
+ knod_vset32(¶m[0], KNOD_AMDGPU_DATA_VREG_LO);
+ knod_vset32(¶m[1], KNOD_AMDGPU_TMP_VREG2_LO);
+ knod_vset32(¶m[2], KNOD_AMDGPU_DATA_VREG_LO);
+ knod_emit(priv, meta, v_add_co_u32, param[0], param[1], param[2]);
+ knod_vset32(¶m[0], KNOD_AMDGPU_DATA_VREG_HI);
+ knod_vset32(¶m[1], KNOD_AMDGPU_TMP_VREG2_HI);
+ knod_vset32(¶m[2], KNOD_AMDGPU_DATA_VREG_HI);
+ knod_emit(priv, meta, v_add_co_ci_u32_e32, param[0], param[1],
+ param[2]);
+
+ if (knod_prog->uses_adjust) {
+ /* Save page_base to PAGE_BASE_VREG before adding off */
+ knod_vset32(¶m[0], KNOD_AMDGPU_PAGE_BASE_VREG_LO);
+ knod_vset32(¶m[1], KNOD_AMDGPU_DATA_VREG_LO);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0], param[1]);
+ knod_vset32(¶m[0], KNOD_AMDGPU_PAGE_BASE_VREG_HI);
+ knod_vset32(¶m[1], KNOD_AMDGPU_DATA_VREG_HI);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0], param[1]);
+ }
+
+ /* extract off (lower 16 bits of TMP_VREG6_LO) */
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG8_LO);
+ knod_iset32(¶m[1], 0xffff);
+ knod_vset32(¶m[2], KNOD_AMDGPU_TMP_VREG6_LO);
+ knod_emit(priv, meta, v_and_b32_e32, param[0], param[1], param[2]);
+
+ /* data += off */
+ knod_vset32(¶m[0], KNOD_AMDGPU_DATA_VREG_LO);
+ knod_vset32(¶m[1], KNOD_AMDGPU_TMP_VREG8_LO);
+ knod_vset32(¶m[2], KNOD_AMDGPU_DATA_VREG_LO);
+ knod_emit(priv, meta, v_add_co_u32, param[0], param[1], param[2]);
+ knod_vset32(¶m[0], KNOD_AMDGPU_DATA_VREG_HI);
+ knod_iset32(¶m[1], 0);
+ knod_emit(priv, meta, v_add_co_ci_u32_e32, param[0], param[1],
+ param[0]);
+
+ /* 9. data_end = data + len (upper 16 bits of TMP_VREG6_LO)
+ * Compute directly into DATA_END_VREG (v66:v67).
+ */
+ knod_vset32(¶m[0], KNOD_AMDGPU_DATA_END_VREG_LO);
+ knod_iset32(¶m[1], 16);
+ knod_vset32(¶m[2], KNOD_AMDGPU_TMP_VREG6_LO);
+ knod_emit(priv, meta, v_lshrrev_b32, param[0], param[1], param[2]);
+ knod_vset32(¶m[0], KNOD_AMDGPU_DATA_END_VREG_LO);
+ knod_vset32(¶m[1], KNOD_AMDGPU_DATA_VREG_LO);
+ knod_vset32(¶m[2], KNOD_AMDGPU_DATA_END_VREG_LO);
+ knod_emit(priv, meta, v_add_co_u32, param[0], param[1], param[2]);
+ knod_vset32(¶m[0], KNOD_AMDGPU_DATA_END_VREG_HI);
+ knod_iset32(¶m[1], 0);
+ knod_vset32(¶m[2], KNOD_AMDGPU_DATA_VREG_HI);
+ knod_emit(priv, meta, v_add_co_ci_u32_e32, param[0], param[1],
+ param[2]);
+
+ /* Step 10 eliminated: data->DATA_VREG, data_end->DATA_END_VREG,
+ * slot_addr->SLOT_VREG computed directly in steps 6/8/9 above.
+ */
+
+ pr_debug("knod_bpf DEBUG: prologue emitted idx=%u (KNOD_META_INSNS=%d)\n",
+ meta->amdgpu_insns, KNOD_META_INSNS);
+ if (WARN_ON(meta->amdgpu_insns > KNOD_META_INSNS))
+ return -ENOSPC;
+ list_add_tail(&meta->l, &knod_prog->pre_insns);
+
+ return 0;
+}
+
+static int knod_prog_prepare(struct knod_bpf_priv *priv,
+ struct knod_prog *knod_prog,
+ const struct bpf_insn *prog,
+ unsigned int cnt)
+{
+ struct knod_insn_meta *meta;
+ unsigned int i;
+
+ /* Pre-scan: detect helper 44/65 to set uses_adjust early */
+ for (i = 0; i < cnt; i++) {
+ if (prog[i].code == (BPF_JMP | BPF_CALL) &&
+ (prog[i].imm == 44 || prog[i].imm == 65)) {
+ knod_prog->uses_adjust = true;
+ break;
+ }
+ }
+
+ knod_vset64(&r64[0], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_vset64(&r64[1], KNOD_AMDGPU_TMP_VREG1_LO);
+ knod_vset64(&r64[2], KNOD_AMDGPU_TMP_VREG2_LO);
+ knod_vset64(&r64[3], KNOD_AMDGPU_TMP_VREG3_LO);
+ knod_vset64(&r64[4], KNOD_AMDGPU_TMP_VREG4_LO);
+ knod_vset64(&r64[5], KNOD_AMDGPU_TMP_VREG5_LO);
+ knod_vset64(&r64[6], KNOD_AMDGPU_TMP_VREG6_LO);
+ knod_vset64(&r64[7], KNOD_AMDGPU_TMP_VREG7_LO);
+ knod_vset64(&r64[8], KNOD_AMDGPU_TMP_VREG8_LO);
+ knod_vset64(&r64[9], KNOD_AMDGPU_TMP_VREG9_LO);
+ knod_vset64(&r64[10], KNOD_AMDGPU_TMP_VREG10_LO);
+ knod_vset64(&r64[11], KNOD_AMDGPU_TMP_VREG11_LO);
+ knod_vset64(&r64[12], KNOD_AMDGPU_TMP_VREG12_LO);
+ knod_vset64(&r64[13], KNOD_AMDGPU_TMP_VREG13_LO);
+ knod_vset64(&r64[14], KNOD_AMDGPU_TMP_VREG14_LO);
+ knod_vset64(&r64[15], KNOD_AMDGPU_TMP_VREG15_LO);
+ knod_vset64(&r64[16], KNOD_AMDGPU_TMP_VREG16_LO);
+ knod_vset64(&r64[17], KNOD_AMDGPU_TMP_VREG17_LO);
+ knod_vset64(&r64[18], KNOD_AMDGPU_TMP_VREG18_LO);
+ knod_vset64(&r64[19], KNOD_AMDGPU_CTX_VREG_LO);
+
+ knod_sset64(&sr64[0], KNOD_AMDGPU_TMP_SREG0_LO);
+ knod_sset64(&sr64[1], KNOD_AMDGPU_TMP_SREG1_LO);
+ knod_sset64(&sr64[2], KNOD_AMDGPU_TMP_SREG2_LO);
+ knod_sset64(&sr64[3], KNOD_AMDGPU_TMP_SREG3_LO);
+ knod_sset64(&sr64[4], KNOD_AMDGPU_TMP_SREG4_LO);
+ knod_sset64(&sr64[5], KNOD_AMDGPU_TMP_SREG5_LO);
+
+ knod_vset64(&bpf_reg64[0], KNOD_AMDGPU_VREG0_LO);
+ knod_vset64(&bpf_reg64[1], KNOD_AMDGPU_VREG1_LO);
+ knod_vset64(&bpf_reg64[2], KNOD_AMDGPU_VREG2_LO);
+ knod_vset64(&bpf_reg64[3], KNOD_AMDGPU_VREG3_LO);
+ knod_vset64(&bpf_reg64[4], KNOD_AMDGPU_VREG4_LO);
+ knod_vset64(&bpf_reg64[5], KNOD_AMDGPU_VREG5_LO);
+ knod_vset64(&bpf_reg64[6], KNOD_AMDGPU_VREG6_LO);
+ knod_vset64(&bpf_reg64[7], KNOD_AMDGPU_VREG7_LO);
+ knod_vset64(&bpf_reg64[8], KNOD_AMDGPU_VREG8_LO);
+ knod_vset64(&bpf_reg64[9], KNOD_AMDGPU_VREG9_LO);
+ knod_vset64(&bpf_reg64[10], KNOD_AMDGPU_FRAME_POINTER_VREG_LO);
+
+ knod_vset32(&r32[0], KNOD_AMDGPU_TMP_VREG0_LO);
+ for (i = 1; i < 40; i++)
+ knod_vset32(&r32[i], r32[i - 1].v + 1);
+
+ if (knod_bpf_pkt_cache) {
+ int pkt_cache_start = knod_prog->uses_adjust ?
+ KNOD_AMDGPU_PKT_CACHE_VREG0 :
+ KNOD_AMDGPU_PAGE_BASE_VREG_LO;
+
+ knod_vset32(&pkt_cache[0], pkt_cache_start);
+ for (i = 1; i < 64; i++)
+ knod_vset32(&pkt_cache[i],
+ pkt_cache[i - 1].v + 1);
+ }
+
+ knod_vset32(&stack[0], KNOD_AMDGPU_STACK_VREG0);
+ for (i = 1; i < 128; i++)
+ knod_vset32(&stack[i], stack[i - 1].v + 1);
+
+ for (i = 0; i < cnt; i++) {
+ meta = kzalloc_obj(*meta, GFP_KERNEL);
+ if (!meta)
+ return -ENOMEM;
+
+ meta->insn = prog[i];
+ meta->bpf_insn_idx = i;
+
+ list_add_tail(&meta->l, &knod_prog->insns);
+ }
+ knod_prog->n_insns = cnt;
+
+ return 0;
+}
+
+static void knod_prog_free(struct knod_prog *knod_prog)
+{
+ struct knod_insn_meta *meta, *tmp;
+
+ //kfree(knod_prog->subprog);
+
+ list_for_each_entry_safe(meta, tmp, &knod_prog->pre_insns, l) {
+ list_del(&meta->l);
+ kfree(meta);
+ }
+ list_for_each_entry_safe(meta, tmp, &knod_prog->insns, l) {
+ list_del(&meta->l);
+ kfree(meta);
+ }
+ list_for_each_entry_safe(meta, tmp, &knod_prog->post_insns, l) {
+ list_del(&meta->l);
+ kfree(meta);
+ }
+ kfree(knod_prog);
+}
+
+static int knod_bpf_verifier_prep(struct bpf_prog *prog)
+{
+ struct knod_prog *knod_prog;
+ struct knod_bpf_priv *priv;
+ int err;
+
+ knod_prog = kzalloc_obj(struct knod_prog, GFP_KERNEL);
+ if (!knod_prog)
+ return -ENOMEM;
+
+ INIT_LIST_HEAD(&knod_prog->insns);
+ INIT_LIST_HEAD(&knod_prog->pre_insns);
+ INIT_LIST_HEAD(&knod_prog->post_insns);
+ prog->aux->offload->dev_priv = knod_prog;
+ priv = bpf_offload_dev_priv(prog->aux->offload->offdev);
+ knod_prog->knodev = priv->knodev;
+ WRITE_ONCE(priv->knod_prog, knod_prog);
+ knod_prog->knod = priv->knod;
+ knod_prog->insn_idx = 0;
+
+ if (priv->isa_version == 10) {
+ knod_prog->done_mask_sreg = 32;
+ knod_prog->exec_save_base = 34;
+ knod_prog->initial_exec_sreg =
+ KNOD_AMDGPU_INITIAL_EXEC_SREG_GFX10;
+ } else {
+ knod_prog->done_mask_sreg = KNOD_AMDGPU_DONE_MASK_SREG;
+ knod_prog->exec_save_base = KNOD_AMDGPU_EXEC_SAVE_SREG_BASE;
+ knod_prog->initial_exec_sreg =
+ KNOD_AMDGPU_INITIAL_EXEC_SREG_GFX9;
+ }
+
+ err = knod_prog_prepare(priv, knod_prog, prog->insnsi, prog->len);
+ if (err)
+ goto err_free;
+
+ knod_prog->meta = knod_prog_first_meta(knod_prog);
+
+ return 0;
+
+err_free:
+ knod_prog_free(knod_prog);
+
+ return err;
+}
+
+static struct knod_insn_meta *knod_bpf_lookup_meta(struct knod_prog *knod_prog,
+ short idx)
+{
+ struct knod_insn_meta *meta;
+
+ list_for_each_entry(meta, &knod_prog->insns, l) {
+ if (meta->amdgpu_insn_idx == AMDGPU_INSN_SKIP)
+ continue;
+ if (meta->bpf_insn_idx == idx)
+ return meta;
+ }
+
+ return NULL;
+}
+
+static void knod_mov64_imm(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ int d, u64 imm64)
+{
+ struct amdgcn_param32 param[2];
+
+ knod_vset32(¶m[0], d);
+ knod_iset32(¶m[1], imm64 & ~0U);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0], param[1]);
+ knod_vset32(¶m[0], d + 1);
+ knod_iset32(¶m[1], imm64 >> 32);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0], param[1]);
+}
+
+static void knod_mov32(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src)
+{
+ knod_emit(priv, meta, v_mov_b32_e32, dst, src);
+}
+
+static void knod_mov64(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src)
+{
+ knod_mov32(priv, meta, dst.lo, src.lo);
+ knod_mov32(priv, meta, dst.hi, src.hi);
+}
+
+static void knod_add64(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src0,
+ struct amdgcn_param64 src1)
+{
+ knod_emit(priv, meta, v_add_co_u32, dst.lo, src0.lo, src1.lo);
+ knod_emit(priv, meta, v_add_co_ci_u32_e32, dst.hi, src0.hi,
+ src1.hi);
+}
+
+/* No carry out/in */
+static void knod_add32(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ knod_emit(priv, meta, v_add_u32, dst, src0, src1);
+}
+
+static void knod_xor32(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ knod_emit(priv, meta, v_xor_b32_e32, dst, src0, src1);
+}
+
+static void knod_alignbit32(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1,
+ struct amdgcn_param32 src2)
+{
+ knod_emit(priv, meta, v_alignbit_b32, dst, src0, src1, src2);
+}
+
+static void knod_bfe32(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1,
+ struct amdgcn_param32 src2)
+{
+ knod_emit(priv, meta, v_bfe_u32, dst, src0, src1, src2);
+}
+
+static void knod_bfi32(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1,
+ struct amdgcn_param32 src2)
+{
+ knod_emit(priv, meta, v_bfi_b32, dst, src0, src1, src2);
+}
+
+static void knod_lshrrev32(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ knod_emit(priv, meta, v_lshrrev_b32, dst, src0, src1);
+}
+
+static void knod_lshrrev64(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src0,
+ struct amdgcn_param64 src1)
+{
+ knod_emit(priv, meta, v_lshrrev_b64, dst, src0, src1);
+}
+
+static void knod_ashrrev32(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ knod_emit(priv, meta, v_ashrrev_i32, dst, src0, src1);
+}
+
+static void knod_ashrrev64(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src0,
+ struct amdgcn_param64 src1)
+{
+ knod_emit(priv, meta, v_ashrrev_i64, dst, src0, src1);
+}
+
+static void knod_lshlrev32(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ knod_emit(priv, meta, v_lshlrev_b32, dst, src0, src1);
+}
+
+static void knod_lshlrev64(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src0,
+ struct amdgcn_param64 src1)
+{
+ knod_emit(priv, meta, v_lshlrev_b64, dst, src0, src1);
+}
+
+/* No carry out/in */
+static void knod_sub32(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ knod_emit(priv, meta, v_sub_u32, dst, src0, src1);
+}
+
+static void knod_and32(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ knod_emit(priv, meta, v_and_b32_e32, dst, src0, src1);
+}
+
+static void knod_and64(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src0,
+ struct amdgcn_param64 src1)
+{
+ knod_and32(priv, meta, dst.lo, src0.lo, src1.lo);
+ knod_and32(priv, meta, dst.hi, src0.hi, src1.hi);
+}
+
+static void knod_or32(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src0,
+ struct amdgcn_param32 src1)
+{
+ knod_emit(priv, meta, v_or_b32_e32, dst, src0, src1);
+}
+
+static void knod_sub64(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src0,
+ struct amdgcn_param64 src1)
+{
+ knod_emit(priv, meta, v_sub_co_u32, dst.lo, src0.lo, src1.lo);
+ knod_emit(priv, meta, v_sub_co_ci_u32_e32, dst.hi, src0.hi,
+ src1.hi);
+}
+
+static void knod_subrev64(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src0,
+ struct amdgcn_param64 src1)
+{
+ knod_emit(priv, meta, v_subrev_co_u32, dst.lo, src0.lo, src1.lo);
+ knod_emit(priv, meta, v_subrev_co_ci_u32_e32, dst.hi, src0.hi,
+ src1.hi);
+}
+
+static void knod_mul_lo32(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src1,
+ struct amdgcn_param32 src2)
+{
+ knod_emit(priv, meta, v_mul_lo_u32, dst, src1, src2);
+}
+
+static void knod_mul_hi32(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ struct amdgcn_param32 dst,
+ struct amdgcn_param32 src1,
+ struct amdgcn_param32 src2)
+{
+ knod_emit(priv, meta, v_mul_hi_u32, dst, src1, src2);
+}
+
+static void knod_mul64(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 src1,
+ struct amdgcn_param64 src2,
+ struct amdgcn_param64 tmp)
+{
+ /*
+ * v_mul_lo_u32 v1, v2, v1
+ * v_mul_hi_u32 v5, v2, v0
+ * v_mul_lo_u32 v3, v3, v0
+ * v_mul_lo_u32 v0, v2, v0
+ * v_add_u32_e32 v1, v5, v1
+ * v_add_u32_e32 v1, v1, v3
+ *
+ * v[0:1] = src1, dst
+ * v[2:3] = src2
+ * v5 = tmp
+ */
+
+ /* v_mul_lo_u32 v1, v2, v1 */
+ knod_mul_lo32(priv, meta, src2.hi, src1.lo, src2.hi);
+ /* v_mul_hi_u32 v5, v2, v0 */
+ knod_mul_hi32(priv, meta, tmp.lo, src1.lo, src2.lo);
+ /* v_mul_lo_u32 v3, v3, v0 */
+ knod_mul_lo32(priv, meta, src1.hi, src1.hi, src2.lo);
+ /* v_mul_lo_u32 v0, v2, v0 */
+ knod_mul_lo32(priv, meta, src1.lo, src1.lo, src2.lo);
+ /* v_add_u32_e32 v1, v5, v1 */
+ knod_add32(priv, meta, src2.lo, tmp.lo, src2.hi);
+ /* v_add_u32_e32 v1, v1, v3 */
+ knod_add32(priv, meta, src1.hi, src2.lo, src1.hi);
+ knod_mov32(priv, meta, dst.lo, src1.lo);
+ knod_mov32(priv, meta, dst.hi, src1.hi);
+}
+
+static void knod_div(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 imm,
+ struct amdgcn_param64 tmp_reg0,
+ struct amdgcn_param64 tmp_reg1,
+ struct amdgcn_param64 tmp_reg2,
+ struct amdgcn_param64 tmp_reg3)
+{
+ struct reciprocal_value_adv rvalue;
+ struct amdgcn_param64 p64[4];
+ u8 pre_shift, exp;
+
+ WARN_ON((imm.lo.type != AMDGCN_PARAM_TYPE_INTEGER_0) &&
+ (imm.lo.type != AMDGCN_PARAM_TYPE_LITERAL_CONST));
+ WARN_ON((imm.hi.type != AMDGCN_PARAM_TYPE_INTEGER_0) &&
+ (imm.hi.type != AMDGCN_PARAM_TYPE_LITERAL_CONST));
+ knod_iset64(&p64[0], 0);
+ knod_iset64(&p64[1], 0);
+ knod_iset64(&p64[2], 0);
+ knod_iset64(&p64[3], 0);
+ /*
+ * dst := imm
+ * n := dst_reg
+ */
+ if (imm.imm > U32_MAX) {
+ knod_mov64(priv, meta, dst, p64[0]);
+ return;
+ }
+
+ if (imm.imm >= 1U << 31) {
+ /* result = n >= dst; */
+ knod_mov64(priv, meta, tmp_reg0, imm);
+ knod_emit(priv, meta, v_cmp_ge_u64, dst, tmp_reg0);
+ return;
+ }
+
+ rvalue = reciprocal_value_adv(imm.lo.v, 32);
+ exp = rvalue.exp;
+ if (rvalue.is_wide_m && !(imm.lo.v & 1)) {
+ pre_shift = fls(imm.lo.v & -imm.lo.v) - 1;
+ rvalue = reciprocal_value_adv(imm.lo.v >> pre_shift,
+ 32 - pre_shift);
+ } else {
+ pre_shift = 0;
+ }
+
+ if (imm.lo.v == 1U << exp) {
+ knod_iset64(&p64[0], exp);
+ /* n = n >> exp */
+ knod_lshrrev64(priv, meta, dst, p64[0], dst);
+ return;
+ } else if (rvalue.is_wide_m) {
+ /*
+ * pre_shift must be zero when reached here.
+ * t = (n * rvalue.m) >> 32;
+ * result = n - t;
+ * result >>= 1;
+ * result += t;
+ * result >>= rvalue.sh - 1;
+ */
+
+ /*
+ * n := VREG0
+ * t := VREG1
+ * rvalue.m := VREG2
+ * tmp := VREG3
+ */
+
+ /* n := TMP_VREG0 */
+ knod_mov64(priv, meta, tmp_reg0, dst);
+
+ knod_iset64(&p64[0], rvalue.m);
+ /* rvalue.m := TMP_VREG2 */
+ knod_mov64(priv, meta, tmp_reg2, p64[0]);
+
+ /* t = n * rvalue.m; */
+ knod_mul64(priv, meta,
+ tmp_reg1, /* t */
+ tmp_reg0, /* n */
+ tmp_reg2, /* rvalue.m */
+ tmp_reg3); /* tmp */
+
+ /* t >>= 32; */
+ knod_iset64(&p64[0], 0);
+ knod_mov32(priv, meta, tmp_reg1.lo, tmp_reg1.hi);
+ knod_mov32(priv, meta, tmp_reg1.hi, p64[0].lo);
+
+ /* result = n - t */
+ knod_sub64(priv, meta, dst, dst, tmp_reg1);
+
+ /* result >>= 1 */
+ knod_iset64(&p64[0], 1);
+ knod_lshrrev64(priv, meta, dst, p64[0], dst);
+
+ /* result += t; */
+ knod_add64(priv, meta,
+ dst,
+ dst, /* result */
+ tmp_reg1); /* t */
+
+ /* result >>= rvalue.sh - 1; */
+ knod_iset64(&p64[0], rvalue.sh - 1);
+ WARN_ON(rvalue.sh - 1 > 31);
+ knod_lshrrev64(priv, meta, dst, p64[0], dst);
+ return;
+ }
+
+ /*
+ * if (pre_shift)
+ * result = n >> pre_shift;
+ * result = ((u64)result * rvalue.m) >> 32;
+ * result >>= rvalue.sh;
+ */
+
+ /*
+ * n := VREG0
+ * <NONE> := VREG1
+ * rvalue.m := VREG2
+ * tmp := VREG3
+ * result := dst * 2
+ */
+
+ /* n := TMP_VREG0 */
+ knod_mov64(priv, meta, tmp_reg0, dst);
+
+ /* rvalue.m := TMP_VREG2 */
+ knod_iset64(&p64[0], rvalue.m);
+ knod_mov64(priv, meta, tmp_reg2, p64[0]);
+
+ if (pre_shift) {
+ /* result = n >> pre_shift; */
+ knod_iset64(&p64[0], pre_shift);
+ knod_lshrrev64(priv, meta, dst, p64[0],
+ tmp_reg0); /* n */
+ } else {
+ /* tmp = 0 */
+ knod_iset64(&p64[0], 0);
+ knod_mov64(priv, meta, tmp_reg0, p64[0]);
+ }
+
+ /* result = result * rvalue.m; */
+ knod_mul64(priv, meta,
+ dst, /* result */
+ dst, /* result */
+ tmp_reg2, /* rvalue.m */
+ tmp_reg3); /* tmp */
+
+ /* result >>= (32 + rvalue.sh); */
+ knod_iset64(&p64[0], 32 + rvalue.sh);
+ knod_lshrrev64(priv, meta, dst, p64[0], dst);
+}
+
+static void knod_mod(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param64 imm,
+ struct amdgcn_param64 tmp_reg0,
+ struct amdgcn_param64 tmp_reg1,
+ struct amdgcn_param64 tmp_reg2,
+ struct amdgcn_param64 tmp_reg3,
+ struct amdgcn_param64 tmp_reg4)
+{
+ WARN_ON((imm.lo.type != AMDGCN_PARAM_TYPE_INTEGER_0) &&
+ (imm.lo.type != AMDGCN_PARAM_TYPE_LITERAL_CONST));
+ WARN_ON((imm.hi.type != AMDGCN_PARAM_TYPE_INTEGER_0) &&
+ (imm.hi.type != AMDGCN_PARAM_TYPE_LITERAL_CONST));
+ /* q := tmp_reg0 */
+ knod_mov64(priv, meta, tmp_reg0, dst);
+ /* q = n / imm */
+ knod_div(priv, meta, tmp_reg0, imm,
+ tmp_reg1, tmp_reg2, tmp_reg3, tmp_reg4);
+
+ /* tmp_reg1 := imm_reg */
+ knod_mov64(priv, meta, tmp_reg1, imm);
+
+ /* imm * q := tmp_reg3 */
+ knod_mul64(priv, meta,
+ tmp_reg3, /* imm * q */
+ tmp_reg0, /* q */
+ tmp_reg1, /* imm_reg */
+ tmp_reg2); /* tmp */
+
+ knod_sub64(priv, meta, dst, dst, tmp_reg3);
+}
+
+/*
+ * Fast constant modulo on the 32-bit value in @dst.lo for divisors of a
+ * special form, avoiding knod_mod's reciprocal divide + 64-bit multiply:
+ * 2^k -> dst & (2^k-1) (mask)
+ * 2^k + 1 -> lo - hi (+C if lo<hi) (Fermat: 2^k = -1 mod C)
+ * 2^k - 1 -> lo + hi (-C while >=C) (Mersenne: 2^k = 1 mod C)
+ * lo/hi are the low/high k-bit halves. One fold is exact for a 32-bit
+ * dividend when 2^k covers the high half (true for e.g. 65537 = 2^16+1,
+ * kondor's per-packet `hash % RING_SIZE`). Returns false for other
+ * divisors (caller falls back to knod_mod). Scratch: r64[0], r64[1].
+ */
+static bool knod_mod_k32(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ struct amdgcn_param64 dst, u32 imm)
+{
+ struct amdgcn_param32 p;
+ int i;
+
+ if (is_power_of_2(imm)) {
+ knod_iset32(&p, imm - 1);
+ knod_and32(priv, meta, dst.lo, p, dst.lo);
+ } else if (is_power_of_2(imm - 1) && (imm - 1) >= (1u << 16)) {
+ knod_iset32(&p, imm - 2); /* mask 2^k-1 */
+ knod_and32(priv, meta, r64[0].lo, p, dst.lo); /* lo */
+ knod_iset32(&p, ilog2(imm - 1)); /* k */
+ knod_emit(priv, meta, v_lshrrev_b32, r64[1].lo, p, dst.lo);
+ /* lo-hi */
+ knod_sub32(priv, meta, dst.lo, r64[0].lo, r64[1].lo);
+ knod_emit(priv, meta, v_cmp_lt_u32, r64[0].lo, r64[1].lo);
+ knod_iset32(&p, imm);
+ knod_add32(priv, meta, r64[1].lo, p, dst.lo); /* +C */
+ knod_emit(priv, meta, v_cndmask_b32_e32, dst.lo, dst.lo,
+ r64[1].lo);
+ } else if (is_power_of_2(imm + 1) && (imm + 1) >= (1u << 16)) {
+ knod_iset32(&p, imm); /* mask 2^k-1 */
+ knod_and32(priv, meta, r64[0].lo, p, dst.lo); /* lo */
+ knod_iset32(&p, ilog2(imm + 1)); /* k */
+ knod_emit(priv, meta, v_lshrrev_b32, r64[1].lo, p, dst.lo);
+ /* lo+hi */
+ knod_add32(priv, meta, dst.lo, r64[0].lo, r64[1].lo);
+ knod_iset32(&p, imm);
+ knod_mov32(priv, meta, r64[0].lo, p); /* C in VGPR */
+ for (i = 0; i < 2; i++) { /* r < 2C */
+ knod_emit(priv, meta, v_cmp_le_u32, r64[0].lo, dst.lo);
+ knod_sub32(priv, meta, r64[1].lo, dst.lo, r64[0].lo);
+ knod_emit(priv, meta, v_cndmask_b32_e32, dst.lo,
+ dst.lo, r64[1].lo);
+ }
+ } else {
+ return false;
+ }
+
+ knod_iset32(&p, 0);
+ knod_mov32(priv, meta, dst.hi, p);
+ return true;
+}
+
+/* Considered to be able to use all temporary vregisters
+ * Also, key is stack pointer, not global
+ */
+static void knod_jhash(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ u32 dst_idx, u32 length, u32 initval)
+{
+ u32 a_reg = TREG32_MAX - 3, b_reg = TREG32_MAX - 2;
+ u32 c_reg = TREG32_MAX - 1, d_reg = TREG32_MAX;
+ u32 key_in_pkt = KEY_IN_PKT_32;
+ struct amdgcn_param32 p32;
+
+ knod_iset32(&p32, JHASH_INITVAL + length + initval);
+ knod_mov32(priv, meta, r32[a_reg], p32);
+ knod_mov32(priv, meta, r32[b_reg], p32);
+ knod_mov32(priv, meta, r32[c_reg], p32);
+ knod_iset32(&p32, 0);
+
+ while (length > 12) {
+ /* a += *key; */
+ knod_add32(priv, meta, r32[a_reg], r32[a_reg],
+ r32[key_in_pkt]);
+ /* b += *(key + 4); */
+ knod_add32(priv, meta, r32[b_reg], r32[b_reg],
+ r32[key_in_pkt + 1]);
+ /* c += *(key + 8); */
+ knod_add32(priv, meta, r32[c_reg], r32[c_reg],
+ r32[key_in_pkt + 2]);
+ /* a -= c; */
+ knod_sub32(priv, meta, r32[a_reg], r32[a_reg], r32[c_reg]);
+ /* a ^= rol32(c, 4); */
+ knod_iset32(&p32, 32 - 4);
+ knod_alignbit32(priv, meta, r32[d_reg], r32[c_reg],
+ r32[c_reg], p32);
+ knod_xor32(priv, meta,
+ r32[a_reg], r32[a_reg], r32[d_reg]);
+ /* c += b; */
+ knod_add32(priv, meta, r32[c_reg], r32[c_reg], r32[b_reg]);
+ /* b -= a; */
+ knod_sub32(priv, meta, r32[b_reg], r32[b_reg], r32[a_reg]);
+ /* b ^= rol32(a, 6); */
+ knod_iset32(&p32, 32 - 6);
+ knod_alignbit32(priv, meta, r32[d_reg], r32[a_reg],
+ r32[a_reg], p32);
+ knod_xor32(priv, meta,
+ r32[b_reg], r32[b_reg], r32[d_reg]);
+ /*a += c; */
+ knod_add32(priv, meta, r32[a_reg], r32[a_reg], r32[c_reg]);
+ /* c -= b; */
+ knod_sub32(priv, meta, r32[c_reg], r32[c_reg], r32[b_reg]);
+ /* c ^= rol32(b, 8); */
+ knod_iset32(&p32, 32 - 8);
+ knod_alignbit32(priv, meta, r32[d_reg], r32[b_reg],
+ r32[b_reg], p32);
+ knod_xor32(priv, meta,
+ r32[c_reg], r32[c_reg], r32[d_reg]);
+ /* b += a; */
+ knod_add32(priv, meta, r32[b_reg], r32[b_reg], r32[a_reg]);
+ /* a -= c; */
+ knod_sub32(priv, meta, r32[a_reg], r32[a_reg], r32[c_reg]);
+ /* a ^= rol32(c, 16); */
+ knod_iset32(&p32, 32 - 16);
+ knod_alignbit32(priv, meta, r32[d_reg], r32[c_reg],
+ r32[c_reg], p32);
+ knod_xor32(priv, meta, r32[a_reg], r32[a_reg], r32[d_reg]);
+ /* c += b; */
+ knod_add32(priv, meta, r32[c_reg], r32[c_reg], r32[b_reg]);
+ /* b -= a; */
+ knod_sub32(priv, meta, r32[b_reg], r32[b_reg], r32[a_reg]);
+ /* b ^= rol32(a, 19); */
+ knod_iset32(&p32, 32 - 19);
+ knod_alignbit32(priv, meta, r32[d_reg], r32[a_reg],
+ r32[a_reg], p32);
+ knod_xor32(priv, meta, r32[b_reg], r32[b_reg], r32[d_reg]);
+ /* a += c; */
+ knod_add32(priv, meta, r32[a_reg], r32[a_reg], r32[c_reg]);
+ /* c -= b; */
+ knod_sub32(priv, meta, r32[c_reg], r32[c_reg], r32[b_reg]);
+ /* c ^= rol32(b, 4); */
+ knod_iset32(&p32, 32 - 4);
+ knod_alignbit32(priv, meta, r32[d_reg], r32[b_reg],
+ r32[b_reg], p32);
+ knod_xor32(priv, meta, r32[c_reg], r32[c_reg], r32[d_reg]);
+ /* b += a; */
+ knod_add32(priv, meta, r32[b_reg], r32[b_reg], r32[a_reg]);
+ length -= 12;
+ key_in_pkt += 3;
+ }
+
+ switch (length) {
+ case 12:
+ /* c += (unsigned int)k[11]<<24; */
+ fallthrough;
+ case 11:
+ /* c += (unsigned int)k[10]<<16; */
+ fallthrough;
+ case 10:
+ /* c += (unsigned int)k[9]<<8; */
+ fallthrough;
+ case 9:
+ /* c += k[8]; */
+ knod_add32(priv, meta, r32[c_reg], r32[c_reg],
+ r32[key_in_pkt + 2]);
+ fallthrough;
+ case 8:
+ /* b += (unsigned int)k[7]<<24; */
+ fallthrough;
+ case 7:
+ /* b += (unsigned int)k[6]<<16; */
+ fallthrough;
+ case 6:
+ /* b += (unsigned int)k[5]<<8; */
+ fallthrough;
+ case 5:
+ /* b += k[4]; */
+ knod_add32(priv, meta, r32[b_reg], r32[b_reg],
+ r32[key_in_pkt + 1]);
+ fallthrough;
+ case 4:
+ /* a += (unsigned int)k[3]<<24; */
+ fallthrough;
+ case 3:
+ /* a += (unsigned int)k[2]<<16; */
+ fallthrough;
+ case 2:
+ /* a += (unsigned int)k[1]<<8; */
+ fallthrough;
+ case 1:
+ /* a += k[0]; */
+ knod_add32(priv, meta, r32[a_reg], r32[a_reg],
+ r32[key_in_pkt]);
+ /* c ^= b; */
+ knod_xor32(priv, meta, r32[c_reg], r32[c_reg], r32[b_reg]);
+ /* c -= rol32(b, 14); */
+ knod_iset32(&p32, 32 - 14);
+ knod_alignbit32(priv, meta, r32[d_reg], r32[b_reg],
+ r32[b_reg], p32);
+ knod_sub32(priv, meta, r32[c_reg], r32[c_reg], r32[d_reg]);
+ /* a ^= c; */
+ knod_xor32(priv, meta, r32[a_reg], r32[a_reg], r32[c_reg]);
+ /* a -= rol32(c, 11); */
+ knod_iset32(&p32, 32 - 11);
+ knod_alignbit32(priv, meta, r32[d_reg], r32[c_reg],
+ r32[c_reg], p32);
+ knod_sub32(priv, meta, r32[a_reg], r32[a_reg], r32[d_reg]);
+ /* b ^= a; */
+ knod_xor32(priv, meta, r32[b_reg], r32[b_reg], r32[a_reg]);
+ /* b -= rol32(a, 25); */
+ knod_iset32(&p32, 32 - 25);
+ knod_alignbit32(priv, meta, r32[d_reg], r32[a_reg],
+ r32[a_reg], p32);
+ knod_sub32(priv, meta, r32[b_reg], r32[b_reg], r32[d_reg]);
+ /* c ^= b; */
+ knod_xor32(priv, meta, r32[c_reg], r32[c_reg], r32[b_reg]);
+ /* c -= rol32(b, 16); */
+ knod_iset32(&p32, 32 - 16);
+ knod_alignbit32(priv, meta, r32[d_reg], r32[b_reg],
+ r32[b_reg], p32);
+ knod_sub32(priv, meta, r32[c_reg], r32[c_reg], r32[d_reg]);
+ /* a ^= c; */
+ knod_xor32(priv, meta, r32[a_reg], r32[a_reg], r32[c_reg]);
+ /* a -= rol32(c, 4); */
+ knod_iset32(&p32, 32 - 4);
+ knod_alignbit32(priv, meta, r32[d_reg], r32[c_reg],
+ r32[c_reg], p32);
+ knod_sub32(priv, meta, r32[a_reg], r32[a_reg], r32[d_reg]);
+ /* b ^= a; */
+ knod_xor32(priv, meta, r32[b_reg], r32[b_reg], r32[a_reg]);
+ /* b -= rol32(a, 14); */
+ knod_iset32(&p32, 32 - 14);
+ knod_alignbit32(priv, meta, r32[d_reg], r32[a_reg],
+ r32[a_reg], p32);
+ knod_sub32(priv, meta, r32[b_reg], r32[b_reg], r32[d_reg]);
+ /* c ^= b; */
+ knod_xor32(priv, meta, r32[c_reg], r32[c_reg], r32[b_reg]);
+ /* c -= rol32(b, 24); */
+ knod_iset32(&p32, 32 - 24);
+ knod_alignbit32(priv, meta, r32[d_reg], r32[b_reg],
+ r32[b_reg], p32);
+ knod_sub32(priv, meta, r32[c_reg], r32[c_reg], r32[d_reg]);
+ break;
+ case 0: /* Nothing left to add */
+ break;
+ }
+
+ knod_mov32(priv, meta, r64[dst_idx].lo, r32[c_reg]);
+}
+
+static u64 knod_bpf_map_gaddr(struct knod_bpf_priv *priv, int id)
+{
+ struct knod_dev *knodev = priv->knodev;
+ struct knod_bpf_map *knod_map;
+ struct knod_mem *mem;
+
+ mutex_lock(&knodev->lock);
+ list_for_each_entry(knod_map, &knodev->accel->xdp.bound_maps, list) {
+ if (knod_map->offmap->map.id == id) {
+ mem = knod_map->mem;
+ mutex_unlock(&knodev->lock);
+ return (u64)mem->gaddr;
+ }
+ }
+ mutex_unlock(&knodev->lock);
+
+ return 0;
+}
+
+static void *knod_bpf_map_kaddr(struct knod_bpf_priv *priv, int id)
+{
+ struct knod_dev *knodev = priv->knodev;
+ struct knod_bpf_map *knod_map;
+ struct knod_mem *mem;
+
+ mutex_lock(&knodev->lock);
+ list_for_each_entry(knod_map, &knodev->accel->xdp.bound_maps, list) {
+ if (knod_map->offmap->map.id == id) {
+ mem = knod_map->mem;
+
+ /* GPUVM */
+ mutex_unlock(&knodev->lock);
+ return (void *)mem->kaddr;
+ }
+ }
+ mutex_unlock(&knodev->lock);
+
+ return NULL;
+}
+
+static u64 knod_bpf_get_map_gaddr(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta1,
+ struct knod_insn_meta *meta2)
+{
+ struct bpf_map *map;
+
+ map = (void *)(unsigned long)((u32)meta1->insn.imm |
+ (u64)meta2->insn.imm << 32);
+
+ return knod_bpf_map_gaddr(priv, map->id);
+}
+
+static int knod_bpf_get_map_id(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta1,
+ struct knod_insn_meta *meta2)
+{
+ struct bpf_map *map;
+
+ map = (void *)(unsigned long)((u32)meta1->insn.imm |
+ (u64)meta2->insn.imm << 32);
+
+ return map->id;
+}
+
+static int knod_bpf_get_amdgpu_insn_idx(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ int t)
+{
+ int i, insn_idx = meta->amdgpu_insn_idx;
+
+ for (i = 0; i < t; i++)
+ insn_idx += meta->amdgpu_insn[i].size / 4;
+
+ return insn_idx;
+}
+
+static void knod_bpf_fixup_branch(struct knod_bpf_priv *priv,
+ struct amdgcn_branch_fixup *fixup)
+{
+ int target_off = knod_bpf_get_amdgpu_insn_idx(priv,
+ fixup->target_label->meta,
+ fixup->target_label->insn_idx);
+ int cur_off = knod_bpf_get_amdgpu_insn_idx(priv,
+ fixup->meta,
+ fixup->insn_idx);
+ cur_off++;
+
+ target_off -= cur_off;
+ emit_branch_fixup(priv->isa_version,
+ &fixup->meta->amdgpu_insn[fixup->insn_idx],
+ target_off);
+ knod_jit_dbg(" target_off was updated to %d\n", target_off);
+}
+
+static void knod_bpf_set_fixup(struct knod_insn_meta *meta,
+ struct amdgcn_branch_fixup *fixup,
+ struct amdgcn_label *target_label,
+ int insn_idx)
+{
+ fixup->meta = meta;
+ fixup->insn_idx = insn_idx;
+ fixup->target_label = target_label;
+}
+
+static void knod_bpf_set_label(struct knod_insn_meta *meta,
+ struct amdgcn_label *label,
+ int insn_idx)
+{
+ label->meta = meta;
+ label->insn_idx = insn_idx;
+}
+
+/*
+ * knod_bpf_emit_offlen_writeback - Write updated off/len to spsc_bd.
+ *
+ * Uses PAGE_BASE_VREG (set once in prologue), computes
+ * new_off = DATA_VREG_LO - PAGE_BASE_VREG_LO and
+ * new_len = DATA_END_VREG_LO - DATA_VREG_LO, packs them as (len<<16)|off,
+ * and stores the result at spsc_bd.off via SLOT_VREG.
+ *
+ * Clobbers: TMP_VREG2 (v26:v27).
+ */
+static void knod_bpf_emit_offlen_writeback(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta)
+{
+ struct amdgcn_param32 s0, s1, data_lo, data_end_lo, pbase_lo, slot_lo;
+ struct amdgcn_param32 imm;
+
+ knod_vset32(&s0, KNOD_AMDGPU_TMP_VREG2_LO);
+ knod_vset32(&s1, KNOD_AMDGPU_TMP_VREG2_HI);
+ knod_vset32(&data_lo, KNOD_AMDGPU_DATA_VREG_LO);
+ knod_vset32(&data_end_lo, KNOD_AMDGPU_DATA_END_VREG_LO);
+ knod_vset32(&pbase_lo, KNOD_AMDGPU_PAGE_BASE_VREG_LO);
+ knod_vset32(&slot_lo, KNOD_AMDGPU_SLOT_VREG_LO);
+
+ /* s0 = len = DATA_END_LO - DATA_LO */
+ knod_sub32(priv, meta, s0, data_end_lo, data_lo);
+
+ /* s0 = len << 16 */
+ knod_iset32(&imm, 16);
+ knod_lshlrev32(priv, meta, s0, imm, s0);
+
+ /* s1 = off = DATA_LO - page_base_lo */
+ knod_sub32(priv, meta, s1, data_lo, pbase_lo);
+
+ /* s0 = (len << 16) | off */
+ knod_or32(priv, meta, s0, s0, s1);
+
+ /* Store packed {off, len} to spsc_bd */
+ knod_emit(priv, meta, global_store_dword, s0, slot_lo,
+ offsetof(struct spsc_bd, off));
+}
+
+/*
+ * knod_bpf_xdp_adjust_head - JIT bpf_xdp_adjust_head (helper 44).
+ *
+ * R2 = delta (signed 32-bit). Adjusts DATA_VREG by delta.
+ * Bounds: page_base <= DATA_VREG <= DATA_END_VREG - ETH_HLEN.
+ * Each bound is checked with its own VOPC, but VCC is captured into
+ * VGPRs via v_cndmask (VALU) rather than SGPRs via s_mov_b64 (SALU).
+ * VALU reads VCC correctly after VOPC; only SALU suffers the GFX10
+ * dual-VOPC stale-read hazard.
+ * page_base is reloaded on demand from param + spsc_bd.
+ * On failure, DATA_VREG is restored and R0 = -EINVAL.
+ * On success, R0 = 0.
+ *
+ * Clobbers: TMP_VREG0 (v22:v23), TMP_VREG1 (v24:v25), TMP_VREG2 (v26:v27),
+ * TMP_SREG0 (s16), TMP_SREG2 (s20:s21).
+ */
+static void knod_bpf_xdp_adjust_head(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta)
+{
+ struct amdgcn_param32 ub_lo, ub_hi, dend_lo, dend_hi, sext_dst;
+ struct amdgcn_param32 shift_amt;
+ struct amdgcn_param32 tmp0_lo, tmp0_hi, data_lo, data_hi, fail_lo;
+ struct amdgcn_param32 fail_hi;
+ struct amdgcn_param64 data_vreg, pbase_vreg, ub;
+ struct amdgcn_param32 r0_lo, r0_hi, imm, delta;
+
+ knod_vset64(&data_vreg, KNOD_AMDGPU_DATA_VREG_LO);
+
+ knod_vset32(&tmp0_lo, KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_vset32(&tmp0_hi, KNOD_AMDGPU_TMP_VREG0_HI);
+ knod_vset32(&data_lo, KNOD_AMDGPU_DATA_VREG_LO);
+ knod_vset32(&data_hi, KNOD_AMDGPU_DATA_VREG_HI);
+ knod_vset32(&r0_lo, KNOD_AMDGPU_VREG0_LO);
+ knod_vset32(&r0_hi, KNOD_AMDGPU_VREG0_HI);
+ knod_vset32(&delta, bpf_reg64[2].lo.v);
+ knod_vset32(&fail_lo, KNOD_AMDGPU_TMP_VREG2_LO);
+ knod_vset32(&fail_hi, KNOD_AMDGPU_TMP_VREG2_HI);
+
+ /* 1. Save original DATA_VREG -> TMP_VREG0 */
+ knod_mov32(priv, meta, tmp0_lo, data_lo);
+ knod_mov32(priv, meta, tmp0_hi, data_hi);
+
+ /* 2. DATA_VREG += delta (R2.lo, sign-extended to 64-bit) */
+ knod_emit(priv, meta, v_add_co_u32, data_lo, delta, data_lo);
+
+ knod_vset32(&sext_dst, KNOD_AMDGPU_TMP_VREG1_LO);
+ knod_iset32(&shift_amt, 31);
+ knod_emit(priv, meta, v_ashrrev_i32, sext_dst, shift_amt, delta);
+
+ knod_emit(priv, meta, v_add_co_ci_u32_e32, data_hi, sext_dst,
+ data_hi);
+
+ /* 3. Lower bound: DATA_VREG < page_base -> VCC = fail */
+ knod_vset64(&pbase_vreg, KNOD_AMDGPU_PAGE_BASE_VREG_LO);
+ knod_emit(priv, meta, v_cmp_lt_u64, data_vreg, pbase_vreg);
+
+ /*
+ * Capture VCC -> VGPR via v_cndmask (VALU reads VCC correctly,
+ * unlike SALU which suffers the dual-VOPC stale-read hazard).
+ */
+ knod_iset32(&imm, 1);
+ knod_mov32(priv, meta, fail_hi, imm);
+ knod_iset32(&imm, 0);
+ knod_emit(priv, meta, v_cndmask_b32_e32, fail_lo, imm, fail_hi);
+
+ /* 5. Upper bound: DATA_VREG > DATA_END_VREG - ETH_HLEN */
+ knod_vset32(&ub_lo, KNOD_AMDGPU_TMP_VREG1_LO);
+ knod_vset32(&ub_hi, KNOD_AMDGPU_TMP_VREG1_HI);
+ knod_vset32(&dend_lo, KNOD_AMDGPU_DATA_END_VREG_LO);
+ knod_vset32(&dend_hi, KNOD_AMDGPU_DATA_END_VREG_HI);
+
+ knod_iset32(&imm, ETH_HLEN);
+ /*
+ * v_sub_co_u32 is VOP2 on GFX9, whose vsrc1 must be a VGPR (a literal
+ * there reads v0). Subtraction is not commutative, so materialise
+ * ETH_HLEN into a scratch VGPR (ub_hi, overwritten by the high half
+ * below) and use it as src1 instead of an immediate.
+ */
+ knod_mov32(priv, meta, ub_hi, imm);
+ knod_emit(priv, meta, v_sub_co_u32, ub_lo, dend_lo, ub_hi);
+ knod_iset32(&imm, 0);
+ knod_mov32(priv, meta, delta, imm);
+ knod_emit(priv, meta, v_sub_co_ci_u32_e32, ub_hi, dend_hi, delta);
+
+ knod_vset64(&ub, KNOD_AMDGPU_TMP_VREG1_LO);
+ knod_emit(priv, meta, v_cmp_gt_u64, data_vreg, ub);
+
+ /* Capture upper_fail via v_cndmask, combine, convert to VCC */
+ knod_iset32(&imm, 0);
+ knod_emit(priv, meta, v_cndmask_b32_e32, fail_hi, imm, fail_hi);
+
+ knod_emit(priv, meta, v_or_b32_e32, fail_lo, fail_lo, fail_hi);
+
+ knod_emit(priv, meta, v_cmp_lt_u32, imm, fail_lo);
+
+ /* 6. Conditional restore: VCC=1(fail) -> original,
+ * VCC=0(pass) -> adjusted
+ */
+ knod_emit(priv, meta, v_cndmask_b32_e32, data_lo, data_lo,
+ tmp0_lo);
+ knod_emit(priv, meta, v_cndmask_b32_e32, data_hi, data_hi,
+ tmp0_hi);
+
+ /* 7. R0 = VCC ? -EINVAL : 0 */
+ knod_iset32(&imm, -EINVAL);
+ knod_mov32(priv, meta, tmp0_lo, imm);
+ knod_iset32(&imm, 0);
+ knod_emit(priv, meta, v_cndmask_b32_e32, r0_lo, imm, tmp0_lo);
+
+ knod_iset32(&imm, -1);
+ knod_mov32(priv, meta, tmp0_hi, imm);
+ knod_iset32(&imm, 0);
+ knod_emit(priv, meta, v_cndmask_b32_e32, r0_hi, imm, tmp0_hi);
+}
+
+/*
+ * knod_bpf_xdp_adjust_tail - JIT bpf_xdp_adjust_tail (helper 65).
+ *
+ * R2 = delta (signed 32-bit). Adjusts DATA_END_VREG by delta.
+ * Bounds: DATA_VREG + ETH_HLEN <= DATA_END_VREG <= page_base + PAGE_SIZE.
+ * Each bound is checked with its own VOPC, but VCC is captured into
+ * VGPRs via v_cndmask (VALU) rather than SGPRs via s_mov_b64 (SALU).
+ * VALU reads VCC correctly after VOPC; only SALU suffers the GFX10
+ * dual-VOPC stale-read hazard.
+ * page_base is reloaded on demand from param + spsc_bd.
+ * On failure, DATA_END_VREG is restored and R0 = -EINVAL.
+ * On success, R0 = 0.
+ *
+ * Clobbers: TMP_VREG0 (v22:v23), TMP_VREG1 (v24:v25), TMP_VREG2 (v26:v27),
+ * TMP_SREG0 (s16), TMP_SREG2 (s20:s21).
+ */
+static void knod_bpf_xdp_adjust_tail(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta)
+{
+ struct amdgcn_param32 tmp0_lo, tmp0_hi, dend_lo, dend_hi, fail_lo;
+ struct amdgcn_param32 fail_hi;
+ struct amdgcn_param32 lb_lo, lb_hi, d_lo, d_hi, sext_dst, shift_amt;
+ struct amdgcn_param32 pb_src_lo, pb_src_hi;
+ struct amdgcn_param32 r0_lo, r0_hi;
+ struct amdgcn_param32 imm, delta;
+ struct amdgcn_param64 dend_vreg, lb;
+
+ knod_vset32(&tmp0_lo, KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_vset32(&tmp0_hi, KNOD_AMDGPU_TMP_VREG0_HI);
+ knod_vset32(&dend_lo, KNOD_AMDGPU_DATA_END_VREG_LO);
+ knod_vset32(&dend_hi, KNOD_AMDGPU_DATA_END_VREG_HI);
+ knod_vset32(&r0_lo, KNOD_AMDGPU_VREG0_LO);
+ knod_vset32(&r0_hi, KNOD_AMDGPU_VREG0_HI);
+ knod_vset32(&delta, bpf_reg64[2].lo.v);
+ knod_vset32(&fail_lo, KNOD_AMDGPU_TMP_VREG2_LO);
+ knod_vset32(&fail_hi, KNOD_AMDGPU_TMP_VREG2_HI);
+ knod_vset64(&dend_vreg, KNOD_AMDGPU_DATA_END_VREG_LO);
+
+ /* 1. Save original DATA_END_VREG -> TMP_VREG0 */
+ knod_mov32(priv, meta, tmp0_lo, dend_lo);
+ knod_mov32(priv, meta, tmp0_hi, dend_hi);
+
+ /* 2. DATA_END_VREG += delta (R2.lo, sign-extended) */
+ knod_emit(priv, meta, v_add_co_u32, dend_lo, delta, dend_lo);
+
+ knod_vset32(&sext_dst, KNOD_AMDGPU_TMP_VREG1_LO);
+ knod_iset32(&shift_amt, 31);
+ knod_emit(priv, meta, v_ashrrev_i32, sext_dst, shift_amt, delta);
+
+ knod_emit(priv, meta, v_add_co_ci_u32_e32, dend_hi, sext_dst,
+ dend_hi);
+
+ /* 3. Lower bound: lb = DATA + ETH_HLEN -> TMP_VREG1 */
+ knod_vset32(&lb_lo, KNOD_AMDGPU_TMP_VREG1_LO);
+ knod_vset32(&lb_hi, KNOD_AMDGPU_TMP_VREG1_HI);
+ knod_vset32(&d_lo, KNOD_AMDGPU_DATA_VREG_LO);
+ knod_vset32(&d_hi, KNOD_AMDGPU_DATA_VREG_HI);
+
+ knod_iset32(&imm, ETH_HLEN);
+ knod_emit(priv, meta, v_add_co_u32, lb_lo, imm, d_lo);
+ knod_iset32(&imm, 0);
+ knod_emit(priv, meta, v_add_co_ci_u32_e32, lb_hi, imm, d_hi);
+
+ /* VOPC#1: DATA_END < lb -> VCC = lower_fail */
+ knod_vset64(&lb, KNOD_AMDGPU_TMP_VREG1_LO);
+ knod_emit(priv, meta, v_cmp_lt_u64, dend_vreg, lb);
+
+ /*
+ * Capture VCC -> VGPR via v_cndmask (VALU reads VCC correctly,
+ * unlike SALU which suffers the GFX10 dual-VOPC stale-read hazard).
+ */
+ knod_iset32(&imm, 1);
+ knod_mov32(priv, meta, fail_hi, imm);
+ knod_iset32(&imm, 0);
+ knod_emit(priv, meta, v_cndmask_b32_e32, fail_lo, imm, fail_hi);
+
+ /* 4. Upper bound: ub = page_base + PAGE_SIZE -> TMP_VREG1 */
+ knod_vset32(&pb_src_lo, KNOD_AMDGPU_PAGE_BASE_VREG_LO);
+ knod_vset32(&pb_src_hi, KNOD_AMDGPU_PAGE_BASE_VREG_HI);
+ knod_mov32(priv, meta, lb_lo, pb_src_lo);
+ knod_mov32(priv, meta, lb_hi, pb_src_hi);
+
+ knod_iset32(&imm, PAGE_SIZE);
+ knod_emit(priv, meta, v_add_co_u32, lb_lo, imm, lb_lo);
+ knod_iset32(&imm, 0);
+ knod_emit(priv, meta, v_add_co_ci_u32_e32, lb_hi, imm, lb_hi);
+
+ /* VOPC#2: DATA_END > ub -> VCC = upper_fail */
+ knod_emit(priv, meta, v_cmp_gt_u64, dend_vreg, lb);
+
+ /* Capture upper_fail via v_cndmask, combine, convert to VCC */
+ knod_iset32(&imm, 0);
+ knod_emit(priv, meta, v_cndmask_b32_e32, fail_hi, imm, fail_hi);
+
+ knod_emit(priv, meta, v_or_b32_e32, fail_lo, fail_lo, fail_hi);
+
+ knod_emit(priv, meta, v_cmp_lt_u32, imm, fail_lo);
+
+ /* 5. Conditional restore: VCC=1(fail) -> original,
+ * VCC=0(pass) -> adjusted
+ */
+ knod_emit(priv, meta, v_cndmask_b32_e32, dend_lo, dend_lo,
+ tmp0_lo);
+ knod_emit(priv, meta, v_cndmask_b32_e32, dend_hi, dend_hi,
+ tmp0_hi);
+
+ /* 6. R0 = VCC ? -EINVAL : 0 */
+ knod_iset32(&imm, -EINVAL);
+ knod_mov32(priv, meta, tmp0_lo, imm);
+ knod_iset32(&imm, 0);
+ knod_emit(priv, meta, v_cndmask_b32_e32, r0_lo, imm, tmp0_lo);
+
+ knod_iset32(&imm, -1);
+ knod_mov32(priv, meta, tmp0_hi, imm);
+ knod_iset32(&imm, 0);
+ knod_emit(priv, meta, v_cndmask_b32_e32, r0_hi, imm, tmp0_hi);
+}
+
+static void knod_bpf_load_size(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ struct amdgcn_param64 *dst,
+ /* packet or stack */
+ struct amdgcn_param32 *cache,
+ int size, int off)
+{
+ struct amdgcn_param32 p32[2];
+
+ knod_jit_dbg(" %d: off = %d off_4 = %d size = %d\n", meta->bpf_insn_idx,
+ off, off%4, size);
+ switch (size) {
+ case sizeof(unsigned long):
+ if ((off % 4) == 0) {
+ knod_mov32(priv, meta, dst->lo, cache[off / 4]);
+ knod_mov32(priv, meta, dst->hi,
+ cache[(off / 4) + 1]);
+ } else if ((off % 4) == 1) {
+ WARN_ON_ONCE(1);
+ } else if ((off % 4) == 2) {
+ WARN_ON_ONCE(1);
+ } else {
+ WARN_ON_ONCE(1);
+ }
+ break;
+ case sizeof(unsigned int):
+ if ((off % 4) == 0) {
+ knod_mov32(priv, meta, dst->lo, cache[off / 4]);
+ } else if ((off % 4) == 1) {
+ knod_iset32(&p32[0], 8);
+ knod_lshrrev32(priv, meta, r32[0], p32[0],
+ cache[off / 4]);
+ knod_iset32(&p32[0], 24);
+ knod_lshlrev32(priv, meta, dst->lo, p32[0],
+ cache[(off / 4) + 1]);
+ knod_or32(priv, meta, dst->lo, dst->lo, r32[0]);
+ } else if ((off % 4) == 2) {
+ knod_iset32(&p32[0], 16);
+ knod_lshrrev32(priv, meta, r32[0], p32[0],
+ cache[off / 4]);
+ knod_lshlrev32(priv, meta, dst->lo, p32[0],
+ cache[(off / 4) + 1]);
+ knod_or32(priv, meta, dst->lo, dst->lo, r32[0]);
+ } else {
+ knod_iset32(&p32[0], 24);
+ knod_lshrrev32(priv, meta, r32[0], p32[0],
+ cache[off / 4]);
+ knod_iset32(&p32[0], 8);
+ knod_lshlrev32(priv, meta, dst->lo, p32[0],
+ cache[(off / 4) + 1]);
+ knod_or32(priv, meta, dst->lo, dst->lo, r32[0]);
+ }
+ break;
+ case sizeof(unsigned short):
+ if ((off % 4) == 3) {
+ knod_iset32(&p32[0], 24);
+ knod_iset32(&p32[1], 8);
+ knod_bfe32(priv, meta, r64[0].lo, cache[off / 4],
+ p32[0], p32[1]);
+ knod_iset32(&p32[0], 0);
+ knod_bfe32(priv, meta, r64[0].hi,
+ cache[(off / 4) + 1], p32[0], p32[1]);
+ /* bpf_reg64[d].lo = (r64[0].hi << 8) | r64[0].lo. */
+ knod_emit(priv, meta, v_lshl_or_b32, dst->lo,
+ r64[0].hi, p32[1], r64[0].lo);
+ } else {
+ if (!(off % 4))
+ knod_iset32(&p32[0], 0);
+ else if ((off % 4) == 1)
+ knod_iset32(&p32[0], 8);
+ else if ((off % 4) == 2)
+ knod_iset32(&p32[0], 16);
+ knod_iset32(&p32[1], 16);
+ knod_bfe32(priv, meta, dst->lo, cache[off / 4],
+ p32[0], p32[1]);
+ }
+ break;
+ case sizeof(unsigned char):
+ if ((off % 4) == 0)
+ knod_iset32(&p32[0], 0);
+ else if ((off % 4) == 1)
+ knod_iset32(&p32[0], 8);
+ else if ((off % 4) == 2)
+ knod_iset32(&p32[0], 16);
+ else
+ knod_iset32(&p32[0], 24);
+ knod_iset32(&p32[1], 8);
+ knod_bfe32(priv, meta, dst->lo, cache[off / 4], p32[0],
+ p32[1]);
+ break;
+ default:
+ WARN_ON_ONCE(1);
+ break;
+ }
+
+ if (size != sizeof(unsigned long)) {
+ knod_iset32(&p32[0], 0);
+ knod_mov32(priv, meta, dst->hi, p32[0]);
+ }
+}
+
+/*
+ * GFX10 (RDNA2) quirk: global_load_{dword,dwordx2,dwordx4} silently
+ * clear the low 2 bits of the effective address, forcing Dword
+ * alignment. For PTR_TO_PACKET loads at a byte offset that is not
+ * Dword-aligned, round the offset down to the nearest 4-byte boundary,
+ * load enough contiguous dwords to cover the requested range, then use
+ * v_alignbit_b32 to extract the byte-aligned result. For size < 4 a
+ * final v_and_b32 masks the result to the correct width.
+ *
+ * Caller is responsible for zeroing dst.hi for size < 8; this helper
+ * only writes dst.lo (and dst.hi when size == 8).
+ *
+ * Scratch: up to 4 contiguous VGPRs at v32..v35
+ * (TMP_VREG5_LO..TMP_VREG6_HI).
+ */
+static void knod_bpf_emit_gfx10_unaligned_load(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ int size,
+ struct amdgcn_param64 dst,
+ struct amdgcn_param32 src_lo,
+ int off)
+{
+ int off_a = off & ~3;
+ int shift_bits = (off - off_a) * 8;
+ int needed = DIV_ROUND_UP((off & 3) + size, 4);
+ struct amdgcn_param32 tmp[4];
+ struct amdgcn_param32 shift_imm, mask_imm;
+
+ knod_vset32(&tmp[0], KNOD_AMDGPU_TMP_VREG5_LO);
+ knod_vset32(&tmp[1], KNOD_AMDGPU_TMP_VREG5_HI);
+ knod_vset32(&tmp[2], KNOD_AMDGPU_TMP_VREG6_LO);
+ knod_vset32(&tmp[3], KNOD_AMDGPU_TMP_VREG6_HI);
+ knod_iset32(&shift_imm, shift_bits);
+
+ if (needed <= 1) {
+ knod_emit(priv, meta, global_load_dword, tmp[0], src_lo,
+ off_a);
+ } else if (needed == 2) {
+ knod_emit(priv, meta, global_load_dwordx2, tmp[0], src_lo,
+ off_a);
+ } else {
+ /* needed == 3: no dwordx3, widen to dwordx4. */
+ knod_emit(priv, meta, global_load_dwordx4, tmp[0], src_lo,
+ off_a);
+ }
+ knod_wait_vmcnt(priv, meta);
+
+ if (size <= 4) {
+ /* v_alignbit_b32 D, S0, S1, S2:
+ * D = ({S0, S1} >> S2)[31:0]
+ * S0 is HIGH, S1 is LOW. tmp[0] holds
+ * bytes[off_a..+4) (memory-low) and tmp[1] holds
+ * bytes[off_a+4..+8) (memory-high), so
+ * src0=tmp[1], src1=tmp[0].
+ */
+ if (shift_bits == 0)
+ knod_mov32(priv, meta, dst.lo, tmp[0]);
+ else
+ knod_alignbit32(priv, meta, dst.lo,
+ tmp[1], tmp[0], shift_imm);
+
+ if (size == 1) {
+ knod_iset32(&mask_imm, 0xff);
+ knod_and32(priv, meta, dst.lo, dst.lo,
+ mask_imm);
+ } else if (size == 2) {
+ knod_iset32(&mask_imm, 0xffff);
+ knod_and32(priv, meta, dst.lo, dst.lo,
+ mask_imm);
+ }
+ } else {
+ /* size == 8: two alignbits for low / high output dwords. */
+ if (shift_bits == 0) {
+ knod_mov32(priv, meta, dst.lo, tmp[0]);
+ knod_mov32(priv, meta, dst.hi, tmp[1]);
+ } else {
+ knod_alignbit32(priv, meta, dst.lo,
+ tmp[1], tmp[0], shift_imm);
+ knod_alignbit32(priv, meta, dst.hi,
+ tmp[2], tmp[1], shift_imm);
+ }
+ }
+}
+
+#define LABEL_NEXT 8
+#define LABEL_OUT 9
+static void knod_bpf_ktime_get_ns(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta)
+{
+ struct amdgcn_param32 p[2];
+
+ knod_sset32(&p[0], KNOD_AMDGPU_TMP_SREG0_LO);
+ knod_sset32(&p[1], KNOD_AMDGPU_PARAM_SREG_LO);
+ knod_emit(priv, meta, s_load_dwordx2, p[0], p[1],
+ offsetof(struct knod_bpf_param, ktime_ns));
+
+ knod_emit(priv, meta, s_waitcnt_lgkmcnt);
+
+ knod_sset32(&p[0], KNOD_AMDGPU_TMP_SREG0_LO);
+ knod_mov32(priv, meta, bpf_reg64[0].lo, p[0]);
+ knod_sset32(&p[0], KNOD_AMDGPU_TMP_SREG0_HI);
+ knod_mov32(priv, meta, bpf_reg64[0].hi, p[0]);
+}
+
+static void knod_bpf_map_lookup(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ int map_id)
+{
+ struct knod_bpf_map_obj *knod_map_obj_k, *knod_map_obj_g;
+ int off, len, _len, idx, key_in_pkt, key_in_map;
+ bool first_cmp;
+ struct amdgcn_branch_fixup fixups[12] = {0,};
+ struct amdgcn_label labels[10] = {0,};
+ u32 stack_off = meta->kreg.stack_off;
+ unsigned long bucket_gaddr;
+ struct amdgcn_param32 p32;
+ int fixup_idx = 0;
+
+ knod_map_obj_k =
+ (struct knod_bpf_map_obj *)knod_bpf_map_kaddr(priv, map_id);
+ knod_map_obj_g =
+ (struct knod_bpf_map_obj *)knod_bpf_map_gaddr(priv, map_id);
+ bucket_gaddr = (unsigned long)knod_map_obj_g +
+ offsetof(struct knod_bpf_map_obj, bucket);
+
+ knod_jit_dbg(" stack_off = %d map_id = %d\n", stack_off, map_id);
+ if (!knod_map_obj_g || !knod_map_obj_k)
+ WARN_ON_ONCE(1);
+
+ knod_bpf_load_size(priv, meta,
+ &r64[2],
+ &stack[0],
+ sizeof(unsigned int),
+ 512 + stack_off);
+ /* reg1 := bucket
+ * NOTE: bucket_gaddr is greater than X
+ */
+ knod_iset64(&p64[0], bucket_gaddr);
+ knod_mov64(priv, meta, r64[1], p64[0]);
+ knod_iset64(&p64[1], 0);
+ knod_mov32(priv, meta, r64[2].hi, p64[1].lo);
+
+ knod_iset64(&p64[1], 0);
+ knod_mov64(priv, meta, bpf_reg64[0], p64[1]);
+
+ /* BPF_REG0 = 0
+ * TMP_REG1 = bucket_gaddr
+ * TMP_REG2 = key
+ */
+
+ if (knod_map_obj_k->map_type == BPF_MAP_TYPE_ARRAY ||
+ knod_map_obj_k->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
+ /* if (key > knod_map_obj_k.max_entries)
+ * NOTE: integer
+ */
+ knod_iset64(&p64[1], knod_map_obj_k->max_entries);
+ knod_mov64(priv, meta, r64[3], p64[1]);
+ knod_emit(priv, meta, v_cmp_ge_u64, r64[2], r64[3]);
+ /* structurized CFG: save OOB lanes, narrow exec */
+ knod_emit(priv, meta, s_and_b64, KNOD_AMDGPU_TMP_SREG3_LO,
+ AMDGCN_SREG_EXEC_LO, AMDGCN_SREG_VCC_LO);
+ knod_emit(priv, meta, s_andn2_b64, AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_EXEC_LO, AMDGCN_SREG_VCC_LO);
+ emit_s_cbranch_execz(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ 0); /* update required */
+ knod_bpf_set_fixup(meta, &fixups[fixup_idx],
+ &labels[LABEL_OUT], meta->amdgpu_insns);
+ debug_insn(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns]);
+ meta->amdgpu_insns++;
+ fixup_idx++;
+ /* PERCPU_ARRAY: bucket += workgroup_id_y * per_instance_size so
+ * each RX queue addresses its own instance and the atomic
+ * update after the lookup has no cross-CU contention. Auto
+ * xgroups keeps PERCPU programs at one workgroup per queue, so
+ * the queue id (workgroup_id_y) is the instance index.
+ */
+ if (knod_map_obj_k->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
+ /* r64[3] is scratch after the bounds check: .lo =
+ * workgroup_id_y, .hi = per_instance_size (too large
+ * for an inline constant, so stage both in VGPRs
+ * first).
+ */
+ knod_sset32(&p32, KNOD_AMDGPU_WORKGROUP_ID_Y_SREG);
+ knod_emit(priv, meta, v_mov_b32_e32, r64[3].lo, p32);
+ knod_iset64(&p64[1],
+ knod_map_obj_k->meta.ameta
+ .per_instance_size);
+ knod_mov32(priv, meta, r64[3].hi, p64[1].lo);
+ emit_v_mad_u64_u32(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ r64[1],
+ sr64[0].lo,
+ r64[3].hi, /* per_instance_size */
+ r64[3].lo, /* workgroup_id_y */
+ r64[1]); /* bucket */
+ debug_insn(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns]);
+ meta->amdgpu_insns++;
+ }
+ /* elem_id = &bucket[key]; */
+ knod_iset64(&p64[1], knod_map_obj_k->value_size);
+ emit_v_mad_u64_u32(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ bpf_reg64[0],
+ sr64[0].lo,
+ p64[1].lo, /* value_size */
+ r64[2].lo, /* key */
+ r64[1]); /* bucket */
+ debug_insn(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns]);
+ meta->amdgpu_insns++;
+ /* structurized CFG: restore OOB lanes */
+ knod_bpf_set_label(meta, &labels[LABEL_OUT],
+ meta->amdgpu_insns);
+ knod_emit(priv, meta, s_or_b64, AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_EXEC_LO, KNOD_AMDGPU_TMP_SREG3_LO);
+ for (idx = 0; idx < fixup_idx; idx++)
+ knod_bpf_fixup_branch(priv, &fixups[idx]);
+ } else if (knod_map_obj_k->map_type == BPF_MAP_TYPE_HASH) {
+ key_in_pkt = KEY_IN_PKT_64;
+ len = knod_map_obj_k->key_size;
+ off = stack_off;
+
+ /* TMP_VREGs(vgpr-pair)
+ * |0|1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|
+ * | | | |K|K|K|K|K|K|K|K |K |K |K |K |K |K |K |K |
+ */
+ while (len) {
+ if (len >= sizeof(unsigned long))
+ _len = sizeof(unsigned long);
+ else
+ _len = len;
+ knod_bpf_load_size(priv, meta,
+ &r64[key_in_pkt],
+ &stack[0],
+ _len,
+ 512 + off);
+ key_in_pkt++;
+ len -= _len;
+ off += _len;
+ }
+
+ knod_jhash(priv, meta,
+ 2,
+ knod_map_obj_k->key_size,
+ knod_map_obj_k->meta.hmeta.hashrnd);
+ /* clear hi register of r64[2] because hash is 32bit */
+ knod_iset64(&p64[0], 0);
+ knod_mov32(priv, meta, r64[2].hi, p64[0].lo);
+
+ /* hash = hash & (n_buckets - 1) before indexing the bucket
+ * array -- jhash returns the full 32-bit hash and the update
+ * and delete emitters mask it too; without this
+ * bucket_gaddr[hash] runs off the end of the bucket array.
+ */
+ knod_iset32(&p64[0].lo,
+ knod_map_obj_k->meta.hmeta.n_buckets - 1);
+ knod_and32(priv, meta, r64[2].lo, p64[0].lo, r64[2].lo);
+
+ /* elem_id = bucket_gaddr[hash]; */
+ knod_iset64(&p64[1], sizeof(int));
+ emit_v_mad_u64_u32(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ r64[2],
+ sr64[0].lo,
+ p64[1].lo, /* sizeof(int) */
+ r64[2].lo, /* hash */
+ r64[1]); /* bucket */
+ debug_insn(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns]);
+ meta->amdgpu_insns++;
+
+ /* bpf_reg64[0] = 0 (default return for not-found lanes) */
+ knod_iset64(&p64[0], 0);
+ knod_mov64(priv, meta, bpf_reg64[0], p64[0]);
+
+ /* structurized CFG: save initial exec for restoring at end */
+ knod_emit(priv, meta, s_mov_b64, KNOD_AMDGPU_TMP_SREG3_LO,
+ AMDGCN_SREG_EXEC_LO);
+
+ knod_bpf_set_label(meta, &labels[LABEL_NEXT],
+ meta->amdgpu_insns);
+ /* load elem_id from elem structure */
+ knod_emit(priv, meta, global_load_dword, r64[2].lo,
+ r64[2].lo, 0);
+ knod_wait_vmcnt(priv, meta);
+
+ /* mask out DELETED bit from next field */
+ knod_iset32(&p32, KNOD_BPF_HASH_NEXT_MASK);
+ knod_emit(priv, meta, v_and_b32_e32, r64[2].lo, p32,
+ r64[2].lo);
+
+ /* if (r64[2].lo == KNOD_BPF_HASH_NEXT_END)
+ * goto out;
+ * VOPC cannot encode literal constants - move to VGPR first.
+ * NEXT_MASK == NEXT_END (0x7FFFFFFF), reuse p32 from v_and
+ * above.
+ */
+ knod_emit(priv, meta, v_mov_b32_e32, r64[0].hi, p32);
+ knod_emit(priv, meta, v_cmp_eq_u32, r64[0].hi, r64[2].lo);
+ /* structurized CFG: remove end-of-chain lanes */
+ knod_emit(priv, meta, s_andn2_b64, AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_EXEC_LO, AMDGCN_SREG_VCC_LO);
+
+ emit_s_cbranch_execz(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ 0); /* update required */
+ knod_bpf_set_fixup(meta, &fixups[fixup_idx],
+ &labels[LABEL_OUT], meta->amdgpu_insns);
+ debug_insn(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns]);
+ meta->amdgpu_insns++;
+ fixup_idx++;
+
+ knod_iset64(&p64[0],
+ (unsigned long)knod_map_obj_k->meta.hmeta.elems);
+ knod_mov64(priv, meta, r64[1], p64[0]);
+ knod_iset64(&p64[0], knod_map_obj_k->meta.hmeta.elem_size);
+ knod_mov64(priv, meta, r64[0], p64[0]);
+
+ key_in_map = KEY_IN_MAP_32;
+ len = knod_map_obj_k->key_size;
+ off = offsetof(struct knod_bpf_hash_elem_obj, kv);
+
+ /* elem = &elems[elem_id]; */
+ emit_v_mad_u64_u32(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ r64[2], /* elem */
+ sr64[0].lo,
+ r64[0].lo, /* elem_size */
+ r64[2].lo, /* elem_id */
+ r64[1]); /* elem_gaddr */
+ debug_insn(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns]);
+ meta->amdgpu_insns++;
+
+ /* load elem.next for DELETED check (parallel with key loads) */
+ knod_emit(priv, meta, global_load_dword, r64[0].hi,
+ r64[2].lo, 0);
+ while (len >= 16) {
+ knod_emit(priv, meta, global_load_dwordx4,
+ r32[key_in_map],
+ r64[2].lo, /* elem ptr */ off);
+ off += 16;
+ len -= 16;
+ key_in_map += 4;
+ }
+
+ if (len >= 8) {
+ knod_emit(priv, meta, global_load_dwordx2,
+ r32[key_in_map],
+ /* elem_id */ r64[2].lo, /* elem ptr */ off);
+ off += 8;
+ len -= 8;
+ key_in_map += 2;
+ }
+
+ if (len >= 4) {
+ knod_emit(priv, meta, global_load_dword,
+ r32[key_in_map],
+ /* elem_id */ r64[2].lo, /* elem ptr */ off);
+ off += 4;
+ len -= 4;
+ key_in_map += 1;
+ }
+
+ /* map key padding was inited to zero, no AND is required */
+ if (len) {
+ knod_emit(priv, meta, global_load_dword,
+ r32[key_in_map],
+ /* elem_id */ r64[2].lo, /* elem ptr */ off);
+ }
+
+ knod_wait_vmcnt(priv, meta);
+
+ /* structurized CFG: accumulate key match into TMP_SREG4
+ * instead of early-exit branching per key part
+ */
+ key_in_map = KEY_IN_MAP_32;
+ key_in_pkt = KEY_IN_PKT_32;
+ len = knod_map_obj_k->key_size;
+ first_cmp = true;
+
+ while (len >= 8) {
+ knod_emit(priv, meta, v_cmp_eq_u64, r32[key_in_map],
+ r32[key_in_pkt]);
+
+ if (first_cmp) {
+ knod_emit(priv, meta, s_and_b64,
+ KNOD_AMDGPU_TMP_SREG4_LO,
+ AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_VCC_LO);
+ first_cmp = false;
+ } else {
+ knod_emit(priv, meta, s_and_b64,
+ KNOD_AMDGPU_TMP_SREG4_LO,
+ KNOD_AMDGPU_TMP_SREG4_LO,
+ AMDGCN_SREG_VCC_LO);
+ }
+
+ key_in_map += 2;
+ key_in_pkt += 2;
+ len -= 8;
+ }
+
+ if (len >= 4) {
+ knod_emit(priv, meta, v_cmp_eq_u32, r32[key_in_map],
+ r32[key_in_pkt]);
+
+ if (first_cmp) {
+ knod_emit(priv, meta, s_and_b64,
+ KNOD_AMDGPU_TMP_SREG4_LO,
+ AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_VCC_LO);
+ first_cmp = false;
+ } else {
+ knod_emit(priv, meta, s_and_b64,
+ KNOD_AMDGPU_TMP_SREG4_LO,
+ KNOD_AMDGPU_TMP_SREG4_LO,
+ AMDGCN_SREG_VCC_LO);
+ }
+
+ key_in_map += 1;
+ key_in_pkt += 1;
+ len -= 4;
+ }
+
+ if (len) {
+ knod_emit(priv, meta, v_cmp_eq_u32, r32[key_in_map],
+ r32[key_in_pkt]);
+
+ if (first_cmp) {
+ knod_emit(priv, meta, s_and_b64,
+ KNOD_AMDGPU_TMP_SREG4_LO,
+ AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_VCC_LO);
+ first_cmp = false;
+ } else {
+ knod_emit(priv, meta, s_and_b64,
+ KNOD_AMDGPU_TMP_SREG4_LO,
+ KNOD_AMDGPU_TMP_SREG4_LO,
+ AMDGCN_SREG_VCC_LO);
+ }
+ }
+
+ /* DELETED check: remove deleted lanes from match result.
+ * r64[0].hi = elem.next (loaded in parallel with key).
+ * Deleted elems have bit 31 set - exclude them from SREG4.
+ */
+ knod_iset32(&p32, KNOD_BPF_HASH_NEXT_DELETED);
+ knod_emit(priv, meta, v_and_b32_e32, r64[0].hi, p32,
+ r64[0].hi);
+ knod_iset32(&p32, 0);
+ knod_emit(priv, meta, v_cmp_eq_u32, p32, r64[0].hi);
+ knod_emit(priv, meta, s_and_b64, KNOD_AMDGPU_TMP_SREG4_LO,
+ KNOD_AMDGPU_TMP_SREG4_LO, AMDGCN_SREG_VCC_LO);
+
+ /* TMP_SREG4 = lanes where key matched AND not deleted.
+ * Save current exec, narrow to matched lanes for value
+ * computation.
+ */
+ knod_emit(priv, meta, s_mov_b64, KNOD_AMDGPU_TMP_SREG5_LO,
+ AMDGCN_SREG_EXEC_LO);
+ knod_emit(priv, meta, s_and_b64, AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_EXEC_LO, KNOD_AMDGPU_TMP_SREG4_LO);
+
+ /* bpf_reg64[0] = value address (only for matched lanes) */
+ knod_iset64(&p64[1],
+ offsetof(struct knod_bpf_hash_elem_obj, kv) +
+ knod_map_obj_k->key_size);
+ knod_add64(priv, meta, bpf_reg64[0], p64[1], r64[2]);
+
+ /* set exec to unmatched lanes for next loop iteration */
+ knod_emit(priv, meta, s_andn2_b64, AMDGCN_SREG_EXEC_LO,
+ KNOD_AMDGPU_TMP_SREG5_LO, KNOD_AMDGPU_TMP_SREG4_LO);
+ /* loop back if any unmatched lanes remain */
+ emit_s_cbranch_execnz(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ 0); /* update required */
+ knod_bpf_set_fixup(meta, &fixups[fixup_idx],
+ &labels[LABEL_NEXT], meta->amdgpu_insns);
+ debug_insn(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns]);
+ meta->amdgpu_insns++;
+ fixup_idx++;
+ /* structurized CFG: restore all original lanes */
+ knod_bpf_set_label(meta, &labels[LABEL_OUT],
+ meta->amdgpu_insns);
+ knod_emit(priv, meta, s_or_b64, AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_EXEC_LO, KNOD_AMDGPU_TMP_SREG3_LO);
+ for (idx = 0; idx < fixup_idx; idx++)
+ knod_bpf_fixup_branch(priv, &fixups[idx]);
+
+ } else {
+ WARN_ON_ONCE(1);
+ }
+}
+
+static void knod_bpf_map_update_array(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ int map_id)
+{
+ struct knod_bpf_map_obj *knod_map_obj_k, *knod_map_obj_g;
+ struct amdgcn_branch_fixup fixups[4] = {0,};
+ u32 key_stack_off = meta->kreg.stack_off;
+ u32 val_stack_off = meta->vreg.stack_off;
+ struct amdgcn_label labels[10] = {0,};
+ int idx, val_off, val_len;
+ unsigned long bucket_gaddr;
+ int fixup_idx = 0;
+
+ knod_map_obj_k =
+ (struct knod_bpf_map_obj *)knod_bpf_map_kaddr(priv, map_id);
+ knod_map_obj_g =
+ (struct knod_bpf_map_obj *)knod_bpf_map_gaddr(priv, map_id);
+ bucket_gaddr = (unsigned long)knod_map_obj_g +
+ offsetof(struct knod_bpf_map_obj, bucket);
+
+ if (!knod_map_obj_g || !knod_map_obj_k)
+ WARN_ON_ONCE(1);
+
+ /* load key from stack -> r64[2] */
+ knod_bpf_load_size(priv, meta,
+ &r64[2],
+ &stack[0],
+ sizeof(unsigned int),
+ 512 + key_stack_off);
+
+ /* r64[1] = bucket_gaddr */
+ knod_iset64(&p64[0], bucket_gaddr);
+ knod_mov64(priv, meta, r64[1], p64[0]);
+
+ /* clear r64[2].hi (key is 32-bit) */
+ knod_iset64(&p64[1], 0);
+ knod_mov32(priv, meta, r64[2].hi, p64[1].lo);
+
+ /* bpf_reg64[0] = 0 (return value) */
+ knod_mov64(priv, meta, bpf_reg64[0], p64[1]);
+
+ /* bounds check: if (key >= max_entries) -> skip */
+ knod_iset64(&p64[1], knod_map_obj_k->max_entries);
+ knod_mov64(priv, meta, r64[3], p64[1]);
+ knod_emit(priv, meta, v_cmp_ge_u64, r64[2], r64[3]);
+
+ /* structurized CFG: save OOB lanes, narrow exec */
+ knod_emit(priv, meta, s_and_b64, KNOD_AMDGPU_TMP_SREG3_LO,
+ AMDGCN_SREG_EXEC_LO, AMDGCN_SREG_VCC_LO);
+ knod_emit(priv, meta, s_andn2_b64, AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_EXEC_LO, AMDGCN_SREG_VCC_LO);
+ emit_s_cbranch_execz(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ 0);
+ knod_bpf_set_fixup(meta, &fixups[fixup_idx],
+ &labels[LABEL_OUT], meta->amdgpu_insns);
+ debug_insn(priv->isa_version, &meta->amdgpu_insn[meta->amdgpu_insns]);
+ meta->amdgpu_insns++;
+ fixup_idx++;
+
+ /* dest = bucket_gaddr + key * value_size -> r64[0] */
+ knod_iset64(&p64[1], knod_map_obj_k->value_size);
+ knod_emit(priv, meta, v_mad_u64_u32, r64[0], sr64[0].lo,
+ p64[1].lo, r64[2].lo, r64[1]);
+
+ /* load value from stack and store to dest */
+ val_off = 0;
+ val_len = knod_map_obj_k->value_size;
+
+ while (val_len >= 16) {
+ knod_bpf_load_size(priv, meta,
+ &r64[3],
+ &stack[0],
+ sizeof(unsigned long),
+ 512 + val_stack_off + val_off);
+ knod_bpf_load_size(priv, meta,
+ &r64[4],
+ &stack[0],
+ sizeof(unsigned long),
+ 512 + val_stack_off + val_off + 8);
+ knod_emit(priv, meta, global_store_dwordx4, r64[3].lo,
+ r64[0].lo, val_off);
+ val_off += 16;
+ val_len -= 16;
+ }
+
+ if (val_len >= 8) {
+ knod_bpf_load_size(priv, meta,
+ &r64[3],
+ &stack[0],
+ sizeof(unsigned long),
+ 512 + val_stack_off + val_off);
+ knod_emit(priv, meta, global_store_dwordx2, r64[3].lo,
+ r64[0].lo, val_off);
+ val_off += 8;
+ val_len -= 8;
+ }
+
+ if (val_len >= 4) {
+ knod_bpf_load_size(priv, meta,
+ &r64[3],
+ &stack[0],
+ sizeof(unsigned int),
+ 512 + val_stack_off + val_off);
+ knod_emit(priv, meta, global_store_dword, r64[3].lo,
+ r64[0].lo, val_off);
+ val_off += 4;
+ val_len -= 4;
+ }
+
+ if (val_len >= 2) {
+ knod_bpf_load_size(priv, meta,
+ &r64[3],
+ &stack[0],
+ sizeof(unsigned short),
+ 512 + val_stack_off + val_off);
+ knod_emit(priv, meta, global_store_short, r64[3].lo,
+ r64[0].lo, val_off);
+ val_off += 2;
+ val_len -= 2;
+ }
+
+ if (val_len >= 1) {
+ knod_bpf_load_size(priv, meta,
+ &r64[3],
+ &stack[0],
+ sizeof(unsigned char),
+ 512 + val_stack_off + val_off);
+ knod_emit(priv, meta, global_store_byte, r64[3].lo,
+ r64[0].lo, val_off);
+ }
+
+ /* structurized CFG: restore OOB lanes */
+ knod_bpf_set_label(meta, &labels[LABEL_OUT], meta->amdgpu_insns);
+ knod_emit(priv, meta, s_or_b64, AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_EXEC_LO, KNOD_AMDGPU_TMP_SREG3_LO);
+
+ for (idx = 0; idx < fixup_idx; idx++)
+ knod_bpf_fixup_branch(priv, &fixups[idx]);
+}
+
+static void knod_bpf_map_update_hash(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ int map_id)
+{
+#define LABEL_BUCKET_LOOP 0
+#define LABEL_LOCK_RETRY 1
+#define LABEL_INSERT_LANE 2
+#define LABEL_CHAIN_NEXT 3
+#define LABEL_ALLOC_INSERT 4
+#define LABEL_LANE_DONE 5
+#define LABEL_UNLOCK 6
+ struct knod_bpf_map_obj *knod_map_obj_k, *knod_map_obj_g;
+ struct amdgcn_param32 s_bucket_lo, v_tmp, v_zero, v_one;
+ int off, len, _len, idx, key_in_pkt, key_in_map;
+ struct amdgcn_param32 s_exec_lo, s_exec_hi, s_elem_id;
+ struct amdgcn_branch_fixup fixups[12] = {0,};
+ u32 key_stack_off = meta->kreg.stack_off;
+ u32 val_stack_off = meta->vreg.stack_off;
+ unsigned long queue_gaddr, elems_gaddr;
+ unsigned long bucket_gaddr, cur_gaddr;
+ struct amdgcn_label labels[12] = {0,};
+ struct amdgcn_param32 v_minus_one;
+ struct amdgcn_param64 sr64_carry;
+ struct amdgcn_param32 p32;
+ unsigned long lock_offset;
+ unsigned int elem_size;
+ int val_off, val_len;
+ int fixup_idx = 0;
+ bool first_cmp;
+ int koff, voff;
+
+ knod_map_obj_k =
+ (struct knod_bpf_map_obj *)knod_bpf_map_kaddr(priv, map_id);
+ knod_map_obj_g =
+ (struct knod_bpf_map_obj *)knod_bpf_map_gaddr(priv, map_id);
+ bucket_gaddr = (unsigned long)knod_map_obj_g +
+ offsetof(struct knod_bpf_map_obj, bucket);
+ cur_gaddr = (unsigned long)knod_map_obj_g +
+ offsetof(struct knod_bpf_map_obj, meta.hmeta.cur);
+ queue_gaddr = (unsigned long)knod_map_obj_k->meta.hmeta.q;
+ elems_gaddr = (unsigned long)knod_map_obj_k->meta.hmeta.elems;
+ elem_size = knod_map_obj_k->meta.hmeta.elem_size;
+
+ if (!knod_map_obj_g || !knod_map_obj_k)
+ WARN_ON_ONCE(1);
+
+ /* ======== Phase 1: Setup ======== */
+
+ /* Load key from stack -> r64[3..9] (KEY_IN_PKT) */
+ key_in_pkt = KEY_IN_PKT_64;
+ len = knod_map_obj_k->key_size;
+ off = key_stack_off;
+ while (len) {
+ if (len >= sizeof(unsigned long))
+ _len = sizeof(unsigned long);
+ else
+ _len = len;
+ knod_bpf_load_size(priv, meta,
+ &r64[key_in_pkt],
+ &stack[0],
+ _len,
+ 512 + off);
+ key_in_pkt++;
+ len -= _len;
+ off += _len;
+ }
+
+ /* jhash -> r64[2].lo = hash */
+ knod_jhash(priv, meta,
+ 2,
+ knod_map_obj_k->key_size,
+ knod_map_obj_k->meta.hmeta.hashrnd);
+ knod_iset64(&p64[0], 0);
+ knod_mov32(priv, meta, r64[2].hi, p64[0].lo);
+
+ /* hash = hash & (n_buckets - 1) */
+ knod_iset32(&p64[0].lo,
+ knod_map_obj_k->meta.hmeta.n_buckets - 1);
+ knod_and32(priv, meta, r64[2].lo, p64[0].lo, r64[2].lo);
+
+ /* r64[1] = bucket_gaddr */
+ knod_iset64(&p64[0], bucket_gaddr);
+ knod_mov64(priv, meta, r64[1], p64[0]);
+
+ /* bucket_addr = bucket_gaddr + hash * sizeof(int) -> r64[2] */
+ knod_iset64(&p64[1], sizeof(int));
+ knod_emit(priv, meta, v_mad_u64_u32, r64[2], sr64[0].lo,
+ p64[1].lo, r64[2].lo, r64[1]);
+
+ /* Save bucket_addr to r64[15] for CAS insert */
+ knod_mov64(priv, meta, r64[15], r64[2]);
+
+ /* SREG3 = initial exec */
+ knod_emit(priv, meta, s_mov_b64, KNOD_AMDGPU_TMP_SREG3_LO,
+ AMDGCN_SREG_EXEC_LO);
+
+ /* ======== Phase 2: Sequential per-lane processing ======== */
+
+ /* SREG5 = exec (all lanes to process, for BUCKET_LOOP) */
+ knod_emit(priv, meta, s_mov_b64, KNOD_AMDGPU_TMP_SREG5_LO,
+ AMDGCN_SREG_EXEC_LO);
+
+ /* ---- BUCKET_LOOP: process one unique bucket per iteration ---- */
+ knod_bpf_set_label(meta, &labels[LABEL_BUCKET_LOOP],
+ meta->amdgpu_insns);
+
+ lock_offset =
+ (unsigned long)knod_map_obj_k->meta.hmeta.n_buckets *
+ sizeof(unsigned int);
+
+ knod_sset32(&s_bucket_lo,
+ KNOD_AMDGPU_TMP_SREG1_HI);
+ knod_vset32(&v_tmp, KNOD_AMDGPU_TMP_VREG0_HI);
+ knod_iset32(&v_zero, 0);
+ knod_iset32(&v_one, 1);
+ knod_sset32(&s_exec_lo, AMDGCN_SREG_EXEC_LO);
+ knod_sset32(&s_exec_hi,
+ AMDGCN_SREG_EXEC_LO + 1);
+ knod_sset32(&s_elem_id,
+ KNOD_AMDGPU_TMP_SREG1_LO);
+ knod_sset64(&sr64_carry,
+ KNOD_AMDGPU_TMP_SREG1_LO);
+
+ /* Pick first active lane's bucket addr */
+ knod_emit(priv, meta, v_readfirstlane_b32, KNOD_AMDGPU_TMP_SREG1_HI,
+ r64[15].lo.v);
+
+ /* vcc = lanes with same bucket */
+ knod_emit(priv, meta, v_cmp_eq_u32, s_bucket_lo, r64[15].lo);
+
+ /* SREG5 = remaining lanes; exec = same-bucket lanes */
+ knod_emit(priv, meta, s_and_saveexec_b64, KNOD_AMDGPU_TMP_SREG5_LO,
+ AMDGCN_SREG_VCC_LO);
+
+ /* SREG0 = same-bucket lanes */
+ knod_emit(priv, meta, s_mov_b64, KNOD_AMDGPU_TMP_SREG0_LO,
+ AMDGCN_SREG_EXEC_LO);
+
+ /* ---- Lock acquire ---- */
+ /* r64[10] = r64[15] + lock_offset */
+ knod_iset64(&p64[0], lock_offset);
+ knod_add64(priv, meta, r64[10], p64[0], r64[15]);
+
+ /* r64[11].lo = 1 (swap data) */
+ knod_emit(priv, meta, v_mov_b32_e32, r64[11].lo, v_one);
+
+ /* First-lane isolation via mbcnt */
+ knod_emit(priv, meta, v_mbcnt_lo_u32_b32, v_tmp, s_exec_lo,
+ v_zero);
+ knod_emit(priv, meta, v_mbcnt_hi_u32_b32, v_tmp, s_exec_hi, v_tmp);
+ knod_emit(priv, meta, v_cmp_eq_u32, v_zero, v_tmp);
+ knod_emit(priv, meta, s_and_b64, AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_EXEC_LO, AMDGCN_SREG_VCC_LO);
+
+ /* LOCK_RETRY: spin until lock acquired */
+ knod_bpf_set_label(meta, &labels[LABEL_LOCK_RETRY], meta->amdgpu_insns);
+
+ knod_emit(priv, meta, global_atomic_swap, r64[11].hi, r64[10].lo,
+ r64[11].lo, 0, 1);
+ knod_wait_vmcnt(priv, meta);
+
+ meta->amdgpu_insn[meta->amdgpu_insns].size =
+ emit_gfx10_v_cmp_ne_u32(
+ &meta->amdgpu_insn[meta->amdgpu_insns].gfx10,
+ v_zero, r64[11].hi);
+ meta->amdgpu_insn[meta->amdgpu_insns].type = AMDGCN_INSN_TYPE_VOPC;
+ debug_insn(priv->isa_version, &meta->amdgpu_insn[meta->amdgpu_insns]);
+ meta->amdgpu_insns++;
+
+ emit_s_cbranch_vccnz(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ 0);
+ knod_bpf_set_fixup(meta, &fixups[fixup_idx],
+ &labels[LABEL_LOCK_RETRY], meta->amdgpu_insns);
+ debug_insn(priv->isa_version, &meta->amdgpu_insn[meta->amdgpu_insns]);
+ meta->amdgpu_insns++;
+ fixup_idx++;
+
+ /* Lock acquired - restore same-bucket lanes */
+ knod_emit(priv, meta, s_mov_b64, AMDGCN_SREG_EXEC_LO,
+ KNOD_AMDGPU_TMP_SREG0_LO);
+
+ /* ---- INSERT_LANE: process one lane at a time ---- */
+ knod_bpf_set_label(meta, &labels[LABEL_INSERT_LANE],
+ meta->amdgpu_insns);
+
+ /* SREG4 = exec (remaining same-bucket lanes) */
+ knod_emit(priv, meta, s_mov_b64, KNOD_AMDGPU_TMP_SREG4_LO,
+ AMDGCN_SREG_EXEC_LO);
+
+ /* Pick first active lane via mbcnt */
+ knod_emit(priv, meta, v_mbcnt_lo_u32_b32, v_tmp, s_exec_lo,
+ v_zero);
+ knod_emit(priv, meta, v_mbcnt_hi_u32_b32, v_tmp, s_exec_hi, v_tmp);
+ knod_emit(priv, meta, v_cmp_eq_u32, v_zero, v_tmp);
+ knod_emit(priv, meta, s_and_b64, AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_EXEC_LO, AMDGCN_SREG_VCC_LO);
+
+ /* SREG2 = exec (single-lane mask) */
+ knod_emit(priv, meta, s_mov_b64, KNOD_AMDGPU_TMP_SREG2_LO,
+ AMDGCN_SREG_EXEC_LO);
+
+ /* r64[2] = r64[15] (bucket_addr for chain walk start) */
+ knod_mov64(priv, meta, r64[2], r64[15]);
+
+ /* ---- CHAIN_NEXT: walk chain ---- */
+ knod_bpf_set_label(meta, &labels[LABEL_CHAIN_NEXT], meta->amdgpu_insns);
+
+ knod_emit(priv, meta, global_load_dword, r64[0].lo, r64[2].lo, 0);
+ knod_wait_vmcnt(priv, meta);
+
+ /* Mask out DELETED bit */
+ knod_iset32(&p32, KNOD_BPF_HASH_NEXT_MASK);
+ knod_emit(priv, meta, v_and_b32_e32, r64[0].lo, p32, r64[0].lo);
+
+ /* End-of-chain check (VOPC literal workaround) */
+ knod_emit(priv, meta, v_mov_b32_e32, r64[0].hi, p32);
+ knod_emit(priv, meta, v_cmp_eq_u32, r64[0].hi, r64[0].lo);
+ knod_emit(priv, meta, s_andn2_b64, AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_EXEC_LO, AMDGCN_SREG_VCC_LO);
+ emit_s_cbranch_execz(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ 0);
+ knod_bpf_set_fixup(meta, &fixups[fixup_idx],
+ &labels[LABEL_ALLOC_INSERT], meta->amdgpu_insns);
+ debug_insn(priv->isa_version, &meta->amdgpu_insn[meta->amdgpu_insns]);
+ meta->amdgpu_insns++;
+ fixup_idx++;
+
+ /* elem_addr = elems + elem_id * elem_size -> r64[2] */
+ knod_iset64(&p64[0], elems_gaddr);
+ knod_mov64(priv, meta, r64[1], p64[0]);
+ knod_iset64(&p64[0], elem_size);
+ knod_mov32(priv, meta, r64[10].lo, p64[0].lo);
+ knod_emit(priv, meta, v_mad_u64_u32, r64[2], sr64_carry.lo,
+ r64[10].lo, r64[0].lo, r64[1]);
+
+ /* Load elem.next for DELETED check */
+ knod_emit(priv, meta, global_load_dword, r64[0].hi, r64[2].lo, 0);
+
+ /* Load key from map element -> KEY_IN_MAP */
+ key_in_map = KEY_IN_MAP_32;
+ len = knod_map_obj_k->key_size;
+ off = offsetof(struct knod_bpf_hash_elem_obj, kv);
+
+ while (len >= 16) {
+ knod_emit(priv, meta, global_load_dwordx4, r32[key_in_map],
+ r64[2].lo, off);
+ off += 16;
+ len -= 16;
+ key_in_map += 4;
+ }
+
+ if (len >= 8) {
+ knod_emit(priv, meta, global_load_dwordx2, r32[key_in_map],
+ r64[2].lo, off);
+ off += 8;
+ len -= 8;
+ key_in_map += 2;
+ }
+
+ if (len >= 4) {
+ knod_emit(priv, meta, global_load_dword, r32[key_in_map],
+ r64[2].lo, off);
+ off += 4;
+ len -= 4;
+ key_in_map += 1;
+ }
+
+ if (len) {
+ knod_emit(priv, meta, global_load_dword, r32[key_in_map],
+ r64[2].lo, off);
+ }
+
+ knod_wait_vmcnt(priv, meta);
+
+ /* Key comparison -> SREG1 */
+ key_in_map = KEY_IN_MAP_32;
+ key_in_pkt = KEY_IN_PKT_32;
+ len = knod_map_obj_k->key_size;
+ first_cmp = true;
+
+ while (len >= 8) {
+ knod_emit(priv, meta, v_cmp_eq_u64, r32[key_in_map],
+ r32[key_in_pkt]);
+
+ if (first_cmp) {
+ emit_s_and_b64(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ KNOD_AMDGPU_TMP_SREG1_LO,
+ AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_VCC_LO);
+ first_cmp = false;
+ } else {
+ emit_s_and_b64(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ KNOD_AMDGPU_TMP_SREG1_LO,
+ KNOD_AMDGPU_TMP_SREG1_LO,
+ AMDGCN_SREG_VCC_LO);
+ }
+ debug_insn(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns]);
+ meta->amdgpu_insns++;
+
+ key_in_map += 2;
+ key_in_pkt += 2;
+ len -= 8;
+ }
+
+ if (len >= 4) {
+ knod_emit(priv, meta, v_cmp_eq_u32, r32[key_in_map],
+ r32[key_in_pkt]);
+
+ if (first_cmp) {
+ emit_s_and_b64(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ KNOD_AMDGPU_TMP_SREG1_LO,
+ AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_VCC_LO);
+ first_cmp = false;
+ } else {
+ emit_s_and_b64(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ KNOD_AMDGPU_TMP_SREG1_LO,
+ KNOD_AMDGPU_TMP_SREG1_LO,
+ AMDGCN_SREG_VCC_LO);
+ }
+ debug_insn(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns]);
+ meta->amdgpu_insns++;
+
+ key_in_map += 1;
+ key_in_pkt += 1;
+ len -= 4;
+ }
+
+ if (len) {
+ knod_emit(priv, meta, v_cmp_eq_u32, r32[key_in_map],
+ r32[key_in_pkt]);
+
+ if (first_cmp) {
+ emit_s_and_b64(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ KNOD_AMDGPU_TMP_SREG1_LO,
+ AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_VCC_LO);
+ first_cmp = false;
+ } else {
+ emit_s_and_b64(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ KNOD_AMDGPU_TMP_SREG1_LO,
+ KNOD_AMDGPU_TMP_SREG1_LO,
+ AMDGCN_SREG_VCC_LO);
+ }
+ debug_insn(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns]);
+ meta->amdgpu_insns++;
+ }
+
+ /* DELETED check: SREG1 &= not_deleted */
+ knod_iset32(&p32,
+ KNOD_BPF_HASH_NEXT_DELETED);
+ knod_emit(priv, meta, v_and_b32_e32, r64[0].hi, p32, r64[0].hi);
+ knod_iset32(&p32, 0);
+ knod_emit(priv, meta, v_cmp_eq_u32, p32, r64[0].hi);
+ knod_emit(priv, meta, s_and_b64, KNOD_AMDGPU_TMP_SREG1_LO,
+ KNOD_AMDGPU_TMP_SREG1_LO, AMDGCN_SREG_VCC_LO);
+
+ /* Narrow exec to matched lane */
+ knod_emit(priv, meta, s_and_b64, AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_EXEC_LO, KNOD_AMDGPU_TMP_SREG1_LO);
+
+ /* Value overwrite for matched lane (exec-masked, skipped if no
+ * match)
+ */
+ knod_iset64(&p64[1],
+ offsetof(struct knod_bpf_hash_elem_obj,
+ kv) +
+ knod_map_obj_k->key_size);
+ knod_add64(priv, meta, r64[0], p64[1], r64[2]);
+
+ val_off = 0;
+ val_len = knod_map_obj_k->value_size;
+
+ while (val_len >= 16) {
+ knod_bpf_load_size(priv, meta,
+ &r64[10], &stack[0],
+ sizeof(unsigned long),
+ 512 + val_stack_off + val_off);
+ knod_bpf_load_size(priv, meta,
+ &r64[11], &stack[0],
+ sizeof(unsigned long),
+ 512 + val_stack_off + val_off + 8);
+ knod_emit(priv, meta, global_store_dwordx4, r64[10].lo,
+ r64[0].lo, val_off);
+ val_off += 16;
+ val_len -= 16;
+ }
+
+ if (val_len >= 8) {
+ knod_bpf_load_size(priv, meta,
+ &r64[10], &stack[0],
+ sizeof(unsigned long),
+ 512 + val_stack_off + val_off);
+ knod_emit(priv, meta, global_store_dwordx2, r64[10].lo,
+ r64[0].lo, val_off);
+ val_off += 8;
+ val_len -= 8;
+ }
+
+ if (val_len >= 4) {
+ knod_bpf_load_size(priv, meta,
+ &r64[10], &stack[0],
+ sizeof(unsigned int),
+ 512 + val_stack_off + val_off);
+ knod_emit(priv, meta, global_store_dword, r64[10].lo,
+ r64[0].lo, val_off);
+ val_off += 4;
+ val_len -= 4;
+ }
+
+ if (val_len >= 2) {
+ knod_bpf_load_size(priv, meta,
+ &r64[10], &stack[0],
+ sizeof(unsigned short),
+ 512 + val_stack_off + val_off);
+ knod_emit(priv, meta, global_store_short, r64[10].lo,
+ r64[0].lo, val_off);
+ val_off += 2;
+ val_len -= 2;
+ }
+
+ if (val_len >= 1) {
+ knod_bpf_load_size(priv, meta,
+ &r64[10], &stack[0],
+ sizeof(unsigned char),
+ 512 + val_stack_off + val_off);
+ knod_emit(priv, meta, global_store_byte, r64[10].lo,
+ r64[0].lo, val_off);
+ }
+
+ /* If matched, done with this lane */
+ emit_s_cbranch_execnz(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ 0);
+ knod_bpf_set_fixup(meta, &fixups[fixup_idx],
+ &labels[LABEL_LANE_DONE], meta->amdgpu_insns);
+ debug_insn(priv->isa_version, &meta->amdgpu_insn[meta->amdgpu_insns]);
+ meta->amdgpu_insns++;
+ fixup_idx++;
+
+ /* No match: restore lane, continue chain walk */
+ knod_emit(priv, meta, s_mov_b64, AMDGCN_SREG_EXEC_LO,
+ KNOD_AMDGPU_TMP_SREG2_LO);
+
+ emit_s_cbranch_execnz(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ 0);
+ knod_bpf_set_fixup(meta, &fixups[fixup_idx],
+ &labels[LABEL_CHAIN_NEXT], meta->amdgpu_insns);
+ debug_insn(priv->isa_version, &meta->amdgpu_insn[meta->amdgpu_insns]);
+ meta->amdgpu_insns++;
+ fixup_idx++;
+
+ /* ---- ALLOC_INSERT: key not found, insert new elem ---- */
+ knod_bpf_set_label(meta, &labels[LABEL_ALLOC_INSERT],
+ meta->amdgpu_insns);
+
+ /* Restore single-lane exec */
+ knod_emit(priv, meta, s_mov_b64, AMDGCN_SREG_EXEC_LO,
+ KNOD_AMDGPU_TMP_SREG2_LO);
+
+ /* Alloc from free pool: atomic_add(cur, -1) */
+ knod_iset32(&v_minus_one, -1);
+ knod_emit(priv, meta, v_mov_b32_e32, r64[11].lo, v_minus_one);
+
+ knod_iset64(&p64[0], cur_gaddr);
+ knod_mov64(priv, meta, r64[1], p64[0]);
+
+ knod_emit(priv, meta, global_atomic_add, r64[11].lo, r64[1].lo,
+ r64[11].lo, 0, 1);
+ knod_wait_vmcnt(priv, meta);
+
+ /* my_cur = old_cur - 1 -> r64[0].lo */
+ knod_emit(priv, meta, v_mov_b32_e32, r64[0].lo, v_one);
+ knod_emit(priv, meta, v_sub_u32, r64[0].lo, r64[11].lo, r64[0].lo);
+
+ /* OOM check: if (my_cur < 0) -> skip insert */
+ knod_emit(priv, meta, v_cmp_gt_i32, v_zero, r64[0].lo);
+ knod_emit(priv, meta, s_andn2_b64, AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_EXEC_LO, AMDGCN_SREG_VCC_LO);
+ emit_s_cbranch_execz(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ 0);
+ knod_bpf_set_fixup(meta, &fixups[fixup_idx],
+ &labels[LABEL_LANE_DONE], meta->amdgpu_insns);
+ debug_insn(priv->isa_version, &meta->amdgpu_insn[meta->amdgpu_insns]);
+ meta->amdgpu_insns++;
+ fixup_idx++;
+
+ /* queue_addr = queue_gaddr + my_cur * 4 -> r64[1] */
+ knod_iset64(&p64[0], queue_gaddr);
+ knod_mov64(priv, meta, r64[1], p64[0]);
+ knod_iset64(&p64[1], sizeof(unsigned int));
+ knod_emit(priv, meta, v_mad_u64_u32, r64[1], sr64_carry.lo,
+ p64[1].lo, r64[0].lo, r64[1]);
+
+ /* elem_id = queue[my_cur] -> r64[0].lo */
+ knod_emit(priv, meta, global_load_dword, r64[0].lo, r64[1].lo, 0);
+ knod_wait_vmcnt(priv, meta);
+
+ /* new_elem_addr = elems + elem_id * elem_size -> r64[2] */
+ knod_iset64(&p64[0], elems_gaddr);
+ knod_mov64(priv, meta, r64[1], p64[0]);
+ knod_iset64(&p64[0], elem_size);
+ knod_mov32(priv, meta, r64[10].lo, p64[0].lo);
+ knod_emit(priv, meta, v_mad_u64_u32, r64[2], sr64_carry.lo,
+ r64[10].lo, r64[0].lo, r64[1]);
+
+ /* Save elem_id to SGPR (v_mad carry already done) */
+ knod_emit(priv, meta, v_readfirstlane_b32, KNOD_AMDGPU_TMP_SREG1_LO,
+ r64[0].lo.v);
+
+ /* Load current bucket head -> r64[1].lo */
+ knod_emit(priv, meta, global_load_dword, r64[1].lo, r64[15].lo, 0);
+ knod_wait_vmcnt(priv, meta);
+
+ /* new_elem.next = old_head */
+ knod_emit(priv, meta, global_store_dword, r64[1].lo, r64[2].lo, 0);
+
+ /* Write key to new element */
+ koff = offsetof(struct knod_bpf_hash_elem_obj, kv);
+
+ key_in_pkt = KEY_IN_PKT_32;
+ len = knod_map_obj_k->key_size;
+
+ while (len >= 8) {
+ knod_emit(priv, meta, global_store_dwordx2, r32[key_in_pkt],
+ r64[2].lo, koff);
+ koff += 8;
+ len -= 8;
+ key_in_pkt += 2;
+ }
+
+ if (len >= 4) {
+ knod_emit(priv, meta, global_store_dword, r32[key_in_pkt],
+ r64[2].lo, koff);
+ koff += 4;
+ len -= 4;
+ key_in_pkt += 1;
+ }
+
+ if (len >= 2) {
+ knod_emit(priv, meta, global_store_short, r32[key_in_pkt],
+ r64[2].lo, koff);
+ koff += 2;
+ len -= 2;
+ }
+
+ if (len >= 1) {
+ knod_emit(priv, meta, global_store_byte, r32[key_in_pkt],
+ r64[2].lo, koff);
+ }
+
+ /* Write value to new element */
+ voff = offsetof(struct knod_bpf_hash_elem_obj, kv) +
+ knod_map_obj_k->key_size;
+
+ val_off = 0;
+ val_len = knod_map_obj_k->value_size;
+
+ knod_iset64(&p64[1], voff);
+ knod_add64(priv, meta, r64[0], p64[1], r64[2]);
+
+ while (val_len >= 16) {
+ knod_bpf_load_size(priv, meta,
+ &r64[10], &stack[0],
+ sizeof(unsigned long),
+ 512 + val_stack_off + val_off);
+ knod_bpf_load_size(priv, meta,
+ &r64[11], &stack[0],
+ sizeof(unsigned long),
+ 512 + val_stack_off + val_off + 8);
+ knod_emit(priv, meta, global_store_dwordx4, r64[10].lo,
+ r64[0].lo, val_off);
+ val_off += 16;
+ val_len -= 16;
+ }
+
+ if (val_len >= 8) {
+ knod_bpf_load_size(priv, meta,
+ &r64[10], &stack[0],
+ sizeof(unsigned long),
+ 512 + val_stack_off + val_off);
+ knod_emit(priv, meta, global_store_dwordx2, r64[10].lo,
+ r64[0].lo, val_off);
+ val_off += 8;
+ val_len -= 8;
+ }
+
+ if (val_len >= 4) {
+ knod_bpf_load_size(priv, meta,
+ &r64[10], &stack[0],
+ sizeof(unsigned int),
+ 512 + val_stack_off + val_off);
+ knod_emit(priv, meta, global_store_dword, r64[10].lo,
+ r64[0].lo, val_off);
+ val_off += 4;
+ val_len -= 4;
+ }
+
+ if (val_len >= 2) {
+ knod_bpf_load_size(priv, meta,
+ &r64[10], &stack[0],
+ sizeof(unsigned short),
+ 512 + val_stack_off + val_off);
+ knod_emit(priv, meta, global_store_short, r64[10].lo,
+ r64[0].lo, val_off);
+ val_off += 2;
+ val_len -= 2;
+ }
+
+ if (val_len >= 1) {
+ knod_bpf_load_size(priv, meta,
+ &r64[10], &stack[0],
+ sizeof(unsigned char),
+ 512 + val_stack_off + val_off);
+ knod_emit(priv, meta, global_store_byte, r64[10].lo,
+ r64[0].lo, val_off);
+ }
+
+ knod_wait_vmcnt(priv, meta);
+
+ /* Update bucket[hash] = new elem_id */
+ knod_emit(priv, meta, v_mov_b32_e32, r64[1].lo, s_elem_id);
+ knod_emit(priv, meta, global_store_dword, r64[1].lo, r64[15].lo,
+ 0);
+ knod_wait_vmcnt(priv, meta);
+
+ /* ---- LANE_DONE: remove this lane, next lane ---- */
+ knod_bpf_set_label(meta, &labels[LABEL_LANE_DONE], meta->amdgpu_insns);
+
+ knod_emit(priv, meta, s_andn2_b64, AMDGCN_SREG_EXEC_LO,
+ KNOD_AMDGPU_TMP_SREG4_LO, KNOD_AMDGPU_TMP_SREG2_LO);
+
+ emit_s_cbranch_execnz(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ 0);
+ knod_bpf_set_fixup(meta, &fixups[fixup_idx],
+ &labels[LABEL_INSERT_LANE], meta->amdgpu_insns);
+ debug_insn(priv->isa_version, &meta->amdgpu_insn[meta->amdgpu_insns]);
+ meta->amdgpu_insns++;
+ fixup_idx++;
+
+ /* ---- UNLOCK: release lock + next bucket ---- */
+ knod_bpf_set_label(meta, &labels[LABEL_UNLOCK], meta->amdgpu_insns);
+
+ knod_emit(priv, meta, s_mov_b64, AMDGCN_SREG_EXEC_LO,
+ KNOD_AMDGPU_TMP_SREG0_LO);
+
+ /* Recompute lock_addr (r64[10] clobbered) */
+ knod_iset64(&p64[0], lock_offset);
+ knod_add64(priv, meta, r64[10], p64[0], r64[15]);
+
+ knod_emit(priv, meta, v_mov_b32_e32, r64[1].lo, v_zero);
+ knod_emit(priv, meta, global_store_dword, r64[1].lo, r64[10].lo,
+ 0);
+ knod_wait_vmcnt(priv, meta);
+
+ /* Next bucket */
+ knod_emit(priv, meta, s_andn2_b64, AMDGCN_SREG_EXEC_LO,
+ KNOD_AMDGPU_TMP_SREG5_LO, KNOD_AMDGPU_TMP_SREG0_LO);
+
+ emit_s_cbranch_execnz(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ 0);
+ knod_bpf_set_fixup(meta, &fixups[fixup_idx],
+ &labels[LABEL_BUCKET_LOOP], meta->amdgpu_insns);
+ debug_insn(priv->isa_version, &meta->amdgpu_insn[meta->amdgpu_insns]);
+ meta->amdgpu_insns++;
+ fixup_idx++;
+
+ /* ======== Phase 5: Restore ======== */
+
+ knod_bpf_set_label(meta, &labels[LABEL_OUT], meta->amdgpu_insns);
+ /* exec = SREG3 (restore all original lanes) */
+ knod_emit(priv, meta, s_or_b64, AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_EXEC_LO, KNOD_AMDGPU_TMP_SREG3_LO);
+
+ /* bpf_reg64[0] = 0 (return value for all lanes) */
+ knod_iset64(&p64[0], 0);
+ knod_mov64(priv, meta, bpf_reg64[0], p64[0]);
+
+ for (idx = 0; idx < fixup_idx; idx++)
+ knod_bpf_fixup_branch(priv, &fixups[idx]);
+
+ return;
+#undef LABEL_BUCKET_LOOP
+#undef LABEL_LOCK_RETRY
+#undef LABEL_INSERT_LANE
+#undef LABEL_CHAIN_NEXT
+#undef LABEL_ALLOC_INSERT
+#undef LABEL_LANE_DONE
+#undef LABEL_UNLOCK
+}
+
+static void knod_bpf_map_delete_hash(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ int map_id)
+{
+#define LABEL_BUCKET_LOOP 0
+#define LABEL_LOCK_RETRY 1
+#define LABEL_DELETE_LANE 2
+#define LABEL_CHAIN_NEXT 3
+#define LABEL_LANE_DONE 4
+#define LABEL_UNLOCK 5
+ struct knod_bpf_map_obj *knod_map_obj_k, *knod_map_obj_g;
+ struct amdgcn_param32 s_bucket_lo, v_tmp, v_zero, v_one;
+ int off, len, _len, idx, key_in_pkt, key_in_map;
+ struct amdgcn_branch_fixup fixups[12] = {0,};
+ unsigned long bucket_gaddr, gc_count_gaddr;
+ struct amdgcn_param32 s_exec_lo, s_exec_hi;
+ unsigned long gc_list_gaddr, elems_gaddr;
+ u32 key_stack_off = meta->kreg.stack_off;
+ struct amdgcn_label labels[12] = {0,};
+ struct amdgcn_param64 sr64_carry;
+ struct amdgcn_param32 v_del;
+ struct amdgcn_param32 p32;
+ unsigned long lock_offset;
+ unsigned int elem_size;
+ int fixup_idx = 0;
+ bool first_cmp;
+
+ knod_map_obj_k =
+ (struct knod_bpf_map_obj *)knod_bpf_map_kaddr(priv, map_id);
+ knod_map_obj_g =
+ (struct knod_bpf_map_obj *)knod_bpf_map_gaddr(priv, map_id);
+ bucket_gaddr = (unsigned long)knod_map_obj_g +
+ offsetof(struct knod_bpf_map_obj, bucket);
+ gc_count_gaddr = (unsigned long)knod_map_obj_g +
+ offsetof(struct knod_bpf_map_obj, meta.hmeta.gc_count);
+ gc_list_gaddr = (unsigned long)knod_map_obj_k->meta.hmeta.gc_list;
+ elems_gaddr = (unsigned long)knod_map_obj_k->meta.hmeta.elems;
+ elem_size = knod_map_obj_k->meta.hmeta.elem_size;
+
+ if (!knod_map_obj_g || !knod_map_obj_k)
+ WARN_ON_ONCE(1);
+
+ /* ======== Phase 1: Setup ======== */
+
+ /* Load key from stack -> r64[3..9] (KEY_IN_PKT) */
+ key_in_pkt = KEY_IN_PKT_64;
+ len = knod_map_obj_k->key_size;
+ off = key_stack_off;
+ while (len) {
+ if (len >= sizeof(unsigned long))
+ _len = sizeof(unsigned long);
+ else
+ _len = len;
+ knod_bpf_load_size(priv, meta,
+ &r64[key_in_pkt],
+ &stack[0],
+ _len,
+ 512 + off);
+ key_in_pkt++;
+ len -= _len;
+ off += _len;
+ }
+
+ /* jhash -> r64[2].lo = hash */
+ knod_jhash(priv, meta,
+ 2,
+ knod_map_obj_k->key_size,
+ knod_map_obj_k->meta.hmeta.hashrnd);
+ knod_iset64(&p64[0], 0);
+ knod_mov32(priv, meta, r64[2].hi, p64[0].lo);
+
+ /* hash = hash & (n_buckets - 1) */
+ knod_iset32(&p64[0].lo,
+ knod_map_obj_k->meta.hmeta.n_buckets - 1);
+ knod_and32(priv, meta, r64[2].lo, p64[0].lo, r64[2].lo);
+
+ /* r64[1] = bucket_gaddr */
+ knod_iset64(&p64[0], bucket_gaddr);
+ knod_mov64(priv, meta, r64[1], p64[0]);
+
+ /* bucket_addr = bucket_gaddr + hash * sizeof(int) -> r64[2] */
+ knod_iset64(&p64[1], sizeof(int));
+ knod_emit(priv, meta, v_mad_u64_u32, r64[2], sr64[0].lo,
+ p64[1].lo, r64[2].lo, r64[1]);
+
+ /* Save bucket_addr to r64[15] */
+ knod_mov64(priv, meta, r64[15], r64[2]);
+
+ /* SREG3 = initial exec */
+ knod_emit(priv, meta, s_mov_b64, KNOD_AMDGPU_TMP_SREG3_LO,
+ AMDGCN_SREG_EXEC_LO);
+
+ /* ======== Phase 2: Sequential per-lane processing ======== */
+
+ /* SREG5 = exec (all lanes to process, for BUCKET_LOOP) */
+ knod_emit(priv, meta, s_mov_b64, KNOD_AMDGPU_TMP_SREG5_LO,
+ AMDGCN_SREG_EXEC_LO);
+
+ /* ---- BUCKET_LOOP: process one unique bucket per iteration ---- */
+ knod_bpf_set_label(meta, &labels[LABEL_BUCKET_LOOP],
+ meta->amdgpu_insns);
+
+ lock_offset =
+ (unsigned long)knod_map_obj_k->meta.hmeta.n_buckets *
+ sizeof(unsigned int);
+
+ knod_sset32(&s_bucket_lo,
+ KNOD_AMDGPU_TMP_SREG1_HI);
+ knod_vset32(&v_tmp, KNOD_AMDGPU_TMP_VREG0_HI);
+ knod_iset32(&v_zero, 0);
+ knod_iset32(&v_one, 1);
+ knod_sset32(&s_exec_lo, AMDGCN_SREG_EXEC_LO);
+ knod_sset32(&s_exec_hi,
+ AMDGCN_SREG_EXEC_LO + 1);
+ knod_sset64(&sr64_carry,
+ KNOD_AMDGPU_TMP_SREG1_LO);
+
+ /* Pick first active lane's bucket addr */
+ knod_emit(priv, meta, v_readfirstlane_b32, KNOD_AMDGPU_TMP_SREG1_HI,
+ r64[15].lo.v);
+
+ /* vcc = lanes with same bucket */
+ knod_emit(priv, meta, v_cmp_eq_u32, s_bucket_lo, r64[15].lo);
+
+ /* SREG5 = remaining lanes; exec = same-bucket lanes */
+ knod_emit(priv, meta, s_and_saveexec_b64, KNOD_AMDGPU_TMP_SREG5_LO,
+ AMDGCN_SREG_VCC_LO);
+
+ /* SREG0 = same-bucket lanes */
+ knod_emit(priv, meta, s_mov_b64, KNOD_AMDGPU_TMP_SREG0_LO,
+ AMDGCN_SREG_EXEC_LO);
+
+ /* ---- Lock acquire ---- */
+ /* r64[10] = r64[15] + lock_offset */
+ knod_iset64(&p64[0], lock_offset);
+ knod_add64(priv, meta, r64[10], p64[0], r64[15]);
+
+ /* r64[11].lo = 1 (swap data) */
+ knod_emit(priv, meta, v_mov_b32_e32, r64[11].lo, v_one);
+
+ /* First-lane isolation via mbcnt */
+ knod_emit(priv, meta, v_mbcnt_lo_u32_b32, v_tmp, s_exec_lo,
+ v_zero);
+ knod_emit(priv, meta, v_mbcnt_hi_u32_b32, v_tmp, s_exec_hi, v_tmp);
+ knod_emit(priv, meta, v_cmp_eq_u32, v_zero, v_tmp);
+ knod_emit(priv, meta, s_and_b64, AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_EXEC_LO, AMDGCN_SREG_VCC_LO);
+
+ /* LOCK_RETRY: spin until lock acquired */
+ knod_bpf_set_label(meta, &labels[LABEL_LOCK_RETRY], meta->amdgpu_insns);
+
+ knod_emit(priv, meta, global_atomic_swap, r64[11].hi, r64[10].lo,
+ r64[11].lo, 0, 1);
+ knod_wait_vmcnt(priv, meta);
+
+ meta->amdgpu_insn[meta->amdgpu_insns].size =
+ emit_gfx10_v_cmp_ne_u32(
+ &meta->amdgpu_insn[meta->amdgpu_insns].gfx10,
+ v_zero, r64[11].hi);
+ meta->amdgpu_insn[meta->amdgpu_insns].type = AMDGCN_INSN_TYPE_VOPC;
+ debug_insn(priv->isa_version, &meta->amdgpu_insn[meta->amdgpu_insns]);
+ meta->amdgpu_insns++;
+
+ emit_s_cbranch_vccnz(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ 0);
+ knod_bpf_set_fixup(meta, &fixups[fixup_idx],
+ &labels[LABEL_LOCK_RETRY], meta->amdgpu_insns);
+ debug_insn(priv->isa_version, &meta->amdgpu_insn[meta->amdgpu_insns]);
+ meta->amdgpu_insns++;
+ fixup_idx++;
+
+ /* Lock acquired - restore same-bucket lanes */
+ knod_emit(priv, meta, s_mov_b64, AMDGCN_SREG_EXEC_LO,
+ KNOD_AMDGPU_TMP_SREG0_LO);
+
+ /* ---- DELETE_LANE: process one lane at a time ---- */
+ knod_bpf_set_label(meta, &labels[LABEL_DELETE_LANE],
+ meta->amdgpu_insns);
+
+ /* SREG4 = exec (remaining same-bucket lanes) */
+ knod_emit(priv, meta, s_mov_b64, KNOD_AMDGPU_TMP_SREG4_LO,
+ AMDGCN_SREG_EXEC_LO);
+
+ /* Pick first active lane via mbcnt */
+ knod_emit(priv, meta, v_mbcnt_lo_u32_b32, v_tmp, s_exec_lo,
+ v_zero);
+ knod_emit(priv, meta, v_mbcnt_hi_u32_b32, v_tmp, s_exec_hi, v_tmp);
+ knod_emit(priv, meta, v_cmp_eq_u32, v_zero, v_tmp);
+ knod_emit(priv, meta, s_and_b64, AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_EXEC_LO, AMDGCN_SREG_VCC_LO);
+
+ /* SREG2 = exec (single-lane mask) */
+ knod_emit(priv, meta, s_mov_b64, KNOD_AMDGPU_TMP_SREG2_LO,
+ AMDGCN_SREG_EXEC_LO);
+
+ /* r64[2] = r64[15] (bucket_addr for chain walk start) */
+ knod_mov64(priv, meta, r64[2], r64[15]);
+
+ /* ---- CHAIN_NEXT: walk chain ---- */
+ knod_bpf_set_label(meta, &labels[LABEL_CHAIN_NEXT], meta->amdgpu_insns);
+
+ knod_emit(priv, meta, global_load_dword, r64[0].lo, r64[2].lo, 0);
+ knod_wait_vmcnt(priv, meta);
+
+ /* Mask out DELETED bit */
+ knod_iset32(&p32, KNOD_BPF_HASH_NEXT_MASK);
+ knod_emit(priv, meta, v_and_b32_e32, r64[0].lo, p32, r64[0].lo);
+
+ /* End-of-chain check (VOPC literal workaround) */
+ knod_emit(priv, meta, v_mov_b32_e32, r64[0].hi, p32);
+ knod_emit(priv, meta, v_cmp_eq_u32, r64[0].hi, r64[0].lo);
+ knod_emit(priv, meta, s_andn2_b64, AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_EXEC_LO, AMDGCN_SREG_VCC_LO);
+ emit_s_cbranch_execz(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ 0);
+ knod_bpf_set_fixup(meta, &fixups[fixup_idx],
+ &labels[LABEL_LANE_DONE], meta->amdgpu_insns);
+ debug_insn(priv->isa_version, &meta->amdgpu_insn[meta->amdgpu_insns]);
+ meta->amdgpu_insns++;
+ fixup_idx++;
+
+ /* elem_addr = elems + elem_id * elem_size -> r64[2] */
+ knod_iset64(&p64[0], elems_gaddr);
+ knod_mov64(priv, meta, r64[1], p64[0]);
+ knod_iset64(&p64[0], elem_size);
+ knod_mov32(priv, meta, r64[10].lo, p64[0].lo);
+ knod_emit(priv, meta, v_mad_u64_u32, r64[2], sr64_carry.lo,
+ r64[10].lo, r64[0].lo, r64[1]);
+
+ /* Load elem.next for DELETED check */
+ knod_emit(priv, meta, global_load_dword, r64[0].hi, r64[2].lo, 0);
+
+ /* Load key from map element -> KEY_IN_MAP */
+ key_in_map = KEY_IN_MAP_32;
+ len = knod_map_obj_k->key_size;
+ off = offsetof(struct knod_bpf_hash_elem_obj, kv);
+
+ while (len >= 16) {
+ knod_emit(priv, meta, global_load_dwordx4, r32[key_in_map],
+ r64[2].lo, off);
+ off += 16;
+ len -= 16;
+ key_in_map += 4;
+ }
+
+ if (len >= 8) {
+ knod_emit(priv, meta, global_load_dwordx2, r32[key_in_map],
+ r64[2].lo, off);
+ off += 8;
+ len -= 8;
+ key_in_map += 2;
+ }
+
+ if (len >= 4) {
+ knod_emit(priv, meta, global_load_dword, r32[key_in_map],
+ r64[2].lo, off);
+ off += 4;
+ len -= 4;
+ key_in_map += 1;
+ }
+
+ if (len) {
+ knod_emit(priv, meta, global_load_dword, r32[key_in_map],
+ r64[2].lo, off);
+ }
+
+ knod_wait_vmcnt(priv, meta);
+
+ /* Key comparison -> SREG1 */
+ key_in_map = KEY_IN_MAP_32;
+ key_in_pkt = KEY_IN_PKT_32;
+ len = knod_map_obj_k->key_size;
+ first_cmp = true;
+
+ while (len >= 8) {
+ knod_emit(priv, meta, v_cmp_eq_u64, r32[key_in_map],
+ r32[key_in_pkt]);
+
+ if (first_cmp) {
+ emit_s_and_b64(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ KNOD_AMDGPU_TMP_SREG1_LO,
+ AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_VCC_LO);
+ first_cmp = false;
+ } else {
+ emit_s_and_b64(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ KNOD_AMDGPU_TMP_SREG1_LO,
+ KNOD_AMDGPU_TMP_SREG1_LO,
+ AMDGCN_SREG_VCC_LO);
+ }
+ debug_insn(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns]);
+ meta->amdgpu_insns++;
+
+ key_in_map += 2;
+ key_in_pkt += 2;
+ len -= 8;
+ }
+
+ if (len >= 4) {
+ knod_emit(priv, meta, v_cmp_eq_u32, r32[key_in_map],
+ r32[key_in_pkt]);
+
+ if (first_cmp) {
+ emit_s_and_b64(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ KNOD_AMDGPU_TMP_SREG1_LO,
+ AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_VCC_LO);
+ first_cmp = false;
+ } else {
+ emit_s_and_b64(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ KNOD_AMDGPU_TMP_SREG1_LO,
+ KNOD_AMDGPU_TMP_SREG1_LO,
+ AMDGCN_SREG_VCC_LO);
+ }
+ debug_insn(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns]);
+ meta->amdgpu_insns++;
+
+ key_in_map += 1;
+ key_in_pkt += 1;
+ len -= 4;
+ }
+
+ if (len) {
+ knod_emit(priv, meta, v_cmp_eq_u32, r32[key_in_map],
+ r32[key_in_pkt]);
+
+ if (first_cmp) {
+ emit_s_and_b64(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ KNOD_AMDGPU_TMP_SREG1_LO,
+ AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_VCC_LO);
+ first_cmp = false;
+ } else {
+ emit_s_and_b64(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ KNOD_AMDGPU_TMP_SREG1_LO,
+ KNOD_AMDGPU_TMP_SREG1_LO,
+ AMDGCN_SREG_VCC_LO);
+ }
+ debug_insn(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns]);
+ meta->amdgpu_insns++;
+ }
+
+ /* DELETED check: SREG1 &= not_deleted */
+ knod_iset32(&p32,
+ KNOD_BPF_HASH_NEXT_DELETED);
+ knod_emit(priv, meta, v_and_b32_e32, r64[0].hi, p32, r64[0].hi);
+ knod_iset32(&p32, 0);
+ knod_emit(priv, meta, v_cmp_eq_u32, p32, r64[0].hi);
+ knod_emit(priv, meta, s_and_b64, KNOD_AMDGPU_TMP_SREG1_LO,
+ KNOD_AMDGPU_TMP_SREG1_LO, AMDGCN_SREG_VCC_LO);
+
+ /* Narrow exec to matched lane */
+ knod_emit(priv, meta, s_and_b64, AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_EXEC_LO, KNOD_AMDGPU_TMP_SREG1_LO);
+
+ /* ---- Match path: set DELETED + append to gc_list ---- */
+
+ /* atomic_or(elem.next, DELETED_BIT) - mark deleted */
+ knod_lset32(&v_del,
+ KNOD_BPF_HASH_NEXT_DELETED);
+ knod_emit(priv, meta, v_mov_b32_e32, r64[11].lo, v_del);
+ knod_emit(priv, meta, global_atomic_or, r64[11].hi, r64[2].lo,
+ r64[11].lo, 0, 0);
+
+ /* atomic_add(gc_count, 1) -> old_count in r64[11].lo */
+ knod_emit(priv, meta, v_mov_b32_e32, r64[11].lo, v_one);
+
+ knod_iset64(&p64[0], gc_count_gaddr);
+ knod_mov64(priv, meta, r64[1], p64[0]);
+
+ knod_emit(priv, meta, global_atomic_add, r64[11].lo, r64[1].lo,
+ r64[11].lo, 0, 1);
+ knod_wait_vmcnt(priv, meta);
+
+ /* Store elem_id to gc_list[old_count] */
+ knod_iset64(&p64[0], gc_list_gaddr);
+ knod_mov64(priv, meta, r64[10], p64[0]);
+
+ knod_iset64(&p64[0], sizeof(unsigned int));
+ knod_emit(priv, meta, v_mad_u64_u32, r64[1], sr64_carry.lo,
+ p64[0].lo, r64[11].lo, r64[10]);
+
+ knod_emit(priv, meta, global_store_dword, r64[0].lo, r64[1].lo, 0);
+ knod_wait_vmcnt(priv, meta);
+
+ /* If matched, done with this lane */
+ emit_s_cbranch_execnz(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ 0);
+ knod_bpf_set_fixup(meta, &fixups[fixup_idx],
+ &labels[LABEL_LANE_DONE], meta->amdgpu_insns);
+ debug_insn(priv->isa_version, &meta->amdgpu_insn[meta->amdgpu_insns]);
+ meta->amdgpu_insns++;
+ fixup_idx++;
+
+ /* No match: restore lane, continue chain walk */
+ knod_emit(priv, meta, s_mov_b64, AMDGCN_SREG_EXEC_LO,
+ KNOD_AMDGPU_TMP_SREG2_LO);
+
+ emit_s_cbranch_execnz(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ 0);
+ knod_bpf_set_fixup(meta, &fixups[fixup_idx],
+ &labels[LABEL_CHAIN_NEXT], meta->amdgpu_insns);
+ debug_insn(priv->isa_version, &meta->amdgpu_insn[meta->amdgpu_insns]);
+ meta->amdgpu_insns++;
+ fixup_idx++;
+
+ /* ---- LANE_DONE: remove this lane, next lane ---- */
+ knod_bpf_set_label(meta, &labels[LABEL_LANE_DONE], meta->amdgpu_insns);
+
+ knod_emit(priv, meta, s_andn2_b64, AMDGCN_SREG_EXEC_LO,
+ KNOD_AMDGPU_TMP_SREG4_LO, KNOD_AMDGPU_TMP_SREG2_LO);
+
+ emit_s_cbranch_execnz(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ 0);
+ knod_bpf_set_fixup(meta, &fixups[fixup_idx],
+ &labels[LABEL_DELETE_LANE], meta->amdgpu_insns);
+ debug_insn(priv->isa_version, &meta->amdgpu_insn[meta->amdgpu_insns]);
+ meta->amdgpu_insns++;
+ fixup_idx++;
+
+ /* ---- UNLOCK: release lock + next bucket ---- */
+ knod_bpf_set_label(meta, &labels[LABEL_UNLOCK], meta->amdgpu_insns);
+
+ knod_emit(priv, meta, s_mov_b64, AMDGCN_SREG_EXEC_LO,
+ KNOD_AMDGPU_TMP_SREG0_LO);
+
+ /* Recompute lock_addr (r64[10] clobbered) */
+ knod_iset64(&p64[0], lock_offset);
+ knod_add64(priv, meta, r64[10], p64[0], r64[15]);
+
+ knod_emit(priv, meta, v_mov_b32_e32, r64[1].lo, v_zero);
+ knod_emit(priv, meta, global_store_dword, r64[1].lo, r64[10].lo,
+ 0);
+ knod_wait_vmcnt(priv, meta);
+
+ /* Next bucket */
+ knod_emit(priv, meta, s_andn2_b64, AMDGCN_SREG_EXEC_LO,
+ KNOD_AMDGPU_TMP_SREG5_LO, KNOD_AMDGPU_TMP_SREG0_LO);
+
+ emit_s_cbranch_execnz(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ 0);
+ knod_bpf_set_fixup(meta, &fixups[fixup_idx],
+ &labels[LABEL_BUCKET_LOOP], meta->amdgpu_insns);
+ debug_insn(priv->isa_version, &meta->amdgpu_insn[meta->amdgpu_insns]);
+ meta->amdgpu_insns++;
+ fixup_idx++;
+
+ /* ======== Phase 3: Restore ======== */
+
+ /* exec = SREG3 (restore all original lanes) */
+ knod_emit(priv, meta, s_or_b64, AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_EXEC_LO, KNOD_AMDGPU_TMP_SREG3_LO);
+
+ /* bpf_reg64[0] = 0 (return value) */
+ knod_iset64(&p64[0], 0);
+ knod_mov64(priv, meta, bpf_reg64[0], p64[0]);
+
+ for (idx = 0; idx < fixup_idx; idx++)
+ knod_bpf_fixup_branch(priv, &fixups[idx]);
+
+ return;
+#undef LABEL_BUCKET_LOOP
+#undef LABEL_LOCK_RETRY
+#undef LABEL_DELETE_LANE
+#undef LABEL_CHAIN_NEXT
+#undef LABEL_LANE_DONE
+#undef LABEL_UNLOCK
+}
+
+static void knod_bpf_map_delete_array(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ int map_id)
+{
+ struct knod_bpf_map_obj *knod_map_obj_k, *knod_map_obj_g;
+ struct amdgcn_branch_fixup fixups[4] = {0,};
+ u32 key_stack_off = meta->kreg.stack_off;
+ struct amdgcn_label labels[10] = {0,};
+ int idx, val_off, val_len;
+ struct amdgcn_param32 v_zero;
+ unsigned long bucket_gaddr;
+ int fixup_idx = 0;
+
+ knod_map_obj_k =
+ (struct knod_bpf_map_obj *)knod_bpf_map_kaddr(priv, map_id);
+ knod_map_obj_g =
+ (struct knod_bpf_map_obj *)knod_bpf_map_gaddr(priv, map_id);
+ bucket_gaddr = (unsigned long)knod_map_obj_g +
+ offsetof(struct knod_bpf_map_obj, bucket);
+
+ if (!knod_map_obj_g || !knod_map_obj_k)
+ WARN_ON_ONCE(1);
+
+ /* load key from stack -> r64[2] */
+ knod_bpf_load_size(priv, meta,
+ &r64[2],
+ &stack[0],
+ sizeof(unsigned int),
+ 512 + key_stack_off);
+
+ /* r64[1] = bucket_gaddr */
+ knod_iset64(&p64[0], bucket_gaddr);
+ knod_mov64(priv, meta, r64[1], p64[0]);
+
+ /* clear r64[2].hi (key is 32-bit) */
+ knod_iset64(&p64[1], 0);
+ knod_mov32(priv, meta, r64[2].hi, p64[1].lo);
+
+ /* bpf_reg64[0] = 0 (return value) */
+ knod_mov64(priv, meta, bpf_reg64[0], p64[1]);
+
+ /* bounds check: if (key >= max_entries) -> skip */
+ knod_iset64(&p64[1], knod_map_obj_k->max_entries);
+ knod_mov64(priv, meta, r64[3], p64[1]);
+ knod_emit(priv, meta, v_cmp_ge_u64, r64[2], r64[3]);
+
+ /* structurized CFG: save OOB lanes, narrow exec */
+ knod_emit(priv, meta, s_and_b64, KNOD_AMDGPU_TMP_SREG3_LO,
+ AMDGCN_SREG_EXEC_LO, AMDGCN_SREG_VCC_LO);
+ knod_emit(priv, meta, s_andn2_b64, AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_EXEC_LO, AMDGCN_SREG_VCC_LO);
+ emit_s_cbranch_execz(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ 0);
+ knod_bpf_set_fixup(meta, &fixups[fixup_idx],
+ &labels[LABEL_OUT], meta->amdgpu_insns);
+ debug_insn(priv->isa_version, &meta->amdgpu_insn[meta->amdgpu_insns]);
+ meta->amdgpu_insns++;
+ fixup_idx++;
+
+ /* dest = bucket_gaddr + key * value_size -> r64[0] */
+ knod_iset64(&p64[1], knod_map_obj_k->value_size);
+ knod_emit(priv, meta, v_mad_u64_u32, r64[0], sr64[0].lo,
+ p64[1].lo, r64[2].lo, r64[1]);
+
+ /* Zero out value at dest */
+ knod_iset32(&v_zero, 0);
+ knod_emit(priv, meta, v_mov_b32_e32, r64[3].lo, v_zero);
+ knod_emit(priv, meta, v_mov_b32_e32, r64[3].hi, v_zero);
+ knod_emit(priv, meta, v_mov_b32_e32, r64[4].lo, v_zero);
+ knod_emit(priv, meta, v_mov_b32_e32, r64[4].hi, v_zero);
+
+ val_off = 0;
+ val_len = knod_map_obj_k->value_size;
+
+ while (val_len >= 16) {
+ knod_emit(priv, meta, global_store_dwordx4, r64[3].lo,
+ r64[0].lo, val_off);
+ val_off += 16;
+ val_len -= 16;
+ }
+
+ if (val_len >= 8) {
+ knod_emit(priv, meta, global_store_dwordx2, r64[3].lo,
+ r64[0].lo, val_off);
+ val_off += 8;
+ val_len -= 8;
+ }
+
+ if (val_len >= 4) {
+ knod_emit(priv, meta, global_store_dword, r64[3].lo,
+ r64[0].lo, val_off);
+ val_off += 4;
+ val_len -= 4;
+ }
+
+ if (val_len >= 2) {
+ knod_emit(priv, meta, global_store_short, r64[3].lo,
+ r64[0].lo, val_off);
+ val_off += 2;
+ val_len -= 2;
+ }
+
+ if (val_len >= 1) {
+ knod_emit(priv, meta, global_store_byte, r64[3].lo,
+ r64[0].lo, val_off);
+ }
+
+ knod_wait_vmcnt(priv, meta);
+
+ /* structurized CFG: restore OOB lanes */
+ knod_bpf_set_label(meta, &labels[LABEL_OUT], meta->amdgpu_insns);
+ knod_emit(priv, meta, s_or_b64, AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_EXEC_LO, KNOD_AMDGPU_TMP_SREG3_LO);
+
+ for (idx = 0; idx < fixup_idx; idx++)
+ knod_bpf_fixup_branch(priv, &fixups[idx]);
+}
+
+static void knod_bpf_store_cache_size(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ struct amdgcn_param64 *src,
+ /* packet or stack */
+ struct amdgcn_param32 *cache,
+ int size, int off)
+{
+ struct amdgcn_param32 p32[2];
+
+ knod_jit_dbg(" %d: off = %d off_4 = %d size = %d\n", meta->bpf_insn_idx,
+ off, off%4, size);
+ WARN_ON(knod_param_is_literal(src->lo) ||
+ knod_param_is_literal(src->hi));
+ switch (size) {
+ case sizeof(unsigned long):
+ if ((off % 4) == 0) {
+ knod_mov32(priv, meta,
+ cache[off / 4],
+ src->lo);
+ knod_mov32(priv, meta,
+ cache[(off / 4) + 1],
+ src->hi);
+ } else if ((off % 4) == 1) {
+ WARN_ON_ONCE(1);
+ } else if ((off % 4) == 2) {
+ WARN_ON_ONCE(1);
+ } else {
+ WARN_ON_ONCE(1);
+ }
+ break;
+ case sizeof(unsigned int):
+ if ((off % 4) == 0) {
+ knod_mov32(priv, meta,
+ cache[off / 4],
+ src->lo);
+ } else if ((off % 4) == 1) {
+ knod_iset64(&p64[0], 8);
+ knod_lshlrev64(priv, meta, r64[0], p64[0], *src);
+
+ knod_iset32(&p32[0], 0xffffff00);
+ knod_mov32(priv, meta, r32[2], p32[0]);
+ knod_bfi32(priv, meta, cache[off / 4],
+ r32[2], r64[0].lo, cache[off / 4]);
+ knod_iset32(&p32[0], 0x000000ff);
+ knod_mov32(priv, meta, r32[2], p32[0]);
+ knod_bfi32(priv, meta, cache[(off / 4) + 1], r32[2],
+ r64[0].hi, cache[(off / 4) + 1]);
+ } else if ((off % 4) == 2) {
+ knod_iset64(&p64[0], 16);
+ knod_lshlrev64(priv, meta, r64[0], p64[0], *src);
+
+ knod_iset32(&p32[0], 0xffff0000);
+ knod_mov32(priv, meta, r32[2], p32[0]);
+ knod_bfi32(priv, meta, cache[off / 4], r32[2],
+ r64[0].lo, cache[off / 4]);
+ knod_iset32(&p32[0], 0x0000ffff);
+ knod_mov32(priv, meta, r32[2], p32[0]);
+ knod_bfi32(priv, meta, cache[(off / 4) + 1], r32[2],
+ r64[0].hi, cache[(off / 4) + 1]);
+ } else {
+ knod_iset64(&p64[0], 24);
+ knod_lshlrev64(priv, meta, r64[0], p64[0], *src);
+
+ knod_iset32(&p32[0], 0xffff0000);
+ knod_mov32(priv, meta, r32[2], p32[0]);
+ knod_bfi32(priv, meta, cache[off / 4], r32[2],
+ r64[0].lo, cache[off / 4]);
+ knod_iset32(&p32[0], 0x00ffffff);
+ knod_mov32(priv, meta, r32[2], p32[0]);
+ knod_bfi32(priv, meta, cache[(off / 4) + 1], r32[2],
+ r64[0].hi, cache[(off / 4) + 1]);
+ }
+ break;
+ case sizeof(unsigned short):
+ if ((off % 4) == 0) {
+ knod_iset32(&p32[0], 0x0000ffff);
+ knod_mov32(priv, meta, r32[2], p32[0]);
+ knod_bfi32(priv, meta, cache[off / 4], r32[2],
+ src->lo, cache[off / 4]);
+ } else if ((off % 4) == 1) {
+ knod_iset32(&p32[0], 8);
+ knod_lshlrev32(priv, meta, r32[0], p32[0],
+ src->lo);
+ knod_iset32(&p32[0], 0x00ffff00);
+ knod_mov32(priv, meta, r32[2], p32[0]);
+ knod_bfi32(priv, meta, cache[off / 4], r32[2],
+ r32[0], cache[off / 4]);
+ } else if ((off % 4) == 2) {
+ knod_iset32(&p32[0], 16);
+ knod_lshlrev32(priv, meta, r32[0], p32[0],
+ src->lo);
+ knod_iset32(&p32[0], 0xffff0000);
+ knod_mov32(priv, meta, r32[2], p32[0]);
+ knod_bfi32(priv, meta, cache[off / 4], r32[2],
+ r32[0], cache[off / 4]);
+ } else {
+ knod_iset64(&p64[0], 24);
+ knod_lshlrev64(priv, meta, r64[0], p64[0], *src);
+
+ knod_iset32(&p32[0], 0xff000000);
+ knod_mov32(priv, meta, r32[2], p32[0]);
+ knod_bfi32(priv, meta, cache[off / 4], r32[2],
+ r64[0].lo, cache[off / 4]);
+ knod_iset32(&p32[0], 0x000000ff);
+ knod_mov32(priv, meta, r32[2], p32[0]);
+ knod_bfi32(priv, meta, cache[(off / 4) + 1], r32[2],
+ r64[0].hi, cache[(off / 4) + 1]);
+ }
+ break;
+ case sizeof(unsigned char):
+ if ((off % 4) == 0) {
+ knod_iset32(&p32[0], 0x000000ff);
+ knod_mov32(priv, meta, r32[2], p32[0]);
+ knod_bfi32(priv, meta, cache[off / 4], r32[2],
+ src->lo, cache[off / 4]);
+ return;
+ } else if ((off % 4) == 1) {
+ knod_iset32(&p32[0], 8);
+ knod_lshlrev32(priv, meta, r32[0], p32[0],
+ src->lo);
+
+ knod_iset32(&p32[0], 0x0000ff00);
+ } else if ((off % 4) == 2) {
+ knod_iset32(&p32[0], 16);
+ knod_lshlrev32(priv, meta, r32[0], p32[0],
+ src->lo);
+
+ knod_iset32(&p32[0], 0x00ff0000);
+ } else {
+ knod_iset32(&p32[0], 24);
+ knod_lshlrev32(priv, meta, r32[0], p32[0],
+ src->lo);
+ knod_iset32(&p32[0], 0xff000000);
+ }
+
+ knod_mov32(priv, meta, r32[2], p32[0]);
+ knod_bfi32(priv, meta, cache[off / 4], r32[2], r32[0],
+ cache[off / 4]);
+ break;
+ default:
+ WARN_ON_ONCE(1);
+ }
+}
+
+static bool knod_meta_is_exit(const struct knod_insn_meta *meta);
+static bool knod_bpf_is_retval_move_to_r0(const struct knod_insn_meta *meta);
+
+/*
+ * knod_bpf_emit_branch_tail - Emit EXEC mask manipulation after v_cmp for
+ * structurized per-lane branching. Replaces the old s_cbranch_vccnz/vccz.
+ *
+ * For FORWARD_SKIP:
+ * Save jumping lanes -> narrow EXEC -> s_cbranch_execz
+ * (skip if no active lanes)
+ *
+ * For DIRECT_EXIT:
+ * Compute exit lanes -> update done_mask -> remove from EXEC (no branch)
+ *
+ * Emits the required EXEC mask manipulation in-place.
+ */
+static void knod_bpf_emit_direct_exit_retval(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *emit_meta,
+ struct knod_insn_meta *target)
+{
+ struct amdgcn_param64 dst, src;
+ s64 imm;
+
+ if (!target || knod_meta_is_exit(target))
+ return;
+
+ if (WARN_ON_ONCE(!knod_bpf_is_retval_move_to_r0(target)))
+ return;
+
+ knod_vset64(&dst, KNOD_AMDGPU_VREG0_LO);
+
+ switch (target->insn.code) {
+ case BPF_ALU | BPF_MOV | BPF_X:
+ case BPF_ALU64 | BPF_MOV | BPF_X:
+ knod_vset64(&src, target->insn.src_reg * 2);
+ knod_mov64(priv, emit_meta, dst, src);
+ break;
+ case BPF_ALU | BPF_MOV | BPF_K:
+ imm = (u32)target->insn.imm;
+ knod_iset64(&src, imm);
+ knod_mov64(priv, emit_meta, dst, src);
+ break;
+ case BPF_ALU64 | BPF_MOV | BPF_K:
+ imm = (s64)(s32)target->insn.imm;
+ knod_iset64(&src, imm);
+ knod_mov64(priv, emit_meta, dst, src);
+ break;
+ default:
+ WARN_ON_ONCE(1);
+ break;
+ }
+}
+
+static void knod_bpf_emit_branch_tail(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ struct knod_prog *knod_prog,
+ short off)
+{
+ switch (meta->branch_type) {
+ case KNOD_BR_FORWARD_SKIP:
+ if (meta->jump_neg_op) {
+ /* JNE: VCC=0 -> jump, VCC=1 -> fall-through.
+ * Save jump lanes (VCC=0): s[n] = exec & ~vcc
+ */
+ knod_emit(priv, meta, s_andn2_b64,
+ meta->exec_save_sreg,
+ AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_VCC_LO);
+
+ /* Keep fall-through (VCC=1): exec = exec & vcc */
+ knod_emit(priv, meta, s_and_b64,
+ AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_VCC_LO);
+ } else {
+ /* Normal: VCC=1 -> jump, VCC=0 -> fall-through.
+ * Save jump lanes (VCC=1): s[n] = exec & vcc
+ */
+ knod_emit(priv, meta, s_and_b64,
+ meta->exec_save_sreg,
+ AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_VCC_LO);
+
+ /* Keep fall-through (VCC=0): exec = exec & ~vcc */
+ knod_emit(priv, meta, s_andn2_b64,
+ AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_VCC_LO);
+ }
+
+ /*
+ * No GPU branch. After the RPO reorder, branch scopes
+ * interleave, so the jumping lanes must flow through every
+ * following block under the EXEC mask and rejoin at their merge
+ * point. An s_cbranch_execz skipping ahead to the merge would
+ * jump over other scopes' merge points and strand their saved
+ * lanes (EXEC never restored -> act=0).
+ */
+ break;
+
+ case KNOD_BR_DIRECT_EXIT:
+ if (meta->jump_neg_op) {
+ /* JNE: VCC=0 -> exit. exit_lanes = exec & ~vcc */
+ knod_emit(priv, meta, s_andn2_b64,
+ KNOD_AMDGPU_TMP_SREG0_LO,
+ AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_VCC_LO);
+ } else {
+ /* Normal: VCC=1 -> exit. exit_lanes = exec & vcc */
+ knod_emit(priv, meta, s_and_b64,
+ KNOD_AMDGPU_TMP_SREG0_LO,
+ AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_VCC_LO);
+ }
+
+ /* Keep the lanes that did not take the exit path. */
+ knod_emit(priv, meta, s_andn2_b64,
+ KNOD_AMDGPU_TMP_SREG1_LO,
+ AMDGCN_SREG_EXEC_LO,
+ KNOD_AMDGPU_TMP_SREG0_LO);
+
+ /* Replay a shared "r0 = action; exit" target under the
+ * exiting lanes before marking them done. Otherwise a direct
+ * branch to the common exit can publish stale r0 scratch state.
+ */
+ knod_emit(priv, meta, s_mov_b64, AMDGCN_SREG_EXEC_LO,
+ KNOD_AMDGPU_TMP_SREG0_LO);
+ knod_bpf_emit_direct_exit_retval(priv, meta, meta->merge_point);
+
+ /* done_mask |= exit_lanes */
+ knod_emit(priv, meta, s_or_b64,
+ knod_prog->done_mask_sreg,
+ knod_prog->done_mask_sreg,
+ AMDGCN_SREG_EXEC_LO);
+
+ /* Continue with the non-exit lanes. */
+ knod_emit(priv, meta, s_mov_b64, AMDGCN_SREG_EXEC_LO,
+ KNOD_AMDGPU_TMP_SREG1_LO);
+
+ /* No branch - fall through with reduced EXEC.
+ * No fixup needed.
+ */
+ meta->jmp_dst = NULL;
+ break;
+
+ default:
+ WARN_ON_ONCE(1);
+ break;
+ }
+}
+
+/*
+ * --- Basic-block CFG analysis (foundation for block reordering) ---
+ *
+ * The emitter is a linear SIMT machine: instructions run in list order under
+ * an EXEC mask. A *forward* jump is realized by masking off the jumping
+ * lanes and restoring them at the merge point. A *backward* jump has no such
+ * realization unless it is a loop (real GPU branch + EXEC convergence, not yet
+ * implemented).
+ *
+ * LLVM tail-sharing and block placement routinely emit jumps that are
+ * backward in BPF byte order but are NOT loops - e.g. a UDP bounds check that
+ * jumps back to a shared XDP_PASS tail. Classifying those as "exit" (the
+ * jmp_off < 0 heuristic in knod_bpf_analyze_cfg) silently miscompiles them:
+ * the jumping lanes exit carrying whatever R0 happened to hold instead of
+ * flowing to the real target.
+ *
+ * The fix is to classify by control-flow, not byte order:
+ * 1. partition the instruction stream into basic blocks,
+ * 2. build the control-flow graph (successor edges),
+ * 3. DFS for a reverse-postorder (RPO) and detect back-edges,
+ * 4. no back-edges (a DAG) -> reorder blocks into RPO so every edge points
+ * forward, then classify by linear position,
+ * 5. a real loop is present -> bail (-EOPNOTSUPP) until loop emission lands.
+ *
+ * Loop emission (step 5) is not implemented yet, so programs containing a
+ * loop are rejected with -EOPNOTSUPP.
+ */
+struct knod_bb {
+ struct knod_insn_meta *leader; /* first instruction of the block */
+ struct knod_insn_meta *last; /* last instruction of the block */
+ /* successors: [0] not-taken, [1] taken */
+ struct knod_bb *succ[2];
+ int n_succ;
+ /* reverse-postorder rank, -1 if unreachable */
+ int rpo;
+ /* DFS color: 0 white, 1 gray, 2 black */
+ int dfs;
+ bool loop_header; /* target of a back-edge */
+ /* scratch: member of the loop being walked */
+ bool in_loop;
+ /* immediate dominator (self for entry) */
+ struct knod_bb *idom;
+};
+
+static bool knod_meta_is_exit(const struct knod_insn_meta *meta)
+{
+ u8 code = meta->insn.code;
+
+ return code == (BPF_JMP | BPF_EXIT) || code == (BPF_JMP32 | BPF_EXIT);
+}
+
+static struct knod_insn_meta *
+knod_bpf_next_meta(struct knod_prog *knod_prog, struct knod_insn_meta *meta)
+{
+ if (!meta || list_is_last(&meta->l, &knod_prog->insns))
+ return NULL;
+
+ return list_next_entry(meta, l);
+}
+
+static bool knod_bpf_is_retval_move_to_r0(const struct knod_insn_meta *meta)
+{
+ u8 code;
+
+ if (!meta || meta->insn.dst_reg != BPF_REG_0)
+ return false;
+
+ code = meta->insn.code;
+ return code == (BPF_ALU | BPF_MOV | BPF_X) ||
+ code == (BPF_ALU64 | BPF_MOV | BPF_X) ||
+ code == (BPF_ALU | BPF_MOV | BPF_K) ||
+ code == (BPF_ALU64 | BPF_MOV | BPF_K);
+}
+
+static bool knod_bpf_is_direct_exit_target(struct knod_prog *knod_prog,
+ struct knod_insn_meta *target)
+{
+ if (knod_meta_is_exit(target))
+ return true;
+
+ if (!knod_bpf_is_retval_move_to_r0(target))
+ return false;
+
+ return knod_meta_is_exit(knod_bpf_next_meta(knod_prog, target));
+}
+
+static bool knod_meta_is_ja(const struct knod_insn_meta *meta)
+{
+ u8 code = meta->insn.code;
+
+ return code == (BPF_JMP | BPF_JA | BPF_K) ||
+ code == (BPF_JMP32 | BPF_JA | BPF_K);
+}
+
+/* A block ends after a terminator; the next instruction starts a new block. */
+static bool knod_meta_is_terminator(const struct knod_insn_meta *meta)
+{
+ return is_mbpf_cond_jump(meta) || knod_meta_is_ja(meta) ||
+ knod_meta_is_exit(meta);
+}
+
+/* Target instruction index of a conditional jump or BPF_JA. */
+static short knod_meta_jump_target_idx(const struct knod_insn_meta *meta)
+{
+ if (meta->insn.code == (BPF_JMP32 | BPF_JA | BPF_K))
+ return meta->bpf_insn_idx + meta->insn.imm + 1;
+ return meta->bpf_insn_idx + meta->insn.off + 1;
+}
+
+static struct knod_bb *knod_bb_of_leader(struct knod_bb *bbs, int n_bbs,
+ const struct knod_insn_meta *meta)
+{
+ int i;
+
+ for (i = 0; i < n_bbs; i++)
+ if (bbs[i].leader == meta)
+ return &bbs[i];
+ return NULL;
+}
+
+/* Resolve the block a conditional jump / BPF_JA at @jmp transfers to. */
+static struct knod_bb *knod_bb_jump_target(struct knod_prog *knod_prog,
+ struct knod_bb *bbs, int n_bbs,
+ const struct knod_insn_meta *jmp)
+{
+ struct knod_insn_meta *tgt;
+
+ tgt = knod_bpf_lookup_meta(knod_prog, knod_meta_jump_target_idx(jmp));
+ return tgt ? knod_bb_of_leader(bbs, n_bbs, tgt) : NULL;
+}
+
+/*
+ * Partition knod_prog->insns into basic blocks. A leader is the first
+ * instruction, any jump target, or the instruction after a terminator.
+ * Returns the block count or a negative errno; @bbs holds >= n_insns blocks.
+ */
+static int knod_bpf_build_bbs(struct knod_prog *knod_prog, struct knod_bb *bbs)
+{
+ struct knod_insn_meta *meta, *tgt;
+ struct knod_bb *cur = NULL;
+ int n_bbs = 0;
+ short tgt_idx;
+
+ /* Pass A: mark every jump target as a leader. */
+ list_for_each_entry(meta, &knod_prog->insns, l)
+ meta->flags &= ~FLAG_INSN_IS_JUMP_DST;
+
+ list_for_each_entry(meta, &knod_prog->insns, l) {
+ if (!is_mbpf_cond_jump(meta) && !knod_meta_is_ja(meta))
+ continue;
+ tgt_idx = knod_meta_jump_target_idx(meta);
+ tgt = knod_bpf_lookup_meta(knod_prog, tgt_idx);
+ if (!tgt) {
+ pr_warn("knod_cfg: bpf#%d jump target %d unresolved\n",
+ meta->bpf_insn_idx, tgt_idx);
+ return -EINVAL;
+ }
+ tgt->flags |= FLAG_INSN_IS_JUMP_DST;
+ }
+
+ /* Pass B: cut the list into blocks. */
+ list_for_each_entry(meta, &knod_prog->insns, l) {
+ if (!cur || (meta->flags & FLAG_INSN_IS_JUMP_DST)) {
+ cur = &bbs[n_bbs++];
+ cur->leader = meta;
+ cur->n_succ = 0;
+ }
+ cur->last = meta;
+
+ if (knod_meta_is_terminator(meta))
+ /* next instruction starts a new block */
+ cur = NULL;
+ }
+
+ return n_bbs;
+}
+
+/* Build successor edges for every block from its terminator. */
+static int knod_bpf_build_edges(struct knod_prog *knod_prog,
+ struct knod_bb *bbs, int n_bbs)
+{
+ struct knod_bb *bb, *fall, *tgt_bb;
+ struct knod_insn_meta *last;
+ int i;
+
+ for (i = 0; i < n_bbs; i++) {
+ bb = &bbs[i];
+ last = bb->last;
+ bb->n_succ = 0;
+
+ if (knod_meta_is_exit(last))
+ continue; /* no successors */
+
+ /* Successor in list order: block led by the next
+ * instruction.
+ */
+ fall = NULL;
+ if (!list_is_last(&last->l, &knod_prog->insns))
+ fall = knod_bb_of_leader(bbs, n_bbs,
+ list_next_entry(last, l));
+
+ if (is_mbpf_cond_jump(last)) {
+ tgt_bb = knod_bb_jump_target(knod_prog, bbs, n_bbs,
+ last);
+ if (!fall || !tgt_bb)
+ return -EINVAL;
+ bb->succ[bb->n_succ++] = fall; /* not taken */
+ bb->succ[bb->n_succ++] = tgt_bb; /* taken */
+ } else if (knod_meta_is_ja(last)) {
+ tgt_bb = knod_bb_jump_target(knod_prog, bbs, n_bbs,
+ last);
+ if (!tgt_bb)
+ return -EINVAL;
+ bb->succ[bb->n_succ++] = tgt_bb;
+ } else {
+ if (!fall) /* fell off the end */
+ return -EINVAL;
+ bb->succ[bb->n_succ++] = fall;
+ }
+ }
+
+ return 0;
+}
+
+/*
+ * Iterative DFS from the entry block. Computes a reverse-postorder rank for
+ * every reachable block and flags back-edge targets as loop headers. Returns
+ * the number of back-edges in *n_back, or a negative errno.
+ */
+static int knod_bpf_compute_rpo(struct knod_bb *bbs, int n_bbs,
+ struct knod_bb *entry, int *n_back)
+{
+ struct knod_bb **stack;
+ int *cursor;
+ int top = 0, post = 0, nb = 0, i;
+
+ for (i = 0; i < n_bbs; i++) {
+ bbs[i].dfs = 0;
+ bbs[i].rpo = -1;
+ bbs[i].loop_header = false;
+ }
+
+ stack = kcalloc(n_bbs, sizeof(*stack), GFP_KERNEL);
+ cursor = kcalloc(n_bbs, sizeof(*cursor), GFP_KERNEL);
+ if (!stack || !cursor) {
+ kfree(stack);
+ kfree(cursor);
+ return -ENOMEM;
+ }
+
+ entry->dfs = 1;
+ stack[top] = entry;
+ cursor[top] = 0;
+ top++;
+
+ while (top > 0) {
+ struct knod_bb *bb = stack[top - 1];
+
+ if (cursor[top - 1] < bb->n_succ) {
+ struct knod_bb *s = bb->succ[cursor[top - 1]++];
+
+ if (s->dfs == 0) { /* tree edge */
+ s->dfs = 1;
+ stack[top] = s;
+ cursor[top] = 0;
+ top++;
+ } else if (s->dfs == 1) { /* gray -> back-edge */
+ s->loop_header = true;
+ nb++;
+ }
+ /* s->dfs == 2 -> forward/cross edge, nothing to do */
+ } else {
+ /* finished: postorder */
+ bb->dfs = 2;
+ bb->rpo = post++;
+ top--;
+ }
+ }
+
+ /* postorder -> reverse-postorder rank */
+ for (i = 0; i < n_bbs; i++)
+ if (bbs[i].rpo >= 0)
+ bbs[i].rpo = post - 1 - bbs[i].rpo;
+
+ kfree(stack);
+ kfree(cursor);
+ *n_back = nb;
+ return 0;
+}
+
+/*
+ * Cooper-Harvey-Kennedy dominator intersect: walk the two fingers up the idom
+ * chain (toward the entry, which has the lowest RPO) until they meet.
+ */
+static struct knod_bb *knod_dom_intersect(struct knod_bb *a, struct knod_bb *b)
+{
+ while (a != b) {
+ while (a->rpo > b->rpo)
+ a = a->idom;
+ while (b->rpo > a->rpo)
+ b = b->idom;
+ }
+ return a;
+}
+
+/*
+ * Compute the immediate dominator of every reachable block (Cooper, Harvey,
+ * Kennedy, "A Simple, Fast Dominance Algorithm"). Iterates over RPO to a
+ * fixpoint; bb->idom is the block's immediate dominator, the entry dominating
+ * itself. Requires bb->rpo from knod_bpf_compute_rpo.
+ */
+static int knod_bpf_compute_dom(struct knod_bb *bbs, int n_bbs,
+ struct knod_bb *entry)
+{
+ struct knod_bb **order;
+ int i, k, n_order = 0;
+ bool changed;
+
+ order = kcalloc(n_bbs, sizeof(*order), GFP_KERNEL);
+ if (!order)
+ return -ENOMEM;
+
+ for (i = 0; i < n_bbs; i++) {
+ bbs[i].idom = NULL;
+ if (bbs[i].rpo >= 0) {
+ order[bbs[i].rpo] = &bbs[i];
+ n_order++;
+ }
+ }
+ entry->idom = entry;
+
+ do {
+ changed = false;
+
+ /* process every reachable block but the entry, in RPO order */
+ for (k = 1; k < n_order; k++) {
+ struct knod_bb *n = order[k];
+ struct knod_bb *new_idom = NULL;
+ int b, s;
+
+ /* intersect over already-processed predecessors */
+ for (b = 0; b < n_bbs; b++) {
+ for (s = 0; s < bbs[b].n_succ; s++) {
+ if (bbs[b].succ[s] != n || !bbs[b].idom)
+ continue;
+ new_idom = new_idom ?
+ knod_dom_intersect(&bbs[b],
+ new_idom) :
+ &bbs[b];
+ }
+ }
+
+ if (new_idom && n->idom != new_idom) {
+ n->idom = new_idom;
+ changed = true;
+ }
+ }
+ } while (changed);
+
+ kfree(order);
+ return 0;
+}
+
+/* Does block @a dominate block @b? Walk @b up the idom chain to the entry. */
+static bool knod_dom_dominates(struct knod_bb *a, struct knod_bb *b)
+{
+ for (;;) {
+ if (b == a)
+ return true;
+ if (b->idom == b) /* reached the entry */
+ return false;
+ b = b->idom;
+ }
+}
+
+/*
+ * Mark the natural loop body of back-edge @latch->@hdr in bb->in_loop: the
+ * header plus every block that reaches the latch without passing through the
+ * header, found by walking predecessors back from the latch. @stack is
+ * caller-provided scratch of at least @n_bbs entries.
+ */
+static void knod_loop_mark_body(struct knod_bb *bbs, int n_bbs,
+ struct knod_bb *latch, struct knod_bb *hdr,
+ struct knod_bb **stack)
+{
+ int b, sp, k, top = 0;
+
+ for (k = 0; k < n_bbs; k++)
+ bbs[k].in_loop = false;
+
+ hdr->in_loop = true;
+ if (latch != hdr) {
+ latch->in_loop = true;
+ stack[top++] = latch;
+ }
+
+ while (top > 0) {
+ struct knod_bb *d = stack[--top];
+
+ for (b = 0; b < n_bbs; b++) {
+ if (bbs[b].in_loop)
+ continue;
+ for (sp = 0; sp < bbs[b].n_succ; sp++) {
+ if (bbs[b].succ[sp] != d)
+ continue;
+ bbs[b].in_loop = true;
+ stack[top++] = &bbs[b];
+ break;
+ }
+ }
+ }
+}
+
+/*
+ * Detect natural loops from the dominator tree and report their structure.
+ *
+ * A back-edge is an edge u->v whose target v dominates its source u - v is
+ * the loop header, u the latch. Its natural loop body is the header plus the
+ * blocks that reach the latch without passing through the header; an exit edge
+ * leaves a body block for a non-body block.
+ *
+ * Loops are still rejected by the reorder (-EOPNOTSUPP); this only reports what
+ * was found (to dmesg, since a rejected program never attaches so /bpf/cfg is
+ * unavailable) so the detection can be verified before emission is built.
+ */
+static int knod_bpf_detect_loops(struct knod_bb *bbs, int n_bbs)
+{
+ struct knod_bb **stack;
+ int u, s, k, n_be = 0;
+
+ stack = kcalloc(n_bbs, sizeof(*stack), GFP_KERNEL);
+ if (!stack)
+ return -ENOMEM;
+
+ for (u = 0; u < n_bbs; u++) {
+ for (s = 0; s < bbs[u].n_succ; s++) {
+ struct knod_bb *hdr = bbs[u].succ[s];
+ int body = 0, exits = 0, sp;
+
+ if (!knod_dom_dominates(hdr, &bbs[u]))
+ continue; /* not a back-edge */
+ n_be++;
+
+ knod_loop_mark_body(bbs, n_bbs, &bbs[u], hdr, stack);
+
+ for (k = 0; k < n_bbs; k++) {
+ if (!bbs[k].in_loop)
+ continue;
+ body++;
+ for (sp = 0; sp < bbs[k].n_succ; sp++)
+ if (!bbs[k].succ[sp]->in_loop)
+ exits++;
+ }
+
+ pr_info("knod_loop: back-edge bpf#%d -> bpf#%d (latch->header) body=%d exits=%d\n",
+ bbs[u].leader->bpf_insn_idx,
+ hdr->leader->bpf_insn_idx, body, exits);
+ }
+ }
+
+ kfree(stack);
+
+ if (n_be)
+ pr_info("knod_loop: %d back-edge(s) - %s\n", n_be,
+ n_be == 1 ? "single loop (simple-shape candidate)" :
+ "nested/multiple loops (complex)");
+ return 0;
+}
+
+/*
+ * Block that lanes fall into in list order when the terminator is not taken:
+ * the not-taken successor of a conditional jump, or the sole successor of a
+ * block that ended only because the next instruction was a leader. BPF_JA and
+ * EXIT have no such successor (control leaves explicitly).
+ */
+static struct knod_bb *knod_bb_fall_succ(struct knod_bb *bb)
+{
+ if (knod_meta_is_exit(bb->last) || knod_meta_is_ja(bb->last))
+ return NULL;
+ return bb->n_succ ? bb->succ[0] : NULL;
+}
+
+/*
+ * Reorder the instruction list into reverse-postorder so every control-flow
+ * edge points forward, and splice in a synthetic BPF_JA wherever a block's
+ * not-taken successor no longer follows it in list order. After this the
+ * emitter's forward-only machinery (FORWARD_SKIP / FORWARD_GOTO) handles the
+ * whole program - including the backward-in-byte-order, non-loop jumps that
+ * the old jmp_off < 0 heuristic miscompiled.
+ *
+ * Loops (back-edges) are rejected with -EOPNOTSUPP until loop emission lands.
+ */
+static int knod_bpf_reorder_rpo(struct knod_prog *knod_prog,
+ struct knod_bb *bbs, int n_bbs, int n_back)
+{
+ struct knod_insn_meta *m, *nx, *sj;
+ struct knod_bb **order;
+ int n_order = 0, r, i, k, idx = 0;
+ LIST_HEAD(new_list);
+
+ if (n_back) {
+ pr_warn("knod_cfg: %d loop back-edge(s) - block reorder cannot lower loops yet (-EOPNOTSUPP)\n",
+ n_back);
+ return -EOPNOTSUPP;
+ }
+
+ order = kcalloc(n_bbs, sizeof(*order), GFP_KERNEL);
+ if (!order)
+ return -ENOMEM;
+
+ /* Reachable blocks in RPO, then any unreachable ones so no instruction
+ * is dropped from the list.
+ */
+ for (r = 0; r < n_bbs; r++)
+ for (i = 0; i < n_bbs; i++)
+ if (bbs[i].rpo == r) {
+ order[n_order++] = &bbs[i];
+ break;
+ }
+ for (i = 0; i < n_bbs; i++)
+ if (bbs[i].rpo < 0)
+ order[n_order++] = &bbs[i];
+
+ for (k = 0; k < n_order; k++) {
+ struct knod_bb *bb = order[k];
+ struct knod_bb *next = (k + 1 < n_order) ? order[k + 1] : NULL;
+ struct knod_bb *fall;
+
+ m = bb->leader;
+ while (true) {
+ nx = (m == bb->last) ? NULL : knod_meta_next(m);
+ list_move_tail(&m->l, &new_list);
+ if (m == bb->last)
+ break;
+ m = nx;
+ }
+
+ fall = knod_bb_fall_succ(bb);
+ if (!fall || (next && next->leader == fall->leader))
+ continue;
+
+ /* Not-taken successor no longer adjacent: route it
+ * explicitly.
+ */
+ sj = kzalloc_obj(*sj, GFP_KERNEL);
+ if (!sj) {
+ list_splice(&new_list, &knod_prog->insns);
+ kfree(order);
+ return -ENOMEM;
+ }
+ sj->insn.code = BPF_JMP | BPF_JA | BPF_K;
+ /* synthetic, never a jump target */
+ sj->bpf_insn_idx = -1;
+ /* consumed by classify_linear */
+ sj->jmp_dst = fall->leader;
+ INIT_LIST_HEAD(&sj->l);
+ list_add_tail(&sj->l, &new_list);
+ }
+
+ list_splice(&new_list, &knod_prog->insns);
+
+ list_for_each_entry(m, &knod_prog->insns, l)
+ m->linear_idx = idx++;
+
+ kfree(order);
+ return 0;
+}
+
+/*
+ * Classify branches by linear position after the RPO reorder. Every edge is
+ * now forward, so a conditional jump is FORWARD_SKIP (or DIRECT_EXIT when it
+ * targets the exit), and every BPF_JA - real or synthetic - is FORWARD_GOTO
+ * (or DIRECT_EXIT).
+ */
+static int knod_bpf_classify_linear(struct knod_prog *knod_prog)
+{
+ struct knod_insn_meta *meta, *target;
+ short ti;
+
+ list_for_each_entry(meta, &knod_prog->insns, l) {
+ if (is_mbpf_cond_jump(meta)) {
+ meta->jump_neg_op = (mbpf_op(meta) == BPF_JNE);
+ ti = knod_meta_jump_target_idx(meta);
+ target = knod_bpf_lookup_meta(knod_prog, ti);
+ } else if (knod_meta_is_ja(meta)) {
+ /* synthetic JA carries its destination in jmp_dst;
+ * a real BPF_JA is resolved from its offset.
+ */
+ if (meta->jmp_dst) {
+ target = meta->jmp_dst;
+ } else {
+ ti = knod_meta_jump_target_idx(meta);
+ target = knod_bpf_lookup_meta(knod_prog, ti);
+ }
+ } else {
+ continue;
+ }
+
+ if (!target) {
+ pr_err("knod_cfg: bpf#%d unresolved branch target\n",
+ meta->bpf_insn_idx);
+ return -EINVAL;
+ }
+
+ if (target->linear_idx <= meta->linear_idx)
+ pr_warn("knod_cfg: bpf#%d -> #%d still backward after reorder (linear %d -> %d)\n",
+ meta->bpf_insn_idx, target->bpf_insn_idx,
+ meta->linear_idx, target->linear_idx);
+
+ if (knod_bpf_is_direct_exit_target(knod_prog, target)) {
+ meta->branch_type = KNOD_BR_DIRECT_EXIT;
+ meta->merge_point = target;
+ continue;
+ }
+
+ meta->branch_type = is_mbpf_cond_jump(meta) ?
+ KNOD_BR_FORWARD_SKIP : KNOD_BR_FORWARD_GOTO;
+ meta->merge_point = target;
+ target->is_merge_point = true;
+ }
+
+ return 0;
+}
+
+/*
+ * Build the basic-block CFG, compute RPO, reorder the instruction list into
+ * RPO and insert synthetic jumps. Returns 0, or a negative errno (a loop
+ * yields -EOPNOTSUPP).
+ */
+static int knod_bpf_build_cfg(struct knod_prog *knod_prog)
+{
+ struct knod_insn_meta *meta;
+ int n_insns = 0, n_bbs, n_back = 0, ret;
+ struct knod_bb *bbs;
+
+ list_for_each_entry(meta, &knod_prog->insns, l)
+ n_insns++;
+ if (!n_insns)
+ return 0;
+
+ bbs = kcalloc(n_insns, sizeof(*bbs), GFP_KERNEL);
+ if (!bbs)
+ return -ENOMEM;
+
+ n_bbs = knod_bpf_build_bbs(knod_prog, bbs);
+ if (n_bbs < 0) {
+ ret = n_bbs;
+ goto out_free;
+ }
+
+ ret = knod_bpf_build_edges(knod_prog, bbs, n_bbs);
+ if (ret)
+ goto out_free;
+
+ ret = knod_bpf_compute_rpo(bbs, n_bbs, &bbs[0], &n_back);
+ if (ret)
+ goto out_free;
+
+ ret = knod_bpf_compute_dom(bbs, n_bbs, &bbs[0]);
+ if (ret)
+ goto out_free;
+
+ if (n_back) {
+ ret = knod_bpf_detect_loops(bbs, n_bbs);
+ if (ret)
+ goto out_free;
+ }
+
+ /* Hand the block array to the prog for the /bpf/cfg view (freed at
+ * teardown); kept even if the reorder below rejects a loop, so the
+ * rejection can be inspected.
+ */
+ kfree(knod_prog->bbs);
+ knod_prog->bbs = bbs;
+ knod_prog->n_bbs = n_bbs;
+ knod_prog->n_back = n_back;
+
+ return knod_bpf_reorder_rpo(knod_prog, bbs, n_bbs, n_back);
+
+out_free:
+ kfree(bbs);
+ return ret;
+}
+
+/*
+ * Assign exec_save SGPR pairs to the forward branches, recycling a pair once
+ * its merge point has been passed. The peak concurrent live count is the
+ * actual SGPR requirement - usually far less than the total branch count.
+ */
+static int knod_bpf_alloc_exec_sregs(struct knod_bpf_priv *priv,
+ struct knod_prog *knod_prog)
+{
+ struct {
+ u8 sreg;
+ struct knod_insn_meta *merge;
+ } live[72];
+ int exec_save_max, max_pairs, n_live, peak, j;
+ struct knod_insn_meta *meta;
+ u8 free_stack[72];
+ int free_top;
+
+ exec_save_max = (priv->isa_version == 10) ?
+ KNOD_AMDGPU_EXEC_SAVE_SREG_MAX_GFX10 :
+ KNOD_AMDGPU_EXEC_SAVE_SREG_MAX_GFX9;
+ max_pairs = (exec_save_max - knod_prog->exec_save_base + 1) / 2;
+
+ for (free_top = 0; free_top < max_pairs; free_top++)
+ free_stack[free_top] = knod_prog->exec_save_base +
+ (max_pairs - 1 - free_top) * 2;
+
+ n_live = 0;
+ peak = 0;
+
+ list_for_each_entry(meta, &knod_prog->insns, l) {
+ /* Reclaim pairs from scopes that merge at this insn */
+ for (j = n_live - 1; j >= 0; j--) {
+ if (live[j].merge == meta) {
+ free_stack[free_top++] = live[j].sreg;
+ live[j] = live[--n_live];
+ }
+ }
+
+ if (meta->branch_type != KNOD_BR_FORWARD_SKIP &&
+ meta->branch_type != KNOD_BR_FORWARD_GOTO)
+ continue;
+
+ if (free_top == 0) {
+ pr_err("knod_cfg: exec_save exhausted, peak %d concurrent scopes (max %d)\n",
+ peak, max_pairs);
+ return -ENOSPC;
+ }
+
+ meta->exec_save_sreg = free_stack[--free_top];
+ live[n_live].sreg = meta->exec_save_sreg;
+ live[n_live].merge = meta->merge_point;
+ n_live++;
+
+ if (n_live > peak)
+ peak = n_live;
+ }
+
+ knod_prog->exec_save_pairs_used = peak;
+ pr_debug("knod_cfg: done, peak %d concurrent scopes (total fwd jumps: %d+%d)\n",
+ peak, peak, n_live);
+ return 0;
+}
+
+/*
+ * knod_bpf_analyze_cfg - Classify branches and allocate SGPRs for
+ * structurized CFG.
+ *
+ * Runs before instruction emission. For each conditional branch:
+ * - Backward jump or jump to EXIT -> DIRECT_EXIT (no SGPR needed)
+ * - Forward jump to non-EXIT -> FORWARD_SKIP, allocate SGPR pair
+ *
+ * The "save jumping lanes" pattern handles crossing scopes correctly:
+ * branch: s_and_b64 s[n], exec, vcc; s_andn2_b64 exec, exec, vcc
+ * merge: s_or_b64 exec, exec, s[n]
+ *
+ * For JNE (jump_neg_op): VCC=0 -> jump, so lanes are swapped.
+ */
+static int knod_bpf_analyze_cfg(struct knod_bpf_priv *priv,
+ struct knod_prog *knod_prog)
+{
+ int ret;
+
+ /* Build the basic-block CFG, reorder the instruction list into RPO so
+ * every branch is forward (inserting synthetic jumps where a not-taken
+ * successor would no longer be adjacent), then classify each branch by
+ * linear position. A loop in the program is rejected (-EOPNOTSUPP).
+ */
+ ret = knod_bpf_build_cfg(knod_prog);
+ if (ret)
+ return ret;
+ ret = knod_bpf_classify_linear(knod_prog);
+ if (ret)
+ return ret;
+
+ return knod_bpf_alloc_exec_sregs(priv, knod_prog);
+}
+
+/*
+ * Shader stores packet address and length into pass_meta_buf slot header.
+ * Host-side SDMA engine does the actual copy to the delivery page.
+ *
+ * At entry:
+ * TMP_VREG10_LO (v42) = old_val * 2 (from pass_indices addressing)
+ * DATA_VREG (v64:v65) = packet source VRAM address
+ * DATA_END_VREG (v66:v67) = packet end address
+ * PARAM_SREG (s28:s29) = param GTT address
+ *
+ * Stores at slot header:
+ * +0: u32 len (DATA_END_LO - DATA_LO)
+ * +8: u64 src_addr (DATA_VREG)
+ */
+static void knod_emit_pass_addr_store(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta)
+{
+ struct amdgcn_param32 p[3];
+
+ /* s_lshl_b32 s18, s15, 3 - queue_idx * 8 for pass_meta_buf_gaddr
+ * stride
+ */
+ knod_sset32(&p[0], KNOD_AMDGPU_TMP_SREG1_LO);
+ knod_sset32(&p[1], KNOD_AMDGPU_WORKGROUP_ID_Y_SREG);
+ knod_iset32(&p[2], 3);
+ knod_emit(priv, meta, s_lshl_b32, p[0], p[1], p[2]);
+
+ /* s_load_dwordx2 s[16:17], s[28:29], offsetof(pass_meta_buf_gaddr)
+ * soffset=s18
+ */
+ knod_sset32(&p[0], KNOD_AMDGPU_TMP_SREG0_LO);
+ knod_sset32(&p[1], KNOD_AMDGPU_PARAM_SREG_LO);
+ knod_emit(priv, meta, s_load_dwordx2_soff, p[0], p[1],
+ offsetof(struct knod_bpf_param, pass_meta_buf_gaddr),
+ KNOD_AMDGPU_TMP_SREG1_LO);
+
+ /* s_waitcnt lgkmcnt(0) */
+ knod_emit(priv, meta, s_waitcnt_lgkmcnt);
+
+ /* Compute slot offset: old_val << 12 = (old_val*2) << 11
+ * v_lshlrev_b32 v44, 11, v42
+ */
+ knod_vset32(&p[0], KNOD_AMDGPU_TMP_VREG11_LO);
+ knod_iset32(&p[1], 11);
+ knod_vset32(&p[2], KNOD_AMDGPU_TMP_VREG10_LO);
+ knod_emit(priv, meta, v_lshlrev_b32, p[0], p[1], p[2]);
+
+ /* v_mov_b32 v45, 0 */
+ knod_vset32(&p[0], KNOD_AMDGPU_TMP_VREG11_HI);
+ knod_iset32(&p[1], 0);
+ knod_emit(priv, meta, v_mov_b32_e32, p[0], p[1]);
+
+ /* v_add_co_u32 v44, s16, v44 */
+ knod_vset32(&p[0], KNOD_AMDGPU_TMP_VREG11_LO);
+ knod_sset32(&p[1], KNOD_AMDGPU_TMP_SREG0_LO);
+ knod_vset32(&p[2], KNOD_AMDGPU_TMP_VREG11_LO);
+ knod_emit(priv, meta, v_add_co_u32, p[0], p[1], p[2]);
+
+ /*
+ * slot_hi = base_hi (NOT base_hi + carry). The slot offset is at
+ * most (pass_pkts_per_queue-1)*KNOD_PASS_SLOT_SIZE and the whole
+ * pass_meta_buf is a single contiguous allocation that never straddles
+ * a 4GiB boundary, so base_lo + offset never wraps and the carry is
+ * always 0. Avoid the v_add_co/v_addc carry chain entirely: on GFX9
+ * the v_addc here was picking up a stale VCC (from the preceding
+ * XDP_PASS v_cmp) instead of the v_add_co carry-out, setting slot_hi=1
+ * and faulting at 0x1_xxxx.
+ * v_mov_b32 v45, s17
+ */
+ knod_vset32(&p[0], KNOD_AMDGPU_TMP_VREG11_HI);
+ knod_sset32(&p[1], KNOD_AMDGPU_TMP_SREG0_HI);
+ knod_emit(priv, meta, v_mov_b32_e32, p[0], p[1]);
+
+ /* v44:v45 = slot_addr in pass_meta_buf */
+
+ /* Store len: v_sub_u32 v0, DATA_END_LO, DATA_LO */
+ knod_vset32(&p[0], KNOD_AMDGPU_VREG0_LO);
+ knod_vset32(&p[1], KNOD_AMDGPU_DATA_END_VREG_LO);
+ knod_vset32(&p[2], KNOD_AMDGPU_DATA_VREG_LO);
+ knod_emit(priv, meta, v_sub_u32, p[0], p[1], p[2]);
+
+ /* global_store_dword [slot+0], len */
+ knod_vset32(&p[0], KNOD_AMDGPU_VREG0_LO);
+ knod_vset32(&p[1], KNOD_AMDGPU_TMP_VREG11_LO);
+ knod_emit(priv, meta, global_store_dword, p[0], p[1],
+ offsetof(struct knod_pass_slot_hdr, len));
+
+ /* global_store_dwordx2 [slot+8], DATA_VREG (src_addr) */
+ knod_vset32(&p[0], KNOD_AMDGPU_DATA_VREG_LO);
+ knod_vset32(&p[1], KNOD_AMDGPU_TMP_VREG11_LO);
+ knod_emit(priv, meta, global_store_dwordx2, p[0], p[1],
+ offsetof(struct knod_pass_slot_hdr, src_addr));
+}
+
+static int knod_bpf_jit(struct knod_dev *knodev,
+ struct knod_prog *knod_prog)
+{
+ struct knod_bpf_priv *priv =
+ (struct knod_bpf_priv *)knodev->accel->xdp.priv;
+ short off, packet_off, stack_off;
+ struct knod_insn_meta *meta, *meta2;
+ struct amdgcn_param64 param64[2];
+ u32 insn_idx = 0, i;
+ struct amdgcn_param32 param[3];
+ struct amdgcn_param32 p32[2];
+ struct amdgcn_param32 p[3];
+ int s, d, imm, imm2;
+ int pass_branch_idx;
+ bool is_dw, fetch;
+ bool skip = false;
+ int pass_dwords;
+ int atomic_op;
+ int j;
+ int map_id;
+ u64 imm64;
+ int ret;
+
+ /* Analyze CFG before instruction emission */
+ ret = knod_bpf_analyze_cfg(priv, knod_prog);
+
+ if (ret)
+ return ret;
+
+ knod_bpf_layout_sregs(priv, knod_prog);
+ ret = knod_prog_prepare_insns(priv, knod_prog);
+ if (ret)
+ return ret;
+
+ knod_prog->max_stack_off = -knod_prog->max_stack_off;
+ knod_prog->max_stack_off = ALIGN(knod_prog->max_stack_off, 4);
+ knod_prog->max_packet_off = ALIGN(knod_prog->max_packet_off, 4);
+ /* NOTE:
+ * packet is accessed with packet_off + size
+ * largest size of it is unsigned long
+ */
+ knod_prog->max_packet_off += sizeof(unsigned long);
+ if (knod_prog->max_packet_off > MAX_PACKET_CACHE) {
+ WARN_ON_ONCE(1);
+ knod_bpf_pkt_cache = 0;
+ }
+
+ /* Initialize all exec_save SGPRs to 0.
+ * Without this, merge points that restore from exec_save SGPRs
+ * of branches that were skipped (by an outer s_cbranch_execz)
+ * would OR garbage into EXEC, enabling invalid lanes.
+ * In the old code, BPF_EXIT used s_endpgm so execution never
+ * reached those merge points; now it does.
+ */
+ if (knod_prog->exec_save_pairs_used > 0) {
+ u8 sreg;
+
+ meta = knod_prog_pre_last_meta(knod_prog);
+
+ for (sreg = knod_prog->exec_save_base;
+ sreg < knod_prog->exec_save_base +
+ knod_prog->exec_save_pairs_used * 2;
+ sreg += 2)
+ knod_emit(priv, meta, s_mov_b64, sreg,
+ AMDGCN_SREG_INTEGER_0);
+ }
+
+ if (knod_bpf_pkt_cache) {
+ meta = knod_prog_pre_last_meta(knod_prog);
+
+ /* ctx->data is in DATA_VREG -> copy to r32[0] via v_mov */
+ knod_vset32(¶m[0], r32[0].v);
+ knod_vset32(¶m[1], KNOD_AMDGPU_DATA_VREG_LO);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0], param[1]);
+ knod_vset32(¶m[0], r32[0].v + 1);
+ knod_vset32(¶m[1], KNOD_AMDGPU_DATA_VREG_HI);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0], param[1]);
+
+ knod_global_load_size_cache(priv, meta,
+ &pkt_cache[0],
+ r32[0],
+ 0, /* dst index */
+ 0, /* start offset */
+ knod_prog->max_packet_off);
+ }
+
+ insn_idx = 0;
+ list_for_each_entry(meta, &knod_prog->pre_insns, l) {
+ for (i = 0; i < meta->amdgpu_insns; i++)
+ insn_idx += (meta->amdgpu_insn[i].size / 4);
+ }
+
+ list_for_each_entry(meta, &knod_prog->insns, l) {
+ if (skip) {
+ skip = false;
+ meta->amdgpu_insn_idx = AMDGPU_INSN_SKIP;
+ continue;
+ }
+ s = meta->insn.src_reg;
+ d = meta->insn.dst_reg;
+ imm = meta->insn.imm;
+ off = meta->insn.off;
+
+ meta->amdgpu_insn_idx = insn_idx;
+ meta->amdgpu_insns = 0;
+
+ /* Structurized CFG: restore EXEC at merge points */
+ if (meta->is_merge_point) {
+ struct knod_insn_meta *br;
+
+ list_for_each_entry(br, &knod_prog->insns, l) {
+ if ((br->branch_type == KNOD_BR_FORWARD_SKIP ||
+ br->branch_type == KNOD_BR_FORWARD_GOTO) &&
+ br->merge_point == meta) {
+ knod_emit(priv, meta, s_or_b64,
+ AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_EXEC_LO,
+ br->exec_save_sreg);
+ }
+ }
+ /* Remove done lanes from restored EXEC */
+ knod_emit(priv, meta, s_andn2_b64, AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_EXEC_LO,
+ knod_prog->done_mask_sreg);
+ }
+
+ switch (meta->insn.code) {
+ /* ALU
+ * If a destination register contains a pointer of STACK,
+ * offset should not be minus.
+ */
+ case BPF_ALU | BPF_MOV | BPF_X:
+ case BPF_ALU64 | BPF_MOV | BPF_X:
+ //r[d] = r[s];
+ knod_mov64(priv, meta, bpf_reg64[d], bpf_reg64[s]);
+ break;
+ case BPF_ALU | BPF_MOV | BPF_K:
+ case BPF_ALU64 | BPF_MOV | BPF_K:
+ //r[d] = imm;
+ knod_iset64(&p64[0], imm);
+ knod_mov64(priv, meta, bpf_reg64[d], p64[0]);
+ break;
+ case BPF_ALU | BPF_XOR | BPF_X:
+ knod_xor32(priv, meta,
+ bpf_reg64[d].lo, bpf_reg64[d].lo,
+ bpf_reg64[s].lo);
+ knod_iset64(&p64[0], 0);
+ knod_mov32(priv, meta, bpf_reg64[d].hi, p64[0].lo);
+ break;
+ case BPF_ALU64 | BPF_XOR | BPF_X:
+ //r[d] ^= r[s];
+ knod_xor32(priv, meta,
+ bpf_reg64[d].lo, bpf_reg64[d].lo,
+ bpf_reg64[s].lo);
+ knod_xor32(priv, meta,
+ bpf_reg64[d].hi, bpf_reg64[d].hi,
+ bpf_reg64[s].hi);
+ break;
+ case BPF_ALU | BPF_XOR | BPF_K:
+ case BPF_ALU64 | BPF_XOR | BPF_K:
+ knod_iset64(&p64[0], imm);
+ knod_mov64(priv, meta, bpf_reg64[d], p64[0]);
+ knod_xor32(priv, meta, bpf_reg64[d].lo,
+ bpf_reg64[d].lo, r64[0].lo);
+ break;
+ //r[d] ^= imm;
+ break;
+ case BPF_ALU | BPF_MOD | BPF_X:
+ case BPF_ALU64 | BPF_MOD | BPF_X:
+ //r[d] %= r[s];
+ knod_iset64(&p64[0], meta->umin_src);
+ knod_mod(priv, meta, bpf_reg64[d], p64[0],
+ r64[0], r64[1], r64[2], r64[3], r64[4]);
+ break;
+ case BPF_ALU | BPF_MOD | BPF_K:
+ case BPF_ALU64 | BPF_MOD | BPF_K:
+ //r[d] %= imm;
+ /* The dividend fits 32 bits (verifier rejects wider
+ * div/mod), so the 32-bit fold is valid even when
+ * clang emitted this as a 64-bit ALU op (e.g. u32
+ * hash % 65537 -> `r2 %= 65537`).
+ */
+ if (meta->umax_dst <= U32_MAX && imm &&
+ knod_mod_k32(priv, meta, bpf_reg64[d], imm))
+ break;
+ knod_iset64(&p64[0], imm);
+ knod_mod(priv, meta, bpf_reg64[d], p64[0],
+ r64[0], r64[1], r64[2], r64[3], r64[4]);
+ break;
+ case BPF_ALU | BPF_AND | BPF_X:
+ knod_and32(priv, meta, bpf_reg64[d].lo,
+ bpf_reg64[d].lo, bpf_reg64[s].lo);
+ knod_iset32(&p32[0], 0);
+ knod_mov32(priv, meta, bpf_reg64[d].hi, p32[0]);
+ break;
+ case BPF_ALU64 | BPF_AND | BPF_X:
+ //r[d] &= r[s];
+ knod_and64(priv, meta, bpf_reg64[d],
+ bpf_reg64[d], bpf_reg64[s]);
+ break;
+ case BPF_ALU | BPF_AND | BPF_K:
+ case BPF_ALU64 | BPF_AND | BPF_K:
+ //r[d] &= imm;
+ knod_iset32(&p32[0], imm);
+ knod_and32(priv, meta, bpf_reg64[d].lo, p32[0],
+ bpf_reg64[d].lo);
+ knod_iset32(&p32[0], 0);
+ knod_mov32(priv, meta, bpf_reg64[d].hi, p32[0]);
+ break;
+ case BPF_ALU | BPF_OR | BPF_X:
+ knod_or32(priv, meta, bpf_reg64[d].lo,
+ bpf_reg64[d].lo, bpf_reg64[s].lo);
+ knod_iset32(&p32[0], 0);
+ knod_mov32(priv, meta, bpf_reg64[d].hi, p32[0]);
+ break;
+ case BPF_ALU64 | BPF_OR | BPF_X:
+ //r[d] |= r[s];
+ knod_or32(priv, meta, bpf_reg64[d].lo,
+ bpf_reg64[d].lo, bpf_reg64[s].lo);
+ knod_or32(priv, meta, bpf_reg64[d].hi,
+ bpf_reg64[d].hi, bpf_reg64[s].hi);
+ break;
+ case BPF_ALU | BPF_OR | BPF_K:
+ case BPF_ALU64 | BPF_OR | BPF_K:
+ //r[d] |= imm;
+ knod_iset32(&p32[0], imm);
+ knod_or32(priv, meta,
+ bpf_reg64[d].lo, p32[0], bpf_reg64[d].lo);
+ knod_iset32(&p32[0], 0);
+ knod_mov32(priv, meta, bpf_reg64[d].hi, p32[0]);
+ break;
+ case BPF_ALU | BPF_ADD | BPF_X:
+ knod_add32(priv, meta, bpf_reg64[d].lo,
+ bpf_reg64[d].lo, bpf_reg64[s].lo);
+ knod_iset32(&p32[0], 0);
+ knod_mov32(priv, meta, bpf_reg64[d].hi, p32[0]);
+ break;
+ case BPF_ALU64 | BPF_ADD | BPF_X:
+ knod_add64(priv, meta, bpf_reg64[d],
+ bpf_reg64[d],
+ bpf_reg64[s]);
+
+ //r[d] += r[s];
+ break;
+ case BPF_ALU | BPF_ADD | BPF_K:
+ case BPF_ALU64 | BPF_ADD | BPF_K:
+ //r[d] += imm;
+ knod_iset32(&p32[0], imm);
+ knod_add32(priv, meta, bpf_reg64[d].lo,
+ p32[0], bpf_reg64[d].lo);
+ /* NOTE:
+ * imm is 24bit.
+ * But should we set hi to 0?
+ */
+ knod_iset32(&p32[0], 0);
+ knod_mov32(priv, meta, bpf_reg64[d].hi, p32[0]);
+ break;
+ case BPF_ALU | BPF_SUB | BPF_X:
+ //r[d] -= r[s];
+ knod_sub32(priv, meta, bpf_reg64[d].lo,
+ bpf_reg64[d].lo, bpf_reg64[s].lo);
+ knod_iset32(&p32[0], 0);
+ knod_mov32(priv, meta, bpf_reg64[d].hi, p32[0]);
+ break;
+ case BPF_ALU64 | BPF_SUB | BPF_X:
+ //r[d] -= r[s];
+
+ knod_sub64(priv, meta, bpf_reg64[d], bpf_reg64[d],
+ bpf_reg64[s]);
+ break;
+ case BPF_ALU | BPF_SUB | BPF_K:
+ case BPF_ALU64 | BPF_SUB | BPF_K:
+ //r[d] -= imm;
+ knod_iset64(&p64[0], imm);
+ knod_subrev64(priv, meta, bpf_reg64[d], p64[0],
+ bpf_reg64[s]);
+ break;
+ case BPF_ALU | BPF_MUL | BPF_X:
+ knod_mul_lo32(priv, meta, bpf_reg64[d].lo,
+ bpf_reg64[d].lo, bpf_reg64[s].lo);
+ break;
+ case BPF_ALU64 | BPF_MUL | BPF_X:
+ //r[d] *= r[s];
+ knod_mov64(priv, meta, r64[0], bpf_reg64[d]);
+ knod_mov64(priv, meta, r64[1], bpf_reg64[s]);
+ knod_mul64(priv, meta,
+ bpf_reg64[d],
+ r64[0],
+ r64[1],
+ r64[2]);
+ break;
+ case BPF_ALU | BPF_MUL | BPF_K:
+ knod_iset32(&p32[0], imm);
+ knod_mul_lo32(priv, meta, bpf_reg64[d].lo,
+ p32[0], bpf_reg64[d].lo);
+ knod_iset32(&p32[0], 0);
+ knod_mov32(priv, meta, bpf_reg64[d].hi, p32[0]);
+ break;
+ case BPF_ALU64 | BPF_MUL | BPF_K:
+ //r[d] *= imm;
+ knod_iset64(&p64[0], imm);
+ knod_mov64(priv, meta, r64[0], bpf_reg64[d]);
+ knod_mov64(priv, meta, r64[1], p64[0]);
+ knod_mul64(priv, meta,
+ bpf_reg64[d],
+ r64[0],
+ r64[1],
+ r64[2]);
+ break;
+ case BPF_ALU | BPF_DIV | BPF_X:
+ case BPF_ALU64 | BPF_DIV | BPF_X:
+ //r[d] /= r[s];
+ knod_iset64(&p64[0], meta->umin_src);
+ knod_div(priv, meta, bpf_reg64[d], p64[0],
+ r64[0], r64[1], r64[2], r64[3]);
+ break;
+ case BPF_ALU | BPF_DIV | BPF_K:
+ case BPF_ALU64 | BPF_DIV | BPF_K:
+ //r[d] /= imm;
+ knod_iset64(&p64[0], imm);
+ knod_div(priv, meta, bpf_reg64[d], p64[0],
+ r64[0], r64[1], r64[2], r64[3]);
+ break;
+ case BPF_ALU | BPF_NEG:
+ knod_iset32(&p32[0], 0);
+ knod_sub32(priv, meta, bpf_reg64[d].lo, p32[0],
+ bpf_reg64[d].lo);
+ knod_iset32(&p32[0], 0);
+ knod_mov32(priv, meta, bpf_reg64[d].hi, p32[0]);
+ break;
+ case BPF_ALU64 | BPF_NEG:
+ //r[d] = -r[d];
+ WARN_ON_ONCE(1);
+ break;
+ case BPF_ALU | BPF_LSH | BPF_X:
+ knod_lshlrev32(priv, meta, bpf_reg64[d].lo,
+ bpf_reg64[s].lo, bpf_reg64[d].lo);
+ knod_iset32(&p32[0], 0);
+ knod_mov32(priv, meta, bpf_reg64[d].hi, p32[0]);
+ break;
+ case BPF_ALU64 | BPF_LSH | BPF_X:
+ //r[d] <<= r[s];
+ knod_lshlrev64(priv, meta, bpf_reg64[d],
+ bpf_reg64[s], bpf_reg64[d]);
+ break;
+ case BPF_ALU | BPF_LSH | BPF_K:
+ knod_iset32(&p32[0], imm);
+ knod_lshlrev32(priv, meta, bpf_reg64[d].lo, p32[0],
+ bpf_reg64[d].lo);
+ knod_iset32(&p32[0], 0);
+ knod_mov32(priv, meta, bpf_reg64[d].hi, p32[0]);
+ break;
+ case BPF_ALU64 | BPF_LSH | BPF_K:
+ //r[d] <<= imm;
+ knod_iset64(&p64[0], imm);
+ knod_lshlrev64(priv, meta, bpf_reg64[d], p64[0],
+ bpf_reg64[d]);
+ break;
+ case BPF_ALU | BPF_RSH | BPF_X:
+ knod_lshrrev32(priv, meta, bpf_reg64[d].lo,
+ bpf_reg64[s].lo,
+ bpf_reg64[d].lo);
+ knod_iset32(&p32[0], 0);
+ knod_mov32(priv, meta, bpf_reg64[d].hi, p32[0]);
+ break;
+ case BPF_ALU64 | BPF_RSH | BPF_X:
+ //r[d] >>= r[s];
+ knod_lshrrev64(priv, meta, bpf_reg64[d],
+ bpf_reg64[s], bpf_reg64[d]);
+ break;
+ case BPF_ALU | BPF_RSH | BPF_K:
+ knod_iset32(&p32[0], imm);
+ knod_lshrrev32(priv, meta, bpf_reg64[d].lo, p32[0],
+ bpf_reg64[d].lo);
+ knod_iset32(&p32[0], 0);
+ knod_mov32(priv, meta, bpf_reg64[d].hi, p32[0]);
+ break;
+ case BPF_ALU64 | BPF_RSH | BPF_K:
+ //r[d] >>= imm;
+ knod_iset64(&p64[0], imm);
+ knod_lshrrev64(priv, meta, bpf_reg64[d],
+ p64[0], bpf_reg64[d]);
+ break;
+ case BPF_ALU | BPF_ARSH | BPF_X:
+ knod_ashrrev32(priv, meta, bpf_reg64[d].lo,
+ bpf_reg64[s].lo, bpf_reg64[d].lo);
+ knod_iset32(&p32[0], 0);
+ knod_mov32(priv, meta, bpf_reg64[d].hi, p32[0]);
+ break;
+ case BPF_ALU64 | BPF_ARSH | BPF_X:
+ //r[d] >>= r[s];
+ knod_ashrrev64(priv, meta, bpf_reg64[d],
+ bpf_reg64[s], bpf_reg64[d]);
+ break;
+ case BPF_ALU | BPF_ARSH | BPF_K:
+ knod_iset32(&p32[0], imm);
+ knod_ashrrev32(priv, meta, bpf_reg64[d].lo,
+ p32[0], bpf_reg64[d].lo);
+ knod_iset32(&p32[0], 0);
+ knod_mov32(priv, meta, bpf_reg64[d].hi, p32[0]);
+ break;
+ case BPF_ALU64 | BPF_ARSH | BPF_K:
+ //r[d] >>= imm;
+ knod_iset64(&p64[0], imm);
+ knod_ashrrev64(priv, meta, bpf_reg64[d],
+ p64[0], bpf_reg64[d]);
+ break;
+ case BPF_LD | BPF_IMM | BPF_DW:
+ meta2 = list_next_entry(meta, l);
+ if (WARN_ON_ONCE(!meta2))
+ return -EINVAL;
+ imm2 = meta2->insn.imm;
+ skip = true;
+ imm64 = (u64)imm2 << 32 | (u32)imm;
+ switch (s) {
+ case 0x00:
+ //r[d] = imm64;
+ knod_mov64_imm(priv, meta, d * 2,
+ imm64);
+
+ break;
+ case 0x01:
+ /* r[d] = param->maps[imm]; */
+ imm64 = knod_bpf_get_map_gaddr(priv,
+ meta,
+ meta2);
+ map_id = knod_bpf_get_map_id(priv,
+ meta,
+ meta2);
+ knod_mov64_imm(priv, meta, d * 2,
+ imm64);
+ break;
+ default:
+ WARN_ON_ONCE(1);
+ break;
+ }
+ break;
+ /* Legacy BPF packet access, not needed */
+ case BPF_LD | BPF_ABS | BPF_B:
+ case BPF_LD | BPF_ABS | BPF_H:
+ case BPF_LD | BPF_ABS | BPF_W:
+ case BPF_LD | BPF_IND | BPF_B:
+ case BPF_LD | BPF_IND | BPF_H:
+ case BPF_LD | BPF_IND | BPF_W:
+ //err = pc | 0x0700;
+ //exit = true;
+ WARN_ON_ONCE(1);
+ break;
+ case BPF_LDX | BPF_MEM | BPF_B:
+ if (meta->ptr.type == PTR_TO_STACK) {
+ stack_off = meta->sreg.stack_off + off;
+ knod_bpf_load_size(priv, meta,
+ &bpf_reg64[d],
+ &stack[0],
+ sizeof(unsigned char),
+ 512 + stack_off);
+ } else if (meta->ptr.type == PTR_TO_CTX) {
+ knod_emit(priv, meta, global_load_ubyte,
+ bpf_reg64[d].lo,
+ bpf_reg64[s].lo, off * 2);
+ } else if (meta->ptr.type == PTR_TO_MAP_VALUE) {
+ knod_emit(priv, meta, global_load_ubyte,
+ bpf_reg64[d].lo,
+ bpf_reg64[s].lo, off);
+ } else if (meta->ptr.type == PTR_TO_MAP_KEY) {
+ knod_emit(priv, meta, global_load_ubyte,
+ bpf_reg64[d].lo,
+ bpf_reg64[s].lo, off);
+ } else if (meta->ptr.type == PTR_TO_PACKET) {
+ if (knod_bpf_pkt_cache) {
+ packet_off = meta->sreg.packet_off +
+ off;
+ knod_bpf_load_size(priv, meta,
+ &bpf_reg64[d],
+ &pkt_cache[0],
+ sizeof(unsigned char),
+ packet_off);
+ } else {
+ knod_emit(priv, meta, global_load_ubyte,
+ bpf_reg64[d].lo,
+ bpf_reg64[s].lo, off);
+ }
+ } else {
+ WARN_ON_ONCE(1);
+ }
+ knod_wait_vmcnt(priv, meta);
+ knod_iset32(&p32[0], 0);
+ knod_mov32(priv, meta, bpf_reg64[d].hi, p32[0]);
+ //ptr = (__global void *)r[s] + off;
+ //r[d] = *(__global unsigned char *)ptr;
+ break;
+ case BPF_LDX | BPF_MEM | BPF_H:
+ if (meta->ptr.type == PTR_TO_STACK) {
+ stack_off = meta->sreg.stack_off + off;
+ knod_bpf_load_size(priv, meta,
+ &bpf_reg64[d],
+ &stack[0],
+ sizeof(unsigned short),
+ 512 + stack_off);
+ } else if (meta->ptr.type == PTR_TO_CTX) {
+ knod_emit(priv, meta, global_load_ushort,
+ bpf_reg64[d].lo,
+ bpf_reg64[s].lo, off * 2);
+ } else if (meta->ptr.type == PTR_TO_MAP_VALUE) {
+ knod_emit(priv, meta, global_load_ushort,
+ bpf_reg64[d].lo,
+ bpf_reg64[s].lo, off);
+ } else if (meta->ptr.type == PTR_TO_MAP_KEY) {
+ knod_emit(priv, meta, global_load_ushort,
+ bpf_reg64[d].lo,
+ bpf_reg64[s].lo, off);
+ } else if (meta->ptr.type == SCALAR_VALUE) {
+ knod_emit(priv, meta, global_load_ushort,
+ bpf_reg64[d].lo,
+ bpf_reg64[s].lo, off);
+ } else if (meta->ptr.type == PTR_TO_PACKET) {
+ if (knod_bpf_pkt_cache) {
+ packet_off = meta->sreg.packet_off +
+ off;
+ knod_bpf_load_size(priv, meta,
+ &bpf_reg64[d],
+ &pkt_cache[0],
+ sizeof(unsigned short),
+ packet_off);
+ } else if (priv->isa_version == 10 &&
+ (off & 1)) {
+ knod_bpf_emit_gfx10_unaligned_load(
+ priv, meta,
+ sizeof(unsigned short),
+ bpf_reg64[d],
+ bpf_reg64[s].lo, off);
+ } else {
+ knod_emit(priv, meta,
+ global_load_ushort,
+ bpf_reg64[d].lo,
+ bpf_reg64[s].lo, off);
+ }
+ } else {
+ knod_jit_err(" type = %d\n", meta->ptr.type);
+ WARN_ON_ONCE(1);
+ }
+ //ptr = (__global void *)r[s] + off;
+ //r[d] = *(__global unsigned short *)ptr;
+ knod_iset32(&p32[0], 0);
+ knod_mov32(priv, meta, bpf_reg64[d].hi, p32[0]);
+ knod_wait_vmcnt(priv, meta);
+ break;
+ case BPF_LDX | BPF_MEM | BPF_W:
+ if (meta->ptr.type == PTR_TO_STACK) {
+ stack_off = meta->sreg.stack_off + off;
+ knod_bpf_load_size(priv, meta,
+ &bpf_reg64[d],
+ &stack[0],
+ sizeof(unsigned int),
+ 512 + stack_off);
+ } else if (meta->ptr.type == PTR_TO_CTX) {
+ if (off == offsetof(struct xdp_md, data)) {
+ knod_mov32(priv, meta,
+ bpf_reg64[d].lo,
+ (struct amdgcn_param32){
+ .v = KNOD_AMDGPU_DATA_VREG_LO,
+ .type = AMDGCN_PARAM_TYPE_VGPR});
+ knod_mov32(priv, meta,
+ bpf_reg64[d].hi,
+ (struct amdgcn_param32){
+ .v = KNOD_AMDGPU_DATA_VREG_HI,
+ .type = AMDGCN_PARAM_TYPE_VGPR});
+ } else if (off == offsetof(struct xdp_md,
+ data_end)) {
+ knod_mov32(priv, meta,
+ bpf_reg64[d].lo,
+ (struct amdgcn_param32){
+ .v = KNOD_AMDGPU_DATA_END_VREG_LO,
+ .type = AMDGCN_PARAM_TYPE_VGPR});
+ knod_mov32(priv, meta,
+ bpf_reg64[d].hi,
+ (struct amdgcn_param32){
+ .v = KNOD_AMDGPU_DATA_END_VREG_HI,
+ .type = AMDGCN_PARAM_TYPE_VGPR});
+ } else {
+ emit_global_load_dwordx2(
+ priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ bpf_reg64[d].lo,
+ bpf_reg64[s].lo,
+ off * 2);
+ debug_insn(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns]);
+ meta->amdgpu_insns++;
+ }
+ } else if (meta->ptr.type == PTR_TO_MAP_VALUE) {
+ knod_emit(priv, meta, global_load_dword,
+ bpf_reg64[d].lo,
+ bpf_reg64[s].lo, off);
+ } else if (meta->ptr.type == PTR_TO_MAP_KEY) {
+ knod_emit(priv, meta, global_load_dword,
+ bpf_reg64[d].lo,
+ bpf_reg64[s].lo, off);
+ } else if (meta->ptr.type == PTR_TO_PACKET) {
+ if (knod_bpf_pkt_cache) {
+ packet_off = meta->sreg.packet_off +
+ off;
+ knod_bpf_load_size(priv, meta,
+ &bpf_reg64[d],
+ &pkt_cache[0],
+ sizeof(unsigned int),
+ packet_off);
+ } else if (priv->isa_version == 10 &&
+ (off & 3)) {
+ knod_bpf_emit_gfx10_unaligned_load(
+ priv, meta,
+ sizeof(unsigned int),
+ bpf_reg64[d],
+ bpf_reg64[s].lo, off);
+ } else {
+ knod_emit(priv, meta, global_load_dword,
+ bpf_reg64[d].lo,
+ bpf_reg64[s].lo, off);
+ }
+ } else {
+ WARN_ON_ONCE(1);
+ }
+ //ptr = (__global void *)r[s] + off;
+ //r[d] = *(__global unsigned int *)ptr;
+ if (meta->ptr.type != PTR_TO_CTX) {
+ knod_iset32(&p32[0], 0);
+ knod_mov32(priv, meta, bpf_reg64[d].hi,
+ p32[0]);
+ }
+ knod_wait_vmcnt(priv, meta);
+ break;
+ case BPF_LDX | BPF_MEM | BPF_DW:
+ if (meta->ptr.type == PTR_TO_STACK) {
+ stack_off = meta->sreg.stack_off + off;
+ knod_bpf_load_size(priv, meta,
+ &bpf_reg64[d],
+ &stack[0],
+ sizeof(unsigned long),
+ 512+stack_off);
+ } else if (meta->ptr.type == PTR_TO_CTX) {
+ if (off == offsetof(struct xdp_md, data)) {
+ knod_mov32(priv, meta,
+ bpf_reg64[d].lo,
+ (struct amdgcn_param32){
+ .v = KNOD_AMDGPU_DATA_VREG_LO,
+ .type = AMDGCN_PARAM_TYPE_VGPR});
+ knod_mov32(priv, meta,
+ bpf_reg64[d].hi,
+ (struct amdgcn_param32){
+ .v = KNOD_AMDGPU_DATA_VREG_HI,
+ .type = AMDGCN_PARAM_TYPE_VGPR});
+ } else if (off == offsetof(struct xdp_md,
+ data_end)) {
+ knod_mov32(priv, meta,
+ bpf_reg64[d].lo,
+ (struct amdgcn_param32){
+ .v = KNOD_AMDGPU_DATA_END_VREG_LO,
+ .type = AMDGCN_PARAM_TYPE_VGPR});
+ knod_mov32(priv, meta,
+ bpf_reg64[d].hi,
+ (struct amdgcn_param32){
+ .v = KNOD_AMDGPU_DATA_END_VREG_HI,
+ .type = AMDGCN_PARAM_TYPE_VGPR});
+ } else {
+ knod_emit(priv, meta,
+ global_load_dwordx2,
+ bpf_reg64[d].lo,
+ bpf_reg64[s].lo, off * 2);
+ }
+ } else if (meta->ptr.type == PTR_TO_MAP_VALUE) {
+ knod_emit(priv, meta, global_load_dwordx2,
+ bpf_reg64[d].lo,
+ bpf_reg64[s].lo, off);
+ } else if (meta->ptr.type == PTR_TO_MAP_KEY) {
+ knod_emit(priv, meta, global_load_dwordx2,
+ bpf_reg64[d].lo,
+ bpf_reg64[s].lo, off);
+ } else if (meta->ptr.type == PTR_TO_PACKET) {
+ if (knod_bpf_pkt_cache) {
+ packet_off = meta->sreg.packet_off +
+ off;
+ knod_bpf_load_size(priv, meta,
+ &bpf_reg64[d],
+ &pkt_cache[0],
+ sizeof(unsigned long),
+ packet_off);
+ } else if (priv->isa_version == 10 &&
+ (off & 3)) {
+ knod_bpf_emit_gfx10_unaligned_load(
+ priv, meta,
+ sizeof(unsigned long),
+ bpf_reg64[d],
+ bpf_reg64[s].lo, off);
+ } else {
+ knod_emit(priv, meta,
+ global_load_dwordx2,
+ bpf_reg64[d].lo,
+ bpf_reg64[s].lo, off);
+ }
+ } else {
+ WARN_ON_ONCE(1);
+ }
+ //ptr = (__global void *)r[s] + off;
+ //r[d] = *(__global unsigned long *)ptr;
+ knod_wait_vmcnt(priv, meta);
+ break;
+ case BPF_STX | BPF_MEM | BPF_B:
+ if (meta->ptr.type == PTR_TO_STACK) {
+ stack_off = meta->dreg.stack_off + off;
+ knod_bpf_store_cache_size(priv, meta,
+ &bpf_reg64[s],
+ &stack[0],
+ sizeof(u8),
+ 512 + stack_off);
+ } else if (meta->ptr.type == PTR_TO_CTX) {
+ knod_emit(priv, meta, global_store_byte,
+ bpf_reg64[s].lo,
+ bpf_reg64[d].lo, off * 2);
+ } else if (meta->ptr.type == PTR_TO_MAP_VALUE) {
+ knod_emit(priv, meta, global_store_byte,
+ bpf_reg64[s].lo,
+ bpf_reg64[d].lo, off);
+ } else if (meta->ptr.type == PTR_TO_MAP_KEY) {
+ knod_emit(priv, meta, global_store_byte,
+ bpf_reg64[s].lo,
+ bpf_reg64[d].lo, off);
+ } else if (meta->ptr.type == PTR_TO_PACKET) {
+ if (knod_bpf_pkt_cache) {
+ packet_off = meta->dreg.packet_off +
+ off;
+ knod_bpf_store_cache_size(priv,
+ meta,
+ &bpf_reg64[s],
+ &pkt_cache[0],
+ sizeof(u8),
+ packet_off);
+ } else {
+ knod_emit(priv, meta, global_store_byte,
+ bpf_reg64[s].lo,
+ bpf_reg64[d].lo, off);
+ }
+ } else {
+ WARN_ON_ONCE(1);
+ }
+ break;
+ case BPF_STX | BPF_MEM | BPF_H:
+ if (meta->ptr.type == PTR_TO_STACK) {
+ stack_off = meta->dreg.stack_off + off;
+ knod_bpf_store_cache_size(priv, meta,
+ &bpf_reg64[s],
+ &stack[0],
+ sizeof(u16),
+ 512 + stack_off);
+ } else if (meta->ptr.type == PTR_TO_CTX) {
+ knod_emit(priv, meta, global_store_short,
+ bpf_reg64[s].lo,
+ bpf_reg64[d].lo, off * 2);
+ } else if (meta->ptr.type == PTR_TO_MAP_VALUE) {
+ knod_emit(priv, meta, global_store_short,
+ bpf_reg64[s].lo,
+ bpf_reg64[d].lo, off);
+ } else if (meta->ptr.type == PTR_TO_MAP_KEY) {
+ knod_emit(priv, meta, global_store_short,
+ bpf_reg64[s].lo,
+ bpf_reg64[d].lo, off);
+ } else if (meta->ptr.type == PTR_TO_PACKET) {
+ if (knod_bpf_pkt_cache) {
+ packet_off = meta->dreg.packet_off +
+ off;
+ knod_bpf_store_cache_size(priv,
+ meta,
+ &bpf_reg64[s],
+ &pkt_cache[0],
+ sizeof(u16),
+ packet_off);
+ } else {
+ knod_emit(priv, meta,
+ global_store_short,
+ bpf_reg64[s].lo,
+ bpf_reg64[d].lo, off);
+ }
+ } else {
+ WARN_ON_ONCE(1);
+ }
+ break;
+ case BPF_STX | BPF_MEM | BPF_W:
+ if (meta->ptr.type == PTR_TO_STACK) {
+ stack_off = meta->dreg.stack_off + off;
+ knod_bpf_store_cache_size(priv, meta,
+ &bpf_reg64[s],
+ &stack[0],
+ sizeof(u32),
+ 512 + stack_off);
+ } else if (meta->ptr.type == PTR_TO_CTX) {
+ knod_emit(priv, meta, global_store_dword,
+ bpf_reg64[s].lo,
+ bpf_reg64[d].lo, off * 2);
+ } else if (meta->ptr.type == PTR_TO_MAP_VALUE) {
+ knod_emit(priv, meta, global_store_dword,
+ bpf_reg64[s].lo,
+ bpf_reg64[d].lo, off);
+ } else if (meta->ptr.type == PTR_TO_MAP_KEY) {
+ knod_emit(priv, meta, global_store_dword,
+ bpf_reg64[s].lo,
+ bpf_reg64[d].lo, off);
+ } else if (meta->ptr.type == PTR_TO_PACKET) {
+ if (knod_bpf_pkt_cache) {
+ packet_off = meta->dreg.packet_off +
+ off;
+ knod_bpf_store_cache_size(priv,
+ meta,
+ &bpf_reg64[s],
+ &pkt_cache[0],
+ sizeof(u32),
+ packet_off);
+ } else {
+ knod_emit(priv, meta,
+ global_store_dword,
+ bpf_reg64[s].lo,
+ bpf_reg64[d].lo, off);
+ }
+ } else {
+ WARN_ON_ONCE(1);
+ }
+ break;
+ case BPF_STX | BPF_MEM | BPF_DW:
+ if (meta->ptr.type == PTR_TO_STACK) {
+ stack_off = meta->dreg.stack_off + off;
+ knod_bpf_store_cache_size(priv, meta,
+ &bpf_reg64[s],
+ &stack[0],
+ sizeof(u64),
+ 512 + stack_off);
+ } else if (meta->ptr.type == PTR_TO_CTX) {
+ knod_emit(priv, meta, global_store_dwordx2,
+ bpf_reg64[s].lo,
+ bpf_reg64[d].lo, off * 2);
+ } else if (meta->ptr.type == PTR_TO_MAP_VALUE) {
+ knod_emit(priv, meta, global_store_dwordx2,
+ bpf_reg64[s].lo,
+ bpf_reg64[d].lo, off);
+ } else if (meta->ptr.type == PTR_TO_MAP_KEY) {
+ knod_emit(priv, meta, global_store_dwordx2,
+ bpf_reg64[s].lo,
+ bpf_reg64[d].lo, off);
+ } else if (meta->ptr.type == PTR_TO_PACKET) {
+ if (knod_bpf_pkt_cache) {
+ packet_off = meta->dreg.packet_off +
+ off;
+ knod_bpf_store_cache_size(priv, meta,
+ &bpf_reg64[s],
+ &pkt_cache[0],
+ sizeof(u64),
+ packet_off);
+ } else {
+ knod_emit(priv, meta,
+ global_store_dwordx2,
+ bpf_reg64[s].lo,
+ bpf_reg64[d].lo, off);
+ }
+ } else {
+ WARN_ON_ONCE(1);
+ }
+ break;
+ case BPF_STX | BPF_ATOMIC | BPF_W:
+ case BPF_STX | BPF_ATOMIC | BPF_DW:
+ is_dw = BPF_SIZE(meta->insn.code) == BPF_DW;
+ atomic_op = imm & ~BPF_FETCH;
+ fetch = imm & BPF_FETCH;
+
+ /*
+ * BPF atomic: *(dst_reg + off) op= src_reg
+ * If BPF_FETCH: src_reg = old value
+ * BPF_CMPXCHG: expect in r0, new in src_reg,
+ * old value returned in r0.
+ *
+ * global_atomic_* with glc=1 returns old value in vdst.
+ * For non-FETCH ops use glc=0 (fire-and-forget).
+ *
+ * 64-bit atomics (global_atomic_*_x2) hang on GFX9
+ * VRAM. GFX10+ supports them.
+ */
+ if (is_dw && priv->isa_version == 9) {
+ pr_err("knod: 64-bit atomic not supported on GFX9\n");
+ return -EOPNOTSUPP;
+ }
+
+ /*
+ * For CMPXCHG/FETCH: drain pending loads so addr/data
+ * VGPRs are ready. For non-fetch ADD wave reduction,
+ * addr was already waited for at map_lookup, and data
+ * is from ALU - no waitcnt needed.
+ */
+ if (imm == BPF_CMPXCHG || fetch)
+ knod_wait_vmcnt(priv, meta);
+
+ if (imm == BPF_CMPXCHG) {
+ /* cmpswap: data = {expect(r0), new(src)}.
+ * AMD cmpswap data reg pair must be
+ * consecutive:
+ * 32-bit: {cmp, new} = 2 consecutive VGPRs
+ * 64-bit: {cmp_lo, cmp_hi, new_lo, new_hi}
+ * Copy r0 and src into TMP consecutive pair.
+ */
+ struct amdgcn_param32 tmp0_lo, tmp0_hi,
+ tmp1_lo, tmp1_hi;
+
+ knod_vset32(&tmp0_lo,
+ KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_vset32(&tmp0_hi,
+ KNOD_AMDGPU_TMP_VREG0_HI);
+ knod_vset32(&tmp1_lo,
+ KNOD_AMDGPU_TMP_VREG1_LO);
+ knod_vset32(&tmp1_hi,
+ KNOD_AMDGPU_TMP_VREG1_HI);
+
+ if (!is_dw) {
+ /* TMP0_LO = r0 (expect),
+ * TMP0_HI = src (new)
+ */
+ knod_mov32(priv, meta,
+ tmp0_lo,
+ bpf_reg64[0].lo);
+ knod_mov32(priv, meta,
+ tmp0_hi,
+ bpf_reg64[s].lo);
+
+ emit_global_atomic_cmpswap(
+ priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ tmp0_lo, bpf_reg64[d].lo,
+ tmp0_lo, off, 1);
+ debug_insn(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns]);
+ meta->amdgpu_insns++;
+ knod_wait_vmcnt(priv, meta);
+ /* Return old value in r0 */
+ knod_mov32(priv, meta,
+ bpf_reg64[0].lo,
+ tmp0_lo);
+ } else {
+ /* 64-bit:
+ * {r0_lo, r0_hi, src_lo, src_hi}
+ */
+ knod_mov32(priv, meta,
+ tmp0_lo,
+ bpf_reg64[0].lo);
+ knod_mov32(priv, meta,
+ tmp0_hi,
+ bpf_reg64[0].hi);
+ knod_mov32(priv, meta,
+ tmp1_lo,
+ bpf_reg64[s].lo);
+ knod_mov32(priv, meta,
+ tmp1_hi,
+ bpf_reg64[s].hi);
+
+ emit_global_atomic_cmpswap_x2(
+ priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ tmp0_lo, bpf_reg64[d].lo,
+ tmp0_lo, off, 1);
+ debug_insn(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns]);
+ meta->amdgpu_insns++;
+ knod_wait_vmcnt(priv, meta);
+ knod_mov32(priv, meta,
+ bpf_reg64[0].lo,
+ tmp0_lo);
+ knod_mov32(priv, meta,
+ bpf_reg64[0].hi,
+ tmp0_hi);
+ }
+ } else if (!fetch && atomic_op == BPF_ADD) {
+ /*
+ * Wave reduction for BPF_ADD (non-fetch):
+ * Instead of all lanes doing atomic_add(val),
+ * count active lanes, multiply by val, and
+ * have a single lane do atomic_add(count*val).
+ *
+ * Assumes src_reg is uniform across all active
+ * lanes (true for constant increments like
+ * +=1).
+ *
+ * s_bcnt1_i32_b64 s_tmp, exec
+ * v_mul_lo_u32 v_tmp, s_tmp, v_src
+ * v_mbcnt_lo v_tmp2, exec_lo, 0
+ * v_mbcnt_hi v_tmp2, exec_hi, v_tmp2
+ * v_cmp_eq_u32 vcc, v_tmp2, 0
+ * s_and_saveexec s_save, vcc
+ * global_atomic_add addr, v_tmp, off
+ * s_waitcnt vmcnt(0)
+ * s_mov_b64 exec, s_save
+ */
+ struct amdgcn_param32 v_tmp, v_tmp2,
+ s_count, s_exec_lo,
+ s_exec_hi, v_zero;
+
+ knod_vset32(&v_tmp,
+ KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_vset32(&v_tmp2,
+ KNOD_AMDGPU_TMP_VREG0_HI);
+ knod_sset32(&s_count,
+ KNOD_AMDGPU_TMP_SREG0_LO);
+ knod_sset32(&s_exec_lo,
+ AMDGCN_SREG_EXEC_LO);
+ knod_sset32(&s_exec_hi,
+ AMDGCN_SREG_EXEC_LO + 1);
+ knod_iset32(&v_zero, 0);
+
+ /* s_bcnt1_i32_b64 s_count, exec */
+ knod_emit(priv, meta, s_bcnt1_i32_b64,
+ KNOD_AMDGPU_TMP_SREG0_LO,
+ AMDGCN_SREG_EXEC_LO);
+
+ /* v_mul_lo_u32 v_tmp, s_count, v_src */
+ knod_emit(priv, meta, v_mul_lo_u32, v_tmp,
+ s_count, bpf_reg64[s].lo);
+
+ /* v_mbcnt_lo_u32_b32 v_tmp2, exec_lo, 0 */
+ knod_emit(priv, meta, v_mbcnt_lo_u32_b32,
+ v_tmp2, s_exec_lo, v_zero);
+
+ /* v_mbcnt_hi_u32_b32 v_tmp2, exec_hi, v_tmp2 */
+ knod_emit(priv, meta, v_mbcnt_hi_u32_b32,
+ v_tmp2, s_exec_hi, v_tmp2);
+
+ /* v_cmp_eq_u32 vcc, 0, v_tmp2 ->
+ * first active lane
+ */
+ knod_emit(priv, meta, v_cmp_eq_u32, v_zero,
+ v_tmp2);
+
+ /* s_and_saveexec_b64 s_save, vcc */
+ knod_emit(priv, meta, s_and_saveexec_b64,
+ KNOD_AMDGPU_TMP_SREG0_LO,
+ AMDGCN_SREG_VCC_LO);
+
+ if (is_dw) {
+ struct amdgcn_param32 v_tmp_hi;
+
+ knod_vset32(&v_tmp_hi,
+ KNOD_AMDGPU_TMP_VREG0_HI);
+ /*
+ * x2 atomics consume a consecutive
+ * VGPR pair, and the 32-bit addend
+ * lands in the host-visible low dword
+ * when it is placed in the second
+ * register.
+ */
+ knod_emit(priv, meta, v_mov_b32_e32,
+ v_tmp_hi, v_tmp);
+ knod_emit(priv, meta, v_mov_b32_e32,
+ v_tmp, v_zero);
+ /* global_atomic_add_x2 addr,
+ * {0, v_tmp_hi}, off
+ */
+ knod_emit(priv, meta,
+ global_atomic_add_x2, v_tmp,
+ bpf_reg64[d].lo, v_tmp, off,
+ 0);
+ } else {
+ /* global_atomic_add addr, v_tmp, off
+ * (single lane)
+ */
+ knod_emit(priv, meta, global_atomic_add,
+ v_tmp,
+ bpf_reg64[d].lo, v_tmp, off,
+ 0);
+ }
+
+ /*
+ * No s_waitcnt needed: glc=0 atomic doesn't
+ * increment vmcnt. The GPU guarantees all
+ * pending ops complete before wave exit.
+ */
+
+ /* s_mov_b64 exec, s_save */
+ knod_emit(priv, meta, s_mov_b64,
+ AMDGCN_SREG_EXEC_LO,
+ KNOD_AMDGPU_TMP_SREG0_LO);
+ } else if (!is_dw) {
+ /* 32-bit: AND, OR, XOR, XCHG, or fetch ops */
+ struct amdgcn_param32 vdst, data_p;
+
+ if (fetch) {
+ vdst = bpf_reg64[s].lo;
+ } else {
+ knod_vset32(&vdst,
+ KNOD_AMDGPU_TMP_VREG0_LO);
+ }
+ data_p = bpf_reg64[s].lo;
+
+ switch (atomic_op) {
+ case BPF_ADD:
+ emit_global_atomic_add(
+ priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ vdst, bpf_reg64[d].lo,
+ data_p, off, fetch);
+ break;
+ case BPF_AND:
+ emit_global_atomic_and(
+ priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ vdst, bpf_reg64[d].lo,
+ data_p, off, fetch);
+ break;
+ case BPF_OR:
+ emit_global_atomic_or(
+ priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ vdst, bpf_reg64[d].lo,
+ data_p, off, fetch);
+ break;
+ case BPF_XOR:
+ emit_global_atomic_xor(
+ priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ vdst, bpf_reg64[d].lo,
+ data_p, off, fetch);
+ break;
+ default: /* BPF_XCHG */
+ emit_global_atomic_swap(
+ priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ vdst, bpf_reg64[d].lo,
+ data_p, off, fetch);
+ break;
+ }
+ } else {
+ /* 64-bit: AND, OR, XOR, XCHG, or fetch ops */
+ struct amdgcn_param32 vdst, data_p;
+
+ if (fetch) {
+ vdst = bpf_reg64[s].lo;
+ } else {
+ knod_vset32(&vdst,
+ KNOD_AMDGPU_TMP_VREG0_LO);
+ }
+ data_p = bpf_reg64[s].lo;
+
+ switch (atomic_op) {
+ case BPF_ADD:
+ emit_global_atomic_add_x2(
+ priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ vdst, bpf_reg64[d].lo,
+ data_p, off, fetch);
+ break;
+ case BPF_AND:
+ emit_global_atomic_and_x2(
+ priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ vdst, bpf_reg64[d].lo,
+ data_p, off, fetch);
+ break;
+ case BPF_OR:
+ emit_global_atomic_or_x2(
+ priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ vdst, bpf_reg64[d].lo,
+ data_p, off, fetch);
+ break;
+ case BPF_XOR:
+ emit_global_atomic_xor_x2(
+ priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ vdst, bpf_reg64[d].lo,
+ data_p, off, fetch);
+ break;
+ default: /* BPF_XCHG */
+ emit_global_atomic_swap_x2(
+ priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns],
+ vdst, bpf_reg64[d].lo,
+ data_p, off, fetch);
+ break;
+ }
+ debug_insn(priv->isa_version,
+ &meta->amdgpu_insn[meta->amdgpu_insns]);
+ meta->amdgpu_insns++;
+ /* Always wait for atomic completion */
+ knod_wait_vmcnt(priv, meta);
+ }
+ break;
+ case BPF_ST | BPF_MEM | BPF_B:
+ knod_iset32(&p32[0], imm);
+ if (meta->ptr.type == PTR_TO_STACK) {
+ stack_off = meta->dreg.stack_off + off;
+ knod_iset64(&p64[0], imm);
+ knod_bpf_store_cache_size(priv, meta,
+ &p64[0],
+ &stack[0],
+ sizeof(u8),
+ 512 + stack_off);
+ } else if (meta->ptr.type == PTR_TO_CTX) {
+ knod_emit(priv, meta, global_store_byte, p32[0],
+ bpf_reg64[d].lo, off * 2);
+ } else if (meta->ptr.type == PTR_TO_MAP_VALUE) {
+ knod_emit(priv, meta, global_store_byte, p32[0],
+ bpf_reg64[d].lo, off);
+ } else if (meta->ptr.type == PTR_TO_MAP_KEY) {
+ knod_emit(priv, meta, global_store_byte, p32[0],
+ bpf_reg64[d].lo, off);
+ } else if (meta->ptr.type == PTR_TO_PACKET) {
+ knod_iset64(&p64[0], imm);
+ if (knod_bpf_pkt_cache) {
+ packet_off = meta->dreg.packet_off +
+ off;
+ knod_bpf_store_cache_size(priv,
+ meta,
+ &p64[0],
+ &pkt_cache[0],
+ sizeof(u8),
+ packet_off);
+ } else {
+ knod_emit(priv, meta, global_store_byte,
+ p64[0].lo,
+ bpf_reg64[d].lo, off);
+ }
+ } else {
+ WARN_ON_ONCE(1);
+ }
+ break;
+ case BPF_ST | BPF_MEM | BPF_H:
+ knod_iset32(&p32[0], imm);
+ if (meta->ptr.type == PTR_TO_STACK) {
+ stack_off = meta->dreg.stack_off + off;
+ knod_iset64(&p64[0], imm);
+ knod_bpf_store_cache_size(priv, meta,
+ &p64[0],
+ &stack[0],
+ sizeof(u16),
+ 512 + stack_off);
+ } else if (meta->ptr.type == PTR_TO_CTX) {
+ knod_emit(priv, meta, global_store_short,
+ p32[0], bpf_reg64[d].lo, off * 2);
+ } else if (meta->ptr.type == PTR_TO_MAP_VALUE) {
+ knod_emit(priv, meta, global_store_short,
+ p32[0], bpf_reg64[d].lo, off);
+ } else if (meta->ptr.type == PTR_TO_MAP_KEY) {
+ knod_emit(priv, meta, global_store_short,
+ p32[0], bpf_reg64[d].lo, off);
+ } else if (meta->ptr.type == PTR_TO_PACKET) {
+ knod_iset64(&p64[0], imm);
+ if (knod_bpf_pkt_cache) {
+ packet_off = meta->dreg.packet_off +
+ off;
+ knod_bpf_store_cache_size(priv,
+ meta,
+ &p64[0],
+ &pkt_cache[0],
+ sizeof(u16),
+ packet_off);
+ } else {
+ knod_emit(priv, meta,
+ global_store_short, p64[0].lo,
+ bpf_reg64[d].lo, off);
+ }
+ } else {
+ WARN_ON_ONCE(1);
+ }
+ break;
+ case BPF_ST | BPF_MEM | BPF_W:
+ knod_iset32(&p32[0], imm);
+ if (meta->ptr.type == PTR_TO_STACK) {
+ stack_off = meta->dreg.stack_off + off;
+ knod_iset64(&p64[0], imm);
+ knod_bpf_store_cache_size(priv, meta,
+ &p64[0],
+ &stack[0],
+ sizeof(u32),
+ 512 + stack_off);
+ } else if (meta->ptr.type == PTR_TO_CTX) {
+ knod_emit(priv, meta, global_store_dword,
+ p32[0], bpf_reg64[d].lo, off * 2);
+ } else if (meta->ptr.type == PTR_TO_MAP_VALUE) {
+ knod_emit(priv, meta, global_store_dword,
+ p32[0], bpf_reg64[d].lo, off);
+ } else if (meta->ptr.type == PTR_TO_MAP_KEY) {
+ knod_emit(priv, meta, global_store_dword,
+ p32[0], bpf_reg64[d].lo, off);
+ } else if (meta->ptr.type == PTR_TO_PACKET) {
+ knod_iset64(&p64[0], imm);
+ if (knod_bpf_pkt_cache) {
+ packet_off = meta->dreg.packet_off +
+ off;
+ knod_bpf_store_cache_size(priv,
+ meta,
+ &p64[0],
+ &pkt_cache[0],
+ sizeof(u32),
+ packet_off);
+ } else {
+ knod_emit(priv, meta,
+ global_store_dword, p64[0].lo,
+ bpf_reg64[d].lo, off);
+ }
+ } else {
+ WARN_ON_ONCE(1);
+ }
+ break;
+ case BPF_ST | BPF_MEM | BPF_DW:
+ knod_iset32(&p32[0], imm);
+ if (meta->ptr.type == PTR_TO_STACK) {
+ stack_off = meta->dreg.stack_off + off;
+ knod_iset64(&p64[0], imm);
+ knod_bpf_store_cache_size(priv, meta,
+ &p64[0],
+ &stack[0],
+ sizeof(u64),
+ 512 + stack_off);
+ } else if (meta->ptr.type == PTR_TO_CTX) {
+ knod_emit(priv, meta, global_store_dwordx2,
+ p32[0], bpf_reg64[d].lo, off * 2);
+ } else if (meta->ptr.type == PTR_TO_MAP_VALUE) {
+ knod_emit(priv, meta, global_store_dwordx2,
+ p32[0], bpf_reg64[d].lo, off);
+ } else if (meta->ptr.type == PTR_TO_MAP_KEY) {
+ knod_emit(priv, meta, global_store_dwordx2,
+ p32[0], bpf_reg64[d].lo, off);
+ } else if (meta->ptr.type == PTR_TO_PACKET) {
+ knod_iset64(&p64[0], imm);
+ if (knod_bpf_pkt_cache) {
+ packet_off = meta->dreg.packet_off +
+ off;
+ knod_bpf_store_cache_size(priv, meta,
+ &p64[0],
+ &pkt_cache[0],
+ sizeof(u64),
+ packet_off);
+ } else {
+ knod_emit(priv, meta,
+ global_store_dwordx2,
+ p64[0].lo,
+ bpf_reg64[d].lo, off);
+ }
+ knod_iset32(&p32[0], imm);
+ } else {
+ WARN_ON_ONCE(1);
+ }
+ break;
+ case BPF_JMP32 | BPF_JA | BPF_K:
+ if (meta->branch_type == KNOD_BR_DIRECT_EXIT) {
+ knod_bpf_emit_direct_exit_retval(priv, meta,
+ meta->merge_point);
+
+ /* Unconditional goto exit:
+ * all active lanes done
+ */
+ knod_emit(priv, meta, s_or_b64,
+ knod_prog->done_mask_sreg,
+ knod_prog->done_mask_sreg,
+ AMDGCN_SREG_EXEC_LO);
+ knod_emit(priv, meta, s_mov_b64,
+ AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_INTEGER_0);
+ } else if (meta->branch_type == KNOD_BR_FORWARD_GOTO) {
+ /* Structurized: save all active lanes, clear
+ * EXEC. Lanes resume at merge_point (target).
+ */
+ knod_emit(priv, meta, s_mov_b64,
+ meta->exec_save_sreg,
+ AMDGCN_SREG_EXEC_LO);
+ knod_emit(priv, meta, s_mov_b64,
+ AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_INTEGER_0);
+ } else {
+ /* Reorder classifies every JA as FORWARD_GOTO
+ * or DIRECT_EXIT; reaching here is a bug.
+ */
+ WARN_ON_ONCE(1);
+ }
+ break;
+ case BPF_JMP | BPF_JA | BPF_K:
+ if (meta->branch_type == KNOD_BR_DIRECT_EXIT) {
+ knod_bpf_emit_direct_exit_retval(priv, meta,
+ meta->merge_point);
+
+ /* Unconditional goto exit:
+ * all active lanes done
+ */
+ knod_emit(priv, meta, s_or_b64,
+ knod_prog->done_mask_sreg,
+ knod_prog->done_mask_sreg,
+ AMDGCN_SREG_EXEC_LO);
+ knod_emit(priv, meta, s_mov_b64,
+ AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_INTEGER_0);
+ } else if (meta->branch_type == KNOD_BR_FORWARD_GOTO) {
+ /* Structurized: save all active lanes, clear
+ * EXEC. Lanes resume at merge_point (target).
+ */
+ knod_emit(priv, meta, s_mov_b64,
+ meta->exec_save_sreg,
+ AMDGCN_SREG_EXEC_LO);
+ knod_emit(priv, meta, s_mov_b64,
+ AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_INTEGER_0);
+ } else {
+ /* Reorder classifies every JA as FORWARD_GOTO
+ * or DIRECT_EXIT; reaching here is a bug.
+ */
+ WARN_ON_ONCE(1);
+ }
+ break;
+ case BPF_JMP32 | BPF_JEQ | BPF_K:
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_iset32(¶m[1], imm);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0],
+ param[1]);
+ knod_vset32(¶m[0], d * 2);
+ knod_vset32(¶m[1], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_emit(priv, meta, v_cmp_eq_u32, param[0],
+ param[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP | BPF_JEQ | BPF_K:
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_iset32(¶m[1], imm);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0],
+ param[1]);
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG0_HI);
+ knod_iset32(¶m[1], 0);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0],
+ param[1]);
+ knod_vset32(¶m[0], d * 2);
+ knod_vset32(¶m[1], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_emit(priv, meta, v_cmp_eq_u64, param[0],
+ param[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP32 | BPF_JEQ | BPF_X:
+ knod_vset32(¶m[0], d * 2);
+ knod_vset32(¶m[1], s * 2);
+ knod_emit(priv, meta, v_cmp_eq_u32, param[0],
+ param[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP | BPF_JEQ | BPF_X:
+ knod_vset32(¶m[0], d * 2);
+ knod_vset32(¶m[1], s * 2);
+ knod_emit(priv, meta, v_cmp_eq_u64, param[0],
+ param[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP32 | BPF_JGT | BPF_K:
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_iset32(¶m[1], imm);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0],
+ param[1]);
+ knod_vset32(¶m[0], d * 2);
+ knod_vset32(¶m[1], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_emit(priv, meta, v_cmp_gt_u32, param[0],
+ param[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP | BPF_JGT | BPF_K:
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_iset32(¶m[1], imm);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0],
+ param[1]);
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG0_HI);
+ knod_iset32(¶m[1], 0);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0],
+ param[1]);
+ knod_vset64(¶m64[0], d * 2);
+ knod_vset64(¶m64[1], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_emit(priv, meta, v_cmp_gt_u64, param64[0],
+ param64[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP32 | BPF_JGT | BPF_X:
+ knod_vset32(¶m[0], d * 2);
+ knod_vset32(¶m[1], s * 2);
+ knod_emit(priv, meta, v_cmp_gt_u32, param[0],
+ param[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP | BPF_JGT | BPF_X:
+ knod_vset64(¶m64[0], d * 2);
+ knod_vset64(¶m64[1], s * 2);
+ knod_emit(priv, meta, v_cmp_gt_u64, param64[0],
+ param64[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP32 | BPF_JGE | BPF_K:
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_iset32(¶m[1], imm);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0],
+ param[1]);
+ knod_vset32(¶m[0], d * 2);
+ knod_vset32(¶m[1], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_emit(priv, meta, v_cmp_ge_u32, param[0],
+ param[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP | BPF_JGE | BPF_K:
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_iset32(¶m[1], imm);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0],
+ param[1]);
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG0_HI);
+ knod_iset32(¶m[1], 0);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0],
+ param[1]);
+ knod_vset64(¶m64[0], d * 2);
+ knod_vset64(¶m64[1], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_emit(priv, meta, v_cmp_ge_u64, param64[0],
+ param64[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP32 | BPF_JGE | BPF_X:
+ knod_vset32(¶m[0], d * 2);
+ knod_vset32(¶m[1], s * 2);
+ knod_emit(priv, meta, v_cmp_ge_u32, param[0],
+ param[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP | BPF_JGE | BPF_X:
+ knod_vset64(¶m64[0], d * 2);
+ knod_vset64(¶m64[1], s * 2);
+ knod_emit(priv, meta, v_cmp_ge_u64, param64[0],
+ param64[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP32 | BPF_JLT | BPF_K:
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_iset32(¶m[1], imm);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0],
+ param[1]);
+ knod_vset32(¶m[0], d * 2);
+ knod_vset32(¶m[1], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_emit(priv, meta, v_cmp_lt_u32, param[0],
+ param[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP | BPF_JLT | BPF_K:
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_iset32(¶m[1], imm);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0],
+ param[1]);
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG0_HI);
+ knod_iset32(¶m[1], 0);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0],
+ param[1]);
+ knod_vset64(¶m64[0], d * 2);
+ knod_vset64(¶m64[1], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_emit(priv, meta, v_cmp_lt_u64, param64[0],
+ param64[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP32 | BPF_JLT | BPF_X:
+ knod_vset32(¶m[0], d * 2);
+ knod_vset32(¶m[1], s * 2);
+ knod_emit(priv, meta, v_cmp_lt_u32, param[0],
+ param[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP | BPF_JLT | BPF_X:
+ knod_vset64(¶m64[0], d * 2);
+ knod_vset64(¶m64[1], s * 2);
+ knod_emit(priv, meta, v_cmp_lt_u64, param64[0],
+ param64[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP32 | BPF_JLE | BPF_K:
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_iset32(¶m[1], imm);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0],
+ param[1]);
+ knod_vset32(¶m[0], d * 2);
+ knod_vset32(¶m[1], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_emit(priv, meta, v_cmp_le_u32, param[0],
+ param[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP | BPF_JLE | BPF_K:
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_iset32(¶m[1], imm);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0],
+ param[1]);
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG0_HI);
+ knod_iset32(¶m[1], 0);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0],
+ param[1]);
+ knod_vset64(¶m64[0], d * 2);
+ knod_vset64(¶m64[1], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_emit(priv, meta, v_cmp_le_u64, param64[0],
+ param64[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP32 | BPF_JLE | BPF_X:
+ knod_vset32(¶m[0], d * 2);
+ knod_vset32(¶m[1], s * 2);
+ knod_emit(priv, meta, v_cmp_le_u32, param[0],
+ param[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP | BPF_JLE | BPF_X:
+ knod_vset64(¶m64[0], d * 2);
+ knod_vset64(¶m64[1], 2 * 2);
+ knod_emit(priv, meta, v_cmp_le_u64, param64[0],
+ param64[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP32 | BPF_JSGT | BPF_K:
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_iset32(¶m[1], imm);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0],
+ param[1]);
+ knod_vset32(¶m[0], d * 2);
+ knod_vset32(¶m[1], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_emit(priv, meta, v_cmp_gt_i32, param[0],
+ param[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP | BPF_JSGT | BPF_K:
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_iset32(¶m[1], imm);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0],
+ param[1]);
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG0_HI);
+ knod_iset32(¶m[1], 0);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0],
+ param[1]);
+ knod_vset64(¶m64[0], d * 2);
+ knod_vset64(¶m64[1], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_emit(priv, meta, v_cmp_gt_i64, param64[0],
+ param64[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP32 | BPF_JSGT | BPF_X:
+ knod_vset32(¶m[0], d * 2);
+ knod_vset32(¶m[1], s * 2);
+ knod_emit(priv, meta, v_cmp_gt_i32, param[0],
+ param[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP | BPF_JSGT | BPF_X:
+ knod_vset64(¶m64[0], d * 2);
+ knod_vset64(¶m64[1], s * 2);
+ knod_emit(priv, meta, v_cmp_gt_i64, param64[0],
+ param64[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP32 | BPF_JSGE | BPF_K:
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_iset32(¶m[1], imm);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0],
+ param[1]);
+ knod_vset32(¶m[0], d * 2);
+ knod_vset32(¶m[1], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_emit(priv, meta, v_cmp_ge_i32, param[0],
+ param[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP | BPF_JSGE | BPF_K:
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_iset32(¶m[1], imm);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0],
+ param[1]);
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG0_HI);
+ knod_iset32(¶m[1], 0);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0],
+ param[1]);
+ knod_vset64(¶m64[0], d * 2);
+ knod_vset64(¶m64[1], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_emit(priv, meta, v_cmp_ge_i64, param64[0],
+ param64[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP32 | BPF_JSGE | BPF_X:
+ knod_vset32(¶m[0], d * 2);
+ knod_vset32(¶m[1], s * 2);
+ knod_emit(priv, meta, v_cmp_ge_i32, param[0],
+ param[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP | BPF_JSGE | BPF_X:
+ knod_vset64(¶m64[0], d * 2);
+ knod_vset64(¶m64[1], s * 2);
+ knod_emit(priv, meta, v_cmp_ge_i64, param64[0],
+ param64[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP32 | BPF_JSLT | BPF_K:
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_iset32(¶m[1], imm);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0],
+ param[1]);
+ knod_vset32(¶m[0], d * 2);
+ knod_vset32(¶m[1], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_emit(priv, meta, v_cmp_lt_i32, param[0],
+ param[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP | BPF_JSLT | BPF_K:
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_iset32(¶m[1], imm);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0],
+ param[1]);
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG0_HI);
+ knod_iset32(¶m[1], 0);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0],
+ param[1]);
+ knod_vset64(¶m64[0], d * 2);
+ knod_vset64(¶m64[1], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_emit(priv, meta, v_cmp_lt_i64, param64[0],
+ param64[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP32 | BPF_JSLT | BPF_X:
+ knod_vset32(¶m[0], d * 2);
+ knod_vset32(¶m[1], s * 2);
+ knod_emit(priv, meta, v_cmp_lt_i32, param[0],
+ param[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP | BPF_JSLT | BPF_X:
+ knod_vset64(¶m64[0], d * 2);
+ knod_vset64(¶m64[1], s * 2);
+ knod_emit(priv, meta, v_cmp_lt_i64, param64[0],
+ param64[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP32 | BPF_JSLE | BPF_K:
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_iset32(¶m[1], imm);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0],
+ param[1]);
+ knod_vset32(¶m[0], d * 2);
+ knod_vset32(¶m[1], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_emit(priv, meta, v_cmp_le_i32, param[0],
+ param[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP | BPF_JSLE | BPF_K:
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_iset32(¶m[1], imm);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0],
+ param[1]);
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG0_HI);
+ knod_iset32(¶m[1], 0);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0],
+ param[1]);
+ knod_vset64(¶m64[0], d * 2);
+ knod_vset64(¶m64[1], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_emit(priv, meta, v_cmp_le_i64, param64[0],
+ param64[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP32 | BPF_JSLE | BPF_X:
+ knod_vset32(¶m[0], d * 2);
+ knod_vset32(¶m[1], s * 2);
+ knod_emit(priv, meta, v_cmp_le_i32, param[0],
+ param[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP | BPF_JSLE | BPF_X:
+ knod_vset64(¶m64[0], d * 2);
+ knod_vset64(¶m64[1], s * 2);
+ knod_emit(priv, meta, v_cmp_le_i64, param64[0],
+ param64[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP32 | BPF_JSET | BPF_K:
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_iset32(¶m[1], imm);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0],
+ param[1]);
+ knod_vset32(¶m[0], d * 2);
+ knod_vset32(¶m[1], d * 2);
+ knod_vset32(¶m[2], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_emit(priv, meta, v_and_b32_e32, param[0],
+ param[1], param[2]);
+ knod_vset32(¶m[0], d * 2);
+ knod_vset32(¶m[1], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_emit(priv, meta, v_cmp_eq_u32, param[0],
+ param[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP | BPF_JSET | BPF_K:
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_iset32(¶m[1], imm);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0],
+ param[1]);
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG0_HI);
+ knod_iset32(¶m[1], 0);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0],
+ param[1]);
+ knod_vset32(¶m[0], d * 2);
+ knod_vset32(¶m[1], d * 2);
+ knod_vset32(¶m[2], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_emit(priv, meta, v_and_b32_e32, param[0],
+ param[1], param[2]);
+ knod_vset32(¶m[0], (d * 2) + 1);
+ knod_vset32(¶m[1], (d * 2) + 1);
+ knod_vset32(¶m[2], KNOD_AMDGPU_TMP_VREG0_HI);
+ knod_emit(priv, meta, v_and_b32_e32, param[0],
+ param[1], param[2]);
+ knod_vset32(¶m[0], d * 2);
+ knod_vset32(¶m[1], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_emit(priv, meta, v_cmp_eq_u64, param[0],
+ param[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP32 | BPF_JSET | BPF_X:
+ knod_vset32(¶m[0], d * 2);
+ knod_vset32(¶m[1], d * 2);
+ knod_vset32(¶m[2], s * 2);
+ knod_emit(priv, meta, v_and_b32_e32, param[0],
+ param[1], param[2]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP | BPF_JSET | BPF_X:
+ knod_vset32(¶m[0], d * 2);
+ knod_vset32(¶m[1], d * 2);
+ knod_vset32(¶m[2], s * 2);
+ knod_emit(priv, meta, v_and_b32_e32, param[0],
+ param[1], param[2]);
+ knod_vset32(¶m[0], (d * 2) + 1);
+ knod_vset32(¶m[1], (d * 2) + 1);
+ knod_vset32(¶m[2], (s * 2) + 1);
+ knod_emit(priv, meta, v_and_b32_e32, param[0],
+ param[1], param[2]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP32 | BPF_JNE | BPF_K:
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_iset32(¶m[1], imm);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0],
+ param[1]);
+ knod_vset32(¶m[0], d * 2);
+ knod_vset32(¶m[1], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_emit(priv, meta, v_cmp_eq_u32, param[0],
+ param[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP | BPF_JNE | BPF_K:
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_iset32(¶m[1], imm);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0],
+ param[1]);
+ knod_vset32(¶m[0], KNOD_AMDGPU_TMP_VREG0_HI);
+ knod_iset32(¶m[1], 0);
+ knod_emit(priv, meta, v_mov_b32_e32, param[0],
+ param[1]);
+ knod_vset32(¶m[0], d * 2);
+ knod_vset32(¶m[1], KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_emit(priv, meta, v_cmp_eq_u64, param[0],
+ param[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP32 | BPF_JNE | BPF_X:
+ knod_vset32(¶m[0], d * 2);
+ knod_vset32(¶m[1], s * 2);
+ knod_emit(priv, meta, v_cmp_eq_u32, param[0],
+ param[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP | BPF_JNE | BPF_X:
+ knod_vset32(¶m[0], d * 2);
+ knod_vset32(¶m[1], s * 2);
+ knod_emit(priv, meta, v_cmp_eq_u64, param[0],
+ param[1]);
+ knod_bpf_emit_branch_tail(priv, meta, knod_prog, off);
+ break;
+ case BPF_JMP32 | BPF_CALL:
+ case BPF_JMP | BPF_CALL:
+ switch (imm) {
+ case 1:
+ if (map_id == -1) {
+ WARN_ON_ONCE(1);
+ break;
+ }
+ knod_bpf_map_lookup(priv, meta, map_id);
+ map_id = -1;
+ break;
+ case 2: {
+ struct knod_bpf_map_obj *_map_obj;
+
+ if (map_id == -1) {
+ WARN_ON_ONCE(1);
+ break;
+ }
+ _map_obj = knod_bpf_map_kaddr(priv, map_id);
+ if (!_map_obj) {
+ WARN_ON_ONCE(1);
+ break;
+ }
+ if (_map_obj->map_type == BPF_MAP_TYPE_ARRAY)
+ knod_bpf_map_update_array(priv, meta,
+ map_id);
+ else if (_map_obj->map_type ==
+ BPF_MAP_TYPE_HASH)
+ knod_bpf_map_update_hash(priv, meta,
+ map_id);
+ else
+ WARN_ON_ONCE(1);
+ map_id = -1;
+ break;
+ }
+ case 3: {
+ struct knod_bpf_map_obj *_map_obj;
+
+ if (map_id == -1) {
+ WARN_ON_ONCE(1);
+ break;
+ }
+ _map_obj = knod_bpf_map_kaddr(priv, map_id);
+ if (!_map_obj) {
+ WARN_ON_ONCE(1);
+ break;
+ }
+ if (_map_obj->map_type == BPF_MAP_TYPE_ARRAY)
+ knod_bpf_map_delete_array(priv, meta,
+ map_id);
+ else if (_map_obj->map_type ==
+ BPF_MAP_TYPE_HASH)
+ knod_bpf_map_delete_hash(priv, meta,
+ map_id);
+ else
+ WARN_ON_ONCE(1);
+ map_id = -1;
+ break;
+ }
+ case 5:
+ knod_bpf_ktime_get_ns(priv, meta);
+ break;
+ case 44:
+ knod_bpf_xdp_adjust_head(priv, meta);
+ break;
+ case 65:
+ knod_bpf_xdp_adjust_tail(priv, meta);
+ break;
+ default:
+ WARN_ON_ONCE(1);
+ break;
+ }
+ break;
+ case BPF_JMP32 | BPF_EXIT:
+ case BPF_JMP | BPF_EXIT:
+ /* Structurized CFG: BPF_EXIT is NOT a terminator.
+ * Mark all active lanes as done and clear EXEC.
+ * Actual exit handling (retval store, pkt_cache,
+ * PASS block, s_endpgm) is in the unified
+ * fallthrough EXIT at the end of the stream.
+ * This follows the LLVM StructurizeCFG model where
+ * all lanes must reach the single exit point.
+ */
+ knod_emit(priv, meta, s_or_b64,
+ knod_prog->done_mask_sreg,
+ knod_prog->done_mask_sreg,
+ AMDGCN_SREG_EXEC_LO);
+
+ knod_emit(priv, meta, s_mov_b64, AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_INTEGER_0);
+ break;
+ case BPF_ALU | BPF_END | BPF_TO_BE: {
+ struct amdgcn_param32 v_dst_lo, v_dst_hi, v_tmp, s_sel;
+
+ knod_vset32(&v_dst_lo, d * 2);
+ knod_vset32(&v_dst_hi, d * 2 + 1);
+ knod_vset32(&v_tmp, KNOD_AMDGPU_TMP_VREG0_LO);
+ knod_sset32(&s_sel, KNOD_AMDGPU_TMP_SREG0_LO);
+
+ switch (imm) {
+ case 16:
+ /* bswap16+zext: {0,0,byte0,byte1} */
+ knod_iset32(¶m[0], 0x0C0C0001);
+ knod_emit(priv, meta, s_mov_b32, s_sel,
+ param[0]);
+
+ knod_emit(priv, meta, v_perm_b32, v_dst_lo,
+ v_dst_lo, v_dst_lo, s_sel);
+
+ knod_iset32(¶m[0], 0);
+ knod_emit(priv, meta, v_mov_b32_e32, v_dst_hi,
+ param[0]);
+ break;
+ case 32:
+ /* bswap32+zext */
+ knod_iset32(¶m[0], 0x00010203);
+ knod_emit(priv, meta, s_mov_b32, s_sel,
+ param[0]);
+
+ knod_emit(priv, meta, v_perm_b32, v_dst_lo,
+ v_dst_lo, v_dst_lo, s_sel);
+
+ knod_iset32(¶m[0], 0);
+ knod_emit(priv, meta, v_mov_b32_e32, v_dst_hi,
+ param[0]);
+ break;
+ case 64: {
+ struct amdgcn_param32 v_src_hi;
+
+ knod_vset32(&v_src_hi, d * 2 + 1);
+
+ /* bswap32 selector */
+ knod_iset32(¶m[0], 0x00010203);
+ knod_emit(priv, meta, s_mov_b32, s_sel,
+ param[0]);
+
+ /* tmp = bswap32(lo) */
+ knod_emit(priv, meta, v_perm_b32, v_tmp,
+ v_dst_lo, v_dst_lo, s_sel);
+
+ /* new_lo = bswap32(hi) */
+ knod_emit(priv, meta, v_perm_b32, v_dst_lo,
+ v_src_hi, v_src_hi, s_sel);
+
+ /* new_hi = tmp (bswap32(old_lo)) */
+ knod_emit(priv, meta, v_mov_b32_e32, v_dst_hi,
+ v_tmp);
+ break;
+ }
+ default:
+ WARN_ON_ONCE(1);
+ break;
+ }
+ break;
+ }
+ case BPF_ALU | BPF_END | BPF_TO_LE: {
+ struct amdgcn_param32 v_dst_lo, v_dst_hi;
+
+ knod_vset32(&v_dst_lo, d * 2);
+ knod_vset32(&v_dst_hi, d * 2 + 1);
+
+ switch (imm) {
+ case 16:
+ knod_iset32(¶m[0], 0xFFFF);
+ knod_emit(priv, meta, v_and_b32_e32, v_dst_lo,
+ param[0], v_dst_lo);
+
+ knod_iset32(¶m[0], 0);
+ knod_emit(priv, meta, v_mov_b32_e32, v_dst_hi,
+ param[0]);
+ break;
+ case 32:
+ knod_iset32(¶m[0], 0);
+ knod_emit(priv, meta, v_mov_b32_e32, v_dst_hi,
+ param[0]);
+ break;
+ case 64:
+ break;
+ default:
+ WARN_ON_ONCE(1);
+ break;
+ }
+ break;
+ }
+ default:
+ WARN_ON_ONCE(1);
+ break;
+ }
+
+ WARN_ON(meta->amdgpu_insns >= KNOD_META_INSNS);
+ for (i = 0; i < meta->amdgpu_insns; i++)
+ insn_idx += (meta->amdgpu_insn[i].size / 4);
+ }
+
+ /* Fallthrough EXIT: publish a verdict for every in-bounds lane.
+ * Lanes that did not reach BPF_EXIT are forced to XDP_DROP below.
+ */
+ meta = kzalloc(sizeof(*meta), GFP_KERNEL);
+ if (!meta)
+ return -ENOMEM;
+ list_add_tail(&meta->l, &knod_prog->post_insns);
+
+ /* Any in-bounds lane outside done_mask gets a conservative DROP
+ * verdict instead of publishing stale VGPR state or leaving the
+ * recycle-time poison in bd->act.
+ */
+ knod_emit(priv, meta, s_andn2_b64, AMDGCN_SREG_EXEC_LO,
+ knod_prog->initial_exec_sreg, knod_prog->done_mask_sreg);
+ knod_vset32(&p[0], KNOD_AMDGPU_VREG0_LO);
+ knod_iset32(&p[1], XDP_DROP);
+ knod_emit(priv, meta, v_mov_b32_e32, p[0], p[1]);
+
+ /* Store bd->act for every lane that participated in this dispatch.
+ * Done lanes retain their low32 action in v0; unfinished lanes publish
+ * the fallback DROP written above.
+ */
+ knod_emit(priv, meta, s_mov_b64, AMDGCN_SREG_EXEC_LO,
+ knod_prog->initial_exec_sreg);
+
+ /* BPF/XDP verdicts are low32; do not spend a second GTT dword per
+ * packet.
+ */
+ knod_vset32(&p[0], KNOD_AMDGPU_VREG0_LO);
+ knod_vset32(&p[1], KNOD_AMDGPU_SLOT_VREG_LO);
+ knod_emit(priv, meta, global_store_dword, p[0], p[1],
+ offsetof(struct spsc_bd, act));
+
+ if (knod_prog->uses_adjust)
+ knod_bpf_emit_offlen_writeback(priv, meta);
+
+ /* pkt_cache writeback: flush modified packet data back to VRAM */
+ if (knod_bpf_pkt_cache) {
+ knod_vset32(&p[0], r32[0].v);
+ knod_vset32(&p[1],
+ KNOD_AMDGPU_DATA_VREG_LO);
+ knod_emit(priv, meta, v_mov_b32_e32, p[0], p[1]);
+ knod_vset32(&p[0], r32[0].v + 1);
+ knod_vset32(&p[1],
+ KNOD_AMDGPU_DATA_VREG_HI);
+ knod_emit(priv, meta, v_mov_b32_e32, p[0], p[1]);
+
+ knod_global_store_size_cache(priv, meta,
+ &pkt_cache[0],
+ r32[0],
+ 0, /* dst index */
+ 0, /* start offset */
+ knod_prog->max_packet_off);
+ }
+
+ /* XDP_PASS detection */
+ knod_iset32(&p[0], XDP_PASS);
+ knod_vset32(&p[1], KNOD_AMDGPU_VREG0_LO);
+ knod_emit(priv, meta, v_cmp_eq_u32, p[0], p[1]);
+
+ pass_branch_idx = meta->amdgpu_insns;
+ knod_emit(priv, meta, s_cbranch_vccz, 0);
+
+ /* EXEC &= VCC (only PASS lanes) */
+ knod_emit(priv, meta, s_and_b64, AMDGCN_SREG_EXEC_LO,
+ AMDGCN_SREG_EXEC_LO, AMDGCN_SREG_VCC_LO);
+
+ /* v_mov param addr to VGPR pair */
+ knod_vset32(&p[0], KNOD_AMDGPU_TMP_VREG9_LO);
+ knod_sset32(&p[1], KNOD_AMDGPU_PARAM_SREG_LO);
+ knod_emit(priv, meta, v_mov_b32_e32, p[0], p[1]);
+
+ knod_vset32(&p[0], KNOD_AMDGPU_TMP_VREG9_HI);
+ knod_sset32(&p[1], KNOD_AMDGPU_PARAM_SREG_HI);
+ knod_emit(priv, meta, v_mov_b32_e32, p[0], p[1]);
+
+ /* Per-queue pass_count: offset TMP_VREG9 by queue_idx * 4 */
+ /* v_mov_b32 v2, s15 (queue_idx -> VGPR) */
+ knod_vset32(&p[0], KNOD_AMDGPU_VREG1_LO);
+ knod_sset32(&p[1],
+ KNOD_AMDGPU_WORKGROUP_ID_Y_SREG);
+ knod_emit(priv, meta, v_mov_b32_e32, p[0], p[1]);
+
+ /* v_lshlrev_b32 v2, 2, v2 (queue_idx * 4) */
+ knod_vset32(&p[0], KNOD_AMDGPU_VREG1_LO);
+ knod_iset32(&p[1], 2);
+ knod_vset32(&p[2], KNOD_AMDGPU_VREG1_LO);
+ knod_emit(priv, meta, v_lshlrev_b32, p[0], p[1], p[2]);
+
+ /* v_add_u32 v40, v2, v40 (param_addr_lo += queue_idx * 4) */
+ knod_vset32(&p[0], KNOD_AMDGPU_TMP_VREG9_LO);
+ knod_vset32(&p[1], KNOD_AMDGPU_VREG1_LO);
+ knod_vset32(&p[2], KNOD_AMDGPU_TMP_VREG9_LO);
+ knod_emit(priv, meta, v_add_u32, p[0], p[1], p[2]);
+
+ /* v_mov TMP10_LO, 1 */
+ knod_vset32(&p[0], KNOD_AMDGPU_TMP_VREG10_LO);
+ knod_iset32(&p[1], 1);
+ knod_emit(priv, meta, v_mov_b32_e32, p[0], p[1]);
+
+ /* global_atomic_add TMP10_LO, TMP9, TMP10_LO,
+ * offsetof(pass_count)
+ * GLC=1 to receive old_val in vdst (needed for per-lane slot
+ * index). With GLC=0 vdst is NOT written, leaving TMP10_LO
+ * as the addend (1) -- every PASS lane then computes slot=1
+ * and races on the same pass_meta_buf entry, leaving slot 0 empty.
+ */
+ knod_vset32(&p[0], KNOD_AMDGPU_TMP_VREG10_LO);
+ knod_vset32(&p[1], KNOD_AMDGPU_TMP_VREG9_LO);
+ knod_vset32(&p[2], KNOD_AMDGPU_TMP_VREG10_LO);
+ knod_emit(priv, meta, global_atomic_add, p[0], p[1], p[2],
+ offsetof(struct knod_bpf_param, pass_count), 1);
+
+ /* s_waitcnt vmcnt(0) */
+ knod_emit(priv, meta, s_waitcnt_vmcnt);
+
+ /* v_sub_u32 v40, v40, v2 (restore param_addr_lo) */
+ knod_vset32(&p[0], KNOD_AMDGPU_TMP_VREG9_LO);
+ knod_vset32(&p[1], KNOD_AMDGPU_TMP_VREG9_LO);
+ knod_vset32(&p[2], KNOD_AMDGPU_VREG1_LO);
+ knod_emit(priv, meta, v_sub_u32, p[0], p[1], p[2]);
+
+ /* old_val * 2 */
+ knod_vset32(&p[0], KNOD_AMDGPU_TMP_VREG10_LO);
+ knod_iset32(&p[1], 1);
+ knod_vset32(&p[2], KNOD_AMDGPU_TMP_VREG10_LO);
+ knod_emit(priv, meta, v_lshlrev_b32, p[0], p[1], p[2]);
+
+ /* addr_lo += old_val * 2 */
+ knod_vset32(&p[0], KNOD_AMDGPU_TMP_VREG9_LO);
+ knod_vset32(&p[1], KNOD_AMDGPU_TMP_VREG10_LO);
+ knod_vset32(&p[2], KNOD_AMDGPU_TMP_VREG9_LO);
+ knod_emit(priv, meta, v_add_u32, p[0], p[1], p[2]);
+
+ /* global_store_short pass_indices[old_val],
+ * BACKLOG_IDX_VREG
+ * dst=data, src=addr in wrapper convention
+ */
+ knod_vset32(&p[0],
+ KNOD_AMDGPU_BACKLOG_IDX_VREG);
+ knod_vset32(&p[1], KNOD_AMDGPU_TMP_VREG9_LO);
+ knod_emit(priv, meta, global_store_short, p[0], p[1],
+ offsetof(struct knod_bpf_param, pass_indices));
+
+ /* Copy PASS packet data (shader mode) or store src addr (SDMA mode) */
+ knod_emit_pass_addr_store(priv, meta);
+
+ /* Patch branch offset */
+ pass_dwords = 0;
+
+ for (j = pass_branch_idx + 1; j < meta->amdgpu_insns; j++)
+ pass_dwords += meta->amdgpu_insn[j].size / 4;
+ emit_s_cbranch_vccz(priv->isa_version,
+ &meta->amdgpu_insn[pass_branch_idx],
+ pass_dwords);
+
+ knod_emit(priv, meta, s_endpgm);
+
+ for (j = 0; j < meta->amdgpu_insns; j++)
+ insn_idx += meta->amdgpu_insn[j].size / 4;
+
+ if (priv->isa_version == 10) {
+ if (insn_idx % 256) {
+ meta = kzalloc_obj(*meta, GFP_KERNEL);
+ if (!meta)
+ return -ENOMEM;
+ list_add_tail(&meta->l, &knod_prog->post_insns);
+ }
+
+ while (insn_idx % 256) {
+ if (meta->amdgpu_insns >= KNOD_META_INSNS) {
+ meta = kzalloc_obj(*meta, GFP_KERNEL);
+ if (!meta)
+ return -ENOMEM;
+ list_add_tail(&meta->l, &knod_prog->post_insns);
+ }
+ knod_emit(priv, meta, s_code_end);
+ insn_idx +=
+ meta->amdgpu_insn[meta->amdgpu_insns - 1].size /
+ 4;
+ }
+ }
+
+ return 0;
+}
+
+static int knod_bpf_translate(struct bpf_prog *prog)
+{
+ struct knod_prog *knod_prog = prog->aux->offload->dev_priv;
+ struct knod_dev *knodev = knod_prog->knodev;
+ int ret;
+
+ knod_bpf_map_setup(prog);
+ ret = knod_bpf_jit(knodev, knod_prog);
+ if (ret < 0) {
+ pr_err("knod: failed to JIT: %d\n", ret);
+ return ret;
+ }
+
+ knod_setup_bpf_prog(prog);
+
+ return 0;
+}
+
+static void knod_bpf_destroy_prog(struct bpf_prog *prog)
+{
+ struct knod_prog *knod_prog = prog->aux->offload->dev_priv;
+ struct knod_dev *knodev = knod_prog->knodev;
+ struct knod_bpf_priv *priv = knodev->accel->xdp.priv;
+
+ /*
+ * Normally the prog was already uninstalled (offload with a NULL prog
+ * flipped back to pass). Guard the abnormal path where the prog is
+ * freed while still tracked: flip to pass first so the worker stops
+ * dispatching this code. The compiled code lives in a kernel slot and
+ * is no longer read once we flip away; knod_prog is CPU-only IR the GPU
+ * never touches, so it is safe to free synchronously.
+ */
+ if (priv && READ_ONCE(priv->prog) == prog) {
+ WRITE_ONCE(priv->prog, NULL);
+ knod_bpf_reload_pass(knodev);
+ }
+ knod_prog_free(knod_prog);
+}
+
+static const struct bpf_prog_offload_ops knod_bpf_dev_ops = {
+ .insn_hook = knod_bpf_verify_insn,
+ .finalize = knod_bpf_finalize,
+ .prepare = knod_bpf_verifier_prep,
+ .translate = knod_bpf_translate,
+ .destroy = knod_bpf_destroy_prog,
+};
+
+static int knod_bpf_setup_prog_hw_checks(struct knod_dev *knodev,
+ struct netdev_bpf *bpf)
+{
+ if (!bpf->prog)
+ return 0;
+
+ return 0;
+}
+
+static int knod_bpf_map_get_next_key(struct bpf_offloaded_map *offmap,
+ void *key, void *next_key)
+{
+ unsigned int *nkey = (unsigned int *)next_key;
+ unsigned int *_key = (unsigned int *)key;
+
+ if (offmap->map.map_type == BPF_MAP_TYPE_ARRAY ||
+ offmap->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
+ if (key == NULL)
+ *nkey = 0;
+ else
+ *nkey = (*_key) + 1;
+
+ if (*nkey >= offmap->map.max_entries)
+ return -ENOENT;
+ } else if (offmap->map.map_type == BPF_MAP_TYPE_HASH) {
+ if (key == NULL)
+ return knod_bpf_map_hash_get_first_key(offmap,
+ next_key);
+ else
+ return knod_bpf_map_hash_get_next_key(offmap, key,
+ nkey);
+ }
+
+ return 0;
+}
+
+static int knod_bpf_map_lookup_elem(struct bpf_offloaded_map *offmap,
+ void *key, void *value)
+{
+ return __knod_bpf_map_lookup_elem(offmap, key, value);
+}
+
+static int knod_bpf_map_update_elem(struct bpf_offloaded_map *offmap,
+ void *key, void *value, u64 flags)
+{
+ return __knod_bpf_map_update_elem(offmap, key, value, flags);
+}
+
+static int knod_bpf_map_delete_elem(struct bpf_offloaded_map *offmap, void *key)
+{
+ return __knod_bpf_map_delete_elem(offmap, key);
+}
+
+static const struct bpf_map_dev_ops knod_bpf_map_ops = {
+ .map_get_next_key = knod_bpf_map_get_next_key,
+ .map_lookup_elem = knod_bpf_map_lookup_elem,
+ .map_update_elem = knod_bpf_map_update_elem,
+ .map_delete_elem = knod_bpf_map_delete_elem,
+};
+
+static int knod_bpf_map_alloc(struct knod_dev *knodev,
+ struct bpf_offloaded_map *offmap)
+{
+ int err;
+
+ if (offmap->map.map_type != BPF_MAP_TYPE_ARRAY &&
+ offmap->map.map_type != BPF_MAP_TYPE_HASH &&
+ offmap->map.map_type != BPF_MAP_TYPE_PERCPU_ARRAY) {
+ knod_jit_dbg(" unsupported map type: %d\n",
+ offmap->map.map_type);
+ return -EOPNOTSUPP;
+ }
+
+ err = __knod_bpf_map_alloc(knodev, offmap);
+ if (err) {
+ knod_jit_dbg(" err = %d\n", err);
+ return err;
+ }
+
+ offmap->dev_ops = &knod_bpf_map_ops;
+ return 0;
+}
+
+static int knod_bpf_xdp_install(struct knod_dev *knodev,
+ struct netdev_bpf *bpf)
+{
+ int err = 0;
+
+ ASSERT_RTNL();
+
+ switch (bpf->command) {
+ case XDP_SETUP_PROG:
+ WARN_ON_ONCE(1);
+ break;
+ case XDP_SETUP_PROG_HW:
+ err = knod_bpf_setup_prog_hw_checks(knodev, bpf);
+ if (err)
+ return err;
+
+ err = knod_bpf_xdp_set_prog(knodev, bpf);
+ break;
+ case BPF_OFFLOAD_MAP_ALLOC:
+ err = knod_bpf_map_alloc(knodev, bpf->offmap);
+ break;
+ case BPF_OFFLOAD_MAP_FREE:
+ knod_bpf_map_free(knodev, bpf->offmap);
+ break;
+ default:
+ knod_jit_dbg(" bpf->command = %d\n", bpf->command);
+ err = -EINVAL;
+ break;
+ }
+
+ return err;
+}
+
+static inline int bpf_debugfs_insn(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ struct seq_file *m,
+ int insn_idx)
+{
+ struct amdgcn_insn *insn = &meta->amdgpu_insn[insn_idx];
+
+ if (priv->isa_version == 10)
+ gfx10_debugfs_insn(insn, m);
+ else if (priv->isa_version == 9)
+ gfx9_debugfs_insn(insn, m);
+ else
+ WARN_ON_ONCE(1);
+
+ return insn->size;
+}
+
+/*
+ * Print one disassembled GPU instruction at @offset, then drop the disasm's
+ * trailing newline and append @tag as a right-hand comment aligned to a fixed
+ * column (tabs expand to 8) so the origin lines up regardless of mnemonic
+ * width. Returns the instruction size in dwords.
+ */
+static int bpf_debugfs_insn_tagged(struct knod_bpf_priv *priv,
+ struct knod_insn_meta *meta,
+ struct seq_file *m, int j,
+ int offset, const char *tag)
+{
+ size_t col, p, line_start = m->count;
+ int sz;
+
+ seq_printf(m, "%d:\t", offset);
+ sz = bpf_debugfs_insn(priv, meta, m, j);
+ if (seq_has_overflowed(m))
+ return sz;
+
+ if (m->count > line_start && m->buf[m->count - 1] == '\n')
+ m->count--;
+ col = 0;
+ for (p = line_start; p < m->count; p++)
+ col = m->buf[p] == '\t' ? (col + 8) & ~(size_t)7 : col + 1;
+ while (col < 96) {
+ seq_putc(m, ' ');
+ col++;
+ }
+ seq_printf(m, " ; %s\n", tag);
+
+ return sz;
+}
+
+/*
+ * Print the instructions a second time, re-sorted into BPF source order so the
+ * dump reads like the program. The offsets are the real (reordered) GPU
+ * offsets, so they appear out of sequence - that shows where the reorder
+ * placed each block. Synthetic jumps have no BPF source insn and are last.
+ */
+static void bpf_insn_show_bpf_order(struct knod_bpf_priv *priv,
+ struct seq_file *m)
+{
+ struct knod_insn_meta *meta;
+ int idx, max_idx = -1, off2, i;
+ bool synth_hdr = false;
+ char tag[24];
+
+ seq_puts(m, "===[INSTRUCTIONS (bpf order)]===\n");
+
+ list_for_each_entry(meta, &priv->knod_prog->insns, l)
+ if (meta->bpf_insn_idx > max_idx)
+ max_idx = meta->bpf_insn_idx;
+
+ for (idx = 0; idx <= max_idx; idx++) {
+ list_for_each_entry(meta, &priv->knod_prog->insns, l) {
+ if (meta->bpf_insn_idx != idx || !meta->amdgpu_insns)
+ continue;
+ scnprintf(tag, sizeof(tag), "bpf#%d", idx);
+ off2 = meta->amdgpu_insn_idx;
+ for (i = 0; i < meta->amdgpu_insns; i++)
+ off2 += bpf_debugfs_insn_tagged(priv, meta, m,
+ i, off2, tag);
+ }
+ }
+
+ list_for_each_entry(meta, &priv->knod_prog->insns, l) {
+ if (meta->bpf_insn_idx >= 0 || !meta->amdgpu_insns)
+ continue;
+ if (!synth_hdr) {
+ seq_puts(m, " [synthetic jumps]\n");
+ synth_hdr = true;
+ }
+ scnprintf(tag, sizeof(tag), "synth JA->#%d",
+ meta->jmp_dst ? meta->jmp_dst->bpf_insn_idx : -1);
+ off2 = meta->amdgpu_insn_idx;
+ for (i = 0; i < meta->amdgpu_insns; i++)
+ off2 += bpf_debugfs_insn_tagged(priv, meta, m,
+ i, off2, tag);
+ }
+}
+
+static int bpf_insn_show(struct seq_file *m, void *v)
+{
+ struct knod_bpf_priv *priv = (struct knod_bpf_priv *)m->private;
+ struct knod_insn_meta *meta;
+ struct knod_prog *kp;
+ int i, insn_idx = 0;
+ bool have_prog;
+
+ if (!priv)
+ return 0;
+
+ /*
+ * Show the kernel the GPU actually dispatches: the XDP prog when one is
+ * attached, otherwise the retained pass-through kernel.
+ */
+ have_prog = READ_ONCE(priv->prog);
+ if (have_prog) {
+ kp = priv->knod_prog;
+ } else {
+ kp = priv->pass_knod_prog;
+ seq_puts(m, "no XDP prog attached -- pass-through kernel:\n");
+ }
+ if (!kp)
+ return 0;
+
+ seq_puts(m, "===[PROLOGUE]===\n");
+ list_for_each_entry(meta, &kp->pre_insns, l) {
+ for (i = 0; i < meta->amdgpu_insns; i++) {
+ seq_printf(m, "%d:\t", insn_idx);
+ insn_idx += bpf_debugfs_insn(priv, meta, m, i);
+ }
+ }
+
+ /* Emission (RPO) order - the actual GPU layout. Each line is tagged
+ * with its origin BPF insn since the reorder makes this differ from the
+ * BPF byte order; synthetic jumps inserted by the reorder have none.
+ */
+ seq_puts(m, "===[INSTRUCTIONS]===\n");
+ list_for_each_entry(meta, &kp->insns, l) {
+ char tag[24];
+
+ if (meta->bpf_insn_idx < 0)
+ scnprintf(tag, sizeof(tag), "synth JA->#%d",
+ meta->jmp_dst ?
+ meta->jmp_dst->bpf_insn_idx : -1);
+ else
+ scnprintf(tag, sizeof(tag), "bpf#%d",
+ meta->bpf_insn_idx);
+
+ for (i = 0; i < meta->amdgpu_insns; i++)
+ insn_idx += bpf_debugfs_insn_tagged(priv, meta, m, i,
+ insn_idx, tag);
+ }
+
+ seq_puts(m, "===[EPILOG]===\n");
+ list_for_each_entry(meta, &kp->post_insns, l) {
+ for (i = 0; i < meta->amdgpu_insns; i++) {
+ seq_printf(m, "%d:\t", insn_idx);
+ insn_idx += bpf_debugfs_insn(priv, meta, m, i);
+ }
+ }
+
+ if (have_prog)
+ bpf_insn_show_bpf_order(priv, m);
+
+ return 0;
+}
+
+static int bpf_insn_open(struct inode *inode, struct file *file)
+{
+ return single_open(file, bpf_insn_show, inode->i_private);
+}
+
+static const struct file_operations bpf_insn_fops = {
+ .owner = THIS_MODULE,
+ .open = bpf_insn_open,
+ .read = seq_read,
+ .llseek = seq_lseek,
+ .release = single_release,
+};
+
+static const char *knod_branch_type_str(enum knod_branch_type type)
+{
+ switch (type) {
+ case KNOD_BR_NONE: return "NONE";
+ case KNOD_BR_DIRECT_EXIT: return "DIRECT_EXIT";
+ case KNOD_BR_FORWARD_SKIP: return "FORWARD_SKIP";
+ case KNOD_BR_FORWARD_GOTO: return "FORWARD_GOTO";
+ default: return "UNKNOWN";
+ }
+}
+
+static int bpf_cfg_show(struct seq_file *m, void *v)
+{
+ struct knod_bpf_priv *priv = (struct knod_bpf_priv *)m->private;
+ struct knod_insn_meta *meta;
+
+ if (!priv || !priv->knod_prog)
+ return 0;
+
+ seq_puts(m, "===[STRUCTURIZED CFG]===\n");
+ seq_printf(m, "exec_save_pairs_used: %u\n",
+ priv->knod_prog->exec_save_pairs_used);
+ seq_printf(m, "done_mask: s[%d:%d]\n",
+ priv->knod_prog->done_mask_sreg,
+ priv->knod_prog->done_mask_sreg + 1);
+ seq_printf(m, "initial_exec: s[%d:%d]\n",
+ priv->knod_prog->initial_exec_sreg,
+ priv->knod_prog->initial_exec_sreg + 1);
+ seq_puts(m, "\n");
+
+ seq_printf(m, "%-6s %-8s %-14s %-10s %-10s %-8s\n",
+ "bpf#", "opcode", "branch_type", "exec_save", "merge_pt",
+ "is_merge");
+
+ list_for_each_entry(meta, &priv->knod_prog->insns, l) {
+ bool is_jmp = is_mbpf_jmp(meta);
+
+ if (!is_jmp && !meta->is_merge_point)
+ continue;
+
+ seq_printf(m, "%-6d 0x%02x ",
+ meta->bpf_insn_idx, meta->insn.code);
+
+ if (meta->branch_type != KNOD_BR_NONE) {
+ seq_printf(m, "%-14s s[%d:%d] ",
+ knod_branch_type_str(meta->branch_type),
+ meta->exec_save_sreg,
+ meta->exec_save_sreg + 1);
+ if (meta->merge_point)
+ seq_printf(m, "%-10d ",
+ meta->merge_point->bpf_insn_idx);
+ else
+ seq_printf(m, "%-10s ", "-");
+ } else if (is_jmp) {
+ seq_printf(m, "%-14s %-10s %-10s ",
+ knod_branch_type_str(KNOD_BR_NONE),
+ "-", "-");
+ } else {
+ seq_printf(m, "%-14s %-10s %-10s ",
+ "", "", "");
+ }
+
+ if (meta->is_merge_point) {
+ struct knod_insn_meta *br;
+
+ seq_puts(m, "YES restore:");
+ list_for_each_entry(br, &priv->knod_prog->insns, l) {
+ if ((br->branch_type == KNOD_BR_FORWARD_SKIP ||
+ br->branch_type == KNOD_BR_FORWARD_GOTO) &&
+ br->merge_point == meta)
+ seq_printf(m, " s[%d:%d](from bpf#%d)",
+ br->exec_save_sreg,
+ br->exec_save_sreg + 1,
+ br->bpf_insn_idx);
+ }
+ seq_puts(m, "\n");
+ } else {
+ seq_puts(m, "\n");
+ }
+ }
+
+ /* Basic-block CFG from the reorder analysis (origin BPF order). */
+ if (priv->knod_prog->bbs) {
+ struct knod_bb *bbs = priv->knod_prog->bbs;
+ int nb = priv->knod_prog->n_bbs;
+ int k, s;
+
+ seq_printf(m, "\n[BASIC BLOCKS] %d blocks, %d back-edge(s) -> %s\n",
+ nb, priv->knod_prog->n_back,
+ priv->knod_prog->n_back ? "HAS LOOP" : "DAG");
+
+ for (k = 0; k < nb; k++) {
+ struct knod_bb *bb = &bbs[k];
+
+ seq_printf(m, "BB%-3d bpf#%d..#%d rpo=%d idom=#%d succ={",
+ k, bb->leader->bpf_insn_idx,
+ bb->last->bpf_insn_idx, bb->rpo,
+ bb->idom ?
+ bb->idom->leader->bpf_insn_idx : -1);
+ for (s = 0; s < bb->n_succ; s++)
+ seq_printf(m, "%s#%d", s ? "," : "",
+ bb->succ[s]->leader->bpf_insn_idx);
+ seq_printf(m, "}%s\n",
+ bb->loop_header ? " LOOP_HDR" : "");
+ }
+ }
+
+ return 0;
+}
+
+DEFINE_SHOW_ATTRIBUTE(bpf_cfg);
+
+static int knod_stats_show(struct seq_file *s, void *unused)
+{
+ struct knod_bpf_priv *priv = s->private;
+ u64 p50 = 0, p99 = 0, p999 = 0, acc;
+ struct knod_bpf_stats *stats;
+ u64 ccnt, dcnt;
+ int i;
+
+ stats = &priv->stats;
+ ccnt = stats->completion_count;
+ dcnt = stats->dispatch_count;
+ seq_printf(s, "enabled: %s\n",
+ static_branch_unlikely(&knod_stats_key) ? "yes" : "no");
+
+ seq_puts(s, "\n--- dispatch ---\n");
+ seq_printf(s, "count: %llu\n", dcnt);
+ seq_printf(s, "avg_ns: %llu\n",
+ dcnt ? stats->dispatch_total_ns / dcnt : 0);
+ seq_printf(s, "max_ns: %llu\n", stats->dispatch_max_ns);
+ seq_printf(s, "backlogs_avg: %llu\n",
+ dcnt ? stats->backlogs_total / dcnt : 0);
+
+ seq_puts(s, "\nbacklogs histogram:\n");
+ for (i = 0; i < KNOD_BL_BUCKETS; i++)
+ seq_printf(s, " %-10s %llu\n",
+ bl_labels[i], stats->backlogs_hist[i]);
+
+ seq_puts(s, "\n--- completion ---\n");
+ seq_printf(s, "count: %llu\n", ccnt);
+ seq_printf(s, "avg_ns: %llu\n",
+ ccnt ? stats->completion_total_ns / ccnt : 0);
+ seq_printf(s, "max_ns: %llu\n",
+ stats->completion_max_ns);
+
+ seq_puts(s, "\nlatency histogram:\n");
+ for (i = 0; i < KNOD_LAT_BUCKETS; i++)
+ seq_printf(s, " %-10s %llu\n",
+ lat_labels[i], stats->completion_hist[i]);
+
+ if (ccnt) {
+ acc = 0;
+ for (i = 0; i < KNOD_LAT_BUCKETS; i++) {
+ acc += stats->completion_hist[i];
+ if (!p50 && acc * 1000 >= ccnt * 500)
+ p50 = i;
+ if (!p99 && acc * 1000 >= ccnt * 990)
+ p99 = i;
+ if (!p999 && acc * 1000 >= ccnt * 999)
+ p999 = i;
+ }
+ seq_printf(s, "\np50: %s\n", lat_labels[p50]);
+ seq_printf(s, "p99: %s\n", lat_labels[p99]);
+ seq_printf(s, "p999: %s\n", lat_labels[p999]);
+ }
+
+ seq_puts(s, "\n--- decode_act ---\n");
+ seq_printf(s, "count: %llu\n", stats->decode_act_count);
+ seq_printf(s, "avg_ns: %llu\n",
+ stats->decode_act_count ?
+ stats->decode_act_total_ns / stats->decode_act_count : 0);
+ seq_printf(s, "max_ns: %llu\n", stats->decode_act_max_ns);
+
+ return 0;
+}
+
+DEFINE_SHOW_ATTRIBUTE(knod_stats);
+
+static ssize_t knod_stats_enable_write(struct file *file,
+ const char __user *buf,
+ size_t count, loff_t *ppos)
+{
+ bool val;
+
+ if (kstrtobool_from_user(buf, count, &val))
+ return -EINVAL;
+
+ if (val)
+ static_branch_enable(&knod_stats_key);
+ else
+ static_branch_disable(&knod_stats_key);
+
+ return count;
+}
+
+static ssize_t knod_stats_enable_read(struct file *file,
+ char __user *buf,
+ size_t count, loff_t *ppos)
+{
+ char tmp[4];
+ int len;
+
+ len = scnprintf(tmp, sizeof(tmp), "%d\n",
+ static_branch_unlikely(&knod_stats_key) ? 1 : 0);
+
+ return simple_read_from_buffer(buf, count, ppos, tmp, len);
+}
+
+static const struct file_operations knod_stats_enable_fops = {
+ .owner = THIS_MODULE,
+ .read = knod_stats_enable_read,
+ .write = knod_stats_enable_write,
+};
+
+static ssize_t knod_stats_reset_write(struct file *file,
+ const char __user *buf,
+ size_t count, loff_t *ppos)
+{
+ struct knod_bpf_priv *priv = file->private_data;
+
+ memset(&priv->stats, 0, sizeof(priv->stats));
+ return count;
+}
+
+static const struct file_operations knod_stats_reset_fops = {
+ .owner = THIS_MODULE,
+ .open = simple_open,
+ .write = knod_stats_reset_write,
+};
+
+static int knod_debugfs_init(struct knod_bpf_priv *priv)
+{
+ struct dentry *dir = priv->knod->debug_dir;
+ struct dentry *bpf_dir;
+
+ if (!dir)
+ return -ENOENT;
+
+ bpf_dir = debugfs_create_dir("bpf", dir);
+ if (IS_ERR(bpf_dir))
+ return PTR_ERR(bpf_dir);
+
+ priv->debug_dir = bpf_dir;
+
+ debugfs_create_file("insn", 0644,
+ bpf_dir, priv, &bpf_insn_fops);
+ debugfs_create_file("cfg", 0444, bpf_dir, priv,
+ &bpf_cfg_fops);
+ debugfs_create_file("stats", 0444, bpf_dir, priv,
+ &knod_stats_fops);
+ debugfs_create_file("stats_enable", 0644, bpf_dir, priv,
+ &knod_stats_enable_fops);
+ debugfs_create_file("stats_reset", 0200, bpf_dir, priv,
+ &knod_stats_reset_fops);
+ debugfs_create_bool("poll_mode", 0644, bpf_dir, &knod_bpf_poll_mode);
+ debugfs_create_u32("dispatch_delay_us", 0644, bpf_dir,
+ &knod_bpf_dispatch_delay_us);
+
+ return 0;
+}
+
+static void knod_debugfs_cleanup(struct knod_bpf_priv *priv)
+{
+ if (!priv->debug_dir)
+ return;
+
+ debugfs_remove_recursive(priv->debug_dir);
+ priv->debug_dir = NULL;
+}
+
+/* Called when attached or module loading time */
+/* attach: allocate the permanent per-attach priv struct. */
+static int knod_accel_xdp_init(struct knod_dev *knodev)
+{
+ struct knod_accel *accel = knodev->accel;
+ struct knod_bpf_priv *priv;
+
+ priv = __knod_accel_xdp_init(accel, knodev);
+ if (IS_ERR(priv))
+ return PTR_ERR(priv);
+ return 0;
+}
+
+/* detach: free the permanent priv struct. */
+static void knod_accel_xdp_exit(struct knod_dev *knodev)
+{
+ struct knod_accel *accel = knodev->accel;
+ struct knod_bpf_priv *priv = accel->xdp.priv;
+
+ __knod_accel_xdp_exit(accel, priv);
+}
+
+/*
+ * Feature select, phase B: register the BPF offload device so user XDP
+ * progs/maps can bind to it. Called after ->activate() set up the GPU
+ * buffers, while xdp_ops already points at the BPF ops.
+ */
+static int knod_bpf_offload_init(struct knod_dev *knodev)
+{
+ struct knod_accel *accel = knodev->accel;
+ struct knod_bpf_priv *priv = accel->xdp.priv;
+ struct bpf_offload_dev *bpf_dev;
+ int err;
+
+ bpf_dev = bpf_offload_dev_create(&knod_bpf_dev_ops, priv);
+ err = PTR_ERR_OR_ZERO(bpf_dev);
+ if (err)
+ return err;
+ err = bpf_offload_dev_netdev_register(bpf_dev, knodev->netdev);
+ if (err) {
+ bpf_offload_dev_destroy(bpf_dev);
+ return err;
+ }
+ knod_debugfs_init(priv);
+ accel->xdp.bpf_dev = bpf_dev;
+ return 0;
+}
+
+/*
+ * Feature deselect, phase 1: unregister the BPF offload device. This
+ * force-frees any user XDP progs/maps still bound; the map-free ndo is
+ * routed back through accel_ops.xdp_ops->xdp_install, so the caller keeps
+ * xdp_ops pointed at the BPF ops until this returns.
+ */
+static void knod_bpf_offload_uninit(struct knod_dev *knodev)
+{
+ struct knod_accel *accel = knodev->accel;
+ struct knod_bpf_priv *priv = accel->xdp.priv;
+
+ knod_debugfs_cleanup(priv);
+ bpf_offload_dev_netdev_unregister(accel->xdp.bpf_dev, knodev->netdev);
+ bpf_offload_dev_destroy(accel->xdp.bpf_dev);
+ accel->xdp.bpf_dev = NULL;
+}
+
+struct knod_accel_xdp_ops accel_xdp_ops = {
+ /* attach/detach: permanent priv struct */
+ .init = &knod_accel_xdp_init,
+ .exit = &knod_accel_xdp_exit,
+ /* feature select: GPU compute buffers (A) + offload dev (B) */
+ .activate = &knod_bpf_activate,
+ .deactivate = &knod_bpf_deactivate,
+ .busy = &knod_bpf_busy,
+ .xdp_offload_init = &knod_bpf_offload_init,
+ .xdp_offload_uninit = &knod_bpf_offload_uninit,
+ /* interface up/down (or feature switch): worker + GPU drain */
+ .start = &knod_bpf_start,
+ .stop = &knod_bpf_stop,
+ .xdp_install = &knod_bpf_xdp_install,
+};
+
+static int __init knod_bpf_init_module(void)
+{
+ pr_info("knod-bpf module load\n");
+
+ /* knod_accel_xdp_register() already calls xdp_ops->init() on every
+ * registered accel, so a second per-accel init loop here would just
+ * re-create the "bpf" debugfs dir ("already exists" warning) and leak
+ * a duplicate offload dev.
+ */
+ knod_dev_lock();
+ knod_accel_xdp_register(&accel_xdp_ops);
+ knod_dev_unlock();
+
+ return 0;
+}
+late_initcall(knod_bpf_init_module);
+
+static void __exit knod_bpf_cleanup_module(void)
+{
+ struct knod_bpf_priv *priv, *tmp;
+ struct knod_accel *accel;
+
+ rtnl_lock();
+ knod_dev_lock();
+ list_for_each_entry_safe(priv, tmp, &priv_list, list) {
+ accel = priv->accel;
+ if (accel->knodev)
+ accel_xdp_ops.exit(accel->knodev);
+ }
+ knod_accel_xdp_unregister();
+ knod_dev_unlock();
+ rtnl_unlock();
+ pr_info("knod-bpf module unload\n");
+}
+module_exit(knod_bpf_cleanup_module);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Taehee Yoo <ap420073@gmail.com>");
+MODULE_DESCRIPTION("AMDGPU BPF offload backend");
+MODULE_VERSION("multi-aql");
diff --git a/drivers/gpu/drm/amd/amdkfd/knod/knod_bpf.h b/drivers/gpu/drm/amd/amdkfd/knod/knod_bpf.h
new file mode 100644
index 000000000000..de6df06c4f2f
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdkfd/knod/knod_bpf.h
@@ -0,0 +1,597 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/* Copyright (c) 2021 Taehee Yoo <ap420073@gmail.com>
+ * Copyright (c) 2021 Hoyeon Lee <hoyeon.rhee@gmail.com>
+ */
+
+#ifndef KFD_BPF_H_INCLUDED
+#define KFD_BPF_H_INCLUDED
+
+#include <uapi/linux/bpf.h>
+#include <net/xdp.h>
+#include <net/netmem.h>
+#include <net/netlink.h>
+#include <net/page_pool/helpers.h>
+#include <net/ip.h>
+#include <net/net_namespace.h>
+#include <net/gro_cells.h>
+#include <net/rtnetlink.h>
+#include <net/protocol.h>
+#include <net/netns/generic.h>
+#include <net/xdp.h>
+#include <net/netdev_lock.h>
+#include <net/spsc_ring.h>
+#include <linux/bpf.h>
+#include <linux/bpf_verifier.h>
+#include <linux/kthread.h>
+#include <linux/sched.h>
+#include <linux/skbuff.h>
+#include <linux/net.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/etherdevice.h>
+#include <linux/hash.h>
+#include <linux/netdevice.h>
+#include <linux/types.h>
+#include <linux/bpf.h>
+#include <linux/bpf_verifier.h>
+#include <linux/debugfs.h>
+#include <linux/kernel.h>
+#include <linux/mutex.h>
+#include <linux/rtnetlink.h>
+#include <linux/workqueue.h>
+#include <linux/ktime.h>
+#include <linux/static_key.h>
+#include "knod_amdgpu_insn.h"
+#include "../../../../../../net/core/devmem.h"
+#include "../amdgpu/amdgpu_vm.h"
+#include "knod_bpf.h"
+#include "kfd_knod.h"
+
+#define KNOD_BPF_BACKLOGS_MAX 65536
+#define KNOD_BPF_INFLIGHT 3 /* triple-buffered dispatches */
+#define KNOD_BPF_WORKGROUPS_DEFAULT 256
+#define KNOD_BPF_WORKGROUPS_MIN 64
+#define KNOD_BPF_WORKGROUPS_MAX 256
+#define KNOD_BPF_EXPIRE_DEFAULT 10
+#define KNOD_BPF_EXPIRE_MIN 1
+#define KNOD_BPF_EXPIRE_MAX 1000
+#define QUEUE_SIZE_DGPU 8192
+#define QUEUE_SIZE_IGPU 2048
+#define KNOD_MAX_BDS (KNOD_BPF_BACKLOGS_MAX / KNOD_SPSC_MAX)
+
+#define MAX_KEY_SIZE 64 /* 64Bytes */
+#define MAX_PACKET_CACHE 256 /* 256Bytes */
+
+#define knod_prog_first_meta(knod_prog) \
+ list_first_entry(&(knod_prog)->insns, struct knod_insn_meta, l)
+#define knod_prog_last_meta(knod_prog) \
+ list_last_entry(&(knod_prog)->insns, struct knod_insn_meta, l)
+#define knod_prog_pre_last_meta(knod_prog) \
+ list_last_entry(&(knod_prog)->pre_insns, struct knod_insn_meta, l)
+#define knod_meta_next(meta) list_next_entry(meta, l)
+#define knod_meta_prev(meta) list_prev_entry(meta, l)
+
+#define knod_for_each_insn_walk2(knod_prog, pos, next) \
+ for (pos = list_first_entry(&(knod_prog)->insns, typeof(*pos), l),\
+ next = list_next_entry(pos, l); \
+ &(knod_prog)->insns != &pos->l && \
+ &(knod_prog)->insns != &next->l; \
+ pos = knod_meta_next(pos), \
+ next = knod_meta_next(pos))
+
+#define knod_for_each_insn_walk3(knod_prog, pos, next, next2) \
+ for (pos = list_first_entry(&(knod_prog)->insns, typeof(*pos), l),\
+ next = list_next_entry(pos, l), \
+ next2 = list_next_entry(next, l); \
+ &(knod_prog)->insns != &pos->l && \
+ &(knod_prog)->insns != &next->l && \
+ &(knod_prog)->insns != &next2->l; \
+ pos = knod_meta_next(pos), \
+ next = knod_meta_next(pos), \
+ next2 = knod_meta_next(next))
+
+struct xdp_md_obj {
+ u64 data;
+ u64 data_end;
+ u64 data_meta;
+ /* Below access go through struct xdp_rxq_info */
+ u64 ingress_ifindex; /* rxq->dev->ifindex */
+ u64 rx_queue_index; /* rxq->queue_index */
+
+ u64 egress_ifindex; /* txq->dev->ifindex */
+ u64 retval;
+};
+
+struct knod_bpf_subparam_obj {
+ struct xdp_md_obj ctx;
+};
+
+#define KNOD_BPF_HASH_NEXT_END 0x7FFFFFFFU
+#define KNOD_BPF_HASH_NEXT_DELETED 0x80000000U
+#define KNOD_BPF_HASH_NEXT_MASK 0x7FFFFFFFU
+
+struct knod_bpf_hash_elem_obj {
+ unsigned int next;
+ unsigned char kv[];
+};
+
+struct knod_bpf_map_hash_meta_obj {
+ unsigned int n_buckets;
+ unsigned int hashrnd;
+ unsigned int cur;
+ unsigned int elem_size;
+ void *q;
+ void *elems;
+ unsigned int gc_count;
+ void *gc_list;
+};
+
+struct knod_bpf_map_array_meta_obj {
+ u32 per_instance_size; /* value_size * max_entries (one instance) */
+ u32 n_instances; /* 1 for ARRAY, num_possible_cpus for PERCPU */
+};
+
+union knod_bpf_map_meta_obj {
+ struct knod_bpf_map_hash_meta_obj hmeta;
+ struct knod_bpf_map_array_meta_obj ameta;
+};
+
+struct knod_bpf_map_obj {
+ enum bpf_map_type map_type;
+ unsigned int key_size;
+ unsigned int value_size;
+ unsigned int max_entries;
+ unsigned int id;
+ unsigned long map_extra; /* any per-map-type extra fields */
+ unsigned int map_flags;
+ union knod_bpf_map_meta_obj meta;
+ int mutex;
+ unsigned char bucket[];
+};
+
+struct knod_bpf_map {
+ struct list_head list;
+ struct knod_mem *mem, *queue_mem, *hash_elems_mem, *gc_mem;
+ /* ptr to mem_k->kaddr */
+ struct knod_bpf_map_obj *knod_map_obj;
+ struct bpf_offloaded_map *offmap;
+ struct knod_bpf_priv *priv;
+};
+
+struct knod_bpf_queue_desc {
+ u64 pool_gaddr; /* SPSC pool GTT address for this queue */
+ u64 base_gaddr; /* dma-buf base address for this queue */
+ u32 count; /* number of packets from this queue */
+ /* was start_idx; kept for global_load_dwordx4 layout */
+ u32 _pad;
+ u32 ring_start; /* acquired cursor at peek time */
+ u32 ring_mask; /* capacity - 1 */
+};
+
+struct knod_bpf_param {
+ u32 nr_backlogs;
+ u32 nr_queues;
+ u32 spsc_stride;
+ u32 _pad0;
+ u64 ktime_ns; /* snapshot of ktime_get_ns() at dispatch */
+ u32 pass_count[KNOD_SPSC_MAX]; /* per-queue atomic XDP_PASS counter */
+ /* per-queue GTT pass_meta_buf GPU addr */
+ u64 pass_meta_buf_gaddr[KNOD_SPSC_MAX];
+ struct knod_bpf_queue_desc queues[KNOD_SPSC_MAX];
+ /* backlog indices of PASS packets */
+ u16 pass_indices[KNOD_BPF_BACKLOGS_MAX];
+ struct knod_bpf_subparam_obj sub[KNOD_BPF_BACKLOGS_MAX];
+};
+
+struct knod_packet {
+ union {
+ netmem_ref netmem;
+ void *kaddr;
+ };
+ u16 len;
+ u16 off;
+};
+
+/* Single Queue Worok */
+struct knod_bpf_work_sq {
+ struct list_head list;
+ struct knod_mem *param;
+ int queue_idx[KNOD_SPSC_MAX];
+ struct spsc_bd *bds[KNOD_BPF_BACKLOGS_MAX];
+ ktime_t dispatch_time;
+ s64 sigval;
+ unsigned long expire;
+ int backlogs;
+};
+
+struct knod_bpf_reg_state {
+ struct bpf_reg_state reg;
+ int stack_off;
+ int packet_off;
+ bool var_off;
+};
+
+/* Structurized CFG branch types */
+enum knod_branch_type {
+ KNOD_BR_NONE = 0, /* not a branch */
+ /* backward jump to exit: inline retval + done_mask update */
+ KNOD_BR_DIRECT_EXIT,
+ KNOD_BR_FORWARD_SKIP, /* forward jump: skip region via EXEC mask */
+ KNOD_BR_FORWARD_GOTO, /* forward jump crossing other branch scopes */
+};
+
+#define KNOD_META_INSNS 1024
+#define AMDGPU_INSN_SKIP -1
+struct knod_insn_meta {
+ struct bpf_insn insn;
+ short bpf_insn_idx;
+
+ struct amdgcn_insn amdgpu_insn[KNOD_META_INSNS];
+ u32 amdgpu_insn_idx;
+ u32 amdgpu_insns;
+
+ union {
+ /* pointer ops (ld/st/xadd) */
+ struct {
+ struct bpf_reg_state ptr;
+ struct bpf_insn *paired_st;
+ s16 ldst_gather_len;
+ bool ptr_not_const;
+ struct {
+ s16 range_start;
+ s16 range_end;
+ bool do_init;
+ } pkt_cache;
+ bool xadd_over_16bit;
+ bool xadd_maybe_16bit;
+ };
+ /* jump */
+ struct {
+ struct knod_insn_meta *jmp_dst;
+ bool jump_neg_op;
+ u32 num_insns_after_br; /* only for BPF-to-BPF calls */
+ /* structurized CFG */
+ enum knod_branch_type branch_type;
+ /* SGPR index for s_and_saveexec_b64 */
+ u8 exec_save_sreg;
+ /* where EXEC is restored */
+ struct knod_insn_meta *merge_point;
+ };
+ /* function calls */
+ struct {
+ u32 func_id;
+ struct bpf_reg_state arg1;
+ struct knod_bpf_reg_state arg2;
+ };
+ /* We are interested in range info for operands of ALU
+ * operations. For example, shift amount, multiplicand and
+ * multiplier etc.
+ */
+ struct {
+ u64 umin_src;
+ u64 umax_src;
+ u64 umin_dst;
+ u64 umax_dst;
+ };
+ };
+
+ struct knod_bpf_reg_state sreg;
+ struct knod_bpf_reg_state dreg;
+ struct knod_bpf_reg_state kreg;
+ struct knod_bpf_reg_state vreg;
+ unsigned int off;
+ unsigned short flags;
+ unsigned short subprog_idx;
+ bool is_merge_point; /* EXEC restore target */
+ u8 restore_sreg; /* SGPR to restore EXEC from at merge point */
+ int linear_idx; /* position in the (reordered) emission list */
+ struct list_head l;
+};
+
+/* Encode one GPU instruction at @meta's running slot and advance it.
+ * @meta->amdgpu_insns is both the cursor during emission and the final
+ * instruction count afterwards. @fn names a knod_amdgpu_insn.h encoder
+ * without its emit_ prefix (e.g. v_add32 for emit_v_add32); the macro
+ * pastes it back, so call sites read knod_emit(priv, meta, v_add32, ...).
+ */
+#define knod_emit(priv, meta, fn, ...) \
+ do { \
+ struct knod_insn_meta *__m = (meta); \
+ \
+ emit_##fn((priv)->isa_version, \
+ &__m->amdgpu_insn[__m->amdgpu_insns], \
+ ##__VA_ARGS__); \
+ debug_insn((priv)->isa_version, \
+ &__m->amdgpu_insn[__m->amdgpu_insns]); \
+ __m->amdgpu_insns++; \
+ } while (0)
+
+/* JIT debug/error trace: auto-prefix with "knod_jit <func>:<line>".
+ * knod_jit_dbg() is a pr_debug(), so it is off by default and toggled
+ * with dynamic debug; knod_jit_err() always fires.
+ */
+#define knod_jit_dbg(fmt, ...) \
+ pr_debug("knod_jit %s:%d" fmt, __func__, __LINE__, ##__VA_ARGS__)
+#define knod_jit_err(fmt, ...) \
+ pr_err("knod_jit %s:%d" fmt, __func__, __LINE__, ##__VA_ARGS__)
+
+#define BPF_SIZE_MASK 0x18
+
+struct knod_bb; /* basic-block CFG analysis (knod_bpf.c) */
+
+struct knod_prog {
+ struct knod *knod;
+ struct knod_dev *knodev;
+
+ u64 *prog;
+ unsigned int prog_len;
+ unsigned int __prog_alloc_len;
+ int max_stack_off;
+ int max_packet_off;
+
+ struct knod_insn_meta *meta;
+ enum bpf_prog_type type;
+ struct list_head pre_insns;
+ struct list_head post_insns;
+ struct list_head insns;
+ unsigned int n_insns;
+ unsigned int pre_n_insns;
+ int insn_idx;
+
+ /* Structurized CFG state */
+ /* GFX9: 34, GFX10: 32 (s[32:33] safe on RDNA) */
+ u8 done_mask_sreg;
+ u8 exec_save_base; /* GFX9: 36, GFX10: 34 */
+ /* in-bounds EXEC snapshot for verdict publish */
+ u8 initial_exec_sreg;
+ /* number of SGPR pairs allocated for EXEC saves */
+ u8 exec_save_pairs_used;
+ bool uses_adjust;
+
+ /* Basic-block CFG analysis, retained for the /bpf/cfg view. */
+ struct knod_bb *bbs;
+ int n_bbs;
+ int n_back;
+};
+
+#define KNOD_XDP_MEMCPY 0
+#define KNOD_XDP_PT 1
+#define KNOD_XDP_NETMEM 2
+#define KNOD_XDP_NONE 3
+#define KNOD_XDP_DEFAULT KNOD_XDP_PT
+
+#define KNOD_LAT_BUCKETS 10
+#define KNOD_BL_BUCKETS 8
+
+struct knod_bpf_stats {
+ u64 dispatch_total_ns;
+ u64 dispatch_count;
+ u64 dispatch_max_ns;
+
+ u64 completion_total_ns;
+ u64 completion_count;
+ u64 completion_max_ns;
+ u64 completion_hist[KNOD_LAT_BUCKETS];
+
+ u64 backlogs_total;
+ u64 backlogs_hist[KNOD_BL_BUCKETS];
+
+ u64 decode_act_total_ns;
+ u64 decode_act_count;
+ u64 decode_act_max_ns;
+};
+
+#define KNOD_PASS_SLOT_SIZE PAGE_SIZE
+
+/* pass_meta_buf slot header, written by the shader (offsetof used by the
+ * codegen). The host read path is gone now that PASS delivery goes via the
+ * NIC act handler + knod_d2h_copy; the shader still stores {len, src_addr}
+ * here pending removal of that store.
+ */
+struct knod_pass_slot_hdr {
+ u32 len; /* packet length */
+ u32 _pad;
+ u64 src_addr; /* VRAM source address (SDMA mode only) */
+};
+
+struct knod_bpf_priv {
+ struct list_head list;
+ struct knod *knod;
+ struct knod_accel *accel;
+ struct knod_dev *knodev;
+ struct net_device *dev;
+ struct knod_prog *knod_prog;
+ /* retained pass IR for debugfs insn dump */
+ struct knod_prog *pass_knod_prog;
+ struct bpf_prog *prog;
+ struct amdgpu_vm *vm;
+ u64 queue_base_gaddr[KNOD_SPSC_MAX];
+ struct knod_bpf_work_sq *inflight[KNOD_BPF_INFLIGHT];
+ unsigned int inflight_cnt;
+ ktime_t next_dispatch_time;
+ struct task_struct *worker_task;
+ struct list_head free_list_sqw;
+ struct mutex map_op_lock;
+ /* maps awaiting deferred free by the worker */
+ struct list_head dead_maps;
+ u32 maps_tick_skip;
+ struct dentry *debug_dir;
+ struct knod_bpf_stats stats;
+ void *prog_buf;
+ void *pass_prog_buf;
+ u32 pass_prog_size;
+ /* descriptor + live shader bytes per kernel slot */
+ u32 kernel_image_len[2];
+ /* knod->kernels[] slot the GPU dispatches */
+ int active_idx;
+ /* knod->kernels[] slot holding the pass kernel */
+ int pass_idx;
+ /*
+ * XDP_PASS shader-to-GTT metadata (shader-written; host read path
+ * removed). GTT metadata: shader-written headers.
+ */
+ struct knod_mem *pass_meta_buf;
+ u32 pass_pkts_per_queue; /* backlogs / nr_works */
+ /* batch size per queue */
+ int batch_size;
+ int nr_works;
+ int isa_version;
+ bool installing_kernel;
+ int start;
+};
+
+static inline u8 mbpf_class(const struct knod_insn_meta *meta)
+{
+ return BPF_CLASS(meta->insn.code);
+}
+
+static inline u8 mbpf_src(const struct knod_insn_meta *meta)
+{
+ return BPF_SRC(meta->insn.code);
+}
+
+static inline u8 mbpf_op(const struct knod_insn_meta *meta)
+{
+ return BPF_OP(meta->insn.code);
+}
+
+static inline u8 mbpf_mode(const struct knod_insn_meta *meta)
+{
+ return BPF_MODE(meta->insn.code);
+}
+
+static inline bool is_mbpf_alu(const struct knod_insn_meta *meta)
+{
+ return mbpf_class(meta) == BPF_ALU64 || mbpf_class(meta) == BPF_ALU;
+}
+
+static inline bool is_mbpf_load(const struct knod_insn_meta *meta)
+{
+ return (meta->insn.code & ~BPF_SIZE_MASK) == (BPF_LDX | BPF_MEM);
+}
+
+static inline bool is_mbpf_jmp32(const struct knod_insn_meta *meta)
+{
+ return mbpf_class(meta) == BPF_JMP32;
+}
+
+static inline bool is_mbpf_jmp64(const struct knod_insn_meta *meta)
+{
+ return mbpf_class(meta) == BPF_JMP;
+}
+
+static inline bool is_mbpf_jmp(const struct knod_insn_meta *meta)
+{
+ return is_mbpf_jmp32(meta) || is_mbpf_jmp64(meta);
+}
+
+static inline bool is_mbpf_store(const struct knod_insn_meta *meta)
+{
+ return (meta->insn.code & ~BPF_SIZE_MASK) == (BPF_STX | BPF_MEM);
+}
+
+static inline bool is_mbpf_load_pkt(const struct knod_insn_meta *meta)
+{
+ return is_mbpf_load(meta) && meta->ptr.type == PTR_TO_PACKET;
+}
+
+static inline bool is_mbpf_store_pkt(const struct knod_insn_meta *meta)
+{
+ return is_mbpf_store(meta) && meta->ptr.type == PTR_TO_PACKET;
+}
+
+static inline bool is_mbpf_classic_load(const struct knod_insn_meta *meta)
+{
+ u8 code = meta->insn.code;
+
+ return BPF_CLASS(code) == BPF_LD &&
+ (BPF_MODE(code) == BPF_ABS || BPF_MODE(code) == BPF_IND);
+}
+
+static inline bool is_mbpf_classic_store(const struct knod_insn_meta *meta)
+{
+ u8 code = meta->insn.code;
+
+ return BPF_CLASS(code) == BPF_ST && BPF_MODE(code) == BPF_MEM;
+}
+
+static inline bool is_mbpf_classic_store_pkt(const struct knod_insn_meta *meta)
+{
+ return is_mbpf_classic_store(meta) && meta->ptr.type == PTR_TO_PACKET;
+}
+
+static inline bool is_mbpf_atomic(const struct knod_insn_meta *meta)
+{
+ return (meta->insn.code & ~BPF_SIZE_MASK) == (BPF_STX | BPF_ATOMIC);
+}
+
+static inline bool is_mbpf_mul(const struct knod_insn_meta *meta)
+{
+ return is_mbpf_alu(meta) && mbpf_op(meta) == BPF_MUL;
+}
+
+static inline bool is_mbpf_div(const struct knod_insn_meta *meta)
+{
+ return is_mbpf_alu(meta) && mbpf_op(meta) == BPF_DIV;
+}
+
+static inline bool is_mbpf_mod(const struct knod_insn_meta *meta)
+{
+ return is_mbpf_alu(meta) && mbpf_op(meta) == BPF_MOD;
+}
+
+static inline bool is_mbpf_cond_jump(const struct knod_insn_meta *meta)
+{
+ u8 op;
+
+ if (is_mbpf_jmp32(meta))
+ return true;
+
+ if (!is_mbpf_jmp64(meta))
+ return false;
+
+ op = mbpf_op(meta);
+ return op != BPF_JA && op != BPF_EXIT && op != BPF_CALL;
+}
+
+static inline bool is_mbpf_helper_call(const struct knod_insn_meta *meta)
+{
+ struct bpf_insn insn = meta->insn;
+
+ return insn.code == (BPF_JMP | BPF_CALL) &&
+ insn.src_reg != BPF_PSEUDO_CALL;
+}
+
+static inline bool is_mbpf_pseudo_call(const struct knod_insn_meta *meta)
+{
+ struct bpf_insn insn = meta->insn;
+
+ return insn.code == (BPF_JMP | BPF_CALL) &&
+ insn.src_reg == BPF_PSEUDO_CALL;
+}
+
+static inline bool is_mbpf_map_call(const struct knod_insn_meta *meta)
+{
+ struct bpf_insn insn = meta->insn;
+
+ return insn.code == (BPF_JMP | BPF_CALL) && insn.imm <= 3;
+}
+
+#define STACK_FRAME_ALIGN 64
+
+#define FLAG_INSN_IS_JUMP_DST BIT(0)
+#define FLAG_INSN_IS_SUBPROG_START BIT(1)
+#define FLAG_INSN_PTR_CALLER_STACK_FRAME BIT(2)
+/* Instruction is pointless, noop even on its own */
+#define FLAG_INSN_SKIP_NOOP BIT(3)
+/* Instruction is optimized out based on preceding instructions */
+#define FLAG_INSN_SKIP_PREC_DEPENDENT BIT(4)
+/* Instruction is optimized by the verifier */
+#define FLAG_INSN_SKIP_VERIFIER_OPT BIT(5)
+/* Instruction needs to zero extend to high 32-bit */
+#define FLAG_INSN_DO_ZEXT BIT(6)
+
+#define FLAG_INSN_SKIP_MASK (FLAG_INSN_SKIP_NOOP | \
+ FLAG_INSN_SKIP_PREC_DEPENDENT | \
+ FLAG_INSN_SKIP_VERIFIER_OPT)
+#endif
--
2.43.0
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox