* [PATCH bpf 2/2] selftests/bpf: make sure signal interrupts BPF_PROG_TEST_RUN
From: Stanislav Fomichev @ 2019-02-12 23:42 UTC (permalink / raw)
To: netdev; +Cc: davem, ast, daniel, Stanislav Fomichev
In-Reply-To: <20190212234239.174386-1-sdf@google.com>
Simple test that I used to reproduce the issue in the previous commit:
Do BPF_PROG_TEST_RUN with max iterations, each program is 4096 simple
move instructions. File alarm in 0.1 second and check that
bpf_prog_test_run is interrupted (i.e. test doesn't hang).
Feel free to ignore it if you feel like that's just a one-off fix and it
doesn't require a test going forward.
Signed-off-by: Stanislav Fomichev <sdf@google.com>
---
tools/testing/selftests/bpf/test_progs.c | 44 ++++++++++++++++++++++++
1 file changed, 44 insertions(+)
diff --git a/tools/testing/selftests/bpf/test_progs.c b/tools/testing/selftests/bpf/test_progs.c
index 25f0083a9b2e..7842e3749b19 100644
--- a/tools/testing/selftests/bpf/test_progs.c
+++ b/tools/testing/selftests/bpf/test_progs.c
@@ -11,6 +11,7 @@
#include <assert.h>
#include <stdlib.h>
#include <time.h>
+#include <signal.h>
#include <linux/types.h>
typedef __u16 __sum16;
@@ -27,6 +28,7 @@ typedef __u16 __sum16;
#include <sys/ioctl.h>
#include <sys/wait.h>
#include <sys/types.h>
+#include <sys/time.h>
#include <fcntl.h>
#include <linux/bpf.h>
@@ -1912,6 +1914,47 @@ static void test_queue_stack_map(int type)
bpf_object__close(obj);
}
+static void sigalrm_handler(int s) {}
+static struct sigaction sigalrm_action = {
+ .sa_handler = sigalrm_handler,
+};
+
+static void test_signal_pending(void)
+{
+ struct bpf_insn prog[4096];
+ struct itimerval timeo = {
+ .it_value.tv_usec = 100000, /* 100ms */
+ };
+ __u32 duration, retval;
+ int prog_fd;
+ int err;
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(prog); i++)
+ prog[i] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 0);
+ prog[ARRAY_SIZE(prog) - 1] = BPF_EXIT_INSN();
+
+ prog_fd = bpf_load_program(BPF_PROG_TYPE_SOCKET_FILTER,
+ prog, ARRAY_SIZE(prog),
+ "GPL", 0, NULL, 0);
+ CHECK(prog_fd < 0, "test-run", "errno %d\n", errno);
+
+ err = sigaction(SIGALRM, &sigalrm_action, NULL);
+ CHECK(err, "test-run-signal-sigaction", "errno %d\n", errno);
+
+ err = setitimer(ITIMER_REAL, &timeo, NULL);
+ CHECK(err, "test-run-signal-timer", "errno %d\n", errno);
+
+ err = bpf_prog_test_run(prog_fd, 0xffffffff, &pkt_v4, sizeof(pkt_v4),
+ NULL, NULL, &retval, &duration);
+ CHECK(err != -1 || errno != EINTR || duration > 1000000000,
+ "test-run-signal-run",
+ "err %d errno %d retval %d\n",
+ err, errno, retval);
+
+ signal(SIGALRM, SIG_DFL);
+}
+
int main(void)
{
srand(time(NULL));
@@ -1939,6 +1982,7 @@ int main(void)
test_reference_tracking();
test_queue_stack_map(QUEUE);
test_queue_stack_map(STACK);
+ test_signal_pending();
printf("Summary: %d PASSED, %d FAILED\n", pass_cnt, error_cnt);
return error_cnt ? EXIT_FAILURE : EXIT_SUCCESS;
--
2.20.1.791.gb4d0f1c61a-goog
^ permalink raw reply related
* [PATCH bpf 1/2] bpf/test_run: fix unkillable BPF_PROG_TEST_RUN
From: Stanislav Fomichev @ 2019-02-12 23:42 UTC (permalink / raw)
To: netdev; +Cc: davem, ast, daniel, Stanislav Fomichev, syzbot
Syzbot found out that running BPF_PROG_TEST_RUN with repeat=0xffffffff
makes process unkillable. The problem is that when CONFIG_PREEMPT is
enabled, we never see need_resched() return true. This is due to the
fact that preempt_enable() (which we do in bpf_test_run_one on each
iteration) now handles resched if it's needed.
Let's disable preemption for the whole run, not per test. In this case
we can properly see whether resched is needed.
Let's also properly return -EINTR to the userspace in case of a signal
interrupt.
See recent discussion:
http://lore.kernel.org/netdev/CAH3MdRWHr4N8jei8jxDppXjmw-Nw=puNDLbu1dQOFQHxfU2onA@mail.gmail.com
I'll follow up with the same fix bpf_prog_test_run_flow_dissector in
bpf-next.
Reported-by: syzbot <syzkaller@googlegroups.com>
Signed-off-by: Stanislav Fomichev <sdf@google.com>
---
net/bpf/test_run.c | 45 ++++++++++++++++++++++++---------------------
1 file changed, 24 insertions(+), 21 deletions(-)
diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c
index fa2644d276ef..e31e1b20f7f4 100644
--- a/net/bpf/test_run.c
+++ b/net/bpf/test_run.c
@@ -13,27 +13,13 @@
#include <net/sock.h>
#include <net/tcp.h>
-static __always_inline u32 bpf_test_run_one(struct bpf_prog *prog, void *ctx,
- struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE])
-{
- u32 ret;
-
- preempt_disable();
- rcu_read_lock();
- bpf_cgroup_storage_set(storage);
- ret = BPF_PROG_RUN(prog, ctx);
- rcu_read_unlock();
- preempt_enable();
-
- return ret;
-}
-
-static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat, u32 *ret,
- u32 *time)
+static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat,
+ u32 *retval, u32 *time)
{
struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE] = { 0 };
enum bpf_cgroup_storage_type stype;
u64 time_start, time_spent = 0;
+ int ret = 0;
u32 i;
for_each_cgroup_storage_type(stype) {
@@ -48,25 +34,42 @@ static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat, u32 *ret,
if (!repeat)
repeat = 1;
+
+ rcu_read_lock();
+ preempt_disable();
time_start = ktime_get_ns();
for (i = 0; i < repeat; i++) {
- *ret = bpf_test_run_one(prog, ctx, storage);
+ bpf_cgroup_storage_set(storage);
+ *retval = BPF_PROG_RUN(prog, ctx);
+
+ if (signal_pending(current)) {
+ ret = -EINTR;
+ break;
+ }
+
if (need_resched()) {
- if (signal_pending(current))
- break;
time_spent += ktime_get_ns() - time_start;
+ preempt_enable();
+ rcu_read_unlock();
+
cond_resched();
+
+ rcu_read_lock();
+ preempt_disable();
time_start = ktime_get_ns();
}
}
time_spent += ktime_get_ns() - time_start;
+ preempt_enable();
+ rcu_read_unlock();
+
do_div(time_spent, repeat);
*time = time_spent > U32_MAX ? U32_MAX : (u32)time_spent;
for_each_cgroup_storage_type(stype)
bpf_cgroup_storage_free(storage[stype]);
- return 0;
+ return ret;
}
static int bpf_test_finish(const union bpf_attr *kattr,
--
2.20.1.791.gb4d0f1c61a-goog
^ permalink raw reply related
* [PATCH 01/13] net: bridge: Allow deying multicast snooping toggling
From: Florian Fainelli @ 2019-02-12 23:40 UTC (permalink / raw)
To: netdev; +Cc: davem, Florian Fainelli
In-Reply-To: <20190212234000.15796-1-f.fainelli@gmail.com>
Some Ethernet switches might not be able to support disabling multicast
flooding globally when e.g: several bridges span the same physical
device.
Introduce a new switchdev attribute:
SWITCHDEV_ATTR_ID_BRIDGE_MC_DISABLED_SUPPORT which is first queried as a
blocking call using switchdev_port_attr_get() and then allow
br_mc_disabled_update() to propagate that return value all the way down
to user-space.
Signed-off-by: Florian Fainelli <f.fainelli@gmail.com>
---
include/net/switchdev.h | 2 ++
net/bridge/br_multicast.c | 36 +++++++++++++++++++++++++++---------
2 files changed, 29 insertions(+), 9 deletions(-)
diff --git a/include/net/switchdev.h b/include/net/switchdev.h
index 5e87b54c5dc5..4531209e6a6a 100644
--- a/include/net/switchdev.h
+++ b/include/net/switchdev.h
@@ -50,6 +50,7 @@ enum switchdev_attr_id {
SWITCHDEV_ATTR_ID_BRIDGE_AGEING_TIME,
SWITCHDEV_ATTR_ID_BRIDGE_VLAN_FILTERING,
SWITCHDEV_ATTR_ID_BRIDGE_MC_DISABLED,
+ SWITCHDEV_ATTR_ID_BRIDGE_MC_DISABLED_SUPPORT,
SWITCHDEV_ATTR_ID_BRIDGE_MROUTER,
};
@@ -67,6 +68,7 @@ struct switchdev_attr {
clock_t ageing_time; /* BRIDGE_AGEING_TIME */
bool vlan_filtering; /* BRIDGE_VLAN_FILTERING */
bool mc_disabled; /* MC_DISABLED */
+ bool mc_disabled_support; /* MC_DISABLED_SUPPORT */
} u;
};
diff --git a/net/bridge/br_multicast.c b/net/bridge/br_multicast.c
index 4a048fd1cbea..97cc5f3ad20a 100644
--- a/net/bridge/br_multicast.c
+++ b/net/bridge/br_multicast.c
@@ -815,20 +815,31 @@ static void br_ip6_multicast_port_query_expired(struct timer_list *t)
}
#endif
-static void br_mc_disabled_update(struct net_device *dev, bool value)
+static int br_mc_disabled_update(struct net_device *dev, bool value)
{
struct switchdev_attr attr = {
.orig_dev = dev,
- .id = SWITCHDEV_ATTR_ID_BRIDGE_MC_DISABLED,
- .flags = SWITCHDEV_F_DEFER,
- .u.mc_disabled = !value,
+ .id = SWITCHDEV_ATTR_ID_BRIDGE_MC_DISABLED_SUPPORT,
+ .flags = SWITCHDEV_F_SKIP_EOPNOTSUPP,
+ .u.mc_disabled_support = !value,
};
+ int err;
+
+ err = switchdev_port_attr_get(dev, &attr);
+ if (err)
+ return err;
- switchdev_port_attr_set(dev, &attr);
+ attr.id = SWITCHDEV_ATTR_ID_BRIDGE_MC_DISABLED;
+ attr.flags |= SWITCHDEV_F_DEFER;
+ attr.u.mc_disabled = !value;
+
+ return switchdev_port_attr_set(dev, &attr);
}
int br_multicast_add_port(struct net_bridge_port *port)
{
+ int ret;
+
port->multicast_router = MDB_RTR_TYPE_TEMP_QUERY;
timer_setup(&port->multicast_router_timer,
@@ -839,8 +850,11 @@ int br_multicast_add_port(struct net_bridge_port *port)
timer_setup(&port->ip6_own_query.timer,
br_ip6_multicast_port_query_expired, 0);
#endif
- br_mc_disabled_update(port->dev,
- br_opt_get(port->br, BROPT_MULTICAST_ENABLED));
+ ret = br_mc_disabled_update(port->dev,
+ br_opt_get(port->br,
+ BROPT_MULTICAST_ENABLED));
+ if (ret)
+ return ret;
port->mcast_stats = netdev_alloc_pcpu_stats(struct bridge_mcast_stats);
if (!port->mcast_stats)
@@ -2057,12 +2071,16 @@ static void br_multicast_start_querier(struct net_bridge *br,
int br_multicast_toggle(struct net_bridge *br, unsigned long val)
{
struct net_bridge_port *port;
+ int err = 0;
spin_lock_bh(&br->multicast_lock);
if (!!br_opt_get(br, BROPT_MULTICAST_ENABLED) == !!val)
goto unlock;
- br_mc_disabled_update(br->dev, val);
+ err = br_mc_disabled_update(br->dev, val);
+ if (err)
+ goto unlock;
+
br_opt_toggle(br, BROPT_MULTICAST_ENABLED, !!val);
if (!br_opt_get(br, BROPT_MULTICAST_ENABLED)) {
br_multicast_leave_snoopers(br);
@@ -2079,7 +2097,7 @@ int br_multicast_toggle(struct net_bridge *br, unsigned long val)
unlock:
spin_unlock_bh(&br->multicast_lock);
- return 0;
+ return err;
}
bool br_multicast_enabled(const struct net_device *dev)
--
2.17.1
From 4de219f5de35f83f6048e2045244cfab6dd7f58a Mon Sep 17 00:00:00 2001
From: Florian Fainelli <f.fainelli@gmail.com>
Date: Mon, 10 Dec 2018 16:11:56 -0800
Subject: [PATCH 02/13] net: dsa: b53: Fix default VLAN ID
We were not consistent in how the default VID of a given port was
defined, b53_br_leave() would make sure the VLAN ID would be either 0/1
depending on the switch generation, but b53_configure_vlan(), which is
the default configuration would unconditionally set it to 1. The correct
value is 1 for 5325/5365 series and 0 otherwise. To avoid repeating that
mistake ever again, introduce a helper function: b53_default_pvid() to
factor that out.
Fixes: 967dd82ffc52 ("net: dsa: b53: Add support for Broadcom RoboSwitch")
Signed-off-by: Florian Fainelli <f.fainelli@gmail.com>
---
drivers/net/dsa/b53/b53_common.c | 29 ++++++++++++++++-------------
1 file changed, 16 insertions(+), 13 deletions(-)
diff --git a/drivers/net/dsa/b53/b53_common.c b/drivers/net/dsa/b53/b53_common.c
index 0e4bbdcc614f..964a9ec4652a 100644
--- a/drivers/net/dsa/b53/b53_common.c
+++ b/drivers/net/dsa/b53/b53_common.c
@@ -632,15 +632,25 @@ static void b53_enable_mib(struct b53_device *dev)
b53_write8(dev, B53_MGMT_PAGE, B53_GLOBAL_CONFIG, gc);
}
+static u16 b53_default_pvid(struct b53_device *dev)
+{
+ if (is5325(dev) || is5365(dev))
+ return 1;
+ else
+ return 0;
+}
+
int b53_configure_vlan(struct dsa_switch *ds)
{
struct b53_device *dev = ds->priv;
struct b53_vlan vl = { 0 };
- int i;
+ int i, def_vid;
+
+ def_vid = b53_default_pvid(dev);
/* clear all vlan entries */
if (is5325(dev) || is5365(dev)) {
- for (i = 1; i < dev->num_vlans; i++)
+ for (i = def_vid; i < dev->num_vlans; i++)
b53_set_vlan_entry(dev, i, &vl);
} else {
b53_do_vlan_op(dev, VTA_CMD_CLEAR);
@@ -650,7 +660,7 @@ int b53_configure_vlan(struct dsa_switch *ds)
b53_for_each_port(dev, i)
b53_write16(dev, B53_VLAN_PAGE,
- B53_VLAN_PORT_DEF_TAG(i), 1);
+ B53_VLAN_PORT_DEF_TAG(i), def_vid);
if (!is5325(dev) && !is5365(dev))
b53_set_jumbo(dev, dev->enable_jumbo, false);
@@ -1326,12 +1336,8 @@ int b53_vlan_del(struct dsa_switch *ds, int port,
vl->members &= ~BIT(port);
- if (pvid == vid) {
- if (is5325(dev) || is5365(dev))
- pvid = 1;
- else
- pvid = 0;
- }
+ if (pvid == vid)
+ pvid = b53_default_pvid(dev);
if (untagged && !dsa_is_cpu_port(ds, port))
vl->untag &= ~(BIT(port));
@@ -1644,10 +1650,7 @@ void b53_br_leave(struct dsa_switch *ds, int port, struct net_device *br)
b53_write16(dev, B53_PVLAN_PAGE, B53_PVLAN_PORT_MASK(port), pvlan);
dev->ports[port].vlan_ctl_mask = pvlan;
- if (is5325(dev) || is5365(dev))
- pvid = 1;
- else
- pvid = 0;
+ pvid = b53_default_pvid(dev);
/* Make this port join all VLANs without VLAN entries */
if (is58xx(dev)) {
--
2.17.1
From f383f69ad8a0da53f4f2d634d4dd4f34babf6d52 Mon Sep 17 00:00:00 2001
From: Florian Fainelli <f.fainelli@gmail.com>
Date: Mon, 10 Dec 2018 13:55:55 -0800
Subject: [PATCH 03/13] net: dsa: b53: Properly account for VLAN filtering
VLAN filtering can be built into the kernel, and also dynamically turned
on/off through the bridge master device. Allow re-configuring the switch
appropriately to account for that by deciding whether VLAN table
(v_table) misses should lead to a drop or forward.
Fixes: a2482d2ce349 ("net: dsa: b53: Plug in VLAN support")
Signed-off-by: Florian Fainelli <f.fainelli@gmail.com>
---
drivers/net/dsa/b53/b53_common.c | 59 +++++++++++++++++++++++++++++---
drivers/net/dsa/b53/b53_priv.h | 3 ++
2 files changed, 57 insertions(+), 5 deletions(-)
diff --git a/drivers/net/dsa/b53/b53_common.c b/drivers/net/dsa/b53/b53_common.c
index 964a9ec4652a..2fef4c564420 100644
--- a/drivers/net/dsa/b53/b53_common.c
+++ b/drivers/net/dsa/b53/b53_common.c
@@ -344,7 +344,8 @@ static void b53_set_forwarding(struct b53_device *dev, int enable)
b53_write8(dev, B53_CTRL_PAGE, B53_SWITCH_CTRL, mgmt);
}
-static void b53_enable_vlan(struct b53_device *dev, bool enable)
+static void b53_enable_vlan(struct b53_device *dev, bool enable,
+ bool enable_filtering)
{
u8 mgmt, vc0, vc1, vc4 = 0, vc5;
@@ -369,8 +370,13 @@ static void b53_enable_vlan(struct b53_device *dev, bool enable)
vc0 |= VC0_VLAN_EN | VC0_VID_CHK_EN | VC0_VID_HASH_VID;
vc1 |= VC1_RX_MCST_UNTAG_EN | VC1_RX_MCST_FWD_EN;
vc4 &= ~VC4_ING_VID_CHECK_MASK;
- vc4 |= VC4_ING_VID_VIO_DROP << VC4_ING_VID_CHECK_S;
- vc5 |= VC5_DROP_VTABLE_MISS;
+ if (enable_filtering) {
+ vc4 |= VC4_ING_VID_VIO_DROP << VC4_ING_VID_CHECK_S;
+ vc5 |= VC5_DROP_VTABLE_MISS;
+ } else {
+ vc4 |= VC4_ING_VID_VIO_FWD << VC4_ING_VID_CHECK_S;
+ vc5 &= ~VC5_DROP_VTABLE_MISS;
+ }
if (is5325(dev))
vc0 &= ~VC0_RESERVED_1;
@@ -420,6 +426,9 @@ static void b53_enable_vlan(struct b53_device *dev, bool enable)
}
b53_write8(dev, B53_CTRL_PAGE, B53_SWITCH_MODE, mgmt);
+
+ dev->vlan_enabled = enable;
+ dev->vlan_filtering_enabled = enable_filtering;
}
static int b53_set_jumbo(struct b53_device *dev, bool enable, bool allow_10_100)
@@ -656,7 +665,7 @@ int b53_configure_vlan(struct dsa_switch *ds)
b53_do_vlan_op(dev, VTA_CMD_CLEAR);
}
- b53_enable_vlan(dev, false);
+ b53_enable_vlan(dev, false, dev->vlan_filtering_enabled);
b53_for_each_port(dev, i)
b53_write16(dev, B53_VLAN_PAGE,
@@ -1265,6 +1274,46 @@ EXPORT_SYMBOL(b53_phylink_mac_link_up);
int b53_vlan_filtering(struct dsa_switch *ds, int port, bool vlan_filtering)
{
+ struct b53_device *dev = ds->priv;
+ struct net_device *bridge_dev;
+ unsigned int i;
+ u16 pvid, new_pvid;
+
+ /* Handle the case were multiple bridges span the same switch device
+ * and one of them has a different setting than what is being requested
+ * which would be breaking filtering semantics for any of the other
+ * bridge devices.
+ */
+ b53_for_each_port(dev, i) {
+ bridge_dev = dsa_to_port(ds, i)->bridge_dev;
+ if (bridge_dev &&
+ bridge_dev != dsa_to_port(ds, port)->bridge_dev &&
+ br_vlan_enabled(bridge_dev) != vlan_filtering) {
+ netdev_err(bridge_dev,
+ "VLAN filtering is global to the switch!\n");
+ return -EINVAL;
+ }
+ }
+
+ b53_read16(dev, B53_VLAN_PAGE, B53_VLAN_PORT_DEF_TAG(port), &pvid);
+ new_pvid = pvid;
+ if (dev->vlan_filtering_enabled && !vlan_filtering) {
+ /* Filtering is currently enabled, use the default PVID since
+ * the bridge does not expect tagging anymore
+ */
+ dev->ports[port].pvid = pvid;
+ new_pvid = b53_default_pvid(dev);
+ } else if (!dev->vlan_filtering_enabled && vlan_filtering) {
+ /* Filtering is currently disabled, restore the previous PVID */
+ new_pvid = dev->ports[port].pvid;
+ }
+
+ if (pvid != new_pvid)
+ b53_write16(dev, B53_VLAN_PAGE, B53_VLAN_PORT_DEF_TAG(port),
+ new_pvid);
+
+ b53_enable_vlan(dev, dev->vlan_enabled, vlan_filtering);
+
return 0;
}
EXPORT_SYMBOL(b53_vlan_filtering);
@@ -1280,7 +1329,7 @@ int b53_vlan_prepare(struct dsa_switch *ds, int port,
if (vlan->vid_end > dev->num_vlans)
return -ERANGE;
- b53_enable_vlan(dev, true);
+ b53_enable_vlan(dev, true, dev->vlan_filtering_enabled);
return 0;
}
diff --git a/drivers/net/dsa/b53/b53_priv.h b/drivers/net/dsa/b53/b53_priv.h
index ec796482792d..4dc7ee38b258 100644
--- a/drivers/net/dsa/b53/b53_priv.h
+++ b/drivers/net/dsa/b53/b53_priv.h
@@ -91,6 +91,7 @@ enum {
struct b53_port {
u16 vlan_ctl_mask;
struct ethtool_eee eee;
+ u16 pvid;
};
struct b53_vlan {
@@ -137,6 +138,8 @@ struct b53_device {
unsigned int num_vlans;
struct b53_vlan *vlans;
+ bool vlan_enabled;
+ bool vlan_filtering_enabled;
unsigned int num_ports;
struct b53_port *ports;
};
--
2.17.1
From adf163ed43eed53888acd5f76e84e0d0db344b4c Mon Sep 17 00:00:00 2001
From: Florian Fainelli <f.fainelli@gmail.com>
Date: Wed, 12 Dec 2018 12:23:59 -0800
Subject: [PATCH 04/13] net: systemport: Fix reception of BPDUs
SYSTEMPORT has its RXCHK parser block that attempts to validate the
packet structures, unfortunately setting the L2 header check bit will
cause Bridge PDUs (BPDUs) to be incorrectly rejected because they look
like LLC/SNAP packets with a non-IPv4 or non-IPv6 Ethernet Type.
Fixes: 4e8aedfe78c7 ("net: systemport: Turn on offloads by default")
Signed-off-by: Florian Fainelli <f.fainelli@gmail.com>
---
drivers/net/ethernet/broadcom/bcmsysport.c | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/drivers/net/ethernet/broadcom/bcmsysport.c b/drivers/net/ethernet/broadcom/bcmsysport.c
index 28c9b0bdf2f6..bc3ac369cbe3 100644
--- a/drivers/net/ethernet/broadcom/bcmsysport.c
+++ b/drivers/net/ethernet/broadcom/bcmsysport.c
@@ -134,6 +134,10 @@ static void bcm_sysport_set_rx_csum(struct net_device *dev,
priv->rx_chk_en = !!(wanted & NETIF_F_RXCSUM);
reg = rxchk_readl(priv, RXCHK_CONTROL);
+ /* Clear L2 header checks, which would prevent BPDUs
+ * from being received.
+ */
+ reg &= ~RXCHK_L2_HDR_DIS;
if (priv->rx_chk_en)
reg |= RXCHK_EN;
else
--
2.17.1
From 149369d20cd22fe66f8b9a499df32c20e466a9c8 Mon Sep 17 00:00:00 2001
From: Florian Fainelli <f.fainelli@gmail.com>
Date: Wed, 5 Dec 2018 17:16:51 -0800
Subject: [PATCH 05/13] net: dsa: b53: Define registers for IGMP snooping
Define all necessary registers in order to implement IGMP snooping later
on, which are mostly comprised of the high-level protocol register
control definitions.
Signed-off-by: Florian Fainelli <f.fainelli@gmail.com>
---
drivers/net/dsa/b53/b53_regs.h | 22 ++++++++++++++++++++++
1 file changed, 22 insertions(+)
diff --git a/drivers/net/dsa/b53/b53_regs.h b/drivers/net/dsa/b53/b53_regs.h
index 2a9f421680aa..b4aecd4552b6 100644
--- a/drivers/net/dsa/b53/b53_regs.h
+++ b/drivers/net/dsa/b53/b53_regs.h
@@ -115,6 +115,8 @@
#define B53_UC_FLOOD_MASK 0x32
#define B53_MC_FLOOD_MASK 0x34
#define B53_IPMC_FLOOD_MASK 0x36
+#define B53_DIS_LEARN 0x3c
+#define B53_SFT_LRN_CTRL 0x3e
/*
* Override Ports 0-7 State on devices with xMII interfaces (8 bit)
@@ -253,6 +255,26 @@
/* Revision ID register (8 bit) */
#define B53_REV_ID 0x40
+/* High-level Protocol Control Register (32 bit) */
+#define B53_HL_PRTC_CTRL 0x50
+#define HL_PRTC_ARP_EN (1 << 0)
+#define HL_PRTC_RARP_EN (1 << 1)
+#define HL_PRTC_DHCP_EN (1 << 2)
+#define HL_PRTC_ICMPV4_EN (1 << 3)
+#define HL_PRTC_ICMPV6_EN (1 << 4)
+#define HL_PRTC_ICMPV6_FWD_MODE (1 << 5)
+#define HL_PRTC_IGMP_DIP_EN (1 << 8)
+#define HL_PRTC_IGMP_RPTLVE_EN (1 << 9)
+#define HL_PRTC_IGMP_RPTVLE_FWD_MODE (1 << 10)
+#define HL_PRTC_IGMP_QRY_EN (1 << 11)
+#define HL_PRTC_IGMP_QRY_FWD_MODE (1 << 12)
+#define HL_PRTC_IGMP_UKN_EN (1 << 13)
+#define HL_PRTC_IGMP_UKN_FWD_MODE (1 << 14)
+#define HL_PRTC_MLD_RPTDONE_EN (1 << 15)
+#define HL_PRTC_MLD_RPTDONE_FWD_MODE (1 << 16)
+#define HL_PRTC_MLD_QRY_EN (1 << 17)
+#define HL_PRTC_MLD_QRY_FWD_MODE (1 << 18)
+
/* Broadcom header RX control (16 bit) */
#define B53_BRCM_HDR_RX_DIS 0x60
--
2.17.1
From b9676fa661680ce69216f59afc3f90c5c5250302 Mon Sep 17 00:00:00 2001
From: Florian Fainelli <f.fainelli@gmail.com>
Date: Thu, 6 Dec 2018 12:13:40 -0800
Subject: [PATCH 06/13] net: dsa: b53: Add support for MDB
In preparation for supporting IGMP snooping with or without the use of
a bridge, add support within b53_common.c to program the ARL entries for
multicast operations. The key difference is that a multicast ARL entry
is comprised of a bitmask of enabled ports, instead of a port number.
Signed-off-by: Florian Fainelli <f.fainelli@gmail.com>
---
drivers/net/dsa/b53/b53_common.c | 62 ++++++++++++++++++++++++++++++--
drivers/net/dsa/b53/b53_priv.h | 8 ++++-
2 files changed, 67 insertions(+), 3 deletions(-)
diff --git a/drivers/net/dsa/b53/b53_common.c b/drivers/net/dsa/b53/b53_common.c
index 2fef4c564420..6c894ad4768a 100644
--- a/drivers/net/dsa/b53/b53_common.c
+++ b/drivers/net/dsa/b53/b53_common.c
@@ -1503,11 +1503,25 @@ static int b53_arl_op(struct b53_device *dev, int op, int port,
idx = 1;
}
- memset(&ent, 0, sizeof(ent));
- ent.port = port;
+ /* For multicast address, the port is a bitmask and the validity
+ * is determined by having at least one port being still active
+ */
+ if (!is_multicast_ether_addr(addr)) {
+ ent.port = port;
+ ent.is_valid = is_valid;
+ } else {
+ if (is_valid)
+ ent.port |= BIT(port);
+ else
+ ent.port &= ~BIT(port);
+
+ ent.is_valid = !!(ent.port);
+ }
+
ent.is_valid = is_valid;
ent.vid = vid;
ent.is_static = true;
+ ent.is_age = false;
memcpy(ent.mac, addr, ETH_ALEN);
b53_arl_from_entry(&mac_vid, &fwd_entry, &ent);
@@ -1626,6 +1640,47 @@ int b53_fdb_dump(struct dsa_switch *ds, int port,
}
EXPORT_SYMBOL(b53_fdb_dump);
+int b53_mdb_prepare(struct dsa_switch *ds, int port,
+ const struct switchdev_obj_port_mdb *mdb)
+{
+ struct b53_device *priv = ds->priv;
+
+ /* 5325 and 5365 require some more massaging, but could
+ * be supported eventually
+ */
+ if (is5325(priv) || is5365(priv))
+ return -EOPNOTSUPP;
+
+ return 0;
+}
+EXPORT_SYMBOL(b53_mdb_prepare);
+
+void b53_mdb_add(struct dsa_switch *ds, int port,
+ const struct switchdev_obj_port_mdb *mdb)
+{
+ struct b53_device *priv = ds->priv;
+ int ret;
+
+ ret = b53_arl_op(priv, 0, port, mdb->addr, mdb->vid, true);
+ if (ret)
+ dev_err(ds->dev, "failed to add MDB entry\n");
+}
+EXPORT_SYMBOL(b53_mdb_add);
+
+int b53_mdb_del(struct dsa_switch *ds, int port,
+ const struct switchdev_obj_port_mdb *mdb)
+{
+ struct b53_device *priv = ds->priv;
+ int ret;
+
+ ret = b53_arl_op(priv, 0, port, mdb->addr, mdb->vid, false);
+ if (ret)
+ dev_err(ds->dev, "failed to delete MDB entry\n");
+
+ return ret;
+}
+EXPORT_SYMBOL(b53_mdb_del);
+
int b53_br_join(struct dsa_switch *ds, int port, struct net_device *br)
{
struct b53_device *dev = ds->priv;
@@ -1969,6 +2024,9 @@ static const struct dsa_switch_ops b53_switch_ops = {
.port_fdb_del = b53_fdb_del,
.port_mirror_add = b53_mirror_add,
.port_mirror_del = b53_mirror_del,
+ .port_mdb_prepare = b53_mdb_prepare,
+ .port_mdb_add = b53_mdb_add,
+ .port_mdb_del = b53_mdb_del,
};
struct b53_chip_data {
diff --git a/drivers/net/dsa/b53/b53_priv.h b/drivers/net/dsa/b53/b53_priv.h
index 4dc7ee38b258..620638ff9338 100644
--- a/drivers/net/dsa/b53/b53_priv.h
+++ b/drivers/net/dsa/b53/b53_priv.h
@@ -251,7 +251,7 @@ b53_build_op(write48, u64);
b53_build_op(write64, u64);
struct b53_arl_entry {
- u8 port;
+ u16 port;
u8 mac[ETH_ALEN];
u16 vid;
u8 is_valid:1;
@@ -350,6 +350,12 @@ int b53_fdb_del(struct dsa_switch *ds, int port,
const unsigned char *addr, u16 vid);
int b53_fdb_dump(struct dsa_switch *ds, int port,
dsa_fdb_dump_cb_t *cb, void *data);
+int b53_mdb_prepare(struct dsa_switch *ds, int port,
+ const struct switchdev_obj_port_mdb *mdb);
+void b53_mdb_add(struct dsa_switch *ds, int port,
+ const struct switchdev_obj_port_mdb *mdb);
+int b53_mdb_del(struct dsa_switch *ds, int port,
+ const struct switchdev_obj_port_mdb *mdb);
int b53_mirror_add(struct dsa_switch *ds, int port,
struct dsa_mall_mirror_tc_entry *mirror, bool ingress);
enum dsa_tag_protocol b53_get_tag_protocol(struct dsa_switch *ds, int port);
--
2.17.1
From 2bb2f9968f6fa9c1593f338c05e696e48103b2a8 Mon Sep 17 00:00:00 2001
From: Florian Fainelli <f.fainelli@gmail.com>
Date: Thu, 6 Dec 2018 12:29:16 -0800
Subject: [PATCH 07/13] net: dsa: Add ability to program multicast filter for
CPU port
When the switch ports operate as individual network devices, the switch
driver might have configured the switch to flood multicast all the way
to the CPU port. This is really undesireable as it can lead to receiving
a lot of unwanted traffic that the network stack needs to filter in
software.
For each valid multicast address, program it into the switch's MDB only
when the host is interested in receiving such traffic, e.g: running an
multicast application.
Signed-off-by: Florian Fainelli <f.fainelli@gmail.com>
---
net/dsa/slave.c | 51 +++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 51 insertions(+)
diff --git a/net/dsa/slave.c b/net/dsa/slave.c
index 2e5e7c04821b..f565b6a595d2 100644
--- a/net/dsa/slave.c
+++ b/net/dsa/slave.c
@@ -68,6 +68,41 @@ static int dsa_slave_get_iflink(const struct net_device *dev)
return dsa_slave_to_master(dev)->ifindex;
}
+static int dsa_slave_sync(struct net_device *dev, const unsigned char *addr)
+{
+ struct switchdev_obj_port_mdb mdb = {
+ .obj.id = SWITCHDEV_OBJ_ID_HOST_MDB,
+ .obj.flags = SWITCHDEV_F_DEFER,
+ };
+
+ ether_addr_copy(mdb.addr, addr);
+
+ return switchdev_port_obj_add(dev, &mdb.obj, NULL);
+}
+
+static int dsa_slave_unsync(struct net_device *dev, const unsigned char *addr)
+{
+ struct switchdev_obj_port_mdb mdb = {
+ .obj.id = SWITCHDEV_OBJ_ID_HOST_MDB,
+ .obj.flags = SWITCHDEV_F_DEFER,
+ };
+
+ ether_addr_copy(mdb.addr, addr);
+
+ return switchdev_port_obj_del(dev, &mdb.obj);
+}
+
+static int dsa_slave_mc_sync(struct net_device *dev)
+{
+ return __hw_addr_sync_dev(&dev->mc, dev, dsa_slave_sync,
+ dsa_slave_unsync);
+}
+
+static void dsa_slave_mc_unsync(struct net_device *dev)
+{
+ __hw_addr_unsync_dev(&dev->mc, dev, dsa_slave_unsync);
+}
+
static int dsa_slave_open(struct net_device *dev)
{
struct net_device *master = dsa_slave_to_master(dev);
@@ -126,6 +161,8 @@ static int dsa_slave_close(struct net_device *dev)
dev_mc_unsync(master, dev);
dev_uc_unsync(master, dev);
+ dsa_slave_mc_unsync(dev);
+
if (dev->flags & IFF_ALLMULTI)
dev_set_allmulti(master, -1);
if (dev->flags & IFF_PROMISC)
@@ -153,7 +190,17 @@ static void dsa_slave_change_rx_flags(struct net_device *dev, int change)
static void dsa_slave_set_rx_mode(struct net_device *dev)
{
struct net_device *master = dsa_slave_to_master(dev);
+ struct dsa_port *dp = dsa_slave_to_port(dev);
+
+ /* If the port is bridged, the bridge takes care of sending
+ * SWITCHDEV_OBJ_ID_HOST_MDB to program the host's MC filter
+ */
+ if (netdev_mc_empty(dev) || dp->bridge_dev)
+ goto out;
+ if (dsa_slave_mc_sync(dev))
+ netdev_err(dev, "failed to synchronize MC addresses\n");
+out:
dev_mc_sync(master, dev);
dev_uc_sync(master, dev);
}
@@ -1405,6 +1452,10 @@ static int dsa_slave_changeupper(struct net_device *dev,
if (netif_is_bridge_master(info->upper_dev)) {
if (info->linking) {
+ /* Remove existing MC addresses that might have been
+ * programmed
+ */
+ dsa_slave_mc_unsync(dev);
err = dsa_port_bridge_join(dp, info->upper_dev);
err = notifier_from_errno(err);
} else {
--
2.17.1
From a18cd91ea44f55b852cd4c771da0abb014ef50ab Mon Sep 17 00:00:00 2001
From: Florian Fainelli <f.fainelli@gmail.com>
Date: Mon, 10 Dec 2018 15:15:17 -0800
Subject: [PATCH 08/13] net: dsa: Add ndo_vlan_rx_{add,kill}_vid implementation
In order to properly support VLAN filtering being enabled/disabled on a
bridge, while having other ports being non bridge port members, we need
to support the ndo_vlan_rx_{add,kill}_vid callbacks in order to make
sure the non-bridge ports can continue receiving VLAN tags, even when
the switch is globally configured to do ingress/egress VID checking.
We don't allow configuring VLAN devices on a bridge port member though,
since the bridge with VLAN awareness should be taking care of that, if
needed.
Since we can call dsa_port_vlan_{add,del} with a bridge_dev pointer
NULL, we now need to check that in these two functions.
Signed-off-by: Florian Fainelli <f.fainelli@gmail.com>
---
net/dsa/port.c | 10 ++++++++--
net/dsa/slave.c | 53 ++++++++++++++++++++++++++++++++++++++++++++++++-
2 files changed, 60 insertions(+), 3 deletions(-)
diff --git a/net/dsa/port.c b/net/dsa/port.c
index 2d7e01b23572..a18d8409acf3 100644
--- a/net/dsa/port.c
+++ b/net/dsa/port.c
@@ -252,7 +252,10 @@ int dsa_port_vlan_add(struct dsa_port *dp,
.vlan = vlan,
};
- if (br_vlan_enabled(dp->bridge_dev))
+ /* Can be called from dsa_slave_port_obj_add() or
+ * dsa_slave_vlan_rx_add_vid()
+ */
+ if (!dp->bridge_dev || br_vlan_enabled(dp->bridge_dev))
return dsa_port_notify(dp, DSA_NOTIFIER_VLAN_ADD, &info);
return 0;
@@ -270,7 +273,10 @@ int dsa_port_vlan_del(struct dsa_port *dp,
if (netif_is_bridge_master(vlan->obj.orig_dev))
return -EOPNOTSUPP;
- if (br_vlan_enabled(dp->bridge_dev))
+ /* Can be called from dsa_slave_port_obj_del() or
+ * dsa_slave_vlan_rx_kill_vid()
+ */
+ if (!dp->bridge_dev || br_vlan_enabled(dp->bridge_dev))
return dsa_port_notify(dp, DSA_NOTIFIER_VLAN_DEL, &info);
return 0;
diff --git a/net/dsa/slave.c b/net/dsa/slave.c
index f565b6a595d2..cc4fb8725fac 100644
--- a/net/dsa/slave.c
+++ b/net/dsa/slave.c
@@ -1037,6 +1037,54 @@ static int dsa_slave_get_ts_info(struct net_device *dev,
return ds->ops->get_ts_info(ds, p->dp->index, ts);
}
+static int dsa_slave_vlan_rx_add_vid(struct net_device *dev, __be16 proto,
+ u16 vid)
+{
+ struct dsa_port *dp = dsa_slave_to_port(dev);
+ struct switchdev_obj_port_vlan vlan = { };
+ int ret = 0;
+
+ /* If the port is bridged and the bridge is VLAN aware, let the bridge
+ * manage VLANs
+ */
+ if (dp->bridge_dev && br_vlan_enabled(dp->bridge_dev))
+ return -EINVAL;
+
+ /* This API only allows programming tagged, non-PVID VIDs */
+ vlan.vid_begin = vid;
+ vlan.vid_end = vid;
+
+ ret = dsa_port_vlan_add(dp, &vlan, NULL);
+ if (ret == -EOPNOTSUPP)
+ ret = 0;
+
+ return ret;
+}
+
+static int dsa_slave_vlan_rx_kill_vid(struct net_device *dev, __be16 proto,
+ u16 vid)
+{
+ struct dsa_port *dp = dsa_slave_to_port(dev);
+ struct switchdev_obj_port_vlan vlan = { };
+ int ret = 0;
+
+ /* If the port is bridged and the bridge is VLAN aware, let the bridge
+ * manage VLANs
+ */
+ if (dp->bridge_dev && br_vlan_enabled(dp->bridge_dev))
+ return -EINVAL;
+
+ /* This API only allows programming tagged, non-PVID VIDs */
+ vlan.vid_begin = vid;
+ vlan.vid_end = vid;
+
+ ret = dsa_port_vlan_del(dp, &vlan);
+ if (ret == -EOPNOTSUPP)
+ ret = 0;
+
+ return ret;
+}
+
static const struct ethtool_ops dsa_slave_ethtool_ops = {
.get_drvinfo = dsa_slave_get_drvinfo,
.get_regs_len = dsa_slave_get_regs_len,
@@ -1102,6 +1150,8 @@ static const struct net_device_ops dsa_slave_netdev_ops = {
.ndo_setup_tc = dsa_slave_setup_tc,
.ndo_get_stats64 = dsa_slave_get_stats64,
.ndo_get_port_parent_id = dsa_slave_get_port_parent_id,
+ .ndo_vlan_rx_add_vid = dsa_slave_vlan_rx_add_vid,
+ .ndo_vlan_rx_kill_vid = dsa_slave_vlan_rx_kill_vid,
};
static const struct switchdev_ops dsa_slave_switchdev_ops = {
@@ -1362,7 +1412,8 @@ int dsa_slave_create(struct dsa_port *port)
if (slave_dev == NULL)
return -ENOMEM;
- slave_dev->features = master->vlan_features | NETIF_F_HW_TC;
+ slave_dev->features = master->vlan_features | NETIF_F_HW_TC |
+ NETIF_F_HW_VLAN_CTAG_FILTER;
slave_dev->hw_features |= NETIF_F_HW_TC;
slave_dev->ethtool_ops = &dsa_slave_ethtool_ops;
eth_hw_addr_inherit(slave_dev, master);
--
2.17.1
From 2d4c257f1b9fd0d1dc2af49c50ebc215c2b561ee Mon Sep 17 00:00:00 2001
From: Florian Fainelli <f.fainelli@gmail.com>
Date: Mon, 7 Jan 2019 15:21:34 -0800
Subject: [PATCH 09/13] net: dsa: Make VLAN filtering use DSA notifiers
In preparation for allowing for global checks that would apply to the
entire switch and not just on a per-port basis, make the VLAN filtering
attribute follow other switchdev attributes/objects and make it use the
DSA notifier infrastructure.
Signed-off-by: Florian Fainelli <f.fainelli@gmail.com>
---
net/dsa/dsa_priv.h | 11 ++++++++++-
net/dsa/port.c | 17 +++++++----------
net/dsa/switch.c | 29 +++++++++++++++++++++++++++++
3 files changed, 46 insertions(+), 11 deletions(-)
diff --git a/net/dsa/dsa_priv.h b/net/dsa/dsa_priv.h
index 1f4972dab9f2..1e3db5f2a699 100644
--- a/net/dsa/dsa_priv.h
+++ b/net/dsa/dsa_priv.h
@@ -26,6 +26,7 @@ enum {
DSA_NOTIFIER_MDB_DEL,
DSA_NOTIFIER_VLAN_ADD,
DSA_NOTIFIER_VLAN_DEL,
+ DSA_NOTIFIER_VLAN_FILTERING,
};
/* DSA_NOTIFIER_AGEING_TIME */
@@ -57,7 +58,7 @@ struct dsa_notifier_mdb_info {
int port;
};
-/* DSA_NOTIFIER_VLAN_* */
+/* DSA_NOTIFIER_VLAN_{ADD,DEL} */
struct dsa_notifier_vlan_info {
const struct switchdev_obj_port_vlan *vlan;
struct switchdev_trans *trans;
@@ -65,6 +66,14 @@ struct dsa_notifier_vlan_info {
int port;
};
+/* DSA_NOTIFIER_VLAN_FILTERING */
+struct dsa_notifier_vlan_filtering_info {
+ bool vlan_filtering;
+ struct switchdev_trans *trans;
+ int sw_index;
+ int port;
+};
+
struct dsa_slave_priv {
/* Copy of CPU port xmit for faster access in slave transmit hot path */
struct sk_buff * (*xmit)(struct sk_buff *skb,
diff --git a/net/dsa/port.c b/net/dsa/port.c
index a18d8409acf3..60c6eca7d964 100644
--- a/net/dsa/port.c
+++ b/net/dsa/port.c
@@ -146,17 +146,14 @@ void dsa_port_bridge_leave(struct dsa_port *dp, struct net_device *br)
int dsa_port_vlan_filtering(struct dsa_port *dp, bool vlan_filtering,
struct switchdev_trans *trans)
{
- struct dsa_switch *ds = dp->ds;
-
- /* bridge skips -EOPNOTSUPP, so skip the prepare phase */
- if (switchdev_trans_ph_prepare(trans))
- return 0;
-
- if (ds->ops->port_vlan_filtering)
- return ds->ops->port_vlan_filtering(ds, dp->index,
- vlan_filtering);
+ struct dsa_notifier_vlan_filtering_info info = {
+ .sw_index = dp->ds->index,
+ .port = dp->index,
+ .trans = trans,
+ .vlan_filtering = vlan_filtering,
+ };
- return 0;
+ return dsa_port_notify(dp, DSA_NOTIFIER_VLAN_FILTERING, &info);
}
int dsa_port_ageing_time(struct dsa_port *dp, clock_t ageing_clock,
diff --git a/net/dsa/switch.c b/net/dsa/switch.c
index 142b294d3446..831334dc5e79 100644
--- a/net/dsa/switch.c
+++ b/net/dsa/switch.c
@@ -235,6 +235,32 @@ static int dsa_switch_vlan_del(struct dsa_switch *ds,
return 0;
}
+static int dsa_switch_vlan_filtering(struct dsa_switch *ds,
+ struct dsa_notifier_vlan_filtering_info *info)
+{
+ struct switchdev_trans *trans = info->trans;
+ bool vlan_filtering = info->vlan_filtering;
+ int port = info->port;
+ int err;
+
+ /* bridge skips -EOPNOTSUPP, so skip the prepare phase */
+ if (switchdev_trans_ph_prepare(trans))
+ return 0;
+
+ /* Build a mask of port members */
+ bitmap_zero(ds->bitmap, ds->num_ports);
+ if (ds->index == info->sw_index)
+ set_bit(port, ds->bitmap);
+
+ for_each_set_bit(port, ds->bitmap, ds->num_ports) {
+ err = ds->ops->port_vlan_filtering(ds, port, vlan_filtering);
+ if (err)
+ return err;
+ }
+
+ return 0;
+}
+
static int dsa_switch_event(struct notifier_block *nb,
unsigned long event, void *info)
{
@@ -269,6 +295,9 @@ static int dsa_switch_event(struct notifier_block *nb,
case DSA_NOTIFIER_VLAN_DEL:
err = dsa_switch_vlan_del(ds, info);
break;
+ case DSA_NOTIFIER_VLAN_FILTERING:
+ err = dsa_switch_vlan_filtering(ds, info);
+ break;
default:
err = -EOPNOTSUPP;
break;
--
2.17.1
From 3f2236e39d2d9b0d7d77d9033becdfeda717c41f Mon Sep 17 00:00:00 2001
From: Florian Fainelli <f.fainelli@gmail.com>
Date: Wed, 12 Dec 2018 15:14:34 -0800
Subject: [PATCH 10/13] net: dsa: Wire up multicast IGMP snooping attribute
notification
The bridge can at runtime be configured with or without IGMP snooping
enabled but we were not processing the switchdev attribute that notifies
about that toggle, do this now.
Signed-off-by: Florian Fainelli <f.fainelli@gmail.com>
---
include/net/dsa.h | 8 ++++++++
net/dsa/dsa_priv.h | 11 +++++++++++
net/dsa/port.c | 13 +++++++++++++
net/dsa/slave.c | 9 +++++++++
net/dsa/switch.c | 28 ++++++++++++++++++++++++++++
5 files changed, 69 insertions(+)
diff --git a/include/net/dsa.h b/include/net/dsa.h
index 7f2a668ef2cc..220f2b1d06d9 100644
--- a/include/net/dsa.h
+++ b/include/net/dsa.h
@@ -425,6 +425,8 @@ struct dsa_switch_ops {
/*
* Multicast database
*/
+ int (*port_mc_disabled)(struct dsa_switch *ds, int port,
+ bool mc_disabled);
int (*port_mdb_prepare)(struct dsa_switch *ds, int port,
const struct switchdev_obj_port_mdb *mdb);
void (*port_mdb_add)(struct dsa_switch *ds, int port,
@@ -467,6 +469,12 @@ struct dsa_switch_ops {
struct sk_buff *clone, unsigned int type);
bool (*port_rxtstamp)(struct dsa_switch *ds, int port,
struct sk_buff *skb, unsigned int type);
+
+ /*
+ * SWITCHDEV integration
+ */
+ int (*port_attr_get)(struct dsa_switch *ds, int port,
+ struct switchdev_attr *attr);
};
struct dsa_switch_driver {
diff --git a/net/dsa/dsa_priv.h b/net/dsa/dsa_priv.h
index 1e3db5f2a699..79dd1dcbcb28 100644
--- a/net/dsa/dsa_priv.h
+++ b/net/dsa/dsa_priv.h
@@ -27,6 +27,7 @@ enum {
DSA_NOTIFIER_VLAN_ADD,
DSA_NOTIFIER_VLAN_DEL,
DSA_NOTIFIER_VLAN_FILTERING,
+ DSA_NOTIFIER_MC_DISABLED,
};
/* DSA_NOTIFIER_AGEING_TIME */
@@ -74,6 +75,14 @@ struct dsa_notifier_vlan_filtering_info {
int port;
};
+/* DSA_NOTIFIER_MC_DISABLED */
+struct dsa_notifier_mc_disabled_info {
+ bool mc_disabled;
+ struct switchdev_trans *trans;
+ int sw_index;
+ int port;
+};
+
struct dsa_slave_priv {
/* Copy of CPU port xmit for faster access in slave transmit hot path */
struct sk_buff * (*xmit)(struct sk_buff *skb,
@@ -155,6 +164,8 @@ int dsa_port_enable(struct dsa_port *dp, struct phy_device *phy);
void dsa_port_disable(struct dsa_port *dp, struct phy_device *phy);
int dsa_port_bridge_join(struct dsa_port *dp, struct net_device *br);
void dsa_port_bridge_leave(struct dsa_port *dp, struct net_device *br);
+int dsa_port_mc_disabled(struct dsa_port *dp, bool mc_disabled,
+ struct switchdev_trans *trans);
int dsa_port_vlan_filtering(struct dsa_port *dp, bool vlan_filtering,
struct switchdev_trans *trans);
int dsa_port_ageing_time(struct dsa_port *dp, clock_t ageing_clock,
diff --git a/net/dsa/port.c b/net/dsa/port.c
index 60c6eca7d964..725824fd88cb 100644
--- a/net/dsa/port.c
+++ b/net/dsa/port.c
@@ -143,6 +143,19 @@ void dsa_port_bridge_leave(struct dsa_port *dp, struct net_device *br)
dsa_port_set_state_now(dp, BR_STATE_FORWARDING);
}
+int dsa_port_mc_disabled(struct dsa_port *dp, bool mc_disabled,
+ struct switchdev_trans *trans)
+{
+ struct dsa_notifier_mc_disabled_info info = {
+ .sw_index = dp->ds->index,
+ .port = dp->index,
+ .trans = trans,
+ .mc_disabled = mc_disabled,
+ };
+
+ return dsa_port_notify(dp, DSA_NOTIFIER_MC_DISABLED, &info);
+}
+
int dsa_port_vlan_filtering(struct dsa_port *dp, bool vlan_filtering,
struct switchdev_trans *trans)
{
diff --git a/net/dsa/slave.c b/net/dsa/slave.c
index cc4fb8725fac..128a8e5b3a42 100644
--- a/net/dsa/slave.c
+++ b/net/dsa/slave.c
@@ -342,6 +342,9 @@ static int dsa_slave_port_attr_set(struct net_device *dev,
case SWITCHDEV_ATTR_ID_BRIDGE_AGEING_TIME:
ret = dsa_port_ageing_time(dp, attr->u.ageing_time, trans);
break;
+ case SWITCHDEV_ATTR_ID_BRIDGE_MC_DISABLED:
+ ret = dsa_port_mc_disabled(dp, attr->u.mc_disabled, trans);
+ break;
default:
ret = -EOPNOTSUPP;
break;
@@ -428,6 +431,12 @@ static int dsa_slave_get_port_parent_id(struct net_device *dev,
static int dsa_slave_port_attr_get(struct net_device *dev,
struct switchdev_attr *attr)
{
+ struct dsa_port *dp = dsa_slave_to_port(dev);
+ struct dsa_switch *ds = dp->ds;
+
+ if (ds->ops->port_attr_get)
+ return ds->ops->port_attr_get(ds, dp->index, attr);
+
switch (attr->id) {
case SWITCHDEV_ATTR_ID_PORT_BRIDGE_FLAGS_SUPPORT:
attr->u.brport_flags_support = 0;
diff --git a/net/dsa/switch.c b/net/dsa/switch.c
index 831334dc5e79..9e7ac39a8dc0 100644
--- a/net/dsa/switch.c
+++ b/net/dsa/switch.c
@@ -261,6 +261,31 @@ static int dsa_switch_vlan_filtering(struct dsa_switch *ds,
return 0;
}
+static int dsa_switch_mc_disabled(struct dsa_switch *ds,
+ struct dsa_notifier_mc_disabled_info *info)
+{
+ struct switchdev_trans *trans = info->trans;
+ bool mc_disabled = info->mc_disabled;
+ int port = info->port;
+ int err;
+
+ if (switchdev_trans_ph_prepare(trans))
+ return ds->ops->port_mc_disabled ? 0 : -EOPNOTSUPP;
+
+ /* Build a mask of port members */
+ bitmap_zero(ds->bitmap, ds->num_ports);
+ if (ds->index == info->sw_index)
+ set_bit(port, ds->bitmap);
+
+ for_each_set_bit(port, ds->bitmap, ds->num_ports) {
+ err = ds->ops->port_mc_disabled(ds, port, mc_disabled);
+ if (err)
+ return err;
+ }
+
+ return 0;
+}
+
static int dsa_switch_event(struct notifier_block *nb,
unsigned long event, void *info)
{
@@ -298,6 +323,9 @@ static int dsa_switch_event(struct notifier_block *nb,
case DSA_NOTIFIER_VLAN_FILTERING:
err = dsa_switch_vlan_filtering(ds, info);
break;
+ case DSA_NOTIFIER_MC_DISABLED:
+ err = dsa_switch_mc_disabled(ds, info);
+ break;
default:
err = -EOPNOTSUPP;
break;
--
2.17.1
From 224cfbb5f7920bc53c5ba7e025301646dca839cf Mon Sep 17 00:00:00 2001
From: Florian Fainelli <f.fainelli@gmail.com>
Date: Wed, 12 Dec 2018 15:35:52 -0800
Subject: [PATCH 11/13] net: dsa: b53: Add support for toggling IGMP snooping
Add the required configuration knobs to honor the turning off of IGMP
snooping (typically through the bridge interface) which means that when
IGMP snooping is off, we must be flooding mutlicast since we do not get
any notifications about IGMP join/leave through the network stack
running on the bridge.
Signed-off-by: Florian Fainelli <f.fainelli@gmail.com>
---
drivers/net/dsa/b53/b53_common.c | 88 ++++++++++++++++++++++++++++++++
drivers/net/dsa/b53/b53_priv.h | 3 ++
2 files changed, 91 insertions(+)
diff --git a/drivers/net/dsa/b53/b53_common.c b/drivers/net/dsa/b53/b53_common.c
index 6c894ad4768a..22a6f6afff7a 100644
--- a/drivers/net/dsa/b53/b53_common.c
+++ b/drivers/net/dsa/b53/b53_common.c
@@ -1640,6 +1640,93 @@ int b53_fdb_dump(struct dsa_switch *ds, int port,
}
EXPORT_SYMBOL(b53_fdb_dump);
+int b53_port_attr_get(struct dsa_switch *ds, int port,
+ struct switchdev_attr *attr)
+{
+ struct b53_device *dev = ds->priv;
+ struct net_device *bridge_dev;
+ const struct dsa_port *dp;
+ bool mc_disabled;
+ unsigned int i;
+
+ if (is5325(dev) || is5365(dev))
+ return -EOPNOTSUPP;
+
+ switch (attr->id) {
+ case SWITCHDEV_ATTR_ID_PORT_BRIDGE_FLAGS_SUPPORT:
+ attr->u.brport_flags_support = 0;
+ return 0;
+ case SWITCHDEV_ATTR_ID_BRIDGE_MC_DISABLED_SUPPORT:
+ mc_disabled = attr->u.mc_disabled_support;
+ break;
+ default:
+ return -EOPNOTSUPP;
+ }
+
+ /* Handle the case were multiple bridges span the same switch device
+ * and one of them has a different setting than what is being requested
+ * which would be breaking filtering semantics for any of the other
+ * bridge devices. We must also take care of non-bridged ports which
+ * expect multicast filtering to remain turned on.
+ */
+ for (i = 0; i < ds->num_ports; i++) {
+ if (dsa_is_unused_port(ds, i) || dsa_is_cpu_port(ds, i))
+ continue;
+
+ if (i == port)
+ continue;
+
+ dp = dsa_to_port(ds, i);
+ bridge_dev = dp->bridge_dev;
+ if ((bridge_dev &&
+ bridge_dev != dsa_to_port(ds, port)->bridge_dev &&
+ br_multicast_enabled(bridge_dev) != !mc_disabled) ||
+ (!bridge_dev && mc_disabled)) {
+ netdev_err(dp->slave,
+ "MC filtering is global to the switch!\n");
+ return -EINVAL;
+ }
+ }
+
+ return 0;
+}
+EXPORT_SYMBOL(b53_port_attr_get);
+
+int b53_mc_disabled(struct dsa_switch *ds, int port, bool mc_disabled)
+{
+ unsigned int cpu_port = dsa_to_port(ds, port)->cpu_dp->index;
+ struct b53_device *dev = ds->priv;
+ u8 port_ctrl;
+ u16 mc_ctrl;
+
+ /* Allow CPU port to receive multicast traffic */
+ b53_read8(dev, B53_CTRL_PAGE, B53_PORT_CTRL(cpu_port), &port_ctrl);
+ if (mc_disabled)
+ port_ctrl |= PORT_CTRL_RX_MCST_EN;
+ else
+ port_ctrl &= ~PORT_CTRL_RX_MCST_EN;
+ b53_write8(dev, B53_CTRL_PAGE, B53_PORT_CTRL(cpu_port), port_ctrl);
+
+ /* Allow port to flood multicast */
+ b53_read16(dev, B53_CTRL_PAGE, B53_MC_FLOOD_MASK, &mc_ctrl);
+ if (mc_disabled)
+ mc_ctrl |= BIT(port);
+ else
+ mc_ctrl &= ~BIT(port);
+ b53_write16(dev, B53_CTRL_PAGE, B53_MC_FLOOD_MASK, mc_ctrl);
+
+ /* And flood IP multicast as well */
+ b53_read16(dev, B53_CTRL_PAGE, B53_IPMC_FLOOD_MASK, &mc_ctrl);
+ if (mc_disabled)
+ mc_ctrl |= BIT(port);
+ else
+ mc_ctrl &= ~BIT(port);
+ b53_write16(dev, B53_CTRL_PAGE, B53_IPMC_FLOOD_MASK, mc_ctrl);
+
+ return 0;
+}
+EXPORT_SYMBOL(b53_mc_disabled);
+
int b53_mdb_prepare(struct dsa_switch *ds, int port,
const struct switchdev_obj_port_mdb *mdb)
{
@@ -2025,6 +2112,7 @@ static const struct dsa_switch_ops b53_switch_ops = {
.port_mirror_add = b53_mirror_add,
.port_mirror_del = b53_mirror_del,
.port_mdb_prepare = b53_mdb_prepare,
+ .port_mc_disabled = b53_mc_disabled,
.port_mdb_add = b53_mdb_add,
.port_mdb_del = b53_mdb_del,
};
diff --git a/drivers/net/dsa/b53/b53_priv.h b/drivers/net/dsa/b53/b53_priv.h
index 620638ff9338..d407512180e3 100644
--- a/drivers/net/dsa/b53/b53_priv.h
+++ b/drivers/net/dsa/b53/b53_priv.h
@@ -350,6 +350,9 @@ int b53_fdb_del(struct dsa_switch *ds, int port,
const unsigned char *addr, u16 vid);
int b53_fdb_dump(struct dsa_switch *ds, int port,
dsa_fdb_dump_cb_t *cb, void *data);
+int b53_port_attr_get(struct dsa_switch *ds, int port,
+ struct switchdev_attr *attr);
+int b53_mc_disabled(struct dsa_switch *ds, int port, bool mc_disabled);
int b53_mdb_prepare(struct dsa_switch *ds, int port,
const struct switchdev_obj_port_mdb *mdb);
void b53_mdb_add(struct dsa_switch *ds, int port,
--
2.17.1
From 7fa263f108e8148a86762d73c74fde048a6e0eb3 Mon Sep 17 00:00:00 2001
From: Florian Fainelli <f.fainelli@gmail.com>
Date: Thu, 6 Dec 2018 12:33:08 -0800
Subject: [PATCH 12/13] net: dsa: bcm_sf2: Enable management mode
Now that we have all the necessary plumbing in place to get notified
when a multicast MAC address must be programmed, configure the switch to
operate in managed mode and let the network stack learn about management
traffic.
During suspend, we need to bring the switch back into unmanaged/dumb
forwarding mode to allow stations to communicate with each other despite
the host CPU being suspended.
Signed-off-by: Florian Fainelli <f.fainelli@gmail.com>
---
drivers/net/dsa/b53/b53_common.c | 39 ++++++++++++++++++-
drivers/net/dsa/b53/b53_priv.h | 1 +
drivers/net/dsa/bcm_sf2.c | 65 +++++++++++++++++++++++++-------
drivers/net/dsa/bcm_sf2_regs.h | 5 +++
4 files changed, 95 insertions(+), 15 deletions(-)
diff --git a/drivers/net/dsa/b53/b53_common.c b/drivers/net/dsa/b53/b53_common.c
index 22a6f6afff7a..2d56e045c7be 100644
--- a/drivers/net/dsa/b53/b53_common.c
+++ b/drivers/net/dsa/b53/b53_common.c
@@ -364,8 +364,6 @@ static void b53_enable_vlan(struct b53_device *dev, bool enable,
b53_read8(dev, B53_VLAN_PAGE, B53_VLAN_CTRL5, &vc5);
}
- mgmt &= ~SM_SW_FWD_MODE;
-
if (enable) {
vc0 |= VC0_VLAN_EN | VC0_VID_CHK_EN | VC0_VID_HASH_VID;
vc1 |= VC1_RX_MCST_UNTAG_EN | VC1_RX_MCST_FWD_EN;
@@ -490,6 +488,43 @@ static int b53_fast_age_vlan(struct b53_device *dev, u16 vid)
return b53_flush_arl(dev, FAST_AGE_VLAN);
}
+void b53_port_learn_setup(struct dsa_switch *ds, int port)
+{
+ struct b53_device *dev = ds->priv;
+ u16 reg;
+
+ /* Enable learning */
+ b53_read16(dev, B53_CTRL_PAGE, B53_DIS_LEARN, ®);
+ reg &= ~BIT(port);
+ b53_write16(dev, B53_CTRL_PAGE, B53_DIS_LEARN, reg);
+
+ /* Software learning control disabled */
+ b53_read16(dev, B53_CTRL_PAGE, B53_SFT_LRN_CTRL, ®);
+ reg &= ~BIT(port);
+ b53_write16(dev, B53_CTRL_PAGE, B53_SFT_LRN_CTRL, reg);
+
+ /* Configure IP multicast, allow Unicast ARL misses to be forwarded */
+ b53_read16(dev, B53_CTRL_PAGE, B53_IP_MULTICAST_CTRL, ®);
+ reg |= B53_IPMC_FWD_EN | B53_UC_FWD_EN;
+ b53_write16(dev, B53_CTRL_PAGE, B53_IP_MULTICAST_CTRL, reg);
+
+ /* Set port in Unicast lookup forward map */
+ b53_read16(dev, B53_CTRL_PAGE, B53_UC_FLOOD_MASK, ®);
+ reg |= BIT(port);
+ b53_write16(dev, B53_CTRL_PAGE, B53_UC_FLOOD_MASK, reg);
+
+ /* Do not set port in Multicast lookup forward map, learn */
+ b53_read16(dev, B53_CTRL_PAGE, B53_MC_FLOOD_MASK, ®);
+ reg &= ~BIT(port);
+ b53_write16(dev, B53_CTRL_PAGE, B53_MC_FLOOD_MASK, reg);
+
+ /* Do not set port in IP multicast lookup formward map, learn */
+ b53_read16(dev, B53_CTRL_PAGE, B53_IPMC_FLOOD_MASK, ®);
+ reg &= ~BIT(port);
+ b53_write16(dev, B53_CTRL_PAGE, B53_IPMC_FLOOD_MASK, reg);
+}
+EXPORT_SYMBOL(b53_port_learn_setup);
+
void b53_imp_vlan_setup(struct dsa_switch *ds, int cpu_port)
{
struct b53_device *dev = ds->priv;
diff --git a/drivers/net/dsa/b53/b53_priv.h b/drivers/net/dsa/b53/b53_priv.h
index d407512180e3..0af394161be7 100644
--- a/drivers/net/dsa/b53/b53_priv.h
+++ b/drivers/net/dsa/b53/b53_priv.h
@@ -309,6 +309,7 @@ static inline int b53_switch_get_reset_gpio(struct b53_device *dev)
#endif
/* Exported functions towards other drivers */
+void b53_port_learn_setup(struct dsa_switch *ds, int port);
void b53_imp_vlan_setup(struct dsa_switch *ds, int cpu_port);
int b53_configure_vlan(struct dsa_switch *ds);
void b53_get_strings(struct dsa_switch *ds, int port, u32 stringset,
diff --git a/drivers/net/dsa/bcm_sf2.c b/drivers/net/dsa/bcm_sf2.c
index 5193da67dcdc..05bcfbf26495 100644
--- a/drivers/net/dsa/bcm_sf2.c
+++ b/drivers/net/dsa/bcm_sf2.c
@@ -51,20 +51,25 @@ static void bcm_sf2_imp_setup(struct dsa_switch *ds, int port)
reg &= ~P_TXQ_PSM_VDD(port);
core_writel(priv, reg, CORE_MEM_PSM_VDD_CTRL);
- /* Enable Broadcast, Multicast, Unicast forwarding to IMP port */
- reg = core_readl(priv, CORE_IMP_CTL);
- reg |= (RX_BCST_EN | RX_MCST_EN | RX_UCST_EN);
- reg &= ~(RX_DIS | TX_DIS);
- core_writel(priv, reg, CORE_IMP_CTL);
+ /* Enable forwarding and managed mode */
+ core_writel(priv, SW_FWDG_EN | SW_FWDG_MODE, CORE_SWMODE);
- /* Enable forwarding */
- core_writel(priv, SW_FWDG_EN, CORE_SWMODE);
+ /* Configure port for learning */
+ b53_port_learn_setup(ds, port);
- /* Enable IMP port in dumb mode */
+ /* Disable dumb forwarding mode for IMP port */
reg = core_readl(priv, CORE_SWITCH_CTRL);
- reg |= MII_DUMB_FWDG_EN;
+ reg &= ~MII_DUMB_FWDG_EN;
core_writel(priv, reg, CORE_SWITCH_CTRL);
+ /* Enable IGMP and MLD high-level protocol snooping support */
+ reg = HL_PRTC_IGMP_RPTLVE_EN | HL_PRTC_IGMP_RPTVLE_FWD_MODE |
+ HL_PRTC_IGMP_QRY_EN | HL_PRTC_IGMP_QRY_FWD_MODE |
+ HL_PRTC_IGMP_UKN_EN | HL_PRTC_IGMP_UKN_FWD_MODE |
+ HL_PRTC_MLD_RPTDONE_EN | HL_PRTC_MLD_RPTDONE_FWD_MODE |
+ HL_PRTC_MLD_QRY_EN | HL_PRTC_MLD_QRY_FWD_MODE;
+ b53_write32(priv->dev, B53_MGMT_PAGE, B53_HL_PRTC_CTRL, reg);
+
/* Configure Traffic Class to QoS mapping, allow each priority to map
* to a different queue number
*/
@@ -75,10 +80,26 @@ static void bcm_sf2_imp_setup(struct dsa_switch *ds, int port)
b53_brcm_hdr_setup(ds, port);
+ /* Set IMP0 or IMP1 port to be managed port, enable BPDU */
+ reg = core_readl(priv, CORE_GMNCFGCFG);
+ reg &= ~(FRM_MGNP_MASK << FRM_MGNP_SHIFT);
+ if (port == core_readl(priv, CORE_IMP0_PRT_ID))
+ reg |= FRM_MNGP_IMP0 << FRM_MGNP_SHIFT;
+ if (port == core_readl(priv, CORE_IMP1_PRT_ID))
+ reg |= FRM_MGNP_IMP_DUAL << FRM_MGNP_SHIFT;
+ reg |= RXBPDU_EN;
+ core_writel(priv, reg, CORE_GMNCFGCFG);
+
/* Force link status for IMP port */
reg = core_readl(priv, offset);
reg |= (MII_SW_OR | LINK_STS);
core_writel(priv, reg, offset);
+
+ /* Enable Broadcast, Unicast forwarding to IMP port */
+ reg = core_readl(priv, CORE_IMP_CTL);
+ reg |= (RX_BCST_EN | RX_UCST_EN);
+ reg &= ~(RX_DIS | TX_DIS);
+ core_writel(priv, reg, CORE_IMP_CTL);
}
static void bcm_sf2_gphy_enable_set(struct dsa_switch *ds, bool enable)
@@ -166,10 +187,8 @@ static int bcm_sf2_port_setup(struct dsa_switch *ds, int port,
reg &= ~P_TXQ_PSM_VDD(port);
core_writel(priv, reg, CORE_MEM_PSM_VDD_CTRL);
- /* Enable learning */
- reg = core_readl(priv, CORE_DIS_LEARN);
- reg &= ~BIT(port);
- core_writel(priv, reg, CORE_DIS_LEARN);
+ /* Configure port for learning */
+ b53_port_learn_setup(ds, port);
/* Enable Broadcom tags for that port if requested */
if (priv->brcm_tag_mask & BIT(port))
@@ -683,6 +702,7 @@ static int bcm_sf2_sw_suspend(struct dsa_switch *ds)
{
struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds);
unsigned int port;
+ u32 reg;
bcm_sf2_intr_disable(priv);
@@ -695,6 +715,20 @@ static int bcm_sf2_sw_suspend(struct dsa_switch *ds)
bcm_sf2_port_disable(ds, port, NULL);
}
+ /* Disable management mode since we won't be able to
+ * perform any tasks while being suspended.
+ */
+ reg = core_readl(priv, CORE_SWMODE);
+ reg &= ~SW_FWDG_MODE;
+ core_writel(priv, reg, CORE_SWMODE);
+
+ /* Enable IMP port in dumb switching mode to allow
+ * Magic Packets to reach IMP port.
+ */
+ reg = core_readl(priv, CORE_SWITCH_CTRL);
+ reg |= MII_DUMB_FWDG_EN;
+ core_writel(priv, reg, CORE_SWITCH_CTRL);
+
return 0;
}
@@ -962,6 +996,11 @@ static const struct dsa_switch_ops bcm_sf2_ops = {
.set_rxnfc = bcm_sf2_set_rxnfc,
.port_mirror_add = b53_mirror_add,
.port_mirror_del = b53_mirror_del,
+ .port_mc_disabled = b53_mc_disabled,
+ .port_mdb_prepare = b53_mdb_prepare,
+ .port_mdb_add = b53_mdb_add,
+ .port_mdb_del = b53_mdb_del,
+ .port_attr_get = b53_port_attr_get,
};
struct bcm_sf2_of_data {
diff --git a/drivers/net/dsa/bcm_sf2_regs.h b/drivers/net/dsa/bcm_sf2_regs.h
index 67f056206f37..d95572a175bc 100644
--- a/drivers/net/dsa/bcm_sf2_regs.h
+++ b/drivers/net/dsa/bcm_sf2_regs.h
@@ -222,8 +222,13 @@ enum bcm_sf2_reg_offs {
#define CORE_GMNCFGCFG 0x0800
#define RST_MIB_CNT (1 << 0)
#define RXBPDU_EN (1 << 1)
+#define FRM_MGNP_SHIFT 6
+#define FRM_MGNP_MASK 0x3
+#define FRM_MNGP_IMP0 2
+#define FRM_MGNP_IMP_DUAL 3
#define CORE_IMP0_PRT_ID 0x0804
+#define CORE_IMP1_PRT_ID 0x0808
#define CORE_RST_MIB_CNT_EN 0x0950
--
2.17.1
From 3fd8225c5a60005141d7c5f0954aaa37d49a51b5 Mon Sep 17 00:00:00 2001
From: Florian Fainelli <f.fainelli@gmail.com>
Date: Thu, 31 Jan 2019 17:04:33 -0800
Subject: [PATCH 13/13] DNP: b53: Debugging
Signed-off-by: Florian Fainelli <f.fainelli@gmail.com>
---
drivers/net/dsa/b53/b53_common.c | 19 ++++++++++++++++---
1 file changed, 16 insertions(+), 3 deletions(-)
diff --git a/drivers/net/dsa/b53/b53_common.c b/drivers/net/dsa/b53/b53_common.c
index 2d56e045c7be..2a05ff3fce27 100644
--- a/drivers/net/dsa/b53/b53_common.c
+++ b/drivers/net/dsa/b53/b53_common.c
@@ -281,7 +281,7 @@ static void b53_set_vlan_entry(struct b53_device *dev, u16 vid,
b53_do_vlan_op(dev, VTA_CMD_WRITE);
}
- dev_dbg(dev->ds->dev, "VID: %d, members: 0x%04x, untag: 0x%04x\n",
+ dev_info(dev->ds->dev, "VID: %d, members: 0x%04x, untag: 0x%04x\n",
vid, vlan->members, vlan->untag);
}
@@ -427,6 +427,9 @@ static void b53_enable_vlan(struct b53_device *dev, bool enable,
dev->vlan_enabled = enable;
dev->vlan_filtering_enabled = enable_filtering;
+
+ dev_info(dev->dev, "VLAN: %d, filtering: %d\n",
+ dev->vlan_enabled, dev->vlan_filtering_enabled);
}
static int b53_set_jumbo(struct b53_device *dev, bool enable, bool allow_10_100)
@@ -1343,6 +1346,8 @@ int b53_vlan_filtering(struct dsa_switch *ds, int port, bool vlan_filtering)
new_pvid = dev->ports[port].pvid;
}
+ dev_info(ds->dev, "PVID: %d, new PVID: %d\n", pvid, new_pvid);
+
if (pvid != new_pvid)
b53_write16(dev, B53_VLAN_PAGE, B53_VLAN_PORT_DEF_TAG(port),
new_pvid);
@@ -1385,7 +1390,7 @@ void b53_vlan_add(struct dsa_switch *ds, int port,
b53_get_vlan_entry(dev, vid, vl);
vl->members |= BIT(port);
- if (untagged && !dsa_is_cpu_port(ds, port))
+ if (untagged) //!dsa_is_cpu_port(ds, port))
vl->untag |= BIT(port);
else
vl->untag &= ~BIT(port);
@@ -1394,7 +1399,7 @@ void b53_vlan_add(struct dsa_switch *ds, int port,
b53_fast_age_vlan(dev, vid);
}
- if (pvid) {
+ if (pvid && !dsa_is_cpu_port(ds, port)) {
b53_write16(dev, B53_VLAN_PAGE, B53_VLAN_PORT_DEF_TAG(port),
vlan->vid_end);
b53_fast_age_vlan(dev, vid);
@@ -1758,6 +1763,8 @@ int b53_mc_disabled(struct dsa_switch *ds, int port, bool mc_disabled)
mc_ctrl &= ~BIT(port);
b53_write16(dev, B53_CTRL_PAGE, B53_IPMC_FLOOD_MASK, mc_ctrl);
+ dev_info(dev->dev, "MC: %d, port: %d\n", mc_disabled, port);
+
return 0;
}
EXPORT_SYMBOL(b53_mc_disabled);
@@ -1783,6 +1790,9 @@ void b53_mdb_add(struct dsa_switch *ds, int port,
struct b53_device *priv = ds->priv;
int ret;
+ dev_info(ds->dev, "Adding, port: %d MAC: %pM, VID: %d\n",
+ port, mdb->addr, mdb->vid);
+
ret = b53_arl_op(priv, 0, port, mdb->addr, mdb->vid, true);
if (ret)
dev_err(ds->dev, "failed to add MDB entry\n");
@@ -1795,6 +1805,9 @@ int b53_mdb_del(struct dsa_switch *ds, int port,
struct b53_device *priv = ds->priv;
int ret;
+ dev_info(ds->dev, "Removing, port: %d MAC: %pM, VID: %d\n",
+ port, mdb->addr, mdb->vid);
+
ret = b53_arl_op(priv, 0, port, mdb->addr, mdb->vid, false);
if (ret)
dev_err(ds->dev, "failed to delete MDB entry\n");
--
2.17.1
^ permalink raw reply related
* [PATCH net-next 2/2] staging: fsl-dpaa2: ethsw: Remove unused port_priv variable
From: Florian Fainelli @ 2019-02-12 23:39 UTC (permalink / raw)
To: netdev; +Cc: davem, Florian Fainelli
In-Reply-To: <20190212234000.15796-1-f.fainelli@gmail.com>
After 1b8b589d9103 ("staging: fsl-dpaa2: ethsw: Remove getting
PORT_BRIDGE_FLAGS") we are not accessing any driver private data
anymore, remove port_priv which is unused.
Fixes: 1b8b589d9103 ("staging: fsl-dpaa2: ethsw: Remove getting PORT_BRIDGE_FLAGS")
Signed-off-by: Florian Fainelli <f.fainelli@gmail.com>
---
drivers/staging/fsl-dpaa2/ethsw/ethsw.c | 2 --
1 file changed, 2 deletions(-)
diff --git a/drivers/staging/fsl-dpaa2/ethsw/ethsw.c b/drivers/staging/fsl-dpaa2/ethsw/ethsw.c
index e5a14c7c777f..1b3943b71254 100644
--- a/drivers/staging/fsl-dpaa2/ethsw/ethsw.c
+++ b/drivers/staging/fsl-dpaa2/ethsw/ethsw.c
@@ -643,8 +643,6 @@ static void ethsw_teardown_irqs(struct fsl_mc_device *sw_dev)
static int swdev_port_attr_get(struct net_device *netdev,
struct switchdev_attr *attr)
{
- struct ethsw_port_priv *port_priv = netdev_priv(netdev);
-
switch (attr->id) {
case SWITCHDEV_ATTR_ID_PORT_BRIDGE_FLAGS_SUPPORT:
attr->u.brport_flags_support = BR_LEARNING | BR_FLOOD;
--
2.17.1
^ permalink raw reply related
* [PATCH net-next 1/2] mlxsw: spectrum_switchdev: Remove unused variables
From: Florian Fainelli @ 2019-02-12 23:39 UTC (permalink / raw)
To: netdev; +Cc: davem, Florian Fainelli
In-Reply-To: <20190212234000.15796-1-f.fainelli@gmail.com>
After 1ecb195753a1 ("mlxsw: spectrum_switchdev: Remove getting
PORT_BRIDGE_FLAGS") we are not accessing any driver private data
structure, so the mlxsw_sp_port and mlxsw_sp variables are unused.
Fixes: 1ecb195753a1 ("mlxsw: spectrum_switchdev: Remove getting PORT_BRIDGE_FLAGS")
Signed-off-by: Florian Fainelli <f.fainelli@gmail.com>
---
drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c | 3 ---
1 file changed, 3 deletions(-)
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c
index 4c5780f8f4b2..1f492b7dbea8 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c
@@ -434,9 +434,6 @@ static void mlxsw_sp_bridge_vlan_put(struct mlxsw_sp_bridge_vlan *bridge_vlan)
static int mlxsw_sp_port_attr_get(struct net_device *dev,
struct switchdev_attr *attr)
{
- struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev);
- struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
-
switch (attr->id) {
case SWITCHDEV_ATTR_ID_PORT_BRIDGE_FLAGS_SUPPORT:
attr->u.brport_flags_support = BR_LEARNING | BR_FLOOD |
--
2.17.1
^ permalink raw reply related
* [PATCH net-next 0/2] Remove unused variables
From: Florian Fainelli @ 2019-02-12 23:39 UTC (permalink / raw)
To: netdev; +Cc: davem, Florian Fainelli
Hi David,
This removes unused variables from mlxsw and ethsw after the recent
removal of SWITCHDEV_ATTR_ID_PORT_BRIDGE_FLAGS, build scripts are now
fixed to take care of those warnings :).
Thank you!
Florian Fainelli (2):
mlxsw: spectrum_switchdev: Remove unused variables
staging: fsl-dpaa2: ethsw: Remove unused port_priv variable
drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c | 3 ---
drivers/staging/fsl-dpaa2/ethsw/ethsw.c | 2 --
2 files changed, 5 deletions(-)
--
2.17.1
^ permalink raw reply
* Re: [PATCHv5 iproute2 net-next 2/2] lib/libnetlink: re malloc buff if size is not enough
From: Eric Dumazet @ 2019-02-12 23:32 UTC (permalink / raw)
To: Hangbin Liu, netdev; +Cc: Stephen Hemminger, Michal Kubecek, Phil Sutter
In-Reply-To: <1508982107-28474-2-git-send-email-liuhangbin@gmail.com>
On 10/25/2017 06:41 PM, Hangbin Liu wrote:
> With commit 72b365e8e0fd ("libnetlink: Double the dump buffer size")
> we doubled the buffer size to support more VFs. But the VFs number is
> increasing all the time. Some customers even use more than 200 VFs now.
>
> We could not double it everytime when the buffer is not enough. Let's just
> not hard code the buffer size and malloc the correct number when running.
>
> Introduce function rtnl_recvmsg() to always return a newly allocated buffer.
> The caller need to free it after using.
>
> Signed-off-by: Hangbin Liu <liuhangbin@gmail.com>
> Signed-off-by: Phil Sutter <phil@nwl.cc>
> ---
> lib/libnetlink.c | 114 ++++++++++++++++++++++++++++++++++++++-----------------
> 1 file changed, 80 insertions(+), 34 deletions(-)
>
> diff --git a/lib/libnetlink.c b/lib/libnetlink.c
> index be7ac86..1847c0b 100644
> --- a/lib/libnetlink.c
> +++ b/lib/libnetlink.c
> @@ -402,6 +402,64 @@ static void rtnl_dump_error(const struct rtnl_handle *rth,
> }
> }
>
> +static int __rtnl_recvmsg(int fd, struct msghdr *msg, int flags)
> +{
> + int len;
> +
> + do {
> + len = recvmsg(fd, msg, flags);
> + } while (len < 0 && (errno == EINTR || errno == EAGAIN));
> +
> + if (len < 0) {
> + fprintf(stderr, "netlink receive error %s (%d)\n",
> + strerror(errno), errno);
> + return -errno;
> + }
> +
> + if (len == 0) {
> + fprintf(stderr, "EOF on netlink\n");
> + return -ENODATA;
> + }
> +
> + return len;
> +}
> +
> +static int rtnl_recvmsg(int fd, struct msghdr *msg, char **answer)
> +{
> + struct iovec *iov = msg->msg_iov;
> + char *buf;
> + int len;
> +
> + iov->iov_base = NULL;
> + iov->iov_len = 0;
> +
> + len = __rtnl_recvmsg(fd, msg, MSG_PEEK | MSG_TRUNC);
> + if (len < 0)
> + return len;
> +
> + buf = malloc(len);
> + if (!buf) {
> + fprintf(stderr, "malloc error: not enough buffer\n");
> + return -ENOMEM;
> + }
> +
> + iov->iov_base = buf;
> + iov->iov_len = len;
> +
> + len = __rtnl_recvmsg(fd, msg, 0);
> + if (len < 0) {
> + free(buf);
> + return len;
> + }
> +
> + if (answer)
> + *answer = buf;
> + else
> + free(buf);
> +
> + return len;
> +}
> +
This patch brings a serious performance penalty.
ss command now uses two system calls per ~4KB worth of data
recvmsg(3, {msg_name(12)={sa_family=AF_NETLINK, pid=0, groups=00000000}, msg_iov(1)=[{NULL, 0}], msg_controllen=0, msg_flags=MSG_TRUNC}, MSG_PEEK|MSG_TRUNC) = 3328 <0.000120>
recvmsg(3, {msg_name(12)={sa_family=AF_NETLINK, pid=0, groups=00000000}, msg_iov(1)=[{"h\0\0\0\24\0\2\0@\342\1\0\322\0\6\0\n\1\1\0\250\253\276@&\7\370\260\200\231\16\6"..., 3328}], msg_controllen=0, msg_flags=0}, 0) = 3328 <0.000108>
recvmsg(3, {msg_name(12)={sa_family=AF_NETLINK, pid=0, groups=00000000}, msg_iov(1)=[{NULL, 0}], msg_controllen=0, msg_flags=MSG_TRUNC}, MSG_PEEK|MSG_TRUNC) = 3328 <0.000086>
recvmsg(3, {msg_name(12)={sa_family=AF_NETLINK, pid=0, groups=00000000}, msg_iov(1)=[{"h\0\0\0\24\0\2\0@\342\1\0\322\0\6\0\n\10\2\0002A\266S&\7\370\260\200\231\16\6"..., 3328}], msg_controllen=0, msg_flags=0}, 0) = 3328 <0.000121>
So we are back to a very pessimistic situation.
^ permalink raw reply
* Is it safe to modify the data directly instead of bpf helper function in tc direct-action mode?
From: IMBRIUS AGER @ 2019-02-12 23:05 UTC (permalink / raw)
To: netdev
I tried the 3 line code:
__builtin_memcpy(skb, &ip->saddr, &src, sizeof(src);
__builtin_memcpy(skb, &ip->daddr, &dst, sizeof(dst);
ip->check = 0x12;
To my surprise, tc dit not reject the code in direct-action mode.
IIRC, since skb must stays uncloned, the verifier will detect all the
writes, which means we need to do the 'data + X > data_end' check
after each write.
Did I miss anything?
Best wishes
^ permalink raw reply
* Re: [RFC 17/19] RDMA/irdma: Add ABI definitions
From: Jason Gunthorpe @ 2019-02-12 23:05 UTC (permalink / raw)
To: Shiraz Saleem
Cc: dledford, davem, linux-rdma, netdev, mustafa.ismail,
jeffrey.t.kirsher
In-Reply-To: <20190212214402.23284-18-shiraz.saleem@intel.com>
On Tue, Feb 12, 2019 at 03:44:00PM -0600, Shiraz Saleem wrote:
> +struct irdma_create_cq_req {
> + __aligned_u64 user_cq_buf;
> + __aligned_u64 user_shadow_area;
> +};
This is the right way to use u64 in uapi headers
> +
> +struct irdma_create_qp_req {
> + __u64 user_wqe_bufs;
> + __u64 user_compl_ctx;
> +};
and this is wrong.. See my prior email.
Jason
^ permalink raw reply
* Re: [PATCH] inet_diag: fix reporting cgroup classid and fallback to priority
From: Eric Dumazet @ 2019-02-12 23:03 UTC (permalink / raw)
To: Konstantin Khlebnikov, netdev, linux-kernel, David S. Miller
Cc: Sasha Levin, linux-sctp
In-Reply-To: <154970855279.305165.13649851988934332761.stgit@buzz>
On 02/09/2019 02:35 AM, Konstantin Khlebnikov wrote:
> Field idiag_ext in struct inet_diag_req_v2 used as bitmap of requested
> extensions has only 8 bits. Thus extensions starting from DCTCPINFO
> cannot be requested directly. Some of them included into response
> unconditionally or hook into some of lower 8 bits.
>
> Extension INET_DIAG_CLASS_ID has not way to request from the beginning.
>
> This patch bundle it with INET_DIAG_TCLASS (ipv6 tos), fixes space
> reservation, and documents behavior for other extensions.
>
> Also this patch adds fallback to reporting socket priority. This filed
> is more widely used for traffic classification because ipv4 sockets
> automatically maps TOS to priority and default qdisc pfifo_fast knows
> about that. But priority could be changed via setsockopt SO_PRIORITY so
> INET_DIAG_TOS isn't enough for predicting class.
>
> Also cgroup2 obsoletes net_cls classid (it always zero), but we cannot
> reuse this field for reporting cgroup2 id because it is 64-bit (ino+gen).
>
> So, after this patch INET_DIAG_CLASS_ID will report socket priority
> for most common setup when net_cls isn't set and/or cgroup2 in use.
>
Nice catch Konstantin
Are you planing sending an iproute2/ss patch to output NET_DIAG_CLASS_ID values then ?
Thanks.
^ permalink raw reply
* Re: [PATCH net] dsa: mv88e6xxx: Ensure all pending interrupts are handled prior to exit
From: Russell King - ARM Linux admin @ 2019-02-12 22:55 UTC (permalink / raw)
To: Heiner Kallweit
Cc: Andrew Lunn, John David Anglin, Vivien Didelot, Florian Fainelli,
netdev
In-Reply-To: <5ba5b654-ca61-253f-042a-2a178ff86b36@gmail.com>
On Tue, Feb 12, 2019 at 09:54:55PM +0100, Heiner Kallweit wrote:
> On 12.02.2019 17:30, Russell King - ARM Linux admin wrote:
> > On Tue, Feb 12, 2019 at 07:51:05AM +0100, Heiner Kallweit wrote:
> >> On 12.02.2019 04:58, Andrew Lunn wrote:
> >>> That change means we don't check the PHY device if it caused an
> >>> interrupt when its state is less than UP.
> >>>
> >>> What i'm seeing is that the PHY is interrupting pretty early on after
> >>> a reboot when the previous boot had the interface up.
> >>>
> >> So this means that when going down for reboot the interrupts are not
> >> properly masked / disabled? Because (at least for net-next) we enable
> >> interrupts in phy_start() only.
> >
> [..]
> > In looking at this, I came across this chunk of code:
> >
> > static inline bool __phy_is_started(struct phy_device *phydev)
> > {
> > WARN_ON(!mutex_is_locked(&phydev->lock));
> >
> > return phydev->state >= PHY_UP;
> > }
> >
> > /**
> > * phy_is_started - Convenience function to check whether PHY is started
> > * @phydev: The phy_device struct
> > */
> > static inline bool phy_is_started(struct phy_device *phydev)
> > {
> > bool started;
> >
> > mutex_lock(&phydev->lock);
> > started = __phy_is_started(phydev);
> > mutex_unlock(&phydev->lock);
> >
> > return started;
> > }
> >
> > which looks to me like over-complication. The mutex locking there is
> > completely pointless - what are you trying to achieve with it?
> >
> > Let's go through this. The above is exactly equivalent to:
> >
> > bool phy_is_started(phydev)
> > {
> > int state;
> >
> > mutex_lock(&phydev->lock);
> > state = phydev->state;
> > mutex_unlock(&phydev->lock);
> >
> > return state >= PHY_UP;
> > }
> >
> > since when we do the test is irrelevant. Architectures that Linux
> > runs on are single-copy atomic, which means that reading phydev->state
> > itself is an atomic operation. So, the mutex locking around that
> > doesn't add to the atomicity of the entire operation.
> >
> > How, depending on what you do with the rest of this function depends
> > whether the entire operation is safe or not. For example, let's take
> > this code at the end of phy_state_machine():
> >
> > if (phy_polling_mode(phydev) && phy_is_started(phydev))
> > phy_queue_state_machine(phydev, PHY_STATE_TIME);
> >
> > state = PHY_UP
> > thread 0 thread 1
> > phy_disconnect()
> > +-phy_is_started()
> > phy_is_started() |
> > `-phy_stop()
> > +-phydev->state = PHY_HALTED
> > `-phy_stop_machine()
> > `-cancel_delayed_work_sync()
> > phy_queue_state_machine()
> > `-mod_delayed_work()
> >
> > At this point, the phydev->state_queue() has been added back onto the
> > system workqueue despite phy_stop_machine() having been called and
> > cancel_delayed_work_sync() called on it.
> >
> > The original code in 4.20 did not have this race condition.
> >
> > Basically, the lock inside phy_is_started() does nothing useful, and
> > I'd say is dangerously misleading.
> >
> Then idea would be to first remove the locking from phy_is_started()
> and in a second step do the following to prevent the described race
> (phy_stop() takes phydev->lock too).
>
> diff --git a/drivers/net/phy/phy.c b/drivers/net/phy/phy.c
> index c1ed03800..69dc64a4d 100644
> --- a/drivers/net/phy/phy.c
> +++ b/drivers/net/phy/phy.c
> @@ -957,8 +957,10 @@ void phy_state_machine(struct work_struct *work)
> * state machine would be pointless and possibly error prone when
> * called from phy_disconnect() synchronously.
> */
> + mutex_lock(&phydev->lock);
> if (phy_polling_mode(phydev) && phy_is_started(phydev))
> phy_queue_state_machine(phydev, PHY_STATE_TIME);
> + mutex_unlock(&phydev->lock);
> }
Yep, that approach would certainly be better. I didn't exhaustively
audit the 5.0-rc code though.
--
RMK's Patch system: https://www.armlinux.org.uk/developer/patches/
FTTC broadband for 0.8mile line in suburbia: sync at 12.1Mbps down 622kbps up
According to speedtest.net: 11.9Mbps down 500kbps up
^ permalink raw reply
* Fw: [Bug 202561] BUG: Null pointer dereference in __skb_unlink()
From: Stephen Hemminger @ 2019-02-12 22:45 UTC (permalink / raw)
To: netdev
https://bugzilla.kernel.org/show_bug.cgi?id=202561
Backtrace:
[ 3.589241] BUG: unable to handle kernel NULL pointer dereference at
0000000000000008
[ 3.598006] IP: __skb_try_recv_from_queue+0x4e/0x1b0
[ 3.606376] Oops: 0002 [#1] PREEMPT SMP NOPTI
[ 3.720520] RIP: 0010:__skb_try_recv_from_queue+0x4e/0x1b0
[ 3.726645] RSP: 0018:ffffb03400e53ac8 EFLAGS: 00010046
[ 3.732470] RAX: ffff9040a78988c8 RBX: ffff904096bbef00 RCX: 000000000000000
[ 3.740441] RDX: 0000000000000000 RSI: ffff9040a78988c8 RDI: fff9040a7898800
[ 3.748411] RBP: ffffb03400e53ae8 R08: ffffb03400e53bf8 R09: fffb03400e53bfc
[ 3.756382] R10: ffff904096989d00 R11: 0000000000000000 R12: fff9040a78988dc
[ 3.764345] R13: ffff9040a78988c8 R14: 0000000000000202 R15: fffb03400e53ba8
[ 3.772305] FS: 00007feb1ef2f740(0000) GS:ffff9040bfd00000(0000)
knlGS:0000000000000000
[ 3.781342] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 3.787757] CR2: 0000000000000008 CR3: 00000001e87fa000 CR4:
00000000003406a0
[ 3.795725] Call Trace:
[ 3.798456] ? preempt_count_add+0x22/0x30
[ 3.803027] __skb_try_recv_datagram+0xe2/0x160
[ 3.808086] __skb_recv_datagram+0x8a/0xc0
[ 3.812657] skb_recv_datagram+0x3a/0x50
[ 3.817035] netlink_recvmsg+0x4e/0x3c0
[ 3.821315] ? copy_msghdr_from_user+0xcf/0x150
[ 3.826372] sock_recvmsg+0x3b/0x50
[ 3.830264] ___sys_recvmsg+0xd4/0x180
[ 3.834441] ? poll_select_copy_remaining+0x140/0x140
[ 3.840080] ? poll_select_copy_remaining+0x140/0x140
[ 3.845721] ? __switch_to_asm+0x34/0x70
[ 3.850098] ? __switch_to_asm+0x40/0x70
[ 3.854473] ? __switch_to_asm+0x34/0x70
[ 3.858849] ? __switch_to_asm+0x40/0x70
[ 3.863228] ? __switch_to_asm+0x34/0x70
[ 3.867604] ? __fget+0x71/0xa0
[ 3.871108] __sys_recvmsg+0x4c/0x90
[ 3.875097] ? __sys_recvmsg+0x4c/0x90
[ 3.879280] SyS_recvmsg+0x9/0x10
[ 3.882979] do_syscall_64+0x7e/0x350
[ 3.887064] ? trace_hardirqs_off_thunk+0x1a/0x1c
[ 3.892315] entry_SYSCALL_64_after_hwframe+0x42/0xb7
Original report from sharathkernel@gmail.com:
NULL POINTER DEFERENCE DURING __skb_unlink()
In the function call, __skb_try_recv_from_queue() (net/core/datagram.c),
sbk_queue_walk() walks through the queue without checking if the next member in the queue has valid next pointer/address. When a socket buffer has to unlink, __skb_unlink() is called.
Inside __skb_unlink() function, it doesn't verify if skb->next has a valid address. skb->next is assigned and used, without verifying the value inside it.
What could be probable solution, in this scenario? Should we check if skb->next is not NULL, before calling __skb_unlink()?
--------------------------------------------------------------------------
--------------------------------------------------------------------------
net/core/datagram.c
struct sk_buff *__skb_try_recv_from_queue(struct sock *sk, struct sk_buff_head *queue, unsigned int flags,void (*destructor)(struct sock *sk, struct sk_buff *skb),int *peeked, int *off, int *err, struct sk_buff **last)
{
...
...
...
skb_queue_walk(queue, skb) {
...
...
...
} else {
==> __skb_unlink(skb, queue);
if(destructor){
....
...
...
}
^ permalink raw reply
* Re: [RFC 12/19] RDMA/irdma: Implement device supported verb APIs
From: Jason Gunthorpe @ 2019-02-12 22:27 UTC (permalink / raw)
To: Shiraz Saleem
Cc: dledford, davem, linux-rdma, netdev, mustafa.ismail,
jeffrey.t.kirsher
In-Reply-To: <20190212214402.23284-13-shiraz.saleem@intel.com>
On Tue, Feb 12, 2019 at 03:43:55PM -0600, Shiraz Saleem wrote:
> +/**
> + * irdma_disassociate_ucontext - Disassociate user context
> + * @context: ib user context
> + */
> +static void irdma_disassociate_ucontext(struct ib_ucontext *context)
> +{
> + struct irdma_ucontext *ucontext = to_ucontext(context);
> +
> + struct irdma_vma_data *vma_data, *n;
> + struct vm_area_struct *vma;
> +
> + irdma_dev_info(&ucontext->iwdev->rf->sc_dev, "called\n");
> + mutex_lock(&ucontext->vma_list_mutex);
> + list_for_each_entry_safe(vma_data, n, &ucontext->vma_list, list) {
> + vma = vma_data->vma;
> + zap_vma_ptes(vma, vma->vm_start, PAGE_SIZE);
> +
> + vma->vm_flags &= ~(VM_SHARED | VM_MAYSHARE);
> + vma->vm_ops = NULL;
> + list_del(&vma_data->list);
> + kfree(vma_data);
> + }
> + mutex_unlock(&ucontext->vma_list_mutex);
> +}
You need to study all the changes that have been done in the core code
and make sure this driver is using all the latest stuff, I do not want
to review a driver and find it is full of obsolete APIs like this
above.
Jason
^ permalink raw reply
* Re: [PATCH net-next] rocker: Remove port_attr_bridge_flags_get assignment
From: Florian Fainelli @ 2019-02-12 21:56 UTC (permalink / raw)
To: David Miller; +Cc: netdev, jiri, linux-kernel
In-Reply-To: <20190212.135401.857430040745833995.davem@davemloft.net>
On 2/12/2019 1:54 PM, David Miller wrote:
> From: Florian Fainelli <f.fainelli@gmail.com>
> Date: Tue, 12 Feb 2019 13:39:05 -0800
>
>> After 610d2b601bba ("rocker: Remove getting PORT_BRIDGE_FLAGS") we no
>> longer have a port_attr_bridge_flags_get member in the rocker_world_ops
>> structre, fix that.
>>
>> Fixes: 610d2b601bba ("rocker: Remove getting PORT_BRIDGE_FLAGS")
>> Signed-off-by: Florian Fainelli <f.fainelli@gmail.com>
>
> Applied, I'm also getting this:
>
> drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c: In function ‘mlxsw_sp_port_attr_get’:
> drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c:438:19: warning: unused variable ‘mlxsw_sp’ [-Wunused-variable]
> struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
> ^~~~~~~~
>
> Related?
Unfortunately yes, that whole patch series was used as test for my build
scripts, which clearly failed miserably here.. Follow-up patches in the
next hours.
--
Florian
^ permalink raw reply
* Re: [PATCH v2] rhashtable: make walk safe from softirq context
From: Bob Copeland @ 2019-02-12 21:54 UTC (permalink / raw)
To: Johannes Berg; +Cc: David Miller, linux-wireless, netdev, j, tgraf, herbert
In-Reply-To: <ea6479e089dffc46bb9b09e28d364db4c071998f.camel@sipsolutions.net>
On Tue, Feb 12, 2019 at 08:03:17PM +0100, Johannes Berg wrote:
> commit 60854fd94573f0d3b80b55b40cf0140a0430f3ab
> Author: Bob Copeland <me@bobcopeland.com>
> Date: Wed Mar 2 10:09:20 2016 -0500
>
> mac80211: mesh: convert path table to rhashtable
>
> which is kinda old. Not sure why this didn't surface before, because the
> spinlock was introduced *before*, otherwise certainly the mutex would've
> caused us to not be able to do this code to start with (commit
> c6ff5268293 - rhashtable: Fix walker list corruption).
>
> That commit also just converted an existing hashtable walk to
> rhashtable, so not sure that counts as having introduced the problem :-)
Yeah, I didn't change any of the contexts in which the iterations were
called, except making it possible to replace the previous mesh-specific
RCU hashtable with rhashtable by introducing this. That said, it would
certainly be nice to not have to walk the table to find paths that
use a specific station as nexthop.
> But I guess we should also ask Bob first:
> 1) do you think it'd be easy to maintain a separate list or avoid the
> iteration in some otherway, and make that a small enough patch to be
> applicable for stable?
I'm not sure, it's been a while. I guess most of the difficulties would
be around station removal?
> 2) or do you think maybe the mesh_plink_broken() call could just be
> lifted into a workqueue instead?
As far as I know, no reason this can't wait until later; we might send
a few frames to the wrong destination but that can happen anyway. But
there are a couple more walks in there like mesh_path_flush_by_nexthop().
--
Bob Copeland %% https://bobcopeland.com/
^ permalink raw reply
* Re: [PATCH net-next] rocker: Remove port_attr_bridge_flags_get assignment
From: David Miller @ 2019-02-12 21:54 UTC (permalink / raw)
To: f.fainelli; +Cc: netdev, jiri, linux-kernel
In-Reply-To: <20190212213905.26559-1-f.fainelli@gmail.com>
From: Florian Fainelli <f.fainelli@gmail.com>
Date: Tue, 12 Feb 2019 13:39:05 -0800
> After 610d2b601bba ("rocker: Remove getting PORT_BRIDGE_FLAGS") we no
> longer have a port_attr_bridge_flags_get member in the rocker_world_ops
> structre, fix that.
>
> Fixes: 610d2b601bba ("rocker: Remove getting PORT_BRIDGE_FLAGS")
> Signed-off-by: Florian Fainelli <f.fainelli@gmail.com>
Applied, I'm also getting this:
drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c: In function ‘mlxsw_sp_port_attr_get’:
drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c:438:19: warning: unused variable ‘mlxsw_sp’ [-Wunused-variable]
struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
^~~~~~~~
Related?
^ permalink raw reply
* Re: [PATCH] isdn: i4l: isdn_tty: Mark expected switch fall-through
From: Gustavo A. R. Silva @ 2019-02-12 21:32 UTC (permalink / raw)
To: David Miller; +Cc: isdn, netdev, linux-kernel, keescook
In-Reply-To: <20190212.102949.565787841270910392.davem@davemloft.net>
Thanks, Dave.
--
Gustavo
On 2/12/19 12:29 PM, David Miller wrote:
> From: "Gustavo A. R. Silva" <gustavo@embeddedor.com>
> Date: Mon, 11 Feb 2019 16:38:21 -0600
>
>> In preparation to enabling -Wimplicit-fallthrough, mark switch
>> cases where we are expecting to fall through.
>>
>> This patch fixes the following warnings:
>>
>> drivers/isdn/i4l/isdn_tty.c: In function ‘isdn_tty_edit_at’:
>> drivers/isdn/i4l/isdn_tty.c:3644:18: warning: this statement may fall through [-Wimplicit-fallthrough=]
>> m->mdmcmdl = 0;
>> ~~~~~~~~~~~^~~
>> drivers/isdn/i4l/isdn_tty.c:3646:5: note: here
>> case 0:
>> ^~~~
>>
>> Warning level 3 was used: -Wimplicit-fallthrough=3
>>
>> Notice that, in this particular case, the code comment is modified
>> in accordance with what GCC is expecting to find.
>>
>> This patch is part of the ongoing efforts to enable
>> -Wimplicit-fallthrough.
>>
>> Signed-off-by: Gustavo A. R. Silva <gustavo@embeddedor.com>
>
> Applied.
>
^ permalink raw reply
* [RFC 05/19] RDMA/irdma: Implement device initialization definitions
From: Shiraz Saleem @ 2019-02-12 21:43 UTC (permalink / raw)
To: dledford, jgg, davem
Cc: linux-rdma, netdev, mustafa.ismail, jeffrey.t.kirsher,
Shiraz Saleem
In-Reply-To: <20190212214402.23284-1-shiraz.saleem@intel.com>
From: Mustafa Ismail <mustafa.ismail@intel.com>
Implement device initialization routines, interrupt set-up,
and allocate object bit-map tracking structures.
Also, add device specific attributes and register definitions.
Signed-off-by: Mustafa Ismail <mustafa.ismail@intel.com>
Signed-off-by: Shiraz Saleem <shiraz.saleem@intel.com>
---
drivers/infiniband/hw/irdma/hw.c | 2484 +++++++++++++++++++++++++++++++
drivers/infiniband/hw/irdma/i40iw_hw.c | 207 +++
drivers/infiniband/hw/irdma/i40iw_hw.h | 164 ++
drivers/infiniband/hw/irdma/icrdma_hw.c | 74 +
drivers/infiniband/hw/irdma/icrdma_hw.h | 60 +
5 files changed, 2989 insertions(+)
create mode 100644 drivers/infiniband/hw/irdma/hw.c
create mode 100644 drivers/infiniband/hw/irdma/i40iw_hw.c
create mode 100644 drivers/infiniband/hw/irdma/i40iw_hw.h
create mode 100644 drivers/infiniband/hw/irdma/icrdma_hw.c
create mode 100644 drivers/infiniband/hw/irdma/icrdma_hw.h
diff --git a/drivers/infiniband/hw/irdma/hw.c b/drivers/infiniband/hw/irdma/hw.c
new file mode 100644
index 0000000..208905e
--- /dev/null
+++ b/drivers/infiniband/hw/irdma/hw.c
@@ -0,0 +1,2484 @@
+// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
+/* Copyright (c) 2019, Intel Corporation. */
+
+#include "main.h"
+
+static struct irdma_rsrc_limits rsrc_limits_table[] = {
+ [0] = {
+ .qplimit = 16384,
+ },
+ [1] = {
+ .qplimit = 128,
+ },
+ [2] = {
+ .qplimit = 1024,
+ },
+ [3] = {
+ .qplimit = 65536,
+ },
+};
+
+/* types of hmc objects */
+static enum irdma_hmc_rsrc_type iw_hmc_obj_types[] = {
+ IRDMA_HMC_IW_QP,
+ IRDMA_HMC_IW_CQ,
+ IRDMA_HMC_IW_HTE,
+ IRDMA_HMC_IW_ARP,
+ IRDMA_HMC_IW_APBVT_ENTRY,
+ IRDMA_HMC_IW_MR,
+ IRDMA_HMC_IW_XF,
+ IRDMA_HMC_IW_XFFL,
+ IRDMA_HMC_IW_Q1,
+ IRDMA_HMC_IW_Q1FL,
+ IRDMA_HMC_IW_TIMER,
+ IRDMA_HMC_IW_FSIMC,
+ IRDMA_HMC_IW_FSIAV,
+ IRDMA_HMC_IW_RRF,
+ IRDMA_HMC_IW_RRFFL,
+ IRDMA_HMC_IW_HDR,
+ IRDMA_HMC_IW_MD,
+ IRDMA_HMC_IW_OOISC,
+ IRDMA_HMC_IW_OOISCFFL,
+};
+
+/**
+ * irdma_enable_intr - set up device interrupts
+ * @dev: hardware control device structure
+ * @msix_id: id of the interrupt to be enabled
+ */
+static void irdma_ena_intr(struct irdma_sc_dev *dev, u32 msix_id)
+{
+ dev->irq_ops->irdma_en_irq(dev, msix_id);
+}
+
+/**
+ * irdma_dpc - tasklet for aeq and ceq 0
+ * @data: RDMA PCI function
+ */
+static void irdma_dpc(unsigned long data)
+{
+ struct irdma_pci_f *rf = (struct irdma_pci_f *)data;
+
+ if (rf->msix_shared)
+ irdma_process_ceq(rf, rf->ceqlist);
+ irdma_process_aeq(rf);
+ irdma_ena_intr(&rf->sc_dev, rf->iw_msixtbl[0].idx);
+}
+
+/**
+ * irdma_ceq_dpc - dpc handler for CEQ
+ * @data: data points to CEQ
+ */
+static void irdma_ceq_dpc(unsigned long data)
+{
+ struct irdma_ceq *iwceq = (struct irdma_ceq *)data;
+ struct irdma_pci_f *rf = iwceq->rf;
+
+ irdma_process_ceq(rf, iwceq);
+ irdma_ena_intr(&rf->sc_dev, iwceq->msix_idx);
+}
+
+/**
+ * irdma_save_msix_info - copy msix vector information to iwarp device
+ * @rf: RDMA PCI function
+ * @ldev: lan device information
+ *
+ * Allocate iwdev msix table and copy the ldev msix info to the table
+ * Return 0 if successful, otherwise return error
+ */
+static enum irdma_status_code irdma_save_msix_info(struct irdma_pci_f *rf)
+{
+ struct irdma_priv_ldev *ldev = &rf->ldev;
+ struct irdma_qvlist_info *iw_qvlist;
+ struct irdma_qv_info *iw_qvinfo;
+ struct msix_entry *pmsix;
+ u32 ceq_idx;
+ u32 i;
+ u32 size;
+
+ if (!ldev->msix_count) {
+ irdma_pr_err("No MSI-X vectors for RDMA\n");
+ return IRDMA_ERR_CFG;
+ }
+
+ rf->msix_count = ldev->msix_count;
+ size = sizeof(struct irdma_msix_vector) * rf->msix_count;
+ size += sizeof(struct irdma_qvlist_info);
+ size += sizeof(struct irdma_qv_info) * rf->msix_count - 1;
+ rf->iw_msixtbl = kzalloc(size, GFP_KERNEL);
+ if (!rf->iw_msixtbl)
+ return IRDMA_ERR_NO_MEMORY;
+
+ rf->iw_qvlist = (struct irdma_qvlist_info *)
+ (&rf->iw_msixtbl[rf->msix_count]);
+ iw_qvlist = rf->iw_qvlist;
+ iw_qvinfo = iw_qvlist->qv_info;
+ iw_qvlist->num_vectors = rf->msix_count;
+ if (rf->msix_count <= num_online_cpus())
+ rf->msix_shared = true;
+
+ for (i = 0, ceq_idx = 0, pmsix = ldev->msix_entries;
+ i < rf->msix_count; i++, iw_qvinfo++, pmsix++) {
+ rf->iw_msixtbl[i].idx = pmsix->entry;
+ rf->iw_msixtbl[i].irq = pmsix->vector;
+ rf->iw_msixtbl[i].cpu_affinity = ceq_idx;
+ if (!i) {
+ iw_qvinfo->aeq_idx = 0;
+ if (rf->msix_shared)
+ iw_qvinfo->ceq_idx = ceq_idx++;
+ else
+ iw_qvinfo->ceq_idx = IRDMA_Q_INVALID_IDX;
+ } else {
+ iw_qvinfo->aeq_idx = IRDMA_Q_INVALID_IDX;
+ iw_qvinfo->ceq_idx = ceq_idx++;
+ }
+ iw_qvinfo->itr_idx = 3;
+ iw_qvinfo->v_idx = rf->iw_msixtbl[i].idx;
+ }
+
+ return 0;
+}
+
+/**
+ * irdma_pble_initialize_lock - initialize lock for pble resource
+ * @pble_rsrc: pble resource management structure pointer
+ */
+static void irdma_pble_initialize_lock(struct irdma_hmc_pble_rsrc *pble_rsrc)
+{
+ spin_lock_init(&pble_rsrc->pble_lock);
+}
+
+/**
+ * irdma_irq_handler - interrupt handler for aeq and ceq0
+ * @irq: Interrupt request number
+ * @data: RDMA PCI function
+ */
+static irqreturn_t irdma_irq_handler(int irq, void *data)
+{
+ struct irdma_pci_f *rf = data;
+
+ tasklet_schedule(&rf->dpc_tasklet);
+ return IRQ_HANDLED;
+}
+
+/**
+ * irdma_ceq_handler - interrupt handler for ceq
+ * @data: ceq pointer
+ */
+static irqreturn_t irdma_ceq_handler(int irq, void *data)
+{
+ struct irdma_ceq *iwceq = (struct irdma_ceq *)data;
+
+ if (iwceq->irq != irq)
+ irdma_dev_err(&iwceq->rf->sc_dev,
+ "expected irq = %d received irq = %d\n",
+ iwceq->irq, irq);
+ tasklet_schedule(&iwceq->dpc_tasklet);
+
+ return IRQ_HANDLED;
+}
+
+/**
+ * irdma_destroy_irq - destroy device interrupts
+ * @rf: RDMA PCI function
+ * @msic_vec: msix vector to disable irq
+ * @dev_id: parameter to pass to free_irq (used during irq setup)
+ *
+ * The function is called when destroying aeq/ceq
+ */
+static void irdma_destroy_irq(struct irdma_pci_f *rf,
+ struct irdma_msix_vector *msix_vec,
+ void *dev_id)
+{
+ struct irdma_sc_dev *dev = &rf->sc_dev;
+
+ dev->irq_ops->irdma_dis_irq(dev, msix_vec->idx);
+ irq_set_affinity_hint(msix_vec->irq, NULL);
+ free_irq(msix_vec->irq, dev_id);
+}
+
+/**
+ * irdma_destroy_cqp - destroy control qp
+ * @rf: RDMA PCI function
+ * @create_done: 1 if cqp create poll was success
+ *
+ * Issue destroy cqp request and
+ * free the resources associated with the cqp
+ */
+static void irdma_destroy_cqp(struct irdma_pci_f *rf, bool free_hwcqp)
+{
+ enum irdma_status_code status = 0;
+ struct irdma_sc_dev *dev = &rf->sc_dev;
+ struct irdma_cqp *cqp = &rf->cqp;
+
+ if (free_hwcqp && dev->cqp_ops->cqp_destroy)
+ status = dev->cqp_ops->cqp_destroy(dev->cqp);
+ if (status)
+ irdma_debug(dev, IRDMA_DEBUG_ERR, "Destroy CQP failed %d\n",
+ status);
+
+ irdma_cleanup_pending_cqp_op(rf);
+ irdma_free_dma_mem(dev->hw, &cqp->sq);
+ kfree(cqp->scratch_array);
+ cqp->scratch_array = NULL;
+ kfree(cqp->cqp_requests);
+ cqp->cqp_requests = NULL;
+}
+
+/**
+ * irdma_destroy_aeq - destroy aeq
+ * @rf: RDMA PCI function
+ *
+ * Issue a destroy aeq request and
+ * free the resources associated with the aeq
+ * The function is called during driver unload
+ */
+static void irdma_destroy_aeq(struct irdma_pci_f *rf)
+{
+ enum irdma_status_code status = IRDMA_ERR_NOT_READY;
+ struct irdma_sc_dev *dev = &rf->sc_dev;
+ struct irdma_aeq *aeq = &rf->aeq;
+
+ if (!rf->msix_shared)
+ irdma_destroy_irq(rf, rf->iw_msixtbl, (void *)rf);
+ if (rf->reset)
+ goto exit;
+
+ if (!dev->aeq_ops->aeq_destroy(&aeq->sc_aeq, 0, 1))
+ status = dev->aeq_ops->aeq_destroy_done(&aeq->sc_aeq);
+ if (status)
+ irdma_debug(dev, IRDMA_DEBUG_ERR, "Destroy AEQ failed %d\n",
+ status);
+
+exit:
+ irdma_free_dma_mem(dev->hw, &aeq->mem);
+}
+
+/**
+ * irdma_destroy_ceq - destroy ceq
+ * @rf: RDMA PCI function
+ * @iwceq: ceq to be destroyed
+ *
+ * Issue a destroy ceq request and
+ * free the resources associated with the ceq
+ */
+static void irdma_destroy_ceq(struct irdma_pci_f *rf, struct irdma_ceq *iwceq)
+{
+ enum irdma_status_code status;
+ struct irdma_sc_dev *dev = &rf->sc_dev;
+
+ if (rf->reset)
+ goto exit;
+
+ status = dev->ceq_ops->ceq_destroy(&iwceq->sc_ceq, 0, 1);
+ if (status) {
+ irdma_debug(dev, IRDMA_DEBUG_ERR,
+ "CEQ destroy command failed %d\n", status);
+ goto exit;
+ }
+
+ status = dev->ceq_ops->cceq_destroy_done(&iwceq->sc_ceq);
+ if (status)
+ irdma_debug(dev, IRDMA_DEBUG_ERR,
+ "CEQ destroy completion failed %d\n", status);
+exit:
+ irdma_free_dma_mem(dev->hw, &iwceq->mem);
+}
+
+/**
+ * irdma_del_ceq_0 - destroy ceq 0
+ * @rf: RDMA PCI function
+ *
+ * Disable the ceq 0 interrupt and destroy the ceq 0
+ */
+static void irdma_del_ceq_0(struct irdma_pci_f *rf)
+{
+ struct irdma_ceq *iwceq = rf->ceqlist;
+ struct irdma_msix_vector *msix_vec;
+
+ if (rf->msix_shared) {
+ msix_vec = &rf->iw_msixtbl[0];
+ irdma_destroy_irq(rf, msix_vec, (void *)rf);
+ } else {
+ msix_vec = &rf->iw_msixtbl[1];
+ irdma_destroy_irq(rf, msix_vec, (void *)iwceq);
+ }
+ irdma_destroy_ceq(rf, iwceq);
+ rf->sc_dev.ceq_valid = false;
+}
+
+/**
+ * irdma_del_ceqs - destroy all ceq's except CEQ 0 // RT mode FSL
+ * @rf: RDMA PCI function
+ *
+ * Go through all of the device ceq's, except 0, and for each
+ * ceq disable the ceq interrupt and destroy the ceq
+ */
+static void irdma_del_ceqs(struct irdma_pci_f *rf)
+{
+ struct irdma_ceq *iwceq = &rf->ceqlist[1];
+ struct irdma_msix_vector *msix_vec;
+ u32 i = 0;
+
+ if (rf->msix_shared)
+ msix_vec = &rf->iw_msixtbl[1];
+ else
+ msix_vec = &rf->iw_msixtbl[2];
+
+ for (i = 0; i < rf->ceqs_count - 1; i++, msix_vec++, iwceq++) {
+ irdma_destroy_irq(rf, msix_vec, (void *)iwceq);
+ irdma_cqp_ceq_cmd(&rf->sc_dev, &iwceq->sc_ceq,
+ IRDMA_OP_CEQ_DESTROY);
+ irdma_free_dma_mem(rf->sc_dev.hw, &iwceq->mem);
+ }
+}
+
+/**
+ * irdma_destroy_ccq - destroy control cq
+ * @rf: RDMA PCI function
+ *
+ * Issue destroy ccq request and
+ * free the resources associated with the ccq
+ */
+static void irdma_destroy_ccq(struct irdma_pci_f *rf)
+{
+ struct irdma_sc_dev *dev = &rf->sc_dev;
+ struct irdma_ccq *ccq = &rf->ccq;
+ enum irdma_status_code status = 0;
+
+ if (!rf->reset)
+ status = dev->ccq_ops->ccq_destroy(dev->ccq, 0, true);
+ if (status)
+ irdma_debug(dev, IRDMA_DEBUG_ERR,
+ "CCQ destroy failed %d\n", status);
+ irdma_free_dma_mem(dev->hw, &ccq->mem_cq);
+}
+
+/**
+ * irdma_close_hmc_objects_type - delete hmc objects of a given type
+ * @iwdev: iwarp device
+ * @obj_type: the hmc object type to be deleted
+ * @is_pf: true if the function is PF otherwise false
+ * @reset: true if called before reset
+ */
+static void irdma_close_hmc_objects_type(struct irdma_sc_dev *dev,
+ enum irdma_hmc_rsrc_type obj_type,
+ struct irdma_hmc_info *hmc_info,
+ bool is_pf,
+ bool reset)
+{
+ struct irdma_hmc_del_obj_info info = {};
+
+ info.hmc_info = hmc_info;
+ info.rsrc_type = obj_type;
+ info.count = hmc_info->hmc_obj[obj_type].cnt;
+ info.is_pf = is_pf;
+ if (dev->hmc_ops->del_hmc_object(dev, &info, reset))
+ irdma_debug(dev, IRDMA_DEBUG_ERR,
+ "del HMC obj of type %d failed\n", obj_type);
+}
+
+/**
+ * irdma_del_hmc_objects - remove all device hmc objects
+ * @dev: iwarp device
+ * @hmc_info: hmc_info to free
+ * @is_pf: true if hmc_info belongs to PF, not vf nor allocated
+ * by PF on behalf of VF
+ * @reset: true if called before reset
+ */
+static void irdma_del_hmc_objects(struct irdma_sc_dev *dev,
+ struct irdma_hmc_info *hmc_info,
+ bool is_pf,
+ bool reset,
+ enum irdma_vers vers)
+{
+ unsigned int i;
+
+ for (i = 0; i < IW_HMC_OBJ_TYPE_NUM; i++) {
+ if (dev->hmc_info->hmc_obj[iw_hmc_obj_types[i]].cnt)
+ irdma_close_hmc_objects_type(dev,
+ iw_hmc_obj_types[i],
+ hmc_info,
+ is_pf,
+ reset);
+ if (vers == IRDMA_GEN_1 && i == IRDMA_HMC_IW_TIMER)
+ break;
+ }
+}
+
+/**
+ * irdma_create_hmc_obj_type - create hmc object of a given type
+ * @dev: hardware control device structure
+ * @info: information for the hmc object to create
+ */
+static enum irdma_status_code
+irdma_create_hmc_obj_type(struct irdma_sc_dev *dev,
+ struct irdma_hmc_create_obj_info *info)
+{
+ return dev->hmc_ops->create_hmc_object(dev, info);
+}
+
+/**
+ * irdma_create_hmc_objs - create all hmc objects for the device
+ * @rf: RDMA PCI function
+ * @is_pf: true if the function is PF otherwise false
+ *
+ * Create the device hmc objects and allocate hmc pages
+ * Return 0 if successful, otherwise clean up and return error
+ */
+static enum irdma_status_code irdma_create_hmc_objs(struct irdma_pci_f *rf,
+ bool is_pf,
+ enum irdma_vers vers)
+{
+ struct irdma_sc_dev *dev = &rf->sc_dev;
+ struct irdma_hmc_create_obj_info info = {};
+ enum irdma_status_code status = 0;
+ int i;
+
+ info.hmc_info = dev->hmc_info;
+ info.is_pf = is_pf;
+ info.entry_type = rf->sd_type;
+
+ for (i = 0; i < IW_HMC_OBJ_TYPE_NUM; i++) {
+ if (dev->hmc_info->hmc_obj[iw_hmc_obj_types[i]].cnt) {
+ info.rsrc_type = iw_hmc_obj_types[i];
+ info.count = dev->hmc_info->hmc_obj[info.rsrc_type].cnt;
+ info.add_sd_cnt = 0;
+ status = irdma_create_hmc_obj_type(dev, &info);
+ if (status) {
+ irdma_debug(dev, IRDMA_DEBUG_ERR,
+ "create obj type %d status = %d\n",
+ iw_hmc_obj_types[i], status);
+ break;
+ }
+ }
+ if (vers == IRDMA_GEN_1 && i == IRDMA_HMC_IW_TIMER)
+ break;
+ }
+
+ if (!status)
+ return dev->hmc_ops->static_hmc_pages_allocated(dev->cqp,
+ 0,
+ dev->hmc_fn_id,
+ true,
+ true);
+
+ while (i) {
+ i--;
+ /* destroy the hmc objects of a given type */
+ irdma_close_hmc_objects_type(dev,
+ iw_hmc_obj_types[i],
+ dev->hmc_info,
+ is_pf,
+ false);
+ }
+
+ return status;
+}
+
+/**
+ * irdma_obj_aligned_mem - get aligned memory from device allocated memory
+ * @rf: RDMA PCI function
+ * @memptr: points to the memory addresses
+ * @size: size of memory needed
+ * @mask: mask for the aligned memory
+ *
+ * Get aligned memory of the requested size and
+ * update the memptr to point to the new aligned memory
+ * Return 0 if successful, otherwise return no memory error
+ */
+static enum irdma_status_code irdma_obj_aligned_mem(struct irdma_pci_f *rf,
+ struct irdma_dma_mem *memptr,
+ u32 size,
+ u32 mask)
+{
+ unsigned long va, newva;
+ unsigned long extra;
+
+ va = (unsigned long)rf->obj_next.va;
+ newva = va;
+ if (mask)
+ newva = ALIGN(va, ((unsigned long)mask + 1ULL));
+ extra = newva - va;
+ memptr->va = (u8 *)va + extra;
+ memptr->pa = rf->obj_next.pa + extra;
+ memptr->size = size;
+ if ((memptr->va + size) > (rf->obj_mem.va + rf->obj_mem.size))
+ return IRDMA_ERR_NO_MEMORY;
+
+ rf->obj_next.va = memptr->va + size;
+ rf->obj_next.pa = memptr->pa + size;
+
+ return 0;
+}
+
+/**
+ * irdma_create_cqp - create control qp
+ * @rf: RDMA PCI function
+ *
+ * Return 0, if the cqp and all the resources associated with it
+ * are successfully created, otherwise return error
+ */
+static enum irdma_status_code irdma_create_cqp(struct irdma_pci_f *rf)
+{
+ enum irdma_status_code status;
+ u32 sqsize = IRDMA_CQP_SW_SQSIZE_2048;
+ struct irdma_dma_mem mem;
+ struct irdma_sc_dev *dev = &rf->sc_dev;
+ struct irdma_cqp_init_info cqp_init_info = {};
+ struct irdma_cqp *cqp = &rf->cqp;
+ u16 maj_err, min_err;
+ int i;
+
+ cqp->cqp_requests = kcalloc(sqsize, sizeof(*cqp->cqp_requests), GFP_KERNEL);
+ if (!cqp->cqp_requests)
+ return IRDMA_ERR_NO_MEMORY;
+
+ cqp->scratch_array = kcalloc(sqsize, sizeof(*cqp->scratch_array), GFP_KERNEL);
+ if (!cqp->scratch_array) {
+ kfree(cqp->cqp_requests);
+ return IRDMA_ERR_NO_MEMORY;
+ }
+
+ dev->cqp = &cqp->sc_cqp;
+ dev->cqp->dev = dev;
+ status = irdma_allocate_dma_mem(dev->hw, &cqp->sq,
+ (sizeof(struct irdma_cqp_sq_wqe) * sqsize),
+ IRDMA_CQP_ALIGNMENT);
+ if (status)
+ goto exit;
+
+ status = irdma_obj_aligned_mem(rf, &mem, sizeof(struct irdma_cqp_ctx),
+ IRDMA_HOST_CTX_ALIGNMENT_M);
+ if (status)
+ goto exit;
+
+ dev->cqp->host_ctx_pa = mem.pa;
+ dev->cqp->host_ctx = mem.va;
+ /* populate the cqp init info */
+ cqp_init_info.dev = dev;
+ cqp_init_info.sq_size = sqsize;
+ cqp_init_info.sq = cqp->sq.va;
+ cqp_init_info.sq_pa = cqp->sq.pa;
+ cqp_init_info.host_ctx_pa = mem.pa;
+ cqp_init_info.host_ctx = mem.va;
+ cqp_init_info.hmc_profile = rf->rsrc_profile;
+ cqp_init_info.ena_vf_count = rf->max_rdma_vfs;
+ cqp_init_info.scratch_array = cqp->scratch_array;
+ cqp_init_info.disable_packed = true;
+ cqp_init_info.protocol_used = rf->protocol_used;
+ status = dev->cqp_ops->cqp_init(dev->cqp, &cqp_init_info);
+ if (status) {
+ irdma_debug(dev, IRDMA_DEBUG_ERR, "cqp init status %d\n",
+ status);
+ goto exit;
+ }
+
+ status = dev->cqp_ops->cqp_create(dev->cqp, &maj_err, &min_err);
+ if (status) {
+ irdma_debug(dev, IRDMA_DEBUG_ERR,
+ "cqp create failed - status %d maj_err %d min_err %d\n",
+ status, maj_err, min_err);
+ goto exit;
+ }
+
+ spin_lock_init(&cqp->req_lock);
+ spin_lock_init(&cqp->compl_lock);
+ INIT_LIST_HEAD(&cqp->cqp_avail_reqs);
+ INIT_LIST_HEAD(&cqp->cqp_pending_reqs);
+ sema_init(&cqp->cqp_compl_sem, 0);
+
+ /* init the waitqueue of the cqp_requests and add them to the list */
+ for (i = 0; i < sqsize; i++) {
+ init_waitqueue_head(&cqp->cqp_requests[i].waitq);
+ list_add_tail(&cqp->cqp_requests[i].list, &cqp->cqp_avail_reqs);
+ }
+ init_waitqueue_head(&cqp->remove_wq);
+ return 0;
+
+exit:
+ irdma_destroy_cqp(rf, false);
+
+ return status;
+}
+
+/**
+ * irdma_create_ccq - create control cq
+ * @rf: RDMA PCI function
+ *
+ * Return 0, if the ccq and the resources associated with it
+ * are successfully created, otherwise return error
+ */
+static enum irdma_status_code irdma_create_ccq(struct irdma_pci_f *rf)
+{
+ struct irdma_sc_dev *dev = &rf->sc_dev;
+ enum irdma_status_code status;
+ struct irdma_ccq_init_info info = {};
+ struct irdma_ccq *ccq = &rf->ccq;
+
+ dev->ccq = &ccq->sc_cq;
+ dev->ccq->dev = dev;
+ info.dev = dev;
+ ccq->shadow_area.size = sizeof(struct irdma_cq_shadow_area);
+ ccq->mem_cq.size = sizeof(struct irdma_cqe) * IW_CCQ_SIZE;
+ status = irdma_allocate_dma_mem(dev->hw, &ccq->mem_cq,
+ ccq->mem_cq.size, IRDMA_CQ0_ALIGNMENT);
+ if (status)
+ goto exit;
+
+ status = irdma_obj_aligned_mem(rf, &ccq->shadow_area,
+ ccq->shadow_area.size,
+ IRDMA_SHADOWAREA_M);
+ if (status)
+ goto exit;
+
+ ccq->sc_cq.back_cq = (void *)ccq;
+ /* populate the ccq init info */
+ info.cq_base = ccq->mem_cq.va;
+ info.cq_pa = ccq->mem_cq.pa;
+ info.num_elem = IW_CCQ_SIZE;
+ info.shadow_area = ccq->shadow_area.va;
+ info.shadow_area_pa = ccq->shadow_area.pa;
+ info.ceqe_mask = false;
+ info.ceq_id_valid = true;
+ info.shadow_read_threshold = 16;
+ info.vsi = &rf->default_vsi;
+ status = dev->ccq_ops->ccq_init(dev->ccq, &info);
+ if (!status)
+ status = dev->ccq_ops->ccq_create(dev->ccq, 0, true, true);
+exit:
+ if (status)
+ irdma_free_dma_mem(dev->hw, &ccq->mem_cq);
+
+ return status;
+}
+
+/**
+ * irdma_alloc_set_mac - set up a mac address table entry
+ * @iwdev: device
+ * @macaddr: pointer to mac address
+ *
+ * Allocate a mac ip entry and add it to the hw table Return 0
+ * if successful, otherwise return error
+ */
+static enum irdma_status_code irdma_alloc_set_mac(struct irdma_device *iwdev)
+{
+ enum irdma_status_code status;
+
+ status = irdma_alloc_local_mac_entry(iwdev->rf, &iwdev->mac_ip_table_idx);
+ if (!status) {
+ status = irdma_add_local_mac_entry(iwdev->rf,
+ (u8 *)iwdev->netdev->dev_addr,
+ (u8)iwdev->mac_ip_table_idx);
+ if (status)
+ irdma_del_local_mac_entry(iwdev->rf, (u8)iwdev->mac_ip_table_idx);
+ }
+ return status;
+}
+
+/**
+ * irdma_configure_ceq_vector - set up the msix interrupt vector for ceq
+ * @rf: RDMA PCI function
+ * @iwceq: ceq associated with the vector
+ * @ceq_id: the id number of the iwceq
+ * @msix_vec: interrupt vector information
+ *
+ * Allocate interrupt resources and enable irq handling
+ * Return 0 if successful, otherwise return error
+ */
+static enum irdma_status_code
+irdma_cfg_ceq_vector(struct irdma_pci_f *rf,
+ struct irdma_ceq *iwceq,
+ u32 ceq_id,
+ struct irdma_msix_vector *msix_vec)
+{
+ int status;
+
+ if (rf->msix_shared && !ceq_id) {
+ tasklet_init(&rf->dpc_tasklet, irdma_dpc, (unsigned long)rf);
+ status = request_irq(msix_vec->irq, irdma_irq_handler, 0,
+ "AEQCEQ", rf);
+ } else {
+ tasklet_init(&iwceq->dpc_tasklet, irdma_ceq_dpc,
+ (unsigned long)iwceq);
+
+ status = request_irq(msix_vec->irq, irdma_ceq_handler, 0,
+ "CEQ", iwceq);
+ }
+
+ cpumask_clear(&msix_vec->mask);
+ cpumask_set_cpu(msix_vec->cpu_affinity, &msix_vec->mask);
+ irq_set_affinity_hint(msix_vec->irq, &msix_vec->mask);
+ if (status) {
+ irdma_debug(&rf->sc_dev, IRDMA_DEBUG_ERR, "ceq irq config fail\n");
+ return IRDMA_ERR_CFG;
+ }
+
+ msix_vec->ceq_id = ceq_id;
+ rf->sc_dev.irq_ops->irdma_cfg_ceq(&rf->sc_dev, ceq_id, msix_vec->idx);
+
+ return 0;
+}
+
+/**
+ * irdma_configure_aeq_vector - set up the msix vector for aeq
+ * @rf: RDMA PCI function
+ *
+ * Allocate interrupt resources and enable irq handling
+ * Return 0 if successful, otherwise return error
+ */
+static enum irdma_status_code irdma_cfg_aeq_vector(struct irdma_pci_f *rf)
+{
+ struct irdma_msix_vector *msix_vec = rf->iw_msixtbl;
+ u32 ret = 0;
+
+ if (!rf->msix_shared) {
+ tasklet_init(&rf->dpc_tasklet, irdma_dpc, (unsigned long)rf);
+ ret = request_irq(msix_vec->irq, irdma_irq_handler, 0,
+ "icrdma", rf);
+ }
+ if (ret) {
+ irdma_debug(&rf->sc_dev, IRDMA_DEBUG_ERR, "aeq irq config fail\n");
+ return IRDMA_ERR_CFG;
+ }
+
+ rf->sc_dev.irq_ops->irdma_cfg_aeq(&rf->sc_dev, msix_vec->idx);
+
+ return 0;
+}
+
+/**
+ * irdma_create_ceq - create completion event queue
+ * @rf: RDMA PCI function
+ * @iwceq: pointer to the ceq resources to be created
+ * @ceq_id: the id number of the iwceq
+ * @vsi: SC vsi struct
+ *
+ * Return 0, if the ceq and the resources associated with it
+ * are successfully created, otherwise return error
+ */
+static enum irdma_status_code irdma_create_ceq(struct irdma_pci_f *rf,
+ struct irdma_ceq *iwceq,
+ u32 ceq_id,
+ struct irdma_sc_vsi *vsi)
+{
+ enum irdma_status_code status;
+ struct irdma_ceq_init_info info = {};
+ struct irdma_sc_dev *dev = &rf->sc_dev;
+ u64 scratch;
+
+ info.ceq_id = ceq_id;
+ iwceq->rf = rf;
+ iwceq->mem.size = sizeof(struct irdma_ceqe) *
+ rf->sc_dev.hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt;
+ status = irdma_allocate_dma_mem(dev->hw, &iwceq->mem, iwceq->mem.size,
+ IRDMA_CEQ_ALIGNMENT);
+ if (status)
+ goto exit;
+
+ info.ceq_id = ceq_id;
+ info.ceqe_base = iwceq->mem.va;
+ info.ceqe_pa = iwceq->mem.pa;
+ info.elem_cnt = rf->sc_dev.hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt;
+ iwceq->sc_ceq.ceq_id = ceq_id;
+ info.dev = dev;
+ info.vsi = vsi;
+ scratch = (uintptr_t)&rf->cqp.sc_cqp;
+ status = dev->ceq_ops->ceq_init(&iwceq->sc_ceq, &info);
+ if (!status) {
+ if (dev->ceq_valid)
+ status = irdma_cqp_ceq_cmd(&rf->sc_dev, &iwceq->sc_ceq,
+ IRDMA_OP_CEQ_CREATE);
+ else
+ status = dev->ceq_ops->cceq_create(&iwceq->sc_ceq,
+ scratch);
+ }
+
+exit:
+ if (status)
+ irdma_free_dma_mem(dev->hw, &iwceq->mem);
+
+ return status;
+}
+
+/**
+ * irdma_setup_ceq_0 - create CEQ 0 and it's interrupt resource
+ * @rf: RDMA PCI function
+ *
+ * Allocate a list for all device completion event queues
+ * Create the ceq 0 and configure it's msix interrupt vector
+ * Return 0, if successfully set up, otherwise return error
+ */
+static enum irdma_status_code irdma_setup_ceq_0(struct irdma_pci_f *rf)
+{
+ u32 i;
+ struct irdma_ceq *iwceq;
+ struct irdma_msix_vector *msix_vec;
+ enum irdma_status_code status = 0;
+ u32 num_ceqs;
+
+ num_ceqs = min(rf->msix_count, rf->sc_dev.hmc_fpm_misc.max_ceqs);
+ rf->ceqlist = kcalloc(num_ceqs, sizeof(*rf->ceqlist), GFP_KERNEL);
+ if (!rf->ceqlist) {
+ status = IRDMA_ERR_NO_MEMORY;
+ goto exit;
+ }
+
+ i = (rf->msix_shared) ? 0 : 1;
+ iwceq = &rf->ceqlist[0];
+ status = irdma_create_ceq(rf, iwceq, 0, &rf->default_vsi);
+ if (status) {
+ irdma_debug(&rf->sc_dev, IRDMA_DEBUG_ERR,
+ "create ceq status = %d\n", status);
+ goto exit;
+ }
+
+ msix_vec = &rf->iw_msixtbl[i];
+ iwceq->irq = msix_vec->irq;
+ iwceq->msix_idx = msix_vec->idx;
+ status = irdma_cfg_ceq_vector(rf, iwceq, 0, msix_vec);
+ if (status) {
+ irdma_destroy_ceq(rf, iwceq);
+ goto exit;
+ }
+
+ irdma_ena_intr(&rf->sc_dev, msix_vec->idx);
+ rf->ceqs_count++;
+
+exit:
+ if (status && !rf->ceqs_count) {
+ kfree(rf->ceqlist);
+ rf->ceqlist = NULL;
+ return status;
+ }
+ rf->sc_dev.ceq_valid = true;
+
+ return 0;
+}
+
+/**
+ * irdma_setup_ceqs - manage the device ceq's and their interrupt resources
+ * @rf: RDMA PCI function
+ * @vsi: VSI structure for this CEQ
+ *
+ * Allocate a list for all device completion event queues
+ * Create the ceq's and configure their msix interrupt vectors
+ * Return 0, if at least one ceq is successfully set up, otherwise return error
+ */
+static enum irdma_status_code irdma_setup_ceqs(struct irdma_pci_f *rf,
+ struct irdma_sc_vsi *vsi)
+{
+ u32 i;
+ u32 ceq_id;
+ struct irdma_ceq *iwceq;
+ struct irdma_msix_vector *msix_vec;
+ enum irdma_status_code status = 0;
+ u32 num_ceqs;
+
+ num_ceqs = min(rf->msix_count, rf->sc_dev.hmc_fpm_misc.max_ceqs);
+ i = (rf->msix_shared) ? 1 : 2;
+ for (ceq_id = 1; i < num_ceqs - 1; i++, ceq_id++) {
+ iwceq = &rf->ceqlist[ceq_id];
+ status = irdma_create_ceq(rf, iwceq, ceq_id, vsi);
+ if (status) {
+ irdma_debug(&rf->sc_dev, IRDMA_DEBUG_ERR,
+ "create ceq status = %d\n", status);
+ break;
+ }
+ msix_vec = &rf->iw_msixtbl[i];
+ iwceq->irq = msix_vec->irq;
+ iwceq->msix_idx = msix_vec->idx;
+ status = irdma_cfg_ceq_vector(rf, iwceq, ceq_id, msix_vec);
+ if (status) {
+ irdma_destroy_ceq(rf, iwceq);
+ break;
+ }
+ irdma_ena_intr(&rf->sc_dev, msix_vec->idx);
+ rf->ceqs_count++;
+ }
+
+ return status;
+}
+
+/**
+ * irdma_create_aeq - create async event queue
+ * @rf: RDMA PCI function
+ *
+ * Return 0, if the aeq and the resources associated with it
+ * are successfully created, otherwise return error
+ */
+static enum irdma_status_code irdma_create_aeq(struct irdma_pci_f *rf)
+{
+ enum irdma_status_code status;
+ struct irdma_aeq_init_info info = {};
+ struct irdma_sc_dev *dev = &rf->sc_dev;
+ struct irdma_aeq *aeq = &rf->aeq;
+ struct irdma_hmc_info *hmc_info = rf->sc_dev.hmc_info;
+ u64 scratch = 0;
+ u32 aeq_size;
+
+ aeq_size = 2 * hmc_info->hmc_obj[IRDMA_HMC_IW_QP].cnt +
+ hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt;
+ aeq->mem.size = sizeof(struct irdma_sc_aeqe) * aeq_size;
+ status = irdma_allocate_dma_mem(dev->hw, &aeq->mem, aeq->mem.size,
+ IRDMA_AEQ_ALIGNMENT);
+ if (status)
+ goto exit;
+
+ info.aeqe_base = aeq->mem.va;
+ info.aeq_elem_pa = aeq->mem.pa;
+ info.elem_cnt = aeq_size;
+ info.dev = dev;
+ status = dev->aeq_ops->aeq_init(&aeq->sc_aeq, &info);
+ if (status)
+ goto exit;
+
+ status = dev->aeq_ops->aeq_create(&aeq->sc_aeq, scratch, 1);
+ if (!status)
+ status = dev->aeq_ops->aeq_create_done(&aeq->sc_aeq);
+exit:
+ if (status)
+ irdma_free_dma_mem(dev->hw, &aeq->mem);
+
+ return status;
+}
+
+/**
+ * irdma_setup_aeq - set up the device aeq
+ * @rf: RDMA PCI function
+ *
+ * Create the aeq and configure its msix interrupt vector
+ * Return 0 if successful, otherwise return error
+ */
+static enum irdma_status_code irdma_setup_aeq(struct irdma_pci_f *rf)
+{
+ struct irdma_sc_dev *dev = &rf->sc_dev;
+ enum irdma_status_code status;
+
+ status = irdma_create_aeq(rf);
+ if (status)
+ return status;
+
+ status = irdma_cfg_aeq_vector(rf);
+ if (status) {
+ irdma_destroy_aeq(rf);
+ return status;
+ }
+
+ if (!rf->msix_shared)
+ irdma_ena_intr(dev, rf->iw_msixtbl[0].idx);
+
+ return 0;
+}
+
+/**
+ * irdma_initialize_ilq - create iwarp local queue for cm
+ * @iwdev: iwarp device
+ *
+ * Return 0 if successful, otherwise return error
+ */
+static enum irdma_status_code irdma_initialize_ilq(struct irdma_device *iwdev)
+{
+ struct irdma_puda_rsrc_info info = {};
+ enum irdma_status_code status;
+
+ info.type = IRDMA_PUDA_RSRC_TYPE_ILQ;
+ info.cq_id = 1;
+ info.qp_id = 1;
+ info.count = 1;
+ info.pd_id = 1;
+ info.sq_size = min(iwdev->rf->max_qp / 2, (u32)32768);
+ info.rq_size = info.sq_size;
+ info.buf_size = 1024;
+ info.tx_buf_cnt = 2 * info.sq_size;
+ info.receive = irdma_receive_ilq;
+ info.xmit_complete = irdma_free_sqbuf;
+ status = irdma_puda_create_rsrc(&iwdev->vsi, &info);
+ if (status)
+ irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_ERR,
+ "ilq create fail\n");
+
+ return status;
+}
+
+/**
+ * irdma_initialize_ieq - create iwarp exception queue
+ * @iwdev: iwarp device
+ *
+ * Return 0 if successful, otherwise return error
+ */
+static enum irdma_status_code irdma_initialize_ieq(struct irdma_device *iwdev)
+{
+ struct irdma_puda_rsrc_info info = {};
+ enum irdma_status_code status;
+
+ info.type = IRDMA_PUDA_RSRC_TYPE_IEQ;
+ info.cq_id = 2;
+ info.qp_id = iwdev->vsi.exception_lan_q;
+ info.count = 1;
+ info.pd_id = 2;
+ info.sq_size = min(iwdev->rf->max_qp / 2, (u32)32768);
+ info.rq_size = info.sq_size;
+ info.buf_size = iwdev->vsi.mtu + VLAN_ETH_HLEN;
+ info.tx_buf_cnt = 4096;
+ status = irdma_puda_create_rsrc(&iwdev->vsi, &info);
+ if (status)
+ irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_ERR,
+ "ieq create fail\n");
+
+ return status;
+}
+
+/**
+ * irdma_reinitialize_ieq - destroy and re-create ieq
+ * @vsi: VSI structure
+ */
+void irdma_reinitialize_ieq(struct irdma_sc_vsi *vsi)
+{
+ struct irdma_device *iwdev = vsi->back_vsi;
+
+ irdma_puda_dele_rsrc(vsi, IRDMA_PUDA_RSRC_TYPE_IEQ, false);
+ if (irdma_initialize_ieq(iwdev)) {
+ iwdev->reset = true;
+ irdma_request_reset(iwdev->rf);
+ }
+}
+
+/**
+ * irdma_hmc_setup - create hmc objects for the device
+ * @rf: RDMA PCI function
+ *
+ * Set up the device private memory space for the number and size of
+ * the hmc objects and create the objects
+ * Return 0 if successful, otherwise return error
+ */
+static enum irdma_status_code irdma_hmc_setup(struct irdma_pci_f *rf)
+{
+ enum irdma_status_code status;
+
+ rf->sd_type = IRDMA_SD_TYPE_DIRECT;
+ status = irdma_cfg_fpm_val(&rf->sc_dev,
+ rsrc_limits_table[rf->limits_sel].qplimit);
+ if (status)
+ return status;
+
+ status = irdma_create_hmc_objs(rf, true, rf->rdma_ver);
+
+ return status;
+}
+
+/**
+ * irdma_del_init_mem - deallocate memory resources
+ * @rf: RDMA PCI function
+ */
+static void irdma_del_init_mem(struct irdma_pci_f *rf)
+{
+ struct irdma_sc_dev *dev = &rf->sc_dev;
+
+ kfree(dev->hmc_info->sd_table.sd_entry);
+ dev->hmc_info->sd_table.sd_entry = NULL;
+ kfree(rf->mem_rsrc);
+ rf->mem_rsrc = NULL;
+ irdma_free_dma_mem(&rf->hw, &rf->obj_mem);
+ if (rf->rdma_ver != IRDMA_GEN_1) {
+ kfree(rf->allocated_ws_nodes);
+ rf->allocated_ws_nodes = NULL;
+ }
+ kfree(rf->ceqlist);
+ rf->ceqlist = NULL;
+ kfree(rf->iw_msixtbl);
+ rf->iw_msixtbl = NULL;
+ kfree(rf->hmc_info_mem);
+ rf->hmc_info_mem = NULL;
+}
+
+/**
+ * irdma_initialize_dev - initialize device
+ * @rf: RDMA PCI function
+ * @ldev: lan device information
+ *
+ * Allocate memory for the hmc objects and initialize iwdev
+ * Return 0 if successful, otherwise clean up the resources
+ * and return error
+ */
+static enum irdma_status_code irdma_initialize_dev(struct irdma_pci_f *rf,
+ struct irdma_priv_ldev *ldev)
+{
+ enum irdma_status_code status;
+ struct irdma_sc_dev *dev = &rf->sc_dev;
+ struct irdma_device_init_info info = {};
+ struct irdma_dma_mem mem;
+ u32 size;
+
+ size = sizeof(struct irdma_hmc_pble_rsrc) +
+ sizeof(struct irdma_hmc_info) +
+ (sizeof(struct irdma_hmc_obj_info) * IRDMA_HMC_IW_MAX);
+
+ rf->hmc_info_mem = kzalloc(size, GFP_KERNEL);
+ if (!rf->hmc_info_mem)
+ return IRDMA_ERR_NO_MEMORY;
+
+ rf->pble_rsrc = (struct irdma_hmc_pble_rsrc *)rf->hmc_info_mem;
+ dev->hmc_info = &rf->hw.hmc;
+ dev->hmc_info->hmc_obj = (struct irdma_hmc_obj_info *)
+ (rf->pble_rsrc + 1);
+
+ status = irdma_obj_aligned_mem(rf, &mem, IRDMA_QUERY_FPM_BUF_SIZE,
+ IRDMA_FPM_QUERY_BUF_ALIGNMENT_M);
+ if (status)
+ goto error;
+
+ info.fpm_query_buf_pa = mem.pa;
+ info.fpm_query_buf = mem.va;
+ info.init_hw = rf->init_hw;
+
+ status = irdma_obj_aligned_mem(rf, &mem, IRDMA_COMMIT_FPM_BUF_SIZE,
+ IRDMA_FPM_COMMIT_BUF_ALIGNMENT_M);
+ if (status)
+ goto error;
+
+ info.fpm_commit_buf_pa = mem.pa;
+ info.fpm_commit_buf = mem.va;
+
+ info.bar0 = rf->hw.hw_addr;
+ info.hmc_fn_id = (u8)ldev->fn_num;
+ info.is_pf = !ldev->ftype;
+ info.hw = &rf->hw;
+#ifndef CONFIG_DYNAMIC_DEBUG
+ if (rf->debug)
+ info.debug_mask = rf->debug;
+#endif
+ info.vchnl_send = NULL;
+ status = irdma_sc_ctrl_init(rf->rdma_ver, &rf->sc_dev, &info);
+ if (status)
+ goto error;
+
+ return status;
+error:
+ kfree(rf->hmc_info_mem);
+ rf->hmc_info_mem = NULL;
+
+ return status;
+}
+
+/**
+ * irdma_deinit_rt_device - clean up the device resources
+ * @iwdev: iwarp device
+ *
+ * Destroy the ib device interface, remove the mac ip entry and
+ * ipv4/ipv6 addresses, destroy the device queues and free the
+ * pble and the hmc objects
+ */
+void irdma_deinit_rt_device(struct irdma_device *iwdev)
+{
+ irdma_dev_info(&iwdev->rf->sc_dev, "state = %d\n", iwdev->init_state);
+
+ switch (iwdev->init_state) {
+ case RDMA_DEV_REGISTERED:
+ iwdev->iw_status = 0;
+ irdma_port_ibevent(iwdev);
+ irdma_destroy_rdma_device(iwdev->iwibdev);
+ /* fallthrough */
+ case IP_ADDR_REGISTERED:
+ if (iwdev->rf->sc_dev.hw_attrs.hw_rev == IRDMA_GEN_1)
+ irdma_del_local_mac_entry(iwdev->rf, (u8)iwdev->mac_ip_table_idx);
+ /* fallthrough */
+ case PBLE_CHUNK_MEM:
+ /* fallthrough */
+ case CEQS_CREATED:
+ /* fallthrough */
+ case IEQ_CREATED:
+ irdma_puda_dele_rsrc(&iwdev->vsi,
+ IRDMA_PUDA_RSRC_TYPE_IEQ,
+ iwdev->reset);
+ /* fallthrough */
+ case ILQ_CREATED:
+ if (iwdev->create_ilq)
+ irdma_puda_dele_rsrc(&iwdev->vsi,
+ IRDMA_PUDA_RSRC_TYPE_ILQ,
+ iwdev->reset);
+ break;
+ default:
+ irdma_dev_warn(&iwdev->rf->sc_dev, "bad init_state = %d\n",
+ iwdev->init_state);
+ break;
+ }
+
+ irdma_cleanup_cm_core(&iwdev->cm_core);
+ if (iwdev->vsi.pestat) {
+ irdma_vsi_stats_free(&iwdev->vsi);
+ kfree(iwdev->vsi.pestat);
+ }
+
+ list_del(&iwdev->list);
+}
+
+/**
+ * irdma_setup_init_state - set up the initial device struct
+ * @hdl: handler for iwarp device - one per instance
+ * @ldev: lan device information
+ * @client: iwarp client information, provided during registration
+ *
+ * Initialize the iwarp device and its hdl information
+ * using the ldev and client information
+ * Return 0 if successful, otherwise return error
+ */
+static enum irdma_status_code irdma_setup_init_state(struct irdma_pci_f *rf)
+{
+ struct irdma_priv_ldev *ldev = &rf->ldev;
+ enum irdma_status_code status;
+
+ status = irdma_save_msix_info(rf);
+ if (status)
+ return status;
+
+ rf->hw.dev_context = (void *)rf->pdev;
+ status = irdma_allocate_dma_mem(&rf->hw, &rf->obj_mem, 8192, 4096);
+ if (status)
+ goto exit;
+
+ rf->obj_next = rf->obj_mem;
+ rf->ooo = false;
+ init_waitqueue_head(&rf->vchnl_waitq);
+
+ status = irdma_initialize_dev(rf, ldev);
+exit:
+ if (status) {
+ kfree(rf->iw_msixtbl);
+ irdma_free_dma_mem(&rf->hw, &rf->obj_mem);
+ rf->iw_msixtbl = NULL;
+ }
+
+ return status;
+}
+
+/**
+ * irdma_get_used_rsrc - determine resources used internally
+ * @iwdev: iwarp device
+ *
+ * Called at the end of open to get all internal allocations
+ */
+static void irdma_get_used_rsrc(struct irdma_device *iwdev)
+{
+ iwdev->rf->used_pds = find_next_zero_bit(iwdev->rf->allocated_pds,
+ iwdev->rf->max_pd, 0);
+ iwdev->rf->used_qps = find_next_zero_bit(iwdev->rf->allocated_qps,
+ iwdev->rf->max_qp, 0);
+ iwdev->rf->used_cqs = find_next_zero_bit(iwdev->rf->allocated_cqs,
+ iwdev->rf->max_cq, 0);
+ iwdev->rf->used_mrs = find_next_zero_bit(iwdev->rf->allocated_mrs,
+ iwdev->rf->max_mr, 0);
+}
+
+/**
+ * irdma_deinit_hw - De-initializes RDMA HW
+ * @rf: RDMA device information
+ *
+ */
+void irdma_deinit_ctrl_hw(struct irdma_pci_f *rf)
+{
+ enum init_completion_state state = rf->init_state;
+
+ rf->init_state = INVALID_STATE;
+ if (rf->rsrc_created) {
+ irdma_destroy_pble_prm(rf->pble_rsrc);
+ irdma_del_ceqs(rf);
+ }
+ switch (state) {
+ case CEQ0_CREATED:
+ irdma_del_ceq_0(rf);
+ /* fallthrough */
+ case AEQ_CREATED:
+ irdma_destroy_aeq(rf);
+ /* fallthrough */
+ case CCQ_CREATED:
+ irdma_destroy_ccq(rf);
+ /* fallthrough */
+ case HMC_OBJS_CREATED:
+ irdma_del_hmc_objects(&rf->sc_dev, rf->sc_dev.hmc_info,
+ true, rf->reset, rf->rdma_ver);
+ /* fallthrough */
+ case CQP_CREATED:
+ if (rf->cqp.cqp_compl_thread) {
+ rf->stop_cqp_thread = true;
+ up(&rf->cqp.cqp_compl_sem);
+ kthread_stop(rf->cqp.cqp_compl_thread);
+ }
+ irdma_destroy_cqp(rf, true);
+ /* fallthrough */
+ case INITIAL_STATE:
+ irdma_del_init_mem(rf);
+ break;
+ case INVALID_STATE:
+ /* fallthrough */
+ default:
+ irdma_pr_warn("bad init_state = %d\n", rf->init_state);
+ break;
+ }
+}
+
+enum irdma_status_code irdma_rt_init_hw(struct irdma_pci_f *rf, struct irdma_device *iwdev,
+ struct irdma_l2params *l2params)
+{
+ struct irdma_sc_dev *dev = &rf->sc_dev;
+ enum irdma_status_code status;
+ struct irdma_vsi_init_info vsi_info = {};
+ struct irdma_vsi_stats_info stats_info = {};
+
+ list_add(&iwdev->list, &rf->vsi_dev_list);
+ irdma_sc_rt_init(dev);
+ vsi_info.vm_vf_type = dev->is_pf ? IRDMA_PF_TYPE : IRDMA_VF_TYPE;
+ vsi_info.dev = dev;
+ vsi_info.back_vsi = (void *)iwdev;
+ vsi_info.params = l2params;
+ vsi_info.pf_data_vsi_num = iwdev->vsi_num;
+ vsi_info.exception_lan_q = 2;
+ irdma_sc_vsi_init(&iwdev->vsi, &vsi_info);
+
+ stats_info.pestat = kzalloc(sizeof(*stats_info.pestat), GFP_KERNEL);
+ if (!stats_info.pestat) {
+ status = IRDMA_ERR_NO_MEMORY;
+ goto error;
+ }
+ stats_info.fcn_id = dev->hmc_fn_id;
+ status = irdma_vsi_stats_init(&iwdev->vsi, &stats_info);
+ if (status) {
+ kfree(stats_info.pestat);
+ goto error;
+ }
+ irdma_setup_cm_core(iwdev, rf->rdma_ver);
+ do {
+ if (iwdev->create_ilq) {
+ status = irdma_initialize_ilq(iwdev);
+ if (status)
+ break;
+ iwdev->init_state = ILQ_CREATED;
+ }
+ status = irdma_initialize_ieq(iwdev);
+ if (status)
+ break;
+ iwdev->init_state = IEQ_CREATED;
+ if (!rf->rsrc_created) {
+ status = irdma_setup_ceqs(rf, &iwdev->vsi);
+ if (status)
+ break;
+ iwdev->init_state = CEQS_CREATED;
+
+ iwdev->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY |
+ IB_DEVICE_MEM_WINDOW |
+ IB_DEVICE_MEM_MGT_EXTENSIONS;
+
+ status = irdma_hmc_init_pble(&rf->sc_dev, rf->pble_rsrc);
+ if (status) {
+ irdma_del_ceqs(rf);
+ break;
+ }
+ irdma_pble_initialize_lock(rf->pble_rsrc);
+ iwdev->init_state = PBLE_CHUNK_MEM;
+ rf->rsrc_created = true;
+ }
+ if (iwdev->rf->sc_dev.hw_attrs.hw_rev == IRDMA_GEN_1)
+ irdma_alloc_set_mac(iwdev);
+ irdma_add_ip(iwdev);
+ iwdev->init_state = IP_ADDR_REGISTERED;
+ status = irdma_register_rdma_device(iwdev);
+ if (status)
+ break;
+ iwdev->init_state = RDMA_DEV_REGISTERED;
+ irdma_port_ibevent(iwdev);
+ iwdev->iw_status = 1;
+ irdma_get_used_rsrc(iwdev);
+ init_waitqueue_head(&iwdev->close_wq);
+ init_waitqueue_head(&iwdev->suspend_wq);
+
+ return 0;
+ } while (0);
+
+ irdma_dev_err(dev, "VSI open FAIL status = %d last cmpl = %d\n",
+ status, iwdev->init_state);
+ irdma_deinit_rt_device(iwdev);
+ return status;
+
+error:
+ list_del(&iwdev->list);
+ return status;
+}
+
+/**
+ * irdma_ctrl_init_hw - Initializes RDMA HW
+ *
+ */
+enum irdma_status_code irdma_ctrl_init_hw(struct irdma_pci_f *rf)
+{
+ struct irdma_sc_dev *dev = &rf->sc_dev;
+ enum irdma_status_code status;
+
+ INIT_LIST_HEAD(&rf->vsi_dev_list);
+ do {
+ status = irdma_setup_init_state(rf);
+ if (status)
+ break;
+ rf->init_state = INITIAL_STATE;
+
+ status = irdma_create_cqp(rf);
+ if (status)
+ break;
+ rf->init_state = CQP_CREATED;
+
+ status = irdma_hmc_setup(rf);
+ if (status)
+ break;
+ rf->init_state = HMC_OBJS_CREATED;
+
+ status = irdma_initialize_hw_rsrc(rf);
+ if (status)
+ break;
+
+ status = irdma_create_ccq(rf);
+ if (status)
+ break;
+ rf->init_state = CCQ_CREATED;
+
+ status = irdma_setup_aeq(rf);
+ if (status)
+ break;
+ rf->init_state = AEQ_CREATED;
+ if (rf->rdma_ver != IRDMA_GEN_1)
+ status = irdma_get_rdma_features(&rf->sc_dev);
+
+ if (status)
+ rf->sc_dev.feature_info[IRDMA_FEATURE_FW_INFO] = IRDMA_FW_VER;
+ else if (has_feature_rts_ae(dev->feature_info[IRDMA_FEATURE_FW_INFO],
+ dev->hw_attrs.hw_rev))
+ dev->hw_attrs.feature_flags |= IRDMA_FEATURE_RTS_AE;
+
+ rf->cqp.cqp_compl_thread =
+ kthread_run(cqp_compl_thread, rf, "cqp_compl_thread");
+
+ status = irdma_setup_ceq_0(rf);
+ if (status)
+ break;
+ rf->init_state = CEQ0_CREATED;
+
+ rf->free_qp_wq = create_singlethread_workqueue("free_qp_wq");
+ dev->ccq_ops->ccq_arm(dev->ccq);
+
+ irdma_dev_info(dev, "IRDMA hardware initialization successful\n");
+ return 0;
+ } while (0);
+
+ irdma_pr_err("IRDMA hardware initialization FAILED init_state=%d status=%d\n",
+ rf->init_state, status);
+ irdma_deinit_ctrl_hw(rf);
+ return IRDMA_NOT_SUPPORTED;
+}
+
+/**
+ * irdma_initialize_hw_resources - initialize hw resource during open
+ * @rf: RDMA PCI function
+ */
+u32 irdma_initialize_hw_rsrc(struct irdma_pci_f *rf)
+{
+ unsigned long num_pds;
+ u32 rsrc_size;
+ u32 max_mr;
+ u32 max_qp;
+ u32 max_cq;
+ u32 arp_table_size;
+ u32 mrdrvbits;
+ void *rsrc_ptr;
+ u32 num_ahs;
+ u32 num_mcg;
+
+ if (rf->rdma_ver != IRDMA_GEN_1) {
+ rf->allocated_ws_nodes =
+ kcalloc(BITS_TO_LONGS(IRDMA_MAX_WS_NODES),
+ sizeof(unsigned long),
+ GFP_KERNEL);
+ if (!rf->allocated_ws_nodes)
+ return -ENOMEM;
+
+ set_bit(0, rf->allocated_ws_nodes);
+ rf->max_ws_node_id = IRDMA_MAX_WS_NODES;
+ }
+ max_qp = rf->sc_dev.hmc_info->hmc_obj[IRDMA_HMC_IW_QP].cnt;
+ max_cq = rf->sc_dev.hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt;
+ max_mr = rf->sc_dev.hmc_info->hmc_obj[IRDMA_HMC_IW_MR].cnt;
+ arp_table_size = rf->sc_dev.hmc_info->hmc_obj[IRDMA_HMC_IW_ARP].cnt;
+ rf->max_cqe = rf->sc_dev.hw_attrs.max_hw_cq_size;
+ num_pds = rf->sc_dev.hw_attrs.max_hw_pds;
+ rsrc_size = sizeof(struct irdma_arp_entry) * arp_table_size;
+ rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
+ rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
+ rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);
+ rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);
+ rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);
+ num_ahs = max_qp * 4;
+ rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(num_ahs);
+ num_mcg = max_qp;
+ rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(num_mcg);
+ rsrc_size += sizeof(struct irdma_qp **) * max_qp;
+
+ rf->mem_rsrc = kzalloc(rsrc_size, GFP_KERNEL);
+ if (!rf->mem_rsrc) {
+ kfree(rf->allocated_ws_nodes);
+ rf->allocated_ws_nodes = NULL;
+ return -ENOMEM;
+ }
+
+ rf->max_qp = max_qp;
+ rf->max_mr = max_mr;
+ rf->max_cq = max_cq;
+ rf->max_pd = num_pds;
+ rf->arp_table_size = arp_table_size;
+ rf->arp_table = (struct irdma_arp_entry *)rf->mem_rsrc;
+ rsrc_ptr = rf->mem_rsrc +
+ (sizeof(struct irdma_arp_entry) * arp_table_size);
+
+ rf->max_ah = num_ahs;
+ rf->max_mcg = num_mcg;
+ rf->allocated_qps = rsrc_ptr;
+ rf->allocated_cqs = &rf->allocated_qps[BITS_TO_LONGS(max_qp)];
+ rf->allocated_mrs = &rf->allocated_cqs[BITS_TO_LONGS(max_cq)];
+ rf->allocated_pds = &rf->allocated_mrs[BITS_TO_LONGS(max_mr)];
+ rf->allocated_ahs = &rf->allocated_pds[BITS_TO_LONGS(num_pds)];
+ rf->allocated_mcgs = &rf->allocated_ahs[BITS_TO_LONGS(num_ahs)];
+ rf->allocated_arps = &rf->allocated_mcgs[BITS_TO_LONGS(num_mcg)];
+ rf->qp_table = (struct irdma_qp **)
+ (&rf->allocated_arps[BITS_TO_LONGS(arp_table_size)]);
+
+ set_bit(0, rf->allocated_mrs);
+ set_bit(0, rf->allocated_qps);
+ set_bit(0, rf->allocated_cqs);
+ set_bit(0, rf->allocated_pds);
+ set_bit(0, rf->allocated_arps);
+ set_bit(0, rf->allocated_ahs);
+ set_bit(0, rf->allocated_mcgs);
+ set_bit(2, rf->allocated_qps); /* qp 2 IEQ */
+ set_bit(1, rf->allocated_qps); /* qp 1 ILQ */
+ set_bit(1, rf->allocated_cqs);
+ set_bit(1, rf->allocated_pds);
+ set_bit(2, rf->allocated_cqs);
+ set_bit(2, rf->allocated_pds);
+
+ spin_lock_init(&rf->rsrc_lock);
+ spin_lock_init(&rf->qptable_lock);
+ spin_lock_init(&rf->qh_list_lock);
+
+ INIT_LIST_HEAD(&rf->mc_qht_list.list);
+ /* stag index mask has a minimum of 14 bits */
+ mrdrvbits = 24 - max(get_count_order(rf->max_mr), 14);
+ rf->mr_stagmask = ~(((1 << mrdrvbits) - 1) << (32 - mrdrvbits));
+
+ return 0;
+}
+
+/**
+ * irdma_cqp_ce_handler - handle cqp completions
+ * @rf: RDMA PCI function
+ * @cq: cq for cqp completions
+ */
+void irdma_cqp_ce_handler(struct irdma_pci_f *rf, struct irdma_sc_cq *cq)
+{
+ struct irdma_cqp_request *cqp_request;
+ struct irdma_sc_dev *dev = &rf->sc_dev;
+ u32 cqe_count = 0;
+ struct irdma_ccq_cqe_info info;
+ unsigned long flags;
+ int ret;
+
+ do {
+ memset(&info, 0, sizeof(info));
+ spin_lock_irqsave(&rf->cqp.compl_lock, flags);
+ ret = dev->ccq_ops->ccq_get_cqe_info(cq, &info);
+ spin_unlock_irqrestore(&rf->cqp.compl_lock, flags);
+ if (ret)
+ break;
+
+ cqp_request = (struct irdma_cqp_request *)
+ (unsigned long)info.scratch;
+ if (info.error)
+ irdma_debug(dev, IRDMA_DEBUG_ERR,
+ "opcode = 0x%x maj_err_code = 0x%x min_err_code = 0x%x\n",
+ info.op_code,
+ info.maj_err_code,
+ info.min_err_code);
+ if (cqp_request) {
+ cqp_request->compl_info.maj_err_code =
+ info.maj_err_code;
+ cqp_request->compl_info.min_err_code =
+ info.min_err_code;
+ cqp_request->compl_info.op_ret_val = info.op_ret_val;
+ cqp_request->compl_info.error = info.error;
+
+ if (cqp_request->waiting) {
+ cqp_request->request_done = true;
+ wake_up(&cqp_request->waitq);
+ irdma_put_cqp_request(&rf->cqp, cqp_request);
+ } else {
+ if (cqp_request->callback_fcn)
+ cqp_request->callback_fcn(cqp_request,
+ 1);
+ irdma_put_cqp_request(&rf->cqp, cqp_request);
+ }
+ }
+
+ cqe_count++;
+ } while (1);
+
+ if (cqe_count) {
+ irdma_process_bh(dev);
+ dev->ccq_ops->ccq_arm(cq);
+ }
+}
+
+/**
+ * irdma_iwarp_ce_handler - handle iwarp completions
+ * @iwcp: iwarp cq receiving event
+ */
+static void irdma_iwarp_ce_handler(struct irdma_sc_cq *iwcq)
+{
+ struct irdma_cq *cq = iwcq->back_cq;
+
+ if (cq->ibcq.comp_handler)
+ cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
+}
+
+/**
+ * irdma_puda_ce_handler - handle puda completion events
+ * @rf: RDMA PCI function
+ * @cq: puda completion q for event
+ */
+static void irdma_puda_ce_handler(struct irdma_pci_f *rf,
+ struct irdma_sc_cq *cq)
+{
+ struct irdma_sc_dev *dev = (struct irdma_sc_dev *)&rf->sc_dev;
+ enum irdma_status_code status;
+ u32 compl_error;
+
+ do {
+ status = irdma_puda_poll_cmpl(dev, cq, &compl_error);
+ if (status == IRDMA_ERR_Q_EMPTY)
+ break;
+ if (status) {
+ irdma_debug(dev, IRDMA_DEBUG_ERR,
+ "puda status = %d\n", status);
+ break;
+ }
+ if (compl_error) {
+ irdma_debug(dev, IRDMA_DEBUG_ERR,
+ "puda compl_err =0x%x\n", compl_error);
+ break;
+ }
+ } while (1);
+
+ dev->ccq_ops->ccq_arm(cq);
+}
+
+/**
+ * cqp_thread - Handle cqp completions
+ * @context: Pointer to RDMA PCI Function
+ */
+int cqp_compl_thread(void *context)
+{
+ struct irdma_pci_f *rf = context;
+ struct irdma_sc_cq *cq = &rf->ccq.sc_cq;
+
+ do {
+ if (down_interruptible(&rf->cqp.cqp_compl_sem))
+ return 0;
+ if (rf->stop_cqp_thread)
+ return 0;
+ irdma_cqp_ce_handler(rf, cq);
+ } while (!kthread_should_stop());
+
+ return 0;
+}
+
+/**
+ * irdma_process_ceq - handle ceq for completions
+ * @rf: RDMA PCI function
+ * @ceq: ceq having cq for completion
+ */
+void irdma_process_ceq(struct irdma_pci_f *rf,
+ struct irdma_ceq *ceq)
+{
+ struct irdma_sc_dev *dev = &rf->sc_dev;
+ struct irdma_sc_ceq *sc_ceq;
+ struct irdma_sc_cq *cq;
+
+ sc_ceq = &ceq->sc_ceq;
+ do {
+ cq = dev->ceq_ops->process_ceq(dev, sc_ceq);
+ if (!cq)
+ break;
+
+ if (cq->cq_type == IRDMA_CQ_TYPE_CQP)
+ up(&rf->cqp.cqp_compl_sem);
+ else if (cq->cq_type == IRDMA_CQ_TYPE_IWARP)
+ irdma_iwarp_ce_handler(cq);
+ else if ((cq->cq_type == IRDMA_CQ_TYPE_ILQ) ||
+ (cq->cq_type == IRDMA_CQ_TYPE_IEQ))
+ irdma_puda_ce_handler(rf, cq);
+ } while (1);
+}
+
+/**
+ * irdma_next_iw_state - modify qp state
+ * @iwqp: iwarp qp to modify
+ * @state: next state for qp
+ * @del_hash: del hash
+ * @term: term message
+ * @termlen: length of term message
+ */
+void irdma_next_iw_state(struct irdma_qp *iwqp,
+ u8 state,
+ u8 del_hash,
+ u8 term,
+ u8 termlen)
+{
+ struct irdma_modify_qp_info info = {};
+
+ info.next_iwarp_state = state;
+ info.remove_hash_idx = del_hash;
+ info.cq_num_valid = true;
+ info.arp_cache_idx_valid = true;
+ info.dont_send_term = true;
+ info.dont_send_fin = true;
+ info.termlen = termlen;
+
+ if (term & IRDMAQP_TERM_SEND_TERM_ONLY)
+ info.dont_send_term = false;
+ if (term & IRDMAQP_TERM_SEND_FIN_ONLY)
+ info.dont_send_fin = false;
+ if (iwqp->sc_qp.term_flags && state == IRDMA_QP_STATE_ERROR)
+ info.reset_tcp_conn = true;
+ iwqp->hw_iwarp_state = state;
+ irdma_hw_modify_qp(iwqp->iwdev, iwqp, &info, 0);
+}
+
+/**
+ * irdma_process_aeq - handle aeq events
+ * @rf: RDMA PCI function
+ */
+void irdma_process_aeq(struct irdma_pci_f *rf)
+{
+ struct irdma_sc_dev *dev = &rf->sc_dev;
+ struct irdma_aeq *aeq = &rf->aeq;
+ struct irdma_sc_aeq *sc_aeq = &aeq->sc_aeq;
+ struct irdma_aeqe_info aeinfo;
+ struct irdma_aeqe_info *info = &aeinfo;
+ int ret;
+ struct irdma_qp *iwqp = NULL;
+ struct irdma_sc_cq *cq = NULL;
+ struct irdma_cq *iwcq = NULL;
+ struct irdma_sc_qp *qp = NULL;
+ struct irdma_qp_host_ctx_info *ctx_info = NULL;
+ unsigned long flags;
+
+ u32 aeqcnt = 0;
+ bool roce_mode = false;
+
+ if (!sc_aeq->size)
+ return;
+
+ do {
+ memset(info, 0, sizeof(*info));
+ ret = dev->aeq_ops->get_next_aeqe(sc_aeq, info);
+ if (ret)
+ break;
+
+ aeqcnt++;
+ irdma_debug(dev, IRDMA_DEBUG_AEQ,
+ "ae_id = 0x%x bool qp=%d qp_id = %d\n",
+ info->ae_id, info->qp, info->qp_cq_id);
+ if (info->qp) {
+ spin_lock_irqsave(&rf->qptable_lock, flags);
+ iwqp = rf->qp_table[info->qp_cq_id];
+ if (!iwqp) {
+ spin_unlock_irqrestore(&rf->qptable_lock,
+ flags);
+ irdma_debug(dev, IRDMA_DEBUG_AEQ,
+ "qp_id %d is already freed\n",
+ info->qp_cq_id);
+ continue;
+ }
+ irdma_add_ref(&iwqp->ibqp);
+ spin_unlock_irqrestore(&rf->qptable_lock, flags);
+ qp = &iwqp->sc_qp;
+ spin_lock_irqsave(&iwqp->lock, flags);
+ iwqp->hw_tcp_state = info->tcp_state;
+ iwqp->hw_iwarp_state = info->iwarp_state;
+ iwqp->last_aeq = info->ae_id;
+ spin_unlock_irqrestore(&iwqp->lock, flags);
+ ctx_info = &iwqp->ctx_info;
+ if (rdma_protocol_roce(&iwqp->iwdev->iwibdev->ibdev, 1))
+ roce_mode = true;
+ if (roce_mode)
+ ctx_info->roce_info->err_rq_idx_valid = true;
+ else
+ ctx_info->iwarp_info->err_rq_idx_valid = true;
+ } else {
+ if (info->ae_id != IRDMA_AE_CQ_OPERATION_ERROR)
+ continue;
+ }
+
+ switch (info->ae_id) {
+ struct irdma_cm_node *cm_node;
+ case IRDMA_AE_LLP_CONNECTION_ESTABLISHED:
+ cm_node = (struct irdma_cm_node *)iwqp->cm_node;
+ if (cm_node->accept_pend) {
+ atomic_dec(&cm_node->listener->pend_accepts_cnt);
+ cm_node->accept_pend = 0;
+ }
+ iwqp->rts_ae_rcvd = 1;
+ wake_up_interruptible(&iwqp->waitq);
+ break;
+ case IRDMA_AE_LLP_FIN_RECEIVED:
+ case IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE:
+ if (qp->term_flags)
+ break;
+ if (atomic_inc_return(&iwqp->close_timer_started) == 1) {
+ iwqp->hw_tcp_state = IRDMA_TCP_STATE_CLOSE_WAIT;
+ if (iwqp->hw_tcp_state == IRDMA_TCP_STATE_CLOSE_WAIT &&
+ iwqp->ibqp_state == IB_QPS_RTS) {
+ irdma_next_iw_state(iwqp,
+ IRDMA_QP_STATE_CLOSING,
+ 0, 0, 0);
+ irdma_cm_disconn(iwqp);
+ }
+ iwqp->cm_id->add_ref(iwqp->cm_id);
+ irdma_schedule_cm_timer(iwqp->cm_node,
+ (struct irdma_puda_buf *)iwqp,
+ IRDMA_TIMER_TYPE_CLOSE,
+ 1, 0);
+ }
+ break;
+ case IRDMA_AE_LLP_CLOSE_COMPLETE:
+ if (qp->term_flags)
+ irdma_terminate_done(qp, 0);
+ else
+ irdma_cm_disconn(iwqp);
+ break;
+ case IRDMA_AE_BAD_CLOSE:
+ /* fall through */
+ case IRDMA_AE_RESET_SENT:
+ irdma_next_iw_state(iwqp,
+ IRDMA_QP_STATE_ERROR,
+ 1, 0, 0);
+ irdma_cm_disconn(iwqp);
+ break;
+ case IRDMA_AE_LLP_CONNECTION_RESET:
+ if (atomic_read(&iwqp->close_timer_started))
+ break;
+ irdma_cm_disconn(iwqp);
+ break;
+ case IRDMA_AE_QP_SUSPEND_COMPLETE:
+ atomic_dec(&iwqp->sc_qp.vsi->qp_suspend_reqs);
+ wake_up(&iwqp->iwdev->suspend_wq);
+ break;
+ case IRDMA_AE_TERMINATE_SENT:
+ irdma_terminate_send_fin(qp);
+ break;
+ case IRDMA_AE_LLP_TERMINATE_RECEIVED:
+ irdma_terminate_received(qp, info);
+ break;
+ case IRDMA_AE_CQ_OPERATION_ERROR:
+ irdma_debug(dev, IRDMA_DEBUG_ERR,
+ "Processing an iWARP related AE for CQ misc = 0x%04X\n",
+ info->ae_id);
+ cq = (struct irdma_sc_cq *)
+ (unsigned long)info->compl_ctx;
+
+ iwcq = (struct irdma_cq *)cq->back_cq;
+
+ if (iwcq->ibcq.event_handler) {
+ struct ib_event ibevent;
+
+ ibevent.device = iwcq->ibcq.device;
+ ibevent.event = IB_EVENT_CQ_ERR;
+ ibevent.element.cq = &iwcq->ibcq;
+ iwcq->ibcq.event_handler(&ibevent,
+ iwcq->ibcq.cq_context);
+ }
+ break;
+ case IRDMA_AE_LLP_DOUBT_REACHABILITY:
+ break;
+ case IRDMA_AE_PRIV_OPERATION_DENIED:
+ case IRDMA_AE_STAG_ZERO_INVALID:
+ case IRDMA_AE_IB_RREQ_AND_Q1_FULL:
+ case IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION:
+ case IRDMA_AE_DDP_UBE_INVALID_MO:
+ case IRDMA_AE_DDP_UBE_INVALID_QN:
+ case IRDMA_AE_DDP_NO_L_BIT:
+ case IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION:
+ case IRDMA_AE_RDMAP_ROE_UNEXPECTED_OPCODE:
+ case IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST:
+ case IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP:
+ case IRDMA_AE_INVALID_ARP_ENTRY:
+ case IRDMA_AE_INVALID_TCP_OPTION_RCVD:
+ case IRDMA_AE_STALE_ARP_ENTRY:
+ case IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR:
+ case IRDMA_AE_LLP_SEGMENT_TOO_SMALL:
+ case IRDMA_AE_LLP_SYN_RECEIVED:
+ case IRDMA_AE_LLP_TOO_MANY_RETRIES:
+ case IRDMA_AE_LCE_QP_CATASTROPHIC:
+ case IRDMA_AE_LCE_FUNCTION_CATASTROPHIC:
+ case IRDMA_AE_LCE_CQ_CATASTROPHIC:
+ case IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG:
+ if (roce_mode)
+ ctx_info->roce_info->err_rq_idx_valid = false;
+ else
+ ctx_info->iwarp_info->err_rq_idx_valid = false;
+ default:
+ if (roce_mode) {
+ if (!info->sq && ctx_info->roce_info->err_rq_idx_valid) {
+ ctx_info->roce_info->err_rq_idx = info->wqe_idx;
+ ret = dev->iw_priv_qp_ops->qp_setctx_roce(&iwqp->sc_qp,
+ iwqp->host_ctx.va,
+ ctx_info);
+ }
+ irdma_cm_disconn(iwqp);
+ break;
+ }
+ if (!info->sq && ctx_info->iwarp_info->err_rq_idx_valid) {
+ ctx_info->iwarp_info->err_rq_idx = info->wqe_idx;
+ ctx_info->tcp_info_valid = false;
+ ctx_info->iwarp_info_valid = false;
+ ret = dev->iw_priv_qp_ops->qp_setctx(&iwqp->sc_qp,
+ iwqp->host_ctx.va,
+ ctx_info);
+ }
+ irdma_terminate_connection(qp, info);
+ break;
+ }
+ if (info->qp)
+ irdma_rem_ref(&iwqp->ibqp);
+ } while (1);
+
+ if (aeqcnt)
+ dev->aeq_ops->repost_aeq_entries(dev, aeqcnt);
+}
+
+/**
+ * irdma_del_mac_entry - remove a mac entry from the hw table
+ * @iwdev: iwarp device
+ * @idx: the index of the mac ip address to delete
+ */
+void irdma_del_local_mac_entry(struct irdma_pci_f *rf, u16 idx)
+{
+ struct irdma_cqp *iwcqp = &rf->cqp;
+ struct irdma_cqp_request *cqp_request;
+ struct cqp_cmds_info *cqp_info;
+ enum irdma_status_code status = 0;
+
+ cqp_request = irdma_get_cqp_request(iwcqp, true);
+ if (!cqp_request) {
+ irdma_pr_err("cqp_request memory failed\n");
+ return;
+ }
+
+ cqp_info = &cqp_request->info;
+ cqp_info->cqp_cmd = IRDMA_OP_DELETE_LOCAL_MAC_ENTRY;
+ cqp_info->post_sq = 1;
+ cqp_info->in.u.del_local_mac_entry.cqp = &iwcqp->sc_cqp;
+ cqp_info->in.u.del_local_mac_entry.scratch = (uintptr_t)cqp_request;
+ cqp_info->in.u.del_local_mac_entry.entry_idx = idx;
+ cqp_info->in.u.del_local_mac_entry.ignore_ref_count = 0;
+ status = irdma_handle_cqp_op(rf, cqp_request);
+ if (status)
+ irdma_pr_err("CQP-OP Del MAC entry fail");
+}
+
+/**
+ * irdma_add_mac_entry - add a mac ip address entry to the hw table
+ * @iwdev: iwarp device
+ * @mac_addr: pointer to mac address
+ * @idx: the index of the mac ip address to add
+ */
+int irdma_add_local_mac_entry(struct irdma_pci_f *rf,
+ u8 *mac_addr,
+ u16 idx)
+{
+ struct irdma_local_mac_entry_info *info;
+ struct irdma_cqp *iwcqp = &rf->cqp;
+ struct irdma_cqp_request *cqp_request;
+ struct cqp_cmds_info *cqp_info;
+ enum irdma_status_code status = 0;
+
+ cqp_request = irdma_get_cqp_request(iwcqp, true);
+ if (!cqp_request) {
+ irdma_pr_err("cqp_request memory failed\n");
+ return IRDMA_ERR_NO_MEMORY;
+ }
+
+ cqp_info = &cqp_request->info;
+ cqp_info->post_sq = 1;
+ info = &cqp_info->in.u.add_local_mac_entry.info;
+ ether_addr_copy(info->mac_addr, mac_addr);
+ info->entry_idx = idx;
+ cqp_info->in.u.add_local_mac_entry.scratch = (uintptr_t)cqp_request;
+ cqp_info->cqp_cmd = IRDMA_OP_ADD_LOCAL_MAC_ENTRY;
+ cqp_info->in.u.add_local_mac_entry.cqp = &iwcqp->sc_cqp;
+ cqp_info->in.u.add_local_mac_entry.scratch = (uintptr_t)cqp_request;
+ status = irdma_handle_cqp_op(rf, cqp_request);
+ if (status)
+ irdma_pr_err("CQP-OP Add MAC entry fail");
+
+ return status;
+}
+
+/**
+ * irdma_alloc_local_mac_entry - allocate a mac entry
+ * @iwdev: iwarp device
+ * @mac_ip_tbl_idx: the index of the new mac address
+ *
+ * Allocate a mac address entry and update the mac_tbl_idx
+ * to hold the index of the newly created mac address
+ * Return 0 if successful, otherwise return error
+ */
+int irdma_alloc_local_mac_entry(struct irdma_pci_f *rf,
+ u16 *mac_tbl_idx)
+{
+ struct irdma_cqp *iwcqp = &rf->cqp;
+ struct irdma_cqp_request *cqp_request;
+ struct cqp_cmds_info *cqp_info;
+ enum irdma_status_code status = 0;
+
+ cqp_request = irdma_get_cqp_request(iwcqp, true);
+ if (!cqp_request) {
+ irdma_pr_err("cqp_request memory failed\n");
+ return IRDMA_ERR_NO_MEMORY;
+ }
+
+ /* increment refcount, because we need the cqp request ret value */
+ atomic_inc(&cqp_request->refcount);
+ cqp_info = &cqp_request->info;
+ cqp_info->cqp_cmd = IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY;
+ cqp_info->post_sq = 1;
+ cqp_info->in.u.alloc_local_mac_entry.cqp = &iwcqp->sc_cqp;
+ cqp_info->in.u.alloc_local_mac_entry.scratch = (uintptr_t)cqp_request;
+ status = irdma_handle_cqp_op(rf, cqp_request);
+ if (!status)
+ *mac_tbl_idx = (u16)cqp_request->compl_info.op_ret_val;
+ else
+ irdma_pr_err("CQP-OP Alloc MAC entry fail");
+ /* decrement refcount and free the cqp request, if no longer used */
+ irdma_put_cqp_request(iwcqp, cqp_request);
+
+ return status;
+}
+
+/**
+ * irdma_cqp_manage_apbvt_cmd - send cqp command manage apbvt
+ * @iwdev: iwarp device
+ * @accel_local_port: port for apbvt
+ * @add_port: add ordelete port
+ */
+static enum irdma_status_code
+irdma_cqp_manage_apbvt_cmd(struct irdma_device *iwdev,
+ u16 accel_local_port,
+ bool add_port)
+{
+ struct irdma_apbvt_info *info;
+ struct irdma_cqp_request *cqp_request;
+ struct cqp_cmds_info *cqp_info;
+ enum irdma_status_code status;
+
+ cqp_request = irdma_get_cqp_request(&iwdev->rf->cqp, add_port);
+ if (!cqp_request)
+ return IRDMA_ERR_NO_MEMORY;
+
+ cqp_info = &cqp_request->info;
+ info = &cqp_info->in.u.manage_apbvt_entry.info;
+ memset(info, 0, sizeof(*info));
+ info->add = add_port;
+ info->port = accel_local_port;
+ cqp_info->cqp_cmd = IRDMA_OP_MANAGE_APBVT_ENTRY;
+ cqp_info->post_sq = 1;
+ cqp_info->in.u.manage_apbvt_entry.cqp = &iwdev->rf->cqp.sc_cqp;
+ cqp_info->in.u.manage_apbvt_entry.scratch = (uintptr_t)cqp_request;
+ status = irdma_handle_cqp_op(iwdev->rf, cqp_request);
+ if (status)
+ irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_ERR,
+ "CQP-OP Manage APBVT entry fail");
+
+ return status;
+}
+
+/**
+ * irdma_manage_apbvt - add or delete tcp port
+ * @iwdev: iwarp device
+ * @accel_local_port: port for apbvt
+ * @add_port: add or delete port
+ */
+enum irdma_status_code irdma_manage_apbvt(struct irdma_device *iwdev,
+ u16 accel_local_port,
+ bool add_port)
+{
+ struct irdma_cm_core *cm_core = &iwdev->cm_core;
+ enum irdma_status_code status = 0;
+ unsigned long flags;
+ bool in_use;
+
+ /* apbvt_lock is held across CQP delete APBVT OP (non-waiting) to
+ * protect against race where add APBVT CQP can race ahead of the delete
+ * APBVT for same port.
+ */
+ if (add_port) {
+ spin_lock_irqsave(&cm_core->apbvt_lock, flags);
+ in_use = __test_and_set_bit(accel_local_port,
+ cm_core->ports_in_use);
+ spin_unlock_irqrestore(&cm_core->apbvt_lock, flags);
+ if (in_use)
+ return 0;
+ return irdma_cqp_manage_apbvt_cmd(iwdev, accel_local_port,
+ true);
+ } else {
+ spin_lock_irqsave(&cm_core->apbvt_lock, flags);
+ in_use = irdma_port_in_use(cm_core, accel_local_port);
+ if (in_use) {
+ spin_unlock_irqrestore(&cm_core->apbvt_lock, flags);
+ return 0;
+ }
+ __clear_bit(accel_local_port, cm_core->ports_in_use);
+ status = irdma_cqp_manage_apbvt_cmd(iwdev, accel_local_port,
+ false);
+ spin_unlock_irqrestore(&cm_core->apbvt_lock, flags);
+ return status;
+ }
+}
+
+/**
+ * irdma_manage_arp_cache - manage hw arp cache
+ * @rf: RDMA PCI function
+ * @mac_addr: mac address ptr
+ * @ip_addr: ip addr for arp cache
+ * @action: add, delete or modify
+ */
+void irdma_manage_arp_cache(struct irdma_pci_f *rf,
+ unsigned char *mac_addr,
+ u32 *ip_addr,
+ bool ipv4,
+ u32 action)
+{
+ struct irdma_add_arp_cache_entry_info *info;
+ struct irdma_cqp_request *cqp_request;
+ struct cqp_cmds_info *cqp_info;
+ int arp_index;
+
+ arp_index = irdma_arp_table(rf, ip_addr, ipv4, mac_addr, action);
+ if (arp_index == -1)
+ return;
+
+ cqp_request = irdma_get_cqp_request(&rf->cqp, false);
+ if (!cqp_request)
+ return;
+
+ cqp_info = &cqp_request->info;
+ if (action == IRDMA_ARP_ADD) {
+ cqp_info->cqp_cmd = IRDMA_OP_ADD_ARP_CACHE_ENTRY;
+ info = &cqp_info->in.u.add_arp_cache_entry.info;
+ memset(info, 0, sizeof(*info));
+ info->arp_index = (u16)arp_index;
+ info->permanent = true;
+ ether_addr_copy(info->mac_addr, mac_addr);
+ cqp_info->in.u.add_arp_cache_entry.scratch =
+ (uintptr_t)cqp_request;
+ cqp_info->in.u.add_arp_cache_entry.cqp = &rf->cqp.sc_cqp;
+ } else {
+ cqp_info->cqp_cmd = IRDMA_OP_DELETE_ARP_CACHE_ENTRY;
+ cqp_info->in.u.del_arp_cache_entry.scratch =
+ (uintptr_t)cqp_request;
+ cqp_info->in.u.del_arp_cache_entry.cqp = &rf->cqp.sc_cqp;
+ cqp_info->in.u.del_arp_cache_entry.arp_index = arp_index;
+ }
+
+ cqp_info->in.u.add_arp_cache_entry.cqp = &rf->cqp.sc_cqp;
+ cqp_info->in.u.add_arp_cache_entry.scratch = (uintptr_t)cqp_request;
+ cqp_info->post_sq = 1;
+ if (irdma_handle_cqp_op(rf, cqp_request))
+ irdma_debug(&rf->sc_dev, IRDMA_DEBUG_ERR,
+ "CQP-OP Add/Del Arp Cache entry fail");
+}
+
+/**
+ * irdma_send_syn_cqp_callback - do syn/ack after qhash
+ * @cqp_request: qhash cqp completion
+ * @send_ack: flag send ack
+ */
+static void irdma_send_syn_cqp_callback(struct irdma_cqp_request *cqp_request,
+ u32 send_ack)
+{
+ irdma_send_syn(cqp_request->param, send_ack);
+}
+
+/**
+ * irdma_manage_qhash - add or modify qhash
+ * @iwdev: iwarp device
+ * @cminfo: cm info for qhash
+ * @etype: type (syn or quad)
+ * @mtype: type of qhash
+ * @cmnode: cmnode associated with connection
+ * @wait: wait for completion
+ * @user_pri:user pri of the connection
+ */
+enum irdma_status_code irdma_manage_qhash(struct irdma_device *iwdev,
+ struct irdma_cm_info *cminfo,
+ enum irdma_quad_entry_type etype,
+ enum irdma_quad_hash_manage_type mtype,
+ void *cmnode,
+ bool wait)
+{
+ struct irdma_qhash_table_info *info;
+ struct irdma_sc_dev *dev = &iwdev->rf->sc_dev;
+ enum irdma_status_code status;
+ struct irdma_cqp *iwcqp = &iwdev->rf->cqp;
+ struct irdma_cqp_request *cqp_request;
+ struct cqp_cmds_info *cqp_info;
+
+ cqp_request = irdma_get_cqp_request(iwcqp, wait);
+ if (!cqp_request)
+ return IRDMA_ERR_NO_MEMORY;
+
+ cqp_info = &cqp_request->info;
+ info = &cqp_info->in.u.manage_qhash_table_entry.info;
+ memset(info, 0, sizeof(*info));
+ info->vsi = &iwdev->vsi;
+ info->manage = mtype;
+ info->entry_type = etype;
+ if (cminfo->vlan_id != 0xFFFF) {
+ info->vlan_valid = true;
+ info->vlan_id = cminfo->vlan_id;
+ } else {
+ info->vlan_valid = false;
+ }
+ info->ipv4_valid = cminfo->ipv4;
+ info->user_pri = cminfo->user_pri;
+ ether_addr_copy(info->mac_addr, iwdev->netdev->dev_addr);
+ info->qp_num = cminfo->qh_qpid;
+ info->dest_port = cminfo->loc_port;
+ info->dest_ip[0] = cminfo->loc_addr[0];
+ info->dest_ip[1] = cminfo->loc_addr[1];
+ info->dest_ip[2] = cminfo->loc_addr[2];
+ info->dest_ip[3] = cminfo->loc_addr[3];
+ if (etype == IRDMA_QHASH_TYPE_TCP_ESTABLISHED ||
+ etype == IRDMA_QHASH_TYPE_UDP_UNICAST ||
+ etype == IRDMA_QHASH_TYPE_UDP_MCAST ||
+ etype == IRDMA_QHASH_TYPE_ROCE_MCAST ||
+ etype == IRDMA_QHASH_TYPE_ROCEV2_HW) {
+ info->src_port = cminfo->rem_port;
+ info->src_ip[0] = cminfo->rem_addr[0];
+ info->src_ip[1] = cminfo->rem_addr[1];
+ info->src_ip[2] = cminfo->rem_addr[2];
+ info->src_ip[3] = cminfo->rem_addr[3];
+ }
+ if (cmnode) {
+ cqp_request->callback_fcn = irdma_send_syn_cqp_callback;
+ cqp_request->param = (void *)cmnode;
+ }
+ if (info->ipv4_valid)
+ irdma_debug(dev, IRDMA_DEBUG_CM,
+ "%s IP=%pI4, port=%d, mac=%pM, vlan_id=%d\n",
+ !mtype ? "DELETE" : "ADD",
+ info->dest_ip,
+ info->dest_port, info->mac_addr, cminfo->vlan_id);
+ else
+ irdma_debug(dev, IRDMA_DEBUG_CM,
+ "%s IP=%pI6, port=%d, mac=%pM, vlan_id=%d\n",
+ !mtype ? "DELETE" : "ADD",
+ info->dest_ip,
+ info->dest_port, info->mac_addr, cminfo->vlan_id);
+ cqp_info->in.u.manage_qhash_table_entry.cqp = &iwdev->rf->cqp.sc_cqp;
+ cqp_info->in.u.manage_qhash_table_entry.scratch = (uintptr_t)cqp_request;
+ cqp_info->cqp_cmd = IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY;
+ cqp_info->post_sq = 1;
+ status = irdma_handle_cqp_op(iwdev->rf, cqp_request);
+ if (status)
+ irdma_debug(dev, IRDMA_DEBUG_ERR,
+ "CQP-OP Manage Qhash Entry fail");
+
+ return status;
+}
+
+/**
+ * irdma_hw_flush_wqes - flush qp's wqe
+ * @rf: RDMA PCI function
+ * @qp: hardware control qp
+ * @info: info for flush
+ * @wait: flag wait for completion
+ */
+enum irdma_status_code irdma_hw_flush_wqes(struct irdma_pci_f *rf,
+ struct irdma_sc_qp *qp,
+ struct irdma_qp_flush_info *info,
+ bool wait)
+{
+ enum irdma_status_code status;
+ struct irdma_qp_flush_info *hw_info;
+ struct irdma_cqp_request *cqp_request;
+ struct cqp_cmds_info *cqp_info;
+ struct irdma_qp *iwqp = (struct irdma_qp *)qp->back_qp;
+ unsigned long flags = 0;
+
+ cqp_request = irdma_get_cqp_request(&rf->cqp, wait);
+ if (!cqp_request)
+ return IRDMA_ERR_NO_MEMORY;
+
+ cqp_info = &cqp_request->info;
+ hw_info = &cqp_request->info.in.u.qp_flush_wqes.info;
+ memcpy(hw_info, info, sizeof(*hw_info));
+ cqp_info->cqp_cmd = IRDMA_OP_QP_FLUSH_WQES;
+ cqp_info->post_sq = 1;
+ cqp_info->in.u.qp_flush_wqes.qp = qp;
+ cqp_info->in.u.qp_flush_wqes.scratch = (uintptr_t)cqp_request;
+ status = irdma_handle_cqp_op(rf, cqp_request);
+ if (status) {
+ irdma_debug(&rf->sc_dev, IRDMA_DEBUG_ERR,
+ "CQP-OP Flush WQE's fail");
+ complete(&iwqp->sq_drained);
+ complete(&iwqp->rq_drained);
+ qp->qp_uk.sq_flush_complete = true;
+ qp->qp_uk.rq_flush_complete = true;
+ return status;
+ }
+
+ if (!cqp_request->compl_info.maj_err_code) {
+ if (info->rq) {
+ if (cqp_request->compl_info.min_err_code == IRDMA_CQP_COMPL_SQ_WQE_FLUSHED ||
+ cqp_request->compl_info.min_err_code == 0) {
+ /* RQ WQE flush was requested but did not happen */
+ qp->qp_uk.rq_flush_complete = true;
+ complete(&iwqp->rq_drained);
+ }
+ }
+ if (info->sq) {
+ if (cqp_request->compl_info.min_err_code == IRDMA_CQP_COMPL_RQ_WQE_FLUSHED ||
+ cqp_request->compl_info.min_err_code == 0) {
+ spin_lock_irqsave(&iwqp->lock, flags);
+ /* Handling case where WQE is posted to empty SQ when
+ * flush has not completed
+ */
+ if (IRDMA_RING_MORE_WORK(qp->qp_uk.sq_ring)) {
+ spin_unlock_irqrestore(&iwqp->lock, flags);
+ cqp_request->waiting = false;
+ info->rq = false;
+ qp->flush_sq = false;
+ irdma_handle_cqp_op(rf, cqp_request);
+ } else {
+ /* SQ WQE flush was requested but did not happen */
+ spin_unlock_irqrestore(&iwqp->lock, flags);
+ qp->qp_uk.sq_flush_complete = true;
+ complete(&iwqp->sq_drained);
+ }
+ } else {
+ spin_lock_irqsave(&iwqp->lock, flags);
+ if (!IRDMA_RING_MORE_WORK(qp->qp_uk.sq_ring)) {
+ spin_unlock_irqrestore(&iwqp->lock, flags);
+ qp->qp_uk.sq_flush_complete = true;
+ complete(&iwqp->sq_drained);
+ } else {
+ spin_unlock_irqrestore(&iwqp->lock, flags);
+ }
+ }
+ }
+ }
+
+ return 0;
+}
+
+/**
+ * irdma_gen_ae - generate AE
+ * @rf: RDMA PCI function
+ * @qp: qp associated with AE
+ * @info: info for ae
+ * @wait: wait for completion
+ */
+void irdma_gen_ae(struct irdma_pci_f *rf,
+ struct irdma_sc_qp *qp,
+ struct irdma_gen_ae_info *info,
+ bool wait)
+{
+ struct irdma_gen_ae_info *ae_info;
+ struct irdma_cqp_request *cqp_request;
+ struct cqp_cmds_info *cqp_info;
+
+ cqp_request = irdma_get_cqp_request(&rf->cqp, wait);
+ if (!cqp_request)
+ return;
+
+ cqp_info = &cqp_request->info;
+ ae_info = &cqp_request->info.in.u.gen_ae.info;
+ memcpy(ae_info, info, sizeof(*ae_info));
+ cqp_info->cqp_cmd = IRDMA_OP_GEN_AE;
+ cqp_info->post_sq = 1;
+ cqp_info->in.u.gen_ae.qp = qp;
+ cqp_info->in.u.gen_ae.scratch = (uintptr_t)cqp_request;
+ if (irdma_handle_cqp_op(rf, cqp_request))
+ irdma_debug(&rf->sc_dev, IRDMA_DEBUG_ERR,
+ "CQP OP failed attempting to generate ae_code=0x%x\n",
+ info->ae_code);
+}
+
+/**
+ * irdma_get_ib_wc - return change flush code to IB's
+ * @opcode: iwarp flush code
+ */
+static enum ib_wc_status irdma_get_ib_wc(enum irdma_flush_opcode opcode)
+{
+ switch (opcode) {
+ case FLUSH_PROT_ERR:
+ return IB_WC_LOC_PROT_ERR;
+ case FLUSH_REM_ACCESS_ERR:
+ return IB_WC_REM_ACCESS_ERR;
+ case FLUSH_LOC_QP_OP_ERR:
+ return IB_WC_LOC_QP_OP_ERR;
+ case FLUSH_REM_OP_ERR:
+ return IB_WC_REM_OP_ERR;
+ case FLUSH_LOC_LEN_ERR:
+ return IB_WC_LOC_LEN_ERR;
+ case FLUSH_GENERAL_ERR:
+ return IB_WC_GENERAL_ERR;
+ case FLUSH_FATAL_ERR:
+ default:
+ return IB_WC_FATAL_ERR;
+ }
+}
+
+/**
+ * irdma_set_flush_info - set flush info
+ * @pinfo: set flush info
+ * @min: minor err
+ * @maj: major err
+ * @opcode: flush error code
+ */
+static void irdma_set_flush_info(struct irdma_qp_flush_info *pinfo,
+ u16 *min,
+ u16 *maj,
+ enum irdma_flush_opcode opcode)
+{
+ *min = (u16)irdma_get_ib_wc(opcode);
+ *maj = CQE_MAJOR_DRV;
+ pinfo->userflushcode = true;
+}
+
+/**
+ * irdma_flush_wqes - flush wqe for qp
+ * @rf: RDMA PCI function
+ * @iwqp: qp to flush wqes
+ */
+void irdma_flush_wqes(struct irdma_pci_f *rf,
+ struct irdma_qp *iwqp)
+{
+ struct irdma_qp_flush_info info = {};
+ struct irdma_sc_qp *qp = &iwqp->sc_qp;
+
+ info.sq = true;
+ info.rq = true;
+ if (qp->term_flags) {
+ irdma_set_flush_info(&info, &info.sq_minor_code,
+ &info.sq_major_code, qp->flush_code);
+ irdma_set_flush_info(&info, &info.rq_minor_code,
+ &info.rq_major_code, qp->flush_code);
+ }
+ (void)irdma_hw_flush_wqes(rf, &iwqp->sc_qp, &info, true);
+}
diff --git a/drivers/infiniband/hw/irdma/i40iw_hw.c b/drivers/infiniband/hw/irdma/i40iw_hw.c
new file mode 100644
index 0000000..e258db8
--- /dev/null
+++ b/drivers/infiniband/hw/irdma/i40iw_hw.c
@@ -0,0 +1,207 @@
+// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
+/* Copyright (c) 2019, Intel Corporation. */
+
+#include "osdep.h"
+#include "type.h"
+#include "i40iw_hw.h"
+#include "status.h"
+#include "protos.h"
+
+static u32 i40iw_regs[IRDMA_MAX_REGS] = {
+ I40E_PFPE_CQPTAIL,
+ I40E_PFPE_CQPDB,
+ I40E_PFPE_CCQPSTATUS,
+ I40E_PFPE_CCQPHIGH,
+ I40E_PFPE_CCQPLOW,
+ I40E_PFPE_CQARM,
+ I40E_PFPE_CQACK,
+ I40E_PFPE_AEQALLOC,
+ I40E_PFPE_CQPERRCODES,
+ I40E_PFPE_WQEALLOC,
+ I40E_PFINT_DYN_CTLN(0),
+ I40IW_DB_ADDR_OFFSET,
+
+ I40E_GLPCI_LBARCTRL,
+ I40E_GLPE_CPUSTATUS0,
+ I40E_GLPE_CPUSTATUS1,
+ I40E_GLPE_CPUSTATUS2,
+ I40E_PFINT_AEQCTL,
+ I40E_PFINT_CEQCTL(0),
+ I40E_VSIQF_CTL(0),
+ I40E_PFHMC_PDINV,
+ I40E_GLHMC_VFPDINV(0)
+};
+
+static u32 i40iw_stat_offsets_32[IRDMA_HW_STAT_INDEX_MAX_32] = {
+ I40E_GLPES_PFIP4RXDISCARD(0),
+ I40E_GLPES_PFIP4RXTRUNC(0),
+ I40E_GLPES_PFIP4TXNOROUTE(0),
+ I40E_GLPES_PFIP6RXDISCARD(0),
+ I40E_GLPES_PFIP6RXTRUNC(0),
+ I40E_GLPES_PFIP6TXNOROUTE(0),
+ I40E_GLPES_PFTCPRTXSEG(0),
+ I40E_GLPES_PFTCPRXOPTERR(0),
+ I40E_GLPES_PFTCPRXPROTOERR(0),
+ I40E_GLPES_PFRXVLANERR(0)
+};
+
+static u32 i40iw_stat_offsets_64[IRDMA_HW_STAT_INDEX_MAX_64] = {
+ I40E_GLPES_PFIP4RXOCTSLO(0),
+ I40E_GLPES_PFIP4RXPKTSLO(0),
+ I40E_GLPES_PFIP4RXFRAGSLO(0),
+ I40E_GLPES_PFIP4RXMCPKTSLO(0),
+ I40E_GLPES_PFIP4TXOCTSLO(0),
+ I40E_GLPES_PFIP4TXPKTSLO(0),
+ I40E_GLPES_PFIP4TXFRAGSLO(0),
+ I40E_GLPES_PFIP4TXMCPKTSLO(0),
+ I40E_GLPES_PFIP6RXOCTSLO(0),
+ I40E_GLPES_PFIP6RXPKTSLO(0),
+ I40E_GLPES_PFIP6RXFRAGSLO(0),
+ I40E_GLPES_PFIP6RXMCPKTSLO(0),
+ I40E_GLPES_PFIP6TXOCTSLO(0),
+ I40E_GLPES_PFIP6TXPKTSLO(0),
+ I40E_GLPES_PFIP6TXFRAGSLO(0),
+ I40E_GLPES_PFIP6TXMCPKTSLO(0),
+ I40E_GLPES_PFTCPRXSEGSLO(0),
+ I40E_GLPES_PFTCPTXSEGLO(0),
+ I40E_GLPES_PFRDMARXRDSLO(0),
+ I40E_GLPES_PFRDMARXSNDSLO(0),
+ I40E_GLPES_PFRDMARXWRSLO(0),
+ I40E_GLPES_PFRDMATXRDSLO(0),
+ I40E_GLPES_PFRDMATXSNDSLO(0),
+ I40E_GLPES_PFRDMATXWRSLO(0),
+ I40E_GLPES_PFRDMAVBNDLO(0),
+ I40E_GLPES_PFRDMAVINVLO(0),
+ I40E_GLPES_PFIP4RXMCOCTSLO(0),
+ I40E_GLPES_PFIP4TXMCOCTSLO(0),
+ I40E_GLPES_PFIP6RXMCOCTSLO(0),
+ I40E_GLPES_PFIP6TXMCOCTSLO(0),
+ I40E_GLPES_PFUDPRXPKTSLO(0),
+ I40E_GLPES_PFUDPTXPKTSLO(0)
+};
+
+static u64 i40iw_masks[IRDMA_MAX_MASKS] = {
+ I40E_PFPE_CCQPSTATUS_CCQP_DONE_MASK,
+ I40E_PFPE_CCQPSTATUS_CCQP_ERR_MASK,
+ I40E_CQPSQ_STAG_PDID_MASK,
+ I40E_CQPSQ_CQ_CEQID_MASK,
+};
+
+static u64 i40iw_shifts[IRDMA_MAX_SHIFTS] = {
+ I40E_PFPE_CCQPSTATUS_CCQP_DONE_SHIFT,
+ I40E_PFPE_CCQPSTATUS_CCQP_ERR_SHIFT,
+ I40E_CQPSQ_STAG_PDID_SHIFT,
+ I40E_CQPSQ_CQ_CEQID_SHIFT,
+};
+
+static struct irdma_irq_ops i40iw_irq_ops;
+
+/**
+ * i40iw_config_ceq- Configure CEQ interrupt
+ * @dev: pointer to the device structure
+ * @ceq_id: Completion Event Queue ID
+ * @idx: vector index
+ */
+static void i40iw_config_ceq(struct irdma_sc_dev *dev, u32 ceq_id, u32 idx)
+{
+ u32 reg_val;
+
+ reg_val = (ceq_id << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT);
+ reg_val |= (QUEUE_TYPE_CEQ << I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT);
+ wr32(dev->hw, I40E_PFINT_LNKLSTN(idx - 1), reg_val);
+
+ reg_val = (0x3 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
+ reg_val |= I40E_PFINT_DYN_CTLN_INTENA_MASK;
+ wr32(dev->hw, I40E_PFINT_DYN_CTLN(idx - 1), reg_val);
+
+ reg_val = (IRDMA_GLINT_CEQCTL_CAUSE_ENA_M |
+ (idx << IRDMA_GLINT_CEQCTL_MSIX_INDX_S) |
+ IRDMA_GLINT_CEQCTL_ITR_INDX_M);
+ reg_val |= (NULL_QUEUE_INDEX << I40E_PFINT_CEQCTL_NEXTQ_INDX_SHIFT);
+
+ wr32(dev->hw, dev->hw_regs[IRDMA_GLINT_CEQCTL] + 4 * ceq_id, reg_val);
+}
+
+/**
+* i40iw_ena_irq - Enable interrupt
+* @dev: pointer to the device structure
+* @idx: vector index
+*/
+static void i40iw_ena_irq(struct irdma_sc_dev *dev, u32 idx)
+{
+ u32 val;
+
+ val = IRDMA_GLINT_DYN_CTL_INTENA_M |
+ IRDMA_GLINT_DYN_CTL_CLEARPBA_M |
+ IRDMA_GLINT_DYN_CTL_ITR_INDX_M;
+ wr32(dev->hw, dev->hw_regs[IRDMA_GLINT_DYN_CTL] + 4 * (idx - 1), val);
+}
+
+/**
+ * irdma_disable_irq - Disable interrupt
+ * @dev: pointer to the device structure
+ * @idx: vector index
+ */
+static void i40iw_disable_irq(struct irdma_sc_dev *dev, u32 idx)
+{
+ wr32(dev->hw, dev->hw_regs[IRDMA_GLINT_DYN_CTL] + 4 * (idx - 1), 0);
+}
+
+void i40iw_init_hw(struct irdma_sc_dev *dev)
+{
+ int i;
+
+ for (i = 0; i < IRDMA_MAX_REGS; ++i)
+ dev->hw_regs[i] = i40iw_regs[i];
+
+ for (i = 0; i < IRDMA_HW_STAT_INDEX_MAX_32; ++i)
+ dev->hw_stats_regs_32[i] = i40iw_stat_offsets_32[i];
+
+ for (i = 0; i < IRDMA_HW_STAT_INDEX_MAX_64; ++i)
+ dev->hw_stats_regs_64[i] = i40iw_stat_offsets_64[i];
+
+ for (i = 0; i < IRDMA_MAX_SHIFTS; ++i)
+ dev->hw_shifts[i] = i40iw_shifts[i];
+
+ for (i = 0; i < IRDMA_MAX_MASKS; ++i)
+ dev->hw_masks[i] = i40iw_masks[i];
+
+ dev->wqe_alloc_db = (u32 __iomem *)(irdma_get_hw_addr(dev) +
+ dev->hw_regs[IRDMA_WQEALLOC]);
+ dev->cq_arm_db = (u32 __iomem *)(irdma_get_hw_addr(dev) +
+ dev->hw_regs[IRDMA_CQARM]);
+ dev->aeq_alloc_db = (u32 __iomem *)(irdma_get_hw_addr(dev) +
+ dev->hw_regs[IRDMA_AEQALLOC]);
+ dev->cqp_db = (u32 __iomem *)(irdma_get_hw_addr(dev) +
+ dev->hw_regs[IRDMA_CQPDB]);
+ dev->cq_ack_db = (u32 __iomem *)(irdma_get_hw_addr(dev) +
+ dev->hw_regs[IRDMA_CQACK]);
+ dev->ceq_itr_mask_db = NULL;
+ dev->aeq_itr_mask_db = NULL;
+
+ memcpy(&i40iw_irq_ops, dev->irq_ops, sizeof(i40iw_irq_ops));
+ i40iw_irq_ops.irdma_en_irq = i40iw_ena_irq;
+ i40iw_irq_ops.irdma_dis_irq = i40iw_disable_irq;
+ i40iw_irq_ops.irdma_cfg_ceq = i40iw_config_ceq;
+ dev->irq_ops = &i40iw_irq_ops;
+
+ /* Setup the hardware limits, hmc may limit further */
+ dev->hw_attrs.max_hw_wq_frags = I40IW_MAX_WQ_FRAGMENT_COUNT;
+ dev->hw_attrs.max_hw_read_sges = I40IW_MAX_SGE_RD;
+ dev->hw_attrs.max_hw_device_pages = I40IW_MAX_PUSH_PAGE_COUNT;
+ dev->hw_attrs.first_hw_vf_fpm_id = I40IW_FIRST_VF_FPM_ID;
+ dev->hw_attrs.max_hw_inline = I40IW_MAX_INLINE_DATA_SIZE;
+ dev->hw_attrs.max_hw_push_inline = I40IW_MAX_PUSHMODE_INLINE_DATA_SIZE;
+ dev->hw_attrs.max_hw_ird = I40IW_MAX_IRD_SIZE;
+ dev->hw_attrs.max_hw_ord = I40IW_MAX_ORD_SIZE;
+ dev->hw_attrs.max_hw_wqes = I40IW_MAX_WQ_ENTRIES;
+ dev->hw_attrs.max_hw_sq_quanta = I40IW_QP_SW_MAX_SQ_QUANTA;
+ dev->hw_attrs.max_hw_rq_quanta = I40IW_QP_SW_MAX_RQ_QUANTA;
+ dev->hw_attrs.max_hw_wq_quanta = I40IW_QP_SW_MAX_WQ_QUANTA;
+ dev->hw_attrs.max_hw_sq_chunk = I40IW_MAX_QUANTA_PER_WR;
+ dev->hw_attrs.max_hw_pds = I40IW_MAX_PDS;
+ dev->hw_attrs.max_stat_inst = I40IW_MAX_STATS_COUNT;
+ dev->hw_attrs.max_hw_wq_size = I40IW_QP_WQE_MAX_SIZE;
+ dev->hw_attrs.min_sw_wq_size = I40IW_QP_SW_MIN_WQSIZE;
+ dev->hw_attrs.max_qp_wr = I40IW_MAX_QP_WRS;
+}
diff --git a/drivers/infiniband/hw/irdma/i40iw_hw.h b/drivers/infiniband/hw/irdma/i40iw_hw.h
new file mode 100644
index 0000000..4b502ed
--- /dev/null
+++ b/drivers/infiniband/hw/irdma/i40iw_hw.h
@@ -0,0 +1,164 @@
+/* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB */
+/* Copyright (c) 2019, Intel Corporation. */
+
+#ifndef I40IW_HW_H
+#define I40IW_HW_H
+#define I40E_VFPE_CQPTAIL1 0x0000A000 /* Reset: VFR */
+#define I40E_VFPE_CQPDB1 0x0000BC00 /* Reset: VFR */
+#define I40E_VFPE_CCQPSTATUS1 0x0000B800 /* Reset: VFR */
+#define I40E_VFPE_CCQPHIGH1 0x00009800 /* Reset: VFR */
+#define I40E_VFPE_CCQPLOW1 0x0000AC00 /* Reset: VFR */
+#define I40E_VFPE_CQARM1 0x0000B400 /* Reset: VFR */
+#define I40E_VFPE_CQACK1 0x0000B000 /* Reset: VFR */
+#define I40E_VFPE_AEQALLOC1 0x0000A400 /* Reset: VFR */
+#define I40E_VFPE_CQPERRCODES1 0x00009C00 /* Reset: VFR */
+#define I40E_VFPE_WQEALLOC1 0x0000C000 /* Reset: VFR */
+#define I40E_VFINT_DYN_CTLN(_INTVF) (0x00024800 + ((_INTVF) * 4)) /* _i=0...511 */ /* Reset: VFR */
+
+#define I40E_PFPE_CQPTAIL 0x00008080 /* Reset: PFR */
+
+#define I40E_PFPE_CQPDB 0x00008000 /* Reset: PFR */
+#define I40E_PFPE_CCQPSTATUS 0x00008100 /* Reset: PFR */
+#define I40E_PFPE_CCQPHIGH 0x00008200 /* Reset: PFR */
+#define I40E_PFPE_CCQPLOW 0x00008180 /* Reset: PFR */
+#define I40E_PFPE_CQARM 0x00131080 /* Reset: PFR */
+#define I40E_PFPE_CQACK 0x00131100 /* Reset: PFR */
+#define I40E_PFPE_AEQALLOC 0x00131180 /* Reset: PFR */
+#define I40E_PFPE_CQPERRCODES 0x00008880 /* Reset: PFR */
+#define I40E_PFPE_WQEALLOC 0x00138C00 /* Reset: PFR */
+#define I40E_GLPCI_LBARCTRL 0x000BE484 /* Reset: POR */
+#define I40E_GLPE_CPUSTATUS0 0x0000D040 /* Reset: PE_CORER */
+#define I40E_GLPE_CPUSTATUS1 0x0000D044 /* Reset: PE_CORER */
+#define I40E_GLPE_CPUSTATUS2 0x0000D048 /* Reset: PE_CORER */
+#define I40E_PFHMC_PDINV 0x000C0300 /* Reset: PFR */
+#define I40E_GLHMC_VFPDINV(_i) (0x000C8300 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */
+#define I40E_PFINT_DYN_CTLN(_INTPF) (0x00034800 + ((_INTPF) * 4)) /* _i=0...511 */ /* Reset: PFR */
+#define I40E_PFINT_AEQCTL 0x00038700 /* Reset: CORER */
+
+#define I40E_GLPES_PFIP4RXDISCARD(_i) (0x00010600 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */
+#define I40E_GLPES_PFIP4RXTRUNC(_i) (0x00010700 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */
+#define I40E_GLPES_PFIP4TXNOROUTE(_i) (0x00012E00 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */
+#define I40E_GLPES_PFIP6RXDISCARD(_i) (0x00011200 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */
+#define I40E_GLPES_PFIP6RXTRUNC(_i) (0x00011300 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */
+
+#define I40E_GLPES_PFRDMAVBNDLO(_i) (0x00014800 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
+#define I40E_GLPES_PFIP4TXMCOCTSLO(_i) (0x00012000 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
+#define I40E_GLPES_PFIP6RXMCOCTSLO(_i) (0x00011600 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
+#define I40E_GLPES_PFIP6TXMCOCTSLO(_i) (0x00012A00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
+#define I40E_GLPES_PFUDPRXPKTSLO(_i) (0x00013800 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
+#define I40E_GLPES_PFUDPTXPKTSLO(_i) (0x00013A00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
+
+#define I40E_GLPES_PFIP6TXNOROUTE(_i) (0x00012F00 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */
+#define I40E_GLPES_PFTCPRTXSEG(_i) (0x00013600 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */
+#define I40E_GLPES_PFTCPRXOPTERR(_i) (0x00013200 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */
+#define I40E_GLPES_PFTCPRXPROTOERR(_i) (0x00013300 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */
+#define I40E_GLPES_PFRXVLANERR(_i) (0x00010000 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */
+#define I40E_GLPES_PFIP4RXOCTSLO(_i) (0x00010200 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
+#define I40E_GLPES_PFIP4RXPKTSLO(_i) (0x00010400 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
+#define I40E_GLPES_PFIP4RXFRAGSLO(_i) (0x00010800 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
+#define I40E_GLPES_PFIP4RXMCPKTSLO(_i) (0x00010C00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
+#define I40E_GLPES_PFIP4TXOCTSLO(_i) (0x00011A00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
+#define I40E_GLPES_PFIP4TXPKTSLO(_i) (0x00011C00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
+#define I40E_GLPES_PFIP4TXFRAGSLO(_i) (0x00011E00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
+#define I40E_GLPES_PFIP4TXMCPKTSLO(_i) (0x00012200 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
+#define I40E_GLPES_PFIP6RXOCTSLO(_i) (0x00010E00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
+#define I40E_GLPES_PFIP6RXPKTSLO(_i) (0x00011000 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
+#define I40E_GLPES_PFIP6RXFRAGSLO(_i) (0x00011400 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
+#define I40E_GLPES_PFIP6TXOCTSLO(_i) (0x00012400 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
+#define I40E_GLPES_PFIP6TXPKTSLO(_i) (0x00012600 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
+#define I40E_GLPES_PFIP6TXFRAGSLO(_i) (0x00012800 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
+#define I40E_GLPES_PFIP6TXMCPKTSLO(_i) (0x00012C00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
+#define I40E_GLPES_PFTCPTXSEGLO(_i) (0x00013400 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
+#define I40E_GLPES_PFRDMARXRDSLO(_i) (0x00013E00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
+#define I40E_GLPES_PFRDMARXSNDSLO(_i) (0x00014000 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
+#define I40E_GLPES_PFRDMARXWRSLO(_i) (0x00013C00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
+#define I40E_GLPES_PFRDMATXRDSLO(_i) (0x00014400 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
+#define I40E_GLPES_PFRDMATXSNDSLO(_i) (0x00014600 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
+#define I40E_GLPES_PFRDMATXWRSLO(_i) (0x00014200 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
+#define I40E_GLPES_PFIP4RXMCOCTSLO(_i) (0x00010A00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
+#define I40E_GLPES_PFIP6RXMCPKTSLO(_i) (0x00011800 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
+#define I40E_GLPES_PFTCPRXSEGSLO(_i) (0x00013000 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
+#define I40E_GLPES_PFRDMAVINVLO(_i) (0x00014A00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
+
+#define I40IW_DB_ADDR_OFFSET (4 * 1024 * 1024 - 64 * 1024)
+
+#define I40IW_VF_DB_ADDR_OFFSET (64 * 1024)
+
+#define I40E_PFINT_LNKLSTN(_INTPF) (0x00035000 + ((_INTPF) * 4)) /* _i=0...511 */ /* Reset: PFR */
+#define I40E_PFINT_LNKLSTN_MAX_INDEX 511
+#define I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT 0
+#define I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK I40E_MASK(0x7FF, I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT)
+#define I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT 11
+#define I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_MASK I40E_MASK(0x3, I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT)
+
+#define I40E_PFINT_CEQCTL(_INTPF) (0x00036800 + ((_INTPF) * 4)) /* _i=0...511 */ /* Reset: CORER */
+#define I40E_PFINT_CEQCTL_MAX_INDEX 511
+#define I40E_PFINT_CEQCTL_MSIX_INDX_SHIFT 0
+#define I40E_PFINT_CEQCTL_MSIX_INDX_MASK I40E_MASK(0xFF, I40E_PFINT_CEQCTL_MSIX_INDX_SHIFT)
+#define I40E_PFINT_CEQCTL_ITR_INDX_SHIFT 11
+#define I40E_PFINT_CEQCTL_ITR_INDX_MASK I40E_MASK(0x3, I40E_PFINT_CEQCTL_ITR_INDX_SHIFT)
+#define I40E_PFINT_CEQCTL_MSIX0_INDX_SHIFT 13
+#define I40E_PFINT_CEQCTL_MSIX0_INDX_MASK I40E_MASK(0x7, I40E_PFINT_CEQCTL_MSIX0_INDX_SHIFT)
+#define I40E_PFINT_CEQCTL_NEXTQ_INDX_SHIFT 16
+#define I40E_PFINT_CEQCTL_NEXTQ_INDX_MASK I40E_MASK(0x7FF, I40E_PFINT_CEQCTL_NEXTQ_INDX_SHIFT)
+#define I40E_PFINT_CEQCTL_NEXTQ_TYPE_SHIFT 27
+#define I40E_PFINT_CEQCTL_NEXTQ_TYPE_MASK I40E_MASK(0x3, I40E_PFINT_CEQCTL_NEXTQ_TYPE_SHIFT)
+#define I40E_PFINT_CEQCTL_CAUSE_ENA_SHIFT 30
+#define I40E_PFINT_CEQCTL_CAUSE_ENA_MASK I40E_MASK(0x1, I40E_PFINT_CEQCTL_CAUSE_ENA_SHIFT)
+#define I40E_PFINT_CEQCTL_INTEVENT_SHIFT 31
+#define I40E_PFINT_CEQCTL_INTEVENT_MASK I40E_MASK(0x1, I40E_PFINT_CEQCTL_INTEVENT_SHIFT)
+
+#define I40E_CQPSQ_STAG_PDID_SHIFT 48
+#define I40E_CQPSQ_STAG_PDID_MASK MAKEMASK(0x7FFFULL, I40E_CQPSQ_STAG_PDID_SHIFT)
+
+#define I40E_PFPE_CCQPSTATUS_CCQP_DONE_SHIFT 0
+#define I40E_PFPE_CCQPSTATUS_CCQP_DONE_MASK (0x1 << I40E_PFPE_CCQPSTATUS_CCQP_DONE_SHIFT)
+
+#define I40E_PFPE_CCQPSTATUS_CCQP_ERR_SHIFT 31
+#define I40E_PFPE_CCQPSTATUS_CCQP_ERR_MASK (0x1 << I40E_PFPE_CCQPSTATUS_CCQP_ERR_SHIFT)
+
+#define I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT 3
+#define I40E_PFINT_DYN_CTLN_ITR_INDX_MASK (0x3 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT)
+
+#define I40E_PFINT_DYN_CTLN_INTENA_SHIFT 0
+#define I40E_PFINT_DYN_CTLN_INTENA_MASK (0x1 << I40E_PFINT_DYN_CTLN_INTENA_SHIFT)
+
+#define I40E_CQPSQ_CQ_CEQID_SHIFT 24
+#define I40E_CQPSQ_CQ_CEQID_MASK (0x7fUL << I40E_CQPSQ_CQ_CEQID_SHIFT)
+
+#define I40E_VSIQF_CTL(_VSI) (0x0020D800 + ((_VSI) * 4))
+
+enum i40iw_device_caps_const {
+ I40IW_MAX_WQ_FRAGMENT_COUNT = 3,
+ I40IW_MAX_SGE_RD = 1,
+ I40IW_MAX_PUSH_PAGE_COUNT = 0,
+ I40IW_MAX_INLINE_DATA_SIZE = 48,
+ I40IW_MAX_PUSHMODE_INLINE_DATA_SIZE = 0,
+ I40IW_MAX_IRD_SIZE = 63,
+ I40IW_MAX_ORD_SIZE = 127,
+ I40IW_MAX_WQ_ENTRIES = 2048,
+ I40IW_MAX_WQE_SIZE_RQ = 128,
+ I40IW_MAX_PDS = 32768,
+ I40IW_MAX_STATS_COUNT = 16,
+ I40IW_MAX_CQ_SIZE = 1048575,
+ I40IW_MAX_OUTBOUND_MESSAGE_SIZE = 2147483647,
+};
+
+#define I40IW_FIRST_VF_FPM_ID 16
+
+#define QUEUE_TYPE_CEQ 2
+#define NULL_QUEUE_INDEX 0x7FF
+
+#define I40IW_QP_WQE_MIN_SIZE 32
+#define I40IW_QP_WQE_MAX_SIZE 128
+#define I40IW_QP_SW_MIN_WQSIZE 4
+
+#define I40IW_MAX_RQ_WQE_SHIFT 2
+#define I40IW_MAX_QUANTA_PER_WR 2
+
+#define I40IW_QP_SW_MAX_SQ_QUANTA 2048
+#define I40IW_QP_SW_MAX_RQ_QUANTA 16384
+#define I40IW_QP_SW_MAX_WQ_QUANTA 2048
+#define I40IW_MAX_QP_WRS ((I40IW_QP_SW_MAX_SQ_QUANTA / I40IW_MAX_QUANTA_PER_WR) - 1)
+void i40iw_init_hw(struct irdma_sc_dev *dev);
+#endif /* I40IW_HW_H*/
diff --git a/drivers/infiniband/hw/irdma/icrdma_hw.c b/drivers/infiniband/hw/irdma/icrdma_hw.c
new file mode 100644
index 0000000..f4ac242
--- /dev/null
+++ b/drivers/infiniband/hw/irdma/icrdma_hw.c
@@ -0,0 +1,74 @@
+// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
+/* Copyright (c) 2019, Intel Corporation. */
+
+#include "osdep.h"
+#include "type.h"
+#include "icrdma_hw.h"
+
+static u32 icrdma_regs[IRDMA_MAX_REGS] = {
+ PFPE_CQPTAIL,
+ PFPE_CQPDB,
+ PFPE_CCQPSTATUS,
+ PFPE_CCQPHIGH,
+ PFPE_CCQPLOW,
+ PFPE_CQARM,
+ PFPE_CQACK,
+ PFPE_AEQALLOC,
+ PFPE_CQPERRCODES,
+ PFPE_WQEALLOC,
+ GLINT_DYN_CTL(0),
+ ICRDMA_DB_ADDR_OFFSET,
+
+ GLPCI_LBARCTRL,
+ GLPE_CPUSTATUS0,
+ GLPE_CPUSTATUS1,
+ GLPE_CPUSTATUS2,
+ PFINT_AEQCTL,
+ GLINT_CEQCTL(0),
+ VSIQF_PE_CTL1(0),
+ PFHMC_PDINV,
+ GLHMC_VFPDINV(0)
+};
+
+static u64 icrdma_masks[IRDMA_MAX_MASKS] = {
+ ICRDMA_CCQPSTATUS_CCQP_DONE_M,
+ ICRDMA_CCQPSTATUS_CCQP_ERR_M,
+ ICRDMA_CQPSQ_STAG_PDID_M,
+ ICRDMA_CQPSQ_CQ_CEQID_M,
+};
+
+static u64 icrdma_shifts[IRDMA_MAX_SHIFTS] = {
+ ICRDMA_CCQPSTATUS_CCQP_DONE_S,
+ ICRDMA_CCQPSTATUS_CCQP_ERR_S,
+ ICRDMA_CQPSQ_STAG_PDID_S,
+ ICRDMA_CQPSQ_CQ_CEQID_S,
+};
+
+void icrdma_init_hw(struct irdma_sc_dev *dev)
+{
+ int i;
+
+ for (i = 0; i < IRDMA_MAX_REGS; ++i)
+ dev->hw_regs[i] = icrdma_regs[i];
+
+ for (i = 0; i < IRDMA_MAX_SHIFTS; ++i)
+ dev->hw_shifts[i] = icrdma_shifts[i];
+
+ for (i = 0; i < IRDMA_MAX_MASKS; ++i)
+ dev->hw_masks[i] = icrdma_masks[i];
+
+ dev->wqe_alloc_db = (u32 __iomem *)(irdma_get_hw_addr(dev) +
+ dev->hw_regs[IRDMA_WQEALLOC]);
+ dev->cq_arm_db = (u32 __iomem *)(irdma_get_hw_addr(dev) +
+ dev->hw_regs[IRDMA_CQARM]);
+ dev->aeq_alloc_db = (u32 __iomem *)(irdma_get_hw_addr(dev) +
+ dev->hw_regs[IRDMA_AEQALLOC]);
+ dev->cqp_db = (u32 __iomem *)(irdma_get_hw_addr(dev) +
+ dev->hw_regs[IRDMA_CQPDB]);
+ dev->cq_ack_db = (u32 __iomem *)(irdma_get_hw_addr(dev) +
+ dev->hw_regs[IRDMA_CQACK]);
+ dev->hw_attrs.max_stat_inst = ICRDMA_MAX_STATS_COUNT;
+ dev->hw_attrs.max_hw_wq_size = IRDMA_QP_WQE_MAX_SIZE;
+ dev->hw_attrs.min_sw_wq_size = IRDMA_QP_SW_MIN_WQSIZE;
+ dev->hw_attrs.max_hw_sq_chunk = IRDMA_MAX_QUANTA_PER_WR;
+}
diff --git a/drivers/infiniband/hw/irdma/icrdma_hw.h b/drivers/infiniband/hw/irdma/icrdma_hw.h
new file mode 100644
index 0000000..f52a3bb
--- /dev/null
+++ b/drivers/infiniband/hw/irdma/icrdma_hw.h
@@ -0,0 +1,60 @@
+/* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB */
+/* Copyright (c) 2019, Intel Corporation. */
+
+#ifndef ICRDMA_HW_H
+#define ICRDMA_HW_H
+
+#define VFPE_CQPTAIL1 0x0000a000
+#define VFPE_CQPDB1 0x0000bc00
+#define VFPE_CCQPSTATUS1 0x0000b800
+#define VFPE_CCQPHIGH1 0x00009800
+#define VFPE_CCQPLOW1 0x0000ac00
+#define VFPE_CQARM1 0x0000b400
+#define VFPE_CQARM1 0x0000b400
+#define VFPE_CQACK1 0x0000b000
+#define VFPE_AEQALLOC1 0x0000a400
+#define VFPE_CQPERRCODES1 0x00009c00
+#define VFPE_WQEALLOC1 0x0000c000
+#define VFINT_DYN_CTLN(_i) (0x00003800 + ((_i) * 4)) /* _i=0...63 */
+
+#define PFPE_CQPTAIL 0x00500880
+#define PFPE_CQPDB 0x00500800
+#define PFPE_CCQPSTATUS 0x0050a000
+#define PFPE_CCQPHIGH 0x0050a100
+#define PFPE_CCQPLOW 0x0050a080
+#define PFPE_CQARM 0x00502c00
+#define PFPE_CQACK 0x00502c80
+#define PFPE_AEQALLOC 0x00502d00
+#define GLINT_DYN_CTL(_INT) (0x00160000 + ((_INT) * 4)) /* _i=0...2047 */
+#define GLPCI_LBARCTRL 0x0009de74
+#define GLPE_CPUSTATUS0 0x0050ba5c
+#define GLPE_CPUSTATUS1 0x0050ba60
+#define GLPE_CPUSTATUS2 0x0050ba64
+#define PFINT_AEQCTL 0x0016cb00
+#define PFPE_CQPERRCODES 0x0050a200
+#define PFPE_WQEALLOC 0x00504400
+#define GLINT_CEQCTL(_INT) (0x0015c000 + ((_INT) * 4)) /* _i=0...2047 */
+#define VSIQF_PE_CTL1(_VSI) (0x00414000 + ((_VSI) * 4)) /* _i=0...767 */
+#define PFHMC_PDINV 0x00520300
+#define GLHMC_VFPDINV(_i) (0x00528300 + ((_i) * 4)) /* _i=0...31 */
+
+#define ICRDMA_DB_ADDR_OFFSET (8 * 1024 * 1024 - 64 * 1024)
+
+#define ICRDMA_VF_DB_ADDR_OFFSET (64 * 1024)
+
+/* CCQSTATUS */
+#define ICRDMA_CCQPSTATUS_CCQP_DONE_S 0
+#define ICRDMA_CCQPSTATUS_CCQP_DONE_M MAKEMASK(0x1ULL, ICRDMA_CCQPSTATUS_CCQP_DONE_S)
+#define ICRDMA_CCQPSTATUS_CCQP_ERR_S 31
+#define ICRDMA_CCQPSTATUS_CCQP_ERR_M MAKEMASK(0x1ULL, ICRDMA_CCQPSTATUS_CCQP_ERR_S)
+#define ICRDMA_CQPSQ_STAG_PDID_S 46
+#define ICRDMA_CQPSQ_STAG_PDID_M MAKEMASK(0x3ffffULL, ICRDMA_CQPSQ_STAG_PDID_S)
+#define ICRDMA_CQPSQ_CQ_CEQID_S 22
+#define ICRDMA_CQPSQ_CQ_CEQID_M (0x3ffULL << ICRDMA_CQPSQ_CQ_CEQID_S)
+
+enum icrdma_device_caps_const {
+ ICRDMA_MAX_STATS_COUNT = 128,
+};
+
+void icrdma_init_hw(struct irdma_sc_dev *dev);
+#endif /* ICRDMA_HW_H*/
--
1.8.3.1
^ permalink raw reply related
* [RFC 06/19] RDMA/irdma: Implement HW Admin Queue OPs
From: Shiraz Saleem @ 2019-02-12 21:43 UTC (permalink / raw)
To: dledford, jgg, davem
Cc: linux-rdma, netdev, mustafa.ismail, jeffrey.t.kirsher,
Shiraz Saleem
In-Reply-To: <20190212214402.23284-1-shiraz.saleem@intel.com>
From: Mustafa Ismail <mustafa.ismail@intel.com>
The driver posts privileged commands to the HW
Admin Queue (Control QP or CQP) to request administrative
actions from the HW. Implement create/destroy of CQP
and the supporting functions, data structures and headers
to handle the different CQP commands
Signed-off-by: Mustafa Ismail <mustafa.ismail@intel.com>
Signed-off-by: Shiraz Saleem <shiraz.saleem@intel.com>
---
drivers/infiniband/hw/irdma/ctrl.c | 6165 +++++++++++++++++++++++++++++++++++
drivers/infiniband/hw/irdma/defs.h | 2053 ++++++++++++
drivers/infiniband/hw/irdma/irdma.h | 187 ++
drivers/infiniband/hw/irdma/type.h | 1705 ++++++++++
4 files changed, 10110 insertions(+)
create mode 100644 drivers/infiniband/hw/irdma/ctrl.c
create mode 100644 drivers/infiniband/hw/irdma/defs.h
create mode 100644 drivers/infiniband/hw/irdma/irdma.h
create mode 100644 drivers/infiniband/hw/irdma/type.h
diff --git a/drivers/infiniband/hw/irdma/ctrl.c b/drivers/infiniband/hw/irdma/ctrl.c
new file mode 100644
index 0000000..b6f0a57
--- /dev/null
+++ b/drivers/infiniband/hw/irdma/ctrl.c
@@ -0,0 +1,6165 @@
+// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
+/* Copyright (c) 2019, Intel Corporation. */
+
+#include "osdep.h"
+#include "status.h"
+#include "hmc.h"
+#include "defs.h"
+#include "type.h"
+#include "protos.h"
+#include "ws.h"
+
+/**
+ * irdma_qp_from_entry - Given entry, get to the qp structure
+ * @entry: Points to list of qp structure
+ */
+static struct irdma_sc_qp *irdma_qp_from_entry(struct list_head *entry)
+{
+ if (!entry)
+ return NULL;
+
+ return (struct irdma_sc_qp *)((char *)entry - offsetof(struct irdma_sc_qp, list));
+}
+
+/**
+ * irdma_get_qp_from_list - get next qp from a list
+ * @head: Listhead of qp's
+ * @qp: current qp
+ */
+struct irdma_sc_qp *irdma_get_qp_from_list(struct list_head *head,
+ struct irdma_sc_qp *qp)
+{
+ struct list_head *lastentry;
+ struct list_head *entry = NULL;
+
+ if (list_empty(head))
+ return NULL;
+
+ if (!qp) {
+ entry = head->next;
+ } else {
+ lastentry = &qp->list;
+ entry = (lastentry->next != head) ? lastentry->next : NULL;
+ }
+
+ return irdma_qp_from_entry(entry);
+}
+
+/**
+ * irdma_suspend_qps - suspend all qp's on VSI
+ * @vsi: the VSI struct pointer
+ */
+void irdma_suspend_qps(struct irdma_sc_vsi *vsi)
+{
+ struct irdma_sc_qp *qp = NULL;
+ unsigned long flags;
+ int i;
+
+ for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) {
+ spin_lock_irqsave(&vsi->qos[i].lock, flags);
+ qp = irdma_get_qp_from_list(&vsi->qos[i].qplist, qp);
+ while (qp) {
+ /* issue cqp suspend command */
+ if (!irdma_qp_suspend_resume(qp, true))
+ atomic_inc(&vsi->qp_suspend_reqs);
+ qp = irdma_get_qp_from_list(&vsi->qos[i].qplist, qp);
+ }
+ spin_unlock_irqrestore(&vsi->qos[i].lock, flags);
+ }
+}
+
+/**
+ * irdma_change_l2params - given the new l2 parameters, change all qp
+ * @vsi: RDMA VSI pointer
+ * @l2params: New parameters from l2
+ */
+void irdma_change_l2params(struct irdma_sc_vsi *vsi,
+ struct irdma_l2params *l2params)
+{
+ struct irdma_sc_qp *qp = NULL;
+ u8 i;
+ unsigned long flags;
+
+ if (l2params->mtu_changed && vsi->mtu != l2params->mtu) {
+ vsi->mtu = l2params->mtu;
+ irdma_reinitialize_ieq(vsi);
+ }
+
+ if (!l2params->tc_changed)
+ return;
+
+ vsi->tc_change_pending = false;
+ for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) {
+ vsi->qos[i].traffic_class = l2params->up2tc[i];
+ spin_lock_irqsave(&vsi->qos[i].lock, flags);
+ qp = irdma_get_qp_from_list(&vsi->qos[i].qplist, qp);
+ while (qp) {
+ if (!irdma_ws_add(vsi, i)) {
+ qp->qs_handle = vsi->qos[qp->user_pri].qs_handle;
+ irdma_qp_suspend_resume(qp, false);
+ } else {
+ irdma_qp_suspend_resume(qp, false);
+ spin_unlock_irqrestore(&vsi->qos[i].lock, flags);
+ irdma_modify_qp_to_err(qp);
+ spin_lock_irqsave(&vsi->qos[i].lock, flags);
+ }
+ qp = irdma_get_qp_from_list(&vsi->qos[i].qplist, qp);
+ }
+ spin_unlock_irqrestore(&vsi->qos[i].lock, flags);
+ }
+}
+
+/**
+ * irdma_qp_rem_qos - remove qp from qos lists during destroy qp
+ * @qp: qp to be removed from qos
+ */
+void irdma_qp_rem_qos(struct irdma_sc_qp *qp)
+{
+ struct irdma_sc_vsi *vsi = qp->vsi;
+ unsigned long flags;
+
+ if (!qp->on_qoslist)
+ return;
+
+ spin_lock_irqsave(&vsi->qos[qp->user_pri].lock, flags);
+ list_del(&qp->list);
+ spin_unlock_irqrestore(&vsi->qos[qp->user_pri].lock, flags);
+}
+
+/**
+ * irdma_qp_add_qos - called during setctx for qp to be added to qos
+ * @qp: qp to be added to qos
+ */
+void irdma_qp_add_qos(struct irdma_sc_qp *qp)
+{
+ struct irdma_sc_vsi *vsi = qp->vsi;
+ unsigned long flags;
+
+ if (qp->on_qoslist)
+ return;
+
+ spin_lock_irqsave(&vsi->qos[qp->user_pri].lock, flags);
+ list_add(&qp->list, &vsi->qos[qp->user_pri].qplist);
+ qp->on_qoslist = true;
+ qp->qs_handle = vsi->qos[qp->user_pri].qs_handle;
+ spin_unlock_irqrestore(&vsi->qos[qp->user_pri].lock, flags);
+}
+
+/**
+ * irdma_sc_pd_init - initialize sc pd struct
+ * @dev: sc device struct
+ * @pd: sc pd ptr
+ * @pd_id: pd_id for allocated pd
+ * @abi_ver: ABI version from user context, -1 if not valid
+ */
+static void irdma_sc_pd_init(struct irdma_sc_dev *dev,
+ struct irdma_sc_pd *pd,
+ u32 pd_id,
+ int abi_ver)
+{
+ pd->pd_id = pd_id;
+ pd->abi_ver = abi_ver;
+ pd->dev = dev;
+}
+
+/**
+ * irdma_get_encoded_wqe_size - given wq size, returns hardware encoded size
+ * @wqsize: size of the wq (sq, rq) to encoded_size
+ * @cqpsq: encoded size for sq for cqp as its encoded size is 1+ other wq's
+ */
+u8 irdma_get_encoded_wqe_size(u32 wqsize, bool cqpsq)
+{
+ u8 encoded_size = 0;
+
+ /* cqp sq's hw coded value starts from 1 for size of 4
+ * while it starts from 0 for qp' wq's.
+ */
+ if (cqpsq)
+ encoded_size = 1;
+ wqsize >>= 2;
+ while (wqsize >>= 1)
+ encoded_size++;
+
+ return encoded_size;
+}
+
+/**
+ * irdma_sc_manage_push_page - Handle push page
+ * @cqp: struct for cqp hw
+ * @info: push page info
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code
+irdma_sc_manage_push_page(struct irdma_sc_cqp *cqp,
+ struct irdma_cqp_manage_push_page_info *info,
+ u64 scratch,
+ bool post_sq)
+{
+ __le64 *wqe;
+ u64 hdr;
+
+ if (info->push_idx >= cqp->dev->hw_attrs.max_hw_device_pages)
+ return IRDMA_ERR_INVALID_PUSH_PAGE_INDEX;
+
+ wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+
+ set_64bit_val(wqe, 16, info->qs_handle);
+
+ hdr = LS_64(info->push_idx, IRDMA_CQPSQ_MPP_PPIDX) |
+ LS_64(info->push_page_type, IRDMA_CQPSQ_MPP_PPTYPE) |
+ LS_64(IRDMA_CQP_OP_MANAGE_PUSH_PAGES, IRDMA_CQPSQ_OPCODE) |
+ LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID) |
+ LS_64(info->free_page, IRDMA_CQPSQ_MPP_FREE_PAGE);
+ irdma_insert_wqe_hdr(wqe, hdr);
+
+ irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "MANAGE_PUSH_PAGES WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+ if (post_sq)
+ irdma_sc_cqp_post_sq(cqp);
+
+ return 0;
+}
+
+/**
+ * irdma_sc_add_arp_cache_entry - cqp wqe add arp cache entry
+ * @cqp: struct for cqp hw
+ * @info: arp entry information
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code
+irdma_sc_add_arp_cache_entry(struct irdma_sc_cqp *cqp,
+ struct irdma_add_arp_cache_entry_info *info,
+ u64 scratch,
+ bool post_sq)
+{
+ __le64 *wqe;
+ u64 temp, hdr;
+
+ wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+ set_64bit_val(wqe, 8, info->reach_max);
+
+ temp = info->mac_addr[5] |
+ LS_64_1(info->mac_addr[4], 8) |
+ LS_64_1(info->mac_addr[3], 16) |
+ LS_64_1(info->mac_addr[2], 24) |
+ LS_64_1(info->mac_addr[1], 32) |
+ LS_64_1(info->mac_addr[0], 40);
+ set_64bit_val(wqe, 16, temp);
+
+ hdr = info->arp_index |
+ LS_64(IRDMA_CQP_OP_MANAGE_ARP, IRDMA_CQPSQ_OPCODE) |
+ LS_64((info->permanent ? 1 : 0), IRDMA_CQPSQ_MAT_PERMANENT) |
+ LS_64(1, IRDMA_CQPSQ_MAT_ENTRYVALID) |
+ LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ irdma_insert_wqe_hdr(wqe, hdr);
+
+ irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "ARP_CACHE_ENTRY WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+ if (post_sq)
+ irdma_sc_cqp_post_sq(cqp);
+
+ return 0;
+}
+
+/**
+ * irdma_sc_del_arp_cache_entry - dele arp cache entry
+ * @cqp: struct for cqp hw
+ * @scratch: u64 saved to be used during cqp completion
+ * @arp_index: arp index to delete arp entry
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code
+irdma_sc_del_arp_cache_entry(struct irdma_sc_cqp *cqp,
+ u64 scratch,
+ u16 arp_index,
+ bool post_sq)
+{
+ __le64 *wqe;
+ u64 hdr;
+
+ wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+
+ hdr = arp_index |
+ LS_64(IRDMA_CQP_OP_MANAGE_ARP, IRDMA_CQPSQ_OPCODE) |
+ LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ irdma_insert_wqe_hdr(wqe, hdr);
+
+ irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "ARP_CACHE_DEL_ENTRY WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+ if (post_sq)
+ irdma_sc_cqp_post_sq(cqp);
+
+ return 0;
+}
+
+/**
+ * irdma_sc_query_arp_cache_entry - cqp wqe to query arp and arp index
+ * @cqp: struct for cqp hw
+ * @scratch: u64 saved to be used during cqp completion
+ * @arp_index: arp index to delete arp entry
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code
+irdma_sc_query_arp_cache_entry(struct irdma_sc_cqp *cqp,
+ u64 scratch,
+ u16 arp_index,
+ bool post_sq)
+{
+ __le64 *wqe;
+ u64 hdr;
+
+ wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+
+ hdr = arp_index |
+ LS_64(IRDMA_CQP_OP_MANAGE_ARP, IRDMA_CQPSQ_OPCODE) |
+ LS_64(1, IRDMA_CQPSQ_MAT_QUERY) |
+ LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ irdma_insert_wqe_hdr(wqe, hdr);
+
+ irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "QUERY_ARP_CACHE_ENTRY WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+ if (post_sq)
+ irdma_sc_cqp_post_sq(cqp);
+
+ return 0;
+}
+
+/**
+ * irdma_sc_manage_apbvt_entry - for adding and deleting apbvt entries
+ * @cqp: struct for cqp hw
+ * @info: info for apbvt entry to add or delete
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code
+irdma_sc_manage_apbvt_entry(struct irdma_sc_cqp *cqp,
+ struct irdma_apbvt_info *info,
+ u64 scratch,
+ bool post_sq)
+{
+ __le64 *wqe;
+ u64 hdr;
+
+ wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+
+ set_64bit_val(wqe, 16, info->port);
+
+ hdr = LS_64(IRDMA_CQP_OP_MANAGE_APBVT, IRDMA_CQPSQ_OPCODE) |
+ LS_64(info->add, IRDMA_CQPSQ_MAPT_ADDPORT) |
+ LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ irdma_insert_wqe_hdr(wqe, hdr);
+
+ irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "MANAGE_APBVT WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+ if (post_sq)
+ irdma_sc_cqp_post_sq(cqp);
+
+ return 0;
+}
+
+/**
+ * irdma_sc_manage_qhash_table_entry - manage quad hash entries
+ * @cqp: struct for cqp hw
+ * @info: info for quad hash to manage
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag for cqp db to ring
+ *
+ * This is called before connection establishment is started.
+ * For passive connections, when listener is created, it will
+ * call with entry type of IRDMA_QHASH_TYPE_TCP_SYN with local
+ * ip address and tcp port. When SYN is received (passive
+ * connections) or sent (active connections), this routine is
+ * called with entry type of IRDMA_QHASH_TYPE_TCP_ESTABLISHED
+ * and quad is passed in info.
+ *
+ * When iwarp connection is done and its state moves to RTS, the
+ * quad hash entry in the hardware will point to iwarp's qp
+ * number and requires no calls from the driver.
+ */
+static enum irdma_status_code
+irdma_sc_manage_qhash_table_entry(struct irdma_sc_cqp *cqp,
+ struct irdma_qhash_table_info *info,
+ u64 scratch,
+ bool post_sq)
+{
+ __le64 *wqe;
+ u64 qw1 = 0;
+ u64 qw2 = 0;
+ u64 temp;
+ struct irdma_sc_vsi *vsi = info->vsi;
+
+ wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+ temp = info->mac_addr[5] |
+ LS_64_1(info->mac_addr[4], 8) |
+ LS_64_1(info->mac_addr[3], 16) |
+ LS_64_1(info->mac_addr[2], 24) |
+ LS_64_1(info->mac_addr[1], 32) |
+ LS_64_1(info->mac_addr[0], 40);
+ set_64bit_val(wqe, 0, temp);
+
+ qw1 = LS_64(info->qp_num, IRDMA_CQPSQ_QHASH_QPN) |
+ LS_64(info->dest_port, IRDMA_CQPSQ_QHASH_DEST_PORT);
+ if (info->ipv4_valid) {
+ set_64bit_val(wqe,
+ 48,
+ LS_64(info->dest_ip[0], IRDMA_CQPSQ_QHASH_ADDR3));
+ } else {
+ set_64bit_val(wqe,
+ 56,
+ LS_64(info->dest_ip[0], IRDMA_CQPSQ_QHASH_ADDR0) |
+ LS_64(info->dest_ip[1], IRDMA_CQPSQ_QHASH_ADDR1));
+
+ set_64bit_val(wqe,
+ 48,
+ LS_64(info->dest_ip[2], IRDMA_CQPSQ_QHASH_ADDR2) |
+ LS_64(info->dest_ip[3], IRDMA_CQPSQ_QHASH_ADDR3));
+ }
+ qw2 = LS_64(vsi->qos[info->user_pri].qs_handle, IRDMA_CQPSQ_QHASH_QS_HANDLE);
+ if (info->vlan_valid)
+ qw2 |= LS_64(info->vlan_id, IRDMA_CQPSQ_QHASH_VLANID);
+ set_64bit_val(wqe, 16, qw2);
+ if (info->entry_type == IRDMA_QHASH_TYPE_TCP_ESTABLISHED) {
+ qw1 |= LS_64(info->src_port, IRDMA_CQPSQ_QHASH_SRC_PORT);
+ if (!info->ipv4_valid) {
+ set_64bit_val(wqe,
+ 40,
+ LS_64(info->src_ip[0], IRDMA_CQPSQ_QHASH_ADDR0) |
+ LS_64(info->src_ip[1], IRDMA_CQPSQ_QHASH_ADDR1));
+ set_64bit_val(wqe,
+ 32,
+ LS_64(info->src_ip[2], IRDMA_CQPSQ_QHASH_ADDR2) |
+ LS_64(info->src_ip[3], IRDMA_CQPSQ_QHASH_ADDR3));
+ } else {
+ set_64bit_val(wqe,
+ 32,
+ LS_64(info->src_ip[0], IRDMA_CQPSQ_QHASH_ADDR3));
+ }
+ }
+
+ set_64bit_val(wqe, 8, qw1);
+ temp = LS_64(cqp->polarity, IRDMA_CQPSQ_QHASH_WQEVALID) |
+ LS_64(IRDMA_CQP_OP_MANAGE_QUAD_HASH_TABLE_ENTRY, IRDMA_CQPSQ_QHASH_OPCODE) |
+ LS_64(info->manage, IRDMA_CQPSQ_QHASH_MANAGE) |
+ LS_64(info->ipv4_valid, IRDMA_CQPSQ_QHASH_IPV4VALID) |
+ LS_64(info->vlan_valid, IRDMA_CQPSQ_QHASH_VLANVALID) |
+ LS_64(info->entry_type, IRDMA_CQPSQ_QHASH_ENTRYTYPE);
+ irdma_insert_wqe_hdr(wqe, temp);
+
+ irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "MANAGE_QHASH WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+ if (post_sq)
+ irdma_sc_cqp_post_sq(cqp);
+
+ return 0;
+}
+
+/**
+ * irdma_sc_cqp_nop - send a nop wqe
+ * @cqp: struct for cqp hw
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code irdma_sc_cqp_nop(struct irdma_sc_cqp *cqp,
+ u64 scratch,
+ bool post_sq)
+{
+ __le64 *wqe;
+ u64 hdr;
+
+ wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+ hdr = LS_64(IRDMA_CQP_OP_NOP, IRDMA_CQPSQ_OPCODE) |
+ LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ irdma_insert_wqe_hdr(wqe, hdr);
+
+ irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "NOP WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+ if (post_sq)
+ irdma_sc_cqp_post_sq(cqp);
+
+ return 0;
+}
+
+/**
+ * irdma_sc_qp_init - initialize qp
+ * @qp: sc qp
+ * @info: initialization qp info
+ */
+static enum irdma_status_code
+irdma_sc_qp_init(struct irdma_sc_qp *qp, struct irdma_qp_init_info *info)
+{
+ enum irdma_status_code ret_code;
+ u32 pble_obj_cnt;
+ u16 wqe_size;
+
+ if (info->qp_uk_init_info.max_sq_frag_cnt > info->pd->dev->hw_attrs.max_hw_wq_frags)
+ return IRDMA_ERR_INVALID_FRAG_COUNT;
+
+ if (info->qp_uk_init_info.max_rq_frag_cnt > info->pd->dev->hw_attrs.max_hw_wq_frags)
+ return IRDMA_ERR_INVALID_FRAG_COUNT;
+
+ qp->dev = info->pd->dev;
+ qp->vsi = info->vsi;
+ qp->ieq_qp = info->vsi->exception_lan_q;
+ qp->sq_pa = info->sq_pa;
+ qp->rq_pa = info->rq_pa;
+ qp->hw_host_ctx_pa = info->host_ctx_pa;
+ qp->q2_pa = info->q2_pa;
+ qp->shadow_area_pa = info->shadow_area_pa;
+ qp->q2_buf = info->q2;
+ qp->pd = info->pd;
+ qp->hw_host_ctx = info->host_ctx;
+ info->qp_uk_init_info.wqe_alloc_db = qp->pd->dev->wqe_alloc_db;
+ ret_code = irdma_qp_uk_init(&qp->qp_uk, &info->qp_uk_init_info);
+ if (ret_code)
+ return ret_code;
+
+ qp->virtual_map = info->virtual_map;
+ pble_obj_cnt = info->pd->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;
+
+ if ((info->virtual_map && info->sq_pa >= pble_obj_cnt) ||
+ (info->virtual_map && info->rq_pa >= pble_obj_cnt))
+ return IRDMA_ERR_INVALID_PBLE_INDEX;
+
+ qp->llp_stream_handle = (void *)(-1);
+ qp->qp_type = (info->type) ? info->type : IRDMA_QP_TYPE_IWARP;
+ qp->hw_sq_size = irdma_get_encoded_wqe_size(qp->qp_uk.sq_ring.size, false);
+ irdma_debug(qp->dev, IRDMA_DEBUG_WQE, "hw_sq_size[%04d] sq_ring.size[%04d]\n",
+ qp->hw_sq_size, qp->qp_uk.sq_ring.size);
+ if (qp->qp_uk.hw_attrs->hw_rev == IRDMA_GEN_1)
+ wqe_size = IRDMA_WQE_SIZE_128;
+ else
+ ret_code = irdma_fragcnt_to_wqesize_rq(qp->qp_uk.max_rq_frag_cnt,
+ &wqe_size);
+ if (ret_code)
+ return ret_code;
+
+ qp->hw_rq_size = irdma_get_encoded_wqe_size(qp->qp_uk.rq_size *
+ (wqe_size / IRDMA_QP_WQE_MIN_SIZE), false);
+ irdma_debug(qp->dev, IRDMA_DEBUG_WQE,
+ "hw_rq_size[%04d] qp_uk.rq_size[%04d] wqe_size[%04d]\n",
+ qp->hw_rq_size, qp->qp_uk.rq_size, wqe_size);
+ qp->sq_tph_val = info->sq_tph_val;
+ qp->rq_tph_val = info->rq_tph_val;
+ qp->sq_tph_en = info->sq_tph_en;
+ qp->rq_tph_en = info->rq_tph_en;
+ qp->rcv_tph_en = info->rcv_tph_en;
+ qp->xmit_tph_en = info->xmit_tph_en;
+ qp->qp_uk.first_sq_wq = info->qp_uk_init_info.first_sq_wq;
+ qp->qs_handle = qp->vsi->qos[qp->user_pri].qs_handle;
+
+ return 0;
+}
+
+/**
+ * irdma_sc_qp_create - create qp
+ * @qp: sc qp
+ * @info: qp create info
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code
+irdma_sc_qp_create(struct irdma_sc_qp *qp,
+ struct irdma_create_qp_info *info,
+ u64 scratch,
+ bool post_sq)
+{
+ struct irdma_sc_cqp *cqp;
+ __le64 *wqe;
+ u64 hdr;
+
+ cqp = qp->pd->dev->cqp;
+ if (qp->qp_uk.qp_id < cqp->dev->hw_attrs.min_hw_qp_id ||
+ qp->qp_uk.qp_id > (cqp->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_QP].max_cnt - 1))
+ return IRDMA_ERR_INVALID_QP_ID;
+
+ wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+
+ set_64bit_val(wqe, 16, qp->hw_host_ctx_pa);
+ set_64bit_val(wqe, 40, qp->shadow_area_pa);
+
+ hdr = qp->qp_uk.qp_id |
+ LS_64(IRDMA_CQP_OP_CREATE_QP, IRDMA_CQPSQ_OPCODE) |
+ LS_64((info->ord_valid ? 1 : 0), IRDMA_CQPSQ_QP_ORDVALID) |
+ LS_64(info->tcp_ctx_valid, IRDMA_CQPSQ_QP_TOECTXVALID) |
+ LS_64(info->mac_valid, IRDMA_CQPSQ_QP_MACVALID) |
+ LS_64(qp->qp_type, IRDMA_CQPSQ_QP_QPTYPE) |
+ LS_64(qp->virtual_map, IRDMA_CQPSQ_QP_VQ) |
+ LS_64(info->force_lpb, IRDMA_CQPSQ_QP_FORCELOOPBACK) |
+ LS_64(info->cq_num_valid, IRDMA_CQPSQ_QP_CQNUMVALID) |
+ LS_64(info->arp_cache_idx_valid, IRDMA_CQPSQ_QP_ARPTABIDXVALID) |
+ LS_64(info->next_iwarp_state, IRDMA_CQPSQ_QP_NEXTIWSTATE) |
+ LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ irdma_insert_wqe_hdr(wqe, hdr);
+
+ irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "QP_CREATE WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+ if (post_sq)
+ irdma_sc_cqp_post_sq(cqp);
+
+ return 0;
+}
+
+/**
+ * irdma_sc_qp_modify - modify qp cqp wqe
+ * @qp: sc qp
+ * @info: modify qp info
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code
+irdma_sc_qp_modify(struct irdma_sc_qp *qp,
+ struct irdma_modify_qp_info *info,
+ u64 scratch,
+ bool post_sq)
+{
+ __le64 *wqe;
+ struct irdma_sc_cqp *cqp;
+ u64 hdr;
+ u8 term_actions = 0;
+ u8 term_len = 0;
+
+ cqp = qp->pd->dev->cqp;
+ wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+
+ if (info->next_iwarp_state == IRDMA_QP_STATE_TERMINATE) {
+ if (info->dont_send_fin)
+ term_actions += IRDMAQP_TERM_SEND_TERM_ONLY;
+ if (info->dont_send_term)
+ term_actions += IRDMAQP_TERM_SEND_FIN_ONLY;
+ if (term_actions == IRDMAQP_TERM_SEND_TERM_AND_FIN ||
+ term_actions == IRDMAQP_TERM_SEND_TERM_ONLY)
+ term_len = info->termlen;
+ }
+
+ set_64bit_val(wqe,
+ 8,
+ LS_64(info->new_mss, IRDMA_CQPSQ_QP_NEWMSS) |
+ LS_64(term_len, IRDMA_CQPSQ_QP_TERMLEN));
+ set_64bit_val(wqe, 16, qp->hw_host_ctx_pa);
+ set_64bit_val(wqe, 40, qp->shadow_area_pa);
+
+ hdr = qp->qp_uk.qp_id |
+ LS_64(IRDMA_CQP_OP_MODIFY_QP, IRDMA_CQPSQ_OPCODE) |
+ LS_64(info->ord_valid, IRDMA_CQPSQ_QP_ORDVALID) |
+ LS_64(info->tcp_ctx_valid, IRDMA_CQPSQ_QP_TOECTXVALID) |
+ LS_64(info->cached_var_valid, IRDMA_CQPSQ_QP_CACHEDVARVALID) |
+ LS_64(qp->virtual_map, IRDMA_CQPSQ_QP_VQ) |
+ LS_64(info->force_lpb, IRDMA_CQPSQ_QP_FORCELOOPBACK) |
+ LS_64(info->cq_num_valid, IRDMA_CQPSQ_QP_CQNUMVALID) |
+ LS_64(info->force_lpb, IRDMA_CQPSQ_QP_FORCELOOPBACK) |
+ LS_64(info->mac_valid, IRDMA_CQPSQ_QP_MACVALID) |
+ LS_64(qp->qp_type, IRDMA_CQPSQ_QP_QPTYPE) |
+ LS_64(info->mss_change, IRDMA_CQPSQ_QP_MSSCHANGE) |
+ LS_64(info->remove_hash_idx, IRDMA_CQPSQ_QP_REMOVEHASHENTRY) |
+ LS_64(term_actions, IRDMA_CQPSQ_QP_TERMACT) |
+ LS_64(info->reset_tcp_conn, IRDMA_CQPSQ_QP_RESETCON) |
+ LS_64(info->arp_cache_idx_valid, IRDMA_CQPSQ_QP_ARPTABIDXVALID) |
+ LS_64(info->next_iwarp_state, IRDMA_CQPSQ_QP_NEXTIWSTATE) |
+ LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ irdma_insert_wqe_hdr(wqe, hdr);
+
+ irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "QP_MODIFY WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+ if (post_sq)
+ irdma_sc_cqp_post_sq(cqp);
+
+ return 0;
+}
+
+/**
+ * irdma_sc_qp_destroy - cqp destroy qp
+ * @qp: sc qp
+ * @scratch: u64 saved to be used during cqp completion
+ * @remove_hash_idx: flag if to remove hash idx
+ * @ignore_mw_bnd: memory window bind flag
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code
+irdma_sc_qp_destroy(struct irdma_sc_qp *qp,
+ u64 scratch,
+ bool remove_hash_idx,
+ bool ignore_mw_bnd,
+ bool post_sq)
+{
+ __le64 *wqe;
+ struct irdma_sc_cqp *cqp;
+ u64 hdr;
+
+ irdma_qp_rem_qos(qp);
+ cqp = qp->pd->dev->cqp;
+ wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+
+ set_64bit_val(wqe, 16, qp->hw_host_ctx_pa);
+ set_64bit_val(wqe, 40, qp->shadow_area_pa);
+
+ hdr = qp->qp_uk.qp_id |
+ LS_64(IRDMA_CQP_OP_DESTROY_QP, IRDMA_CQPSQ_OPCODE) |
+ LS_64(qp->qp_type, IRDMA_CQPSQ_QP_QPTYPE) |
+ LS_64(ignore_mw_bnd, IRDMA_CQPSQ_QP_IGNOREMWBOUND) |
+ LS_64(remove_hash_idx, IRDMA_CQPSQ_QP_REMOVEHASHENTRY) |
+ LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ irdma_insert_wqe_hdr(wqe, hdr);
+
+ irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "QP_DESTROY WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+ if (post_sq)
+ irdma_sc_cqp_post_sq(cqp);
+
+ return 0;
+}
+
+/**
+ * irdma_sc_qp_flush_wqes - flush qp's wqe
+ * @qp: sc qp
+ * @info: dlush information
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code
+irdma_sc_qp_flush_wqes(struct irdma_sc_qp *qp,
+ struct irdma_qp_flush_info *info,
+ u64 scratch,
+ bool post_sq)
+{
+ u64 temp = 0;
+ __le64 *wqe;
+ struct irdma_sc_cqp *cqp;
+ u64 hdr;
+ bool flush_sq = false, flush_rq = false;
+
+ if (info->rq && !qp->flush_rq)
+ flush_rq = true;
+ if (info->sq && !qp->flush_sq)
+ flush_sq = true;
+ qp->flush_sq |= flush_sq;
+ qp->flush_rq |= flush_rq;
+
+ if (!flush_sq && !flush_rq) {
+ irdma_debug(qp->dev, IRDMA_DEBUG_CQP, "Additional flush request ignored\n");
+ return 0;
+ }
+
+ cqp = qp->pd->dev->cqp;
+ wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+
+ if (info->userflushcode) {
+ if (flush_rq)
+ temp |= LS_64(info->rq_minor_code, IRDMA_CQPSQ_FWQE_RQMNERR) |
+ LS_64(info->rq_major_code, IRDMA_CQPSQ_FWQE_RQMJERR);
+ if (flush_sq)
+ temp |= LS_64(info->sq_minor_code, IRDMA_CQPSQ_FWQE_SQMNERR) |
+ LS_64(info->sq_major_code, IRDMA_CQPSQ_FWQE_SQMJERR);
+ }
+ set_64bit_val(wqe, 16, temp);
+
+ temp = (info->generate_ae) ?
+ info->ae_code | LS_64(info->ae_src, IRDMA_CQPSQ_FWQE_AESOURCE) : 0;
+ set_64bit_val(wqe, 8, temp);
+
+ hdr = qp->qp_uk.qp_id |
+ LS_64(IRDMA_CQP_OP_FLUSH_WQES, IRDMA_CQPSQ_OPCODE) |
+ LS_64(info->generate_ae, IRDMA_CQPSQ_FWQE_GENERATE_AE) |
+ LS_64(info->userflushcode, IRDMA_CQPSQ_FWQE_USERFLCODE) |
+ LS_64(flush_sq, IRDMA_CQPSQ_FWQE_FLUSHSQ) |
+ LS_64(flush_rq, IRDMA_CQPSQ_FWQE_FLUSHRQ) |
+ LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ irdma_insert_wqe_hdr(wqe, hdr);
+
+ irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "QP_FLUSH WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+ if (post_sq)
+ irdma_sc_cqp_post_sq(cqp);
+
+ return 0;
+}
+
+/**
+ * irdma_sc_gen_ae - generate AE, uses flush WQE CQP OP
+ * @qp: sc qp
+ * @info: gen ae information
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code irdma_sc_gen_ae(struct irdma_sc_qp *qp,
+ struct irdma_gen_ae_info *info,
+ u64 scratch,
+ bool post_sq)
+{
+ u64 temp;
+ __le64 *wqe;
+ struct irdma_sc_cqp *cqp;
+ u64 hdr;
+
+ cqp = qp->pd->dev->cqp;
+ wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+
+ temp = info->ae_code | LS_64(info->ae_src, IRDMA_CQPSQ_FWQE_AESOURCE);
+ set_64bit_val(wqe, 8, temp);
+
+ hdr = qp->qp_uk.qp_id |
+ LS_64(IRDMA_CQP_OP_GEN_AE, IRDMA_CQPSQ_OPCODE) |
+ LS_64(1, IRDMA_CQPSQ_FWQE_GENERATE_AE) |
+ LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ irdma_insert_wqe_hdr(wqe, hdr);
+
+ irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "GEN_AE WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+ if (post_sq)
+ irdma_sc_cqp_post_sq(cqp);
+
+ return 0;
+}
+
+/**
+ * irdma_sc_qp_upload_context - upload qp's context
+ * @dev: sc device struct
+ * @info: upload context info ptr for return
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code
+irdma_sc_qp_upload_context(struct irdma_sc_dev *dev,
+ struct irdma_upload_context_info *info,
+ u64 scratch,
+ bool post_sq)
+{
+ __le64 *wqe;
+ struct irdma_sc_cqp *cqp;
+ u64 hdr;
+
+ cqp = dev->cqp;
+ wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+
+ set_64bit_val(wqe, 16, info->buf_pa);
+
+ hdr = LS_64(info->qp_id, IRDMA_CQPSQ_UCTX_QPID) |
+ LS_64(IRDMA_CQP_OP_UPLOAD_CONTEXT, IRDMA_CQPSQ_OPCODE) |
+ LS_64(info->qp_type, IRDMA_CQPSQ_UCTX_QPTYPE) |
+ LS_64(info->raw_format, IRDMA_CQPSQ_UCTX_RAWFORMAT) |
+ LS_64(info->freeze_qp, IRDMA_CQPSQ_UCTX_FREEZEQP) |
+ LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ irdma_insert_wqe_hdr(wqe, hdr);
+
+ irdma_debug_buf(dev, IRDMA_DEBUG_WQE, "QP_UPLOAD_CTX WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+ if (post_sq)
+ irdma_sc_cqp_post_sq(cqp);
+
+ return 0;
+}
+
+/**
+ * irdma_sc_qp_setctx_roce - set qp's context
+ * @qp: sc qp
+ * @qp_ctx: context ptr
+ * @info: ctx info
+ */
+static enum irdma_status_code
+irdma_sc_qp_setctx_roce(struct irdma_sc_qp *qp,
+ __le64 *qp_ctx,
+ struct irdma_qp_host_ctx_info *info)
+{
+ struct irdma_roce_offload_info *roce_info;
+ struct irdma_udp_offload_info *udp;
+ u64 qw0, qw3, qw7 = 0, qw8 = 0;
+ struct irdma_sc_vsi *vsi;
+ struct irdma_sc_dev *dev;
+ u64 mac;
+
+ roce_info = info->roce_info;
+ udp = info->udp_info;
+ vsi = qp->vsi;
+ dev = qp->dev;
+
+ mac = LS_64_1(roce_info->mac_addr[5], 16) |
+ LS_64_1(roce_info->mac_addr[4], 24) |
+ LS_64_1(roce_info->mac_addr[3], 32) |
+ LS_64_1(roce_info->mac_addr[2], 40) |
+ LS_64_1(roce_info->mac_addr[1], 48) |
+ LS_64_1(roce_info->mac_addr[0], 56);
+
+ if (info->add_to_qoslist) {
+ qp->user_pri = info->user_pri;
+ irdma_qp_add_qos(qp);
+ irdma_debug(qp->dev, IRDMA_DEBUG_DCB, "DCB: qp[%d] UP[%d] qset[%d]\n WQE",
+ qp->qp_uk.qp_id, qp->user_pri, qp->qs_handle);
+ }
+ qw0 = LS_64(qp->qp_uk.rq_wqe_size, IRDMAQPC_RQWQESIZE) |
+ LS_64(qp->rcv_tph_en, IRDMAQPC_RCVTPHEN) |
+ LS_64(qp->xmit_tph_en, IRDMAQPC_XMITTPHEN) |
+ LS_64(qp->rq_tph_en, IRDMAQPC_RQTPHEN) |
+ LS_64(qp->sq_tph_en, IRDMAQPC_SQTPHEN) |
+ LS_64(info->push_idx, IRDMAQPC_PPIDX) |
+ LS_64(info->push_mode_en, IRDMAQPC_PMENA) |
+ LS_64(roce_info->pd_id >> 16, IRDMAQPC_PDIDXHI) |
+ LS_64(roce_info->dctcp_en, IRDMAQPC_DC_TCP_EN) |
+ LS_64(roce_info->err_rq_idx_valid, IRDMAQPC_ERR_RQ_IDX_VALID) |
+ LS_64(roce_info->ecn_en, IRDMAQPC_ECN_EN);
+ set_64bit_val(qp_ctx, 8, qp->sq_pa);
+ set_64bit_val(qp_ctx, 16, qp->rq_pa);
+ qw3 = LS_64(qp->hw_rq_size, IRDMAQPC_RQSIZE) |
+ LS_64(qp->hw_sq_size, IRDMAQPC_SQSIZE);
+ set_64bit_val(qp_ctx,
+ 136,
+ LS_64(info->send_cq_num, IRDMAQPC_TXCQNUM) |
+ LS_64(info->rcv_cq_num, IRDMAQPC_RXCQNUM));
+ set_64bit_val(qp_ctx,
+ 168,
+ LS_64(info->qp_compl_ctx, IRDMAQPC_QPCOMPCTX));
+ set_64bit_val(qp_ctx,
+ 176,
+ LS_64(qp->sq_tph_val, IRDMAQPC_SQTPHVAL) |
+ LS_64(qp->rq_tph_val, IRDMAQPC_RQTPHVAL) |
+ LS_64(qp->qs_handle, IRDMAQPC_QSHANDLE));
+ qw0 |= LS_64(roce_info->is_qp1, IRDMAQPC_ISQP1) |
+ LS_64(roce_info->roce_tver, IRDMAQPC_ROCE_TVER);
+ qw7 |= LS_64(roce_info->p_key, IRDMAQPC_PKEY) |
+ LS_64(roce_info->pd_id, IRDMAQPC_PDIDX) |
+ LS_64(roce_info->ack_credits, IRDMAQPC_ACKCREDITS);
+ qw8 = LS_64(roce_info->qkey, IRDMAQPC_QKEY) |
+ LS_64(roce_info->dest_qp, IRDMAQPC_DESTQP);
+ set_64bit_val(qp_ctx,
+ 160,
+ LS_64(roce_info->ord_size, IRDMAQPC_ORDSIZE) |
+ LS_64(roce_info->ird_size, IRDMAQPC_IRDSIZE) |
+ LS_64(roce_info->wr_rdresp_en, IRDMAQPC_WRRDRSPOK) |
+ LS_64(roce_info->rd_en, IRDMAQPC_RDOK) |
+ LS_64(info->stats_idx_valid, IRDMAQPC_USESTATSINSTANCE) |
+ LS_64(roce_info->bind_en, IRDMAQPC_BINDEN) |
+ LS_64(roce_info->fast_reg_en, IRDMAQPC_FASTREGEN) |
+ LS_64(roce_info->dcqcn_en, IRDMAQPC_DCQCNENABLE) |
+ LS_64(roce_info->rcv_no_icrc, IRDMAQPC_RCVNOICRC) |
+ LS_64(roce_info->fw_cc_enable, IRDMAQPC_FW_CC_ENABLE) |
+ LS_64(roce_info->udprivcq_en, IRDMAQPC_UDPRIVCQENABLE) |
+ LS_64(roce_info->priv_mode_en, IRDMAQPC_PRIVEN) |
+ LS_64(roce_info->timely_en, IRDMAQPC_TIMELYENABLE));
+ qw0 |= LS_64(udp->ipv4, IRDMAQPC_IPV4) |
+ LS_64(udp->insert_vlan_tag, IRDMAQPC_INSERTVLANTAG);
+ qw3 |= LS_64(udp->ttl, IRDMAQPC_TTL) |
+ LS_64(udp->tos, IRDMAQPC_TOS) |
+ LS_64(udp->src_port, IRDMAQPC_SRCPORTNUM) |
+ LS_64(udp->dst_port, IRDMAQPC_DESTPORTNUM);
+ set_64bit_val(qp_ctx,
+ 32,
+ LS_64(udp->dest_ip_addr2, IRDMAQPC_DESTIPADDR2) |
+ LS_64(udp->dest_ip_addr3, IRDMAQPC_DESTIPADDR3));
+ set_64bit_val(qp_ctx,
+ 40,
+ LS_64(udp->dest_ip_addr0, IRDMAQPC_DESTIPADDR0) |
+ LS_64(udp->dest_ip_addr1, IRDMAQPC_DESTIPADDR1));
+ set_64bit_val(qp_ctx,
+ 48,
+ LS_64(udp->snd_mss, IRDMAQPC_SNDMSS) |
+ LS_64(udp->vlan_tag, IRDMAQPC_VLANTAG) |
+ LS_64(udp->arp_idx, IRDMAQPC_ARPIDX));
+ qw7 |= LS_64(udp->flow_label, IRDMAQPC_FLOWLABEL);
+ set_64bit_val(qp_ctx,
+ 80,
+ LS_64(udp->psn_nxt, IRDMAQPC_PSNNXT) |
+ LS_64(udp->lsn, IRDMAQPC_LSN));
+ set_64bit_val(qp_ctx,
+ 88,
+ LS_64(udp->epsn, IRDMAQPC_EPSN));
+ set_64bit_val(qp_ctx,
+ 96,
+ LS_64(udp->psn_max, IRDMAQPC_PSNMAX) |
+ LS_64(udp->psn_una, IRDMAQPC_PSNUNA));
+ set_64bit_val(qp_ctx,
+ 112,
+ LS_64(udp->cwnd, IRDMAQPC_CWNDROCE));
+ set_64bit_val(qp_ctx,
+ 128,
+ LS_64(roce_info->err_rq_idx, IRDMAQPC_ERR_RQ_IDX));
+ set_64bit_val(qp_ctx,
+ 184,
+ LS_64(udp->local_ipaddr3, IRDMAQPC_LOCAL_IPADDR3) |
+ LS_64(udp->local_ipaddr2, IRDMAQPC_LOCAL_IPADDR2));
+ set_64bit_val(qp_ctx,
+ 192,
+ LS_64(udp->local_ipaddr1, IRDMAQPC_LOCAL_IPADDR1) |
+ LS_64(udp->local_ipaddr0, IRDMAQPC_LOCAL_IPADDR0));
+ set_64bit_val(qp_ctx,
+ 128,
+ LS_64(udp->rnr_nak_thresh, IRDMAQPC_RNRNAK_THRESH) |
+ LS_64(udp->rexmit_thresh, IRDMAQPC_REXMIT_THRESH));
+ set_64bit_val(qp_ctx, 0, qw0);
+ set_64bit_val(qp_ctx, 24, qw3);
+ set_64bit_val(qp_ctx, 56, qw7);
+ set_64bit_val(qp_ctx, 64, qw8);
+ set_64bit_val(qp_ctx,
+ 144,
+ LS_64(info->stats_idx, IRDMAQPC_STAT_INDEX));
+ set_64bit_val(qp_ctx, 152, mac);
+ set_64bit_val(qp_ctx,
+ 200,
+ LS_64(roce_info->t_high, IRDMAQPC_THIGH) |
+ LS_64(roce_info->t_low, IRDMAQPC_TLOW));
+
+ irdma_debug_buf(qp->dev, IRDMA_DEBUG_WQE, "QP_HOST CTX WQE",
+ qp_ctx, IRDMA_QP_CTX_SIZE);
+
+ return 0;
+}
+
+/**
+ * irdma_sc_qp_setctx - set qp's context
+ * @qp: sc qp
+ * @qp_ctx: context ptr
+ * @info: ctx info
+ */
+static enum irdma_status_code
+irdma_sc_qp_setctx(struct irdma_sc_qp *qp,
+ __le64 *qp_ctx,
+ struct irdma_qp_host_ctx_info *info)
+{
+ struct irdma_iwarp_offload_info *iw;
+ struct irdma_tcp_offload_info *tcp;
+ struct irdma_sc_vsi *vsi;
+ struct irdma_sc_dev *dev;
+ u64 qw0, qw3, qw7 = 0;
+ u64 mac = 0;
+
+ iw = info->iwarp_info;
+ tcp = info->tcp_info;
+ vsi = qp->vsi;
+ dev = qp->dev;
+
+ if (dev->hw_attrs.hw_rev > IRDMA_GEN_1) {
+ mac = LS_64_1(iw->mac_addr[5], 16) |
+ LS_64_1(iw->mac_addr[4], 24) |
+ LS_64_1(iw->mac_addr[3], 32) |
+ LS_64_1(iw->mac_addr[2], 40) |
+ LS_64_1(iw->mac_addr[1], 48) |
+ LS_64_1(iw->mac_addr[0], 56);
+ }
+
+ if (info->add_to_qoslist) {
+ qp->user_pri = info->user_pri;
+ irdma_qp_add_qos(qp);
+ irdma_debug(qp->dev, IRDMA_DEBUG_DCB, "DCB: qp[%d] UP[%d] qset[%d]\n",
+ qp->qp_uk.qp_id, qp->user_pri, qp->qs_handle);
+ }
+ if (info->add_to_qoslist)
+ qp->user_pri = info->user_pri;
+
+ qw0 = LS_64(qp->qp_uk.rq_wqe_size, IRDMAQPC_RQWQESIZE) |
+ LS_64(iw->err_rq_idx_valid, IRDMAQPC_ERR_RQ_IDX_VALID) |
+ LS_64(qp->rcv_tph_en, IRDMAQPC_RCVTPHEN) |
+ LS_64(qp->xmit_tph_en, IRDMAQPC_XMITTPHEN) |
+ LS_64(qp->rq_tph_en, IRDMAQPC_RQTPHEN) |
+ LS_64(qp->sq_tph_en, IRDMAQPC_SQTPHEN) |
+ LS_64(info->push_idx, IRDMAQPC_PPIDX) |
+ LS_64(info->push_mode_en, IRDMAQPC_PMENA) |
+ LS_64(iw->ib_rd_en, IRDMAQPC_IBRDENABLE) |
+ LS_64(iw->pd_id >> 16, IRDMAQPC_PDIDXHI);
+
+ set_64bit_val(qp_ctx, 8, qp->sq_pa);
+ set_64bit_val(qp_ctx, 16, qp->rq_pa);
+
+ qw3 = LS_64(qp->hw_rq_size, IRDMAQPC_RQSIZE) |
+ LS_64(qp->hw_sq_size, IRDMAQPC_SQSIZE);
+ if (dev->hw_attrs.hw_rev == IRDMA_GEN_1)
+ qw3 |= LS_64(qp->src_mac_addr_idx, IRDMAQPC_GEN1_SRCMACADDRIDX);
+ set_64bit_val(qp_ctx,
+ 128,
+ LS_64(iw->err_rq_idx, IRDMAQPC_ERR_RQ_IDX));
+ set_64bit_val(qp_ctx,
+ 136,
+ LS_64(info->send_cq_num, IRDMAQPC_TXCQNUM) |
+ LS_64(info->rcv_cq_num, IRDMAQPC_RXCQNUM));
+ set_64bit_val(qp_ctx,
+ 168,
+ LS_64(info->qp_compl_ctx, IRDMAQPC_QPCOMPCTX));
+ set_64bit_val(qp_ctx,
+ 176,
+ LS_64(qp->sq_tph_val, IRDMAQPC_SQTPHVAL) |
+ LS_64(qp->rq_tph_val, IRDMAQPC_RQTPHVAL) |
+ LS_64(qp->qs_handle, IRDMAQPC_QSHANDLE) |
+ LS_64(qp->ieq_qp, IRDMAQPC_EXCEPTION_LAN_QUEUE));
+ if (info->iwarp_info_valid) {
+ qw0 |= LS_64(iw->ddp_ver, IRDMAQPC_DDP_VER) |
+ LS_64(iw->rdmap_ver, IRDMAQPC_RDMAP_VER) |
+ LS_64(iw->dctcp_en, IRDMAQPC_DC_TCP_EN) |
+ LS_64(iw->ecn_en, IRDMAQPC_ECN_EN);
+ qw7 |= LS_64(iw->pd_id, IRDMAQPC_PDIDX);
+ set_64bit_val(qp_ctx,
+ 144,
+ LS_64(qp->q2_pa >> 8, IRDMAQPC_Q2ADDR) |
+ LS_64(info->stats_idx, IRDMAQPC_STAT_INDEX));
+ set_64bit_val(qp_ctx,
+ 152,
+ mac | LS_64(iw->last_byte_sent, IRDMAQPC_LASTBYTESENT));
+ set_64bit_val(qp_ctx,
+ 160,
+ LS_64(iw->ord_size, IRDMAQPC_ORDSIZE) |
+ LS_64(iw->ird_size, IRDMAQPC_IRDSIZE) |
+ LS_64(iw->wr_rdresp_en, IRDMAQPC_WRRDRSPOK) |
+ LS_64(iw->rd_en, IRDMAQPC_RDOK) |
+ LS_64(iw->snd_mark_en, IRDMAQPC_SNDMARKERS) |
+ LS_64(iw->bind_en, IRDMAQPC_BINDEN) |
+ LS_64(iw->fast_reg_en, IRDMAQPC_FASTREGEN) |
+ LS_64(iw->priv_mode_en, IRDMAQPC_PRIVEN) |
+ LS_64(info->stats_idx_valid, IRDMAQPC_USESTATSINSTANCE) |
+ LS_64(1, IRDMAQPC_IWARPMODE) |
+ LS_64(iw->rcv_mark_en, IRDMAQPC_RCVMARKERS) |
+ LS_64(iw->align_hdrs, IRDMAQPC_ALIGNHDRS) |
+ LS_64(iw->rcv_no_mpa_crc, IRDMAQPC_RCVNOMPACRC) |
+ LS_64(iw->rcv_mark_offset, IRDMAQPC_RCVMARKOFFSET) |
+ LS_64(iw->snd_mark_offset, IRDMAQPC_SNDMARKOFFSET) |
+ LS_64(iw->timely_en, IRDMAQPC_TIMELYENABLE));
+ }
+ if (info->tcp_info_valid) {
+ qw0 |= LS_64(tcp->ipv4, IRDMAQPC_IPV4) |
+ LS_64(tcp->no_nagle, IRDMAQPC_NONAGLE) |
+ LS_64(tcp->insert_vlan_tag, IRDMAQPC_INSERTVLANTAG) |
+ LS_64(tcp->time_stamp, IRDMAQPC_TIMESTAMP) |
+ LS_64(tcp->cwnd_inc_limit, IRDMAQPC_LIMIT) |
+ LS_64(tcp->drop_ooo_seg, IRDMAQPC_DROPOOOSEG) |
+ LS_64(tcp->dup_ack_thresh, IRDMAQPC_DUPACK_THRESH);
+ qw3 |= LS_64(tcp->ttl, IRDMAQPC_TTL) |
+ LS_64(tcp->avoid_stretch_ack, IRDMAQPC_AVOIDSTRETCHACK) |
+ LS_64(tcp->tos, IRDMAQPC_TOS) |
+ LS_64(tcp->src_port, IRDMAQPC_SRCPORTNUM) |
+ LS_64(tcp->dst_port, IRDMAQPC_DESTPORTNUM);
+ if (dev->hw_attrs.hw_rev == IRDMA_GEN_1) {
+ qw3 |= LS_64(tcp->src_mac_addr_idx, IRDMAQPC_GEN1_SRCMACADDRIDX);
+
+ qp->src_mac_addr_idx = tcp->src_mac_addr_idx;
+ }
+ set_64bit_val(qp_ctx,
+ 32,
+ LS_64(tcp->dest_ip_addr2, IRDMAQPC_DESTIPADDR2) |
+ LS_64(tcp->dest_ip_addr3, IRDMAQPC_DESTIPADDR3));
+ set_64bit_val(qp_ctx,
+ 40,
+ LS_64(tcp->dest_ip_addr0, IRDMAQPC_DESTIPADDR0) |
+ LS_64(tcp->dest_ip_addr1, IRDMAQPC_DESTIPADDR1));
+ set_64bit_val(qp_ctx,
+ 48,
+ LS_64(tcp->snd_mss, IRDMAQPC_SNDMSS) |
+ LS_64(tcp->vlan_tag, IRDMAQPC_VLANTAG) |
+ LS_64(tcp->arp_idx, IRDMAQPC_ARPIDX));
+ qw7 |= LS_64(tcp->flow_label, IRDMAQPC_FLOWLABEL) |
+ LS_64(tcp->wscale, IRDMAQPC_WSCALE) |
+ LS_64(tcp->ignore_tcp_opt, IRDMAQPC_IGNORE_TCP_OPT) |
+ LS_64(tcp->ignore_tcp_uns_opt, IRDMAQPC_IGNORE_TCP_UNS_OPT) |
+ LS_64(tcp->tcp_state, IRDMAQPC_TCPSTATE) |
+ LS_64(tcp->rcv_wscale, IRDMAQPC_RCVSCALE) |
+ LS_64(tcp->snd_wscale, IRDMAQPC_SNDSCALE);
+ set_64bit_val(qp_ctx,
+ 72,
+ LS_64(tcp->time_stamp_recent, IRDMAQPC_TIMESTAMP_RECENT) |
+ LS_64(tcp->time_stamp_age, IRDMAQPC_TIMESTAMP_AGE));
+ set_64bit_val(qp_ctx,
+ 80,
+ LS_64(tcp->snd_nxt, IRDMAQPC_SNDNXT) |
+ LS_64(tcp->snd_wnd, IRDMAQPC_SNDWND));
+ set_64bit_val(qp_ctx,
+ 88,
+ LS_64(tcp->rcv_nxt, IRDMAQPC_RCVNXT) |
+ LS_64(tcp->rcv_wnd, IRDMAQPC_RCVWND));
+ set_64bit_val(qp_ctx,
+ 96,
+ LS_64(tcp->snd_max, IRDMAQPC_SNDMAX) |
+ LS_64(tcp->snd_una, IRDMAQPC_SNDUNA));
+ set_64bit_val(qp_ctx,
+ 104,
+ LS_64(tcp->srtt, IRDMAQPC_SRTT) |
+ LS_64(tcp->rtt_var, IRDMAQPC_RTTVAR));
+ set_64bit_val(qp_ctx,
+ 112,
+ LS_64(tcp->ss_thresh, IRDMAQPC_SSTHRESH) |
+ LS_64(tcp->cwnd, IRDMAQPC_CWND));
+ set_64bit_val(qp_ctx,
+ 120,
+ LS_64(tcp->snd_wl1, IRDMAQPC_SNDWL1) |
+ LS_64(tcp->snd_wl2, IRDMAQPC_SNDWL2));
+ set_64bit_val(qp_ctx,
+ 128,
+ LS_64(tcp->max_snd_window, IRDMAQPC_MAXSNDWND) |
+ LS_64(tcp->rexmit_thresh, IRDMAQPC_REXMIT_THRESH));
+ set_64bit_val(qp_ctx,
+ 184,
+ LS_64(tcp->local_ipaddr3, IRDMAQPC_LOCAL_IPADDR3) |
+ LS_64(tcp->local_ipaddr2, IRDMAQPC_LOCAL_IPADDR2));
+ set_64bit_val(qp_ctx,
+ 192,
+ LS_64(tcp->local_ipaddr1, IRDMAQPC_LOCAL_IPADDR1) |
+ LS_64(tcp->local_ipaddr0, IRDMAQPC_LOCAL_IPADDR0));
+ set_64bit_val(qp_ctx,
+ 200,
+ LS_64(iw->t_high, IRDMAQPC_THIGH) |
+ LS_64(iw->t_low, IRDMAQPC_TLOW));
+ }
+
+ set_64bit_val(qp_ctx, 0, qw0);
+ set_64bit_val(qp_ctx, 24, qw3);
+ set_64bit_val(qp_ctx, 56, qw7);
+
+ irdma_debug_buf(qp->dev, IRDMA_DEBUG_WQE, "QP_HOST CTX",
+ qp_ctx, IRDMA_QP_CTX_SIZE);
+
+ return 0;
+}
+
+/**
+ * irdma_sc_suspend_qp - suspend qp for param change
+ * @cqp: struct for cqp hw
+ * @qp: sc qp struct
+ * @scratch: u64 saved to be used during cqp completion
+ */
+static enum irdma_status_code irdma_sc_suspend_qp(struct irdma_sc_cqp *cqp,
+ struct irdma_sc_qp *qp,
+ u64 scratch)
+{
+ u64 hdr;
+ __le64 *wqe;
+
+ wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+
+ hdr = LS_64(qp->qp_uk.qp_id, IRDMA_CQPSQ_SUSPENDQP_QPID) |
+ LS_64(IRDMA_CQP_OP_SUSPEND_QP, IRDMA_CQPSQ_OPCODE) |
+ LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ irdma_insert_wqe_hdr(wqe, hdr);
+
+ irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "SUSPEND_QP WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+ irdma_sc_cqp_post_sq(cqp);
+
+ return 0;
+}
+
+/**
+ * irdma_sc_resume_qp - resume qp after suspend
+ * @cqp: struct for cqp hw
+ * @qp: sc qp struct
+ * @scratch: u64 saved to be used during cqp completion
+ */
+static enum irdma_status_code irdma_sc_resume_qp(struct irdma_sc_cqp *cqp,
+ struct irdma_sc_qp *qp,
+ u64 scratch)
+{
+ u64 hdr;
+ __le64 *wqe;
+
+ wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+
+ set_64bit_val(wqe,
+ 16,
+ LS_64(qp->qs_handle, IRDMA_CQPSQ_RESUMEQP_QSHANDLE));
+
+ hdr = LS_64(qp->qp_uk.qp_id, IRDMA_CQPSQ_RESUMEQP_QPID) |
+ LS_64(IRDMA_CQP_OP_RESUME_QP, IRDMA_CQPSQ_OPCODE) |
+ LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ irdma_insert_wqe_hdr(wqe, hdr);
+
+ irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "RESUME_QP WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+ irdma_sc_cqp_post_sq(cqp);
+
+ return 0;
+}
+
+/**
+ * irdma_sc_alloc_stag - mr stag alloc
+ * @dev: sc device struct
+ * @info: stag info
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code
+irdma_sc_alloc_stag(struct irdma_sc_dev *dev,
+ struct irdma_allocate_stag_info *info,
+ u64 scratch,
+ bool post_sq)
+{
+ __le64 *wqe;
+ struct irdma_sc_cqp *cqp;
+ u64 hdr;
+ enum irdma_page_size page_size;
+
+ if (info->page_size == 0x40000000)
+ page_size = IRDMA_PAGE_SIZE_1G;
+ else if (info->page_size == 0x200000)
+ page_size = IRDMA_PAGE_SIZE_2M;
+ else
+ page_size = IRDMA_PAGE_SIZE_4K;
+
+ cqp = dev->cqp;
+ wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+
+ set_64bit_val(wqe,
+ 8,
+ FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID) |
+ LS_64(info->total_len, IRDMA_CQPSQ_STAG_STAGLEN));
+ set_64bit_val(wqe,
+ 16,
+ LS_64(info->stag_idx, IRDMA_CQPSQ_STAG_IDX));
+ set_64bit_val(wqe,
+ 40,
+ LS_64(info->hmc_fcn_index, IRDMA_CQPSQ_STAG_HMCFNIDX));
+
+ if (info->chunk_size)
+ set_64bit_val(wqe,
+ 48,
+ LS_64(info->first_pm_pbl_idx, IRDMA_CQPSQ_STAG_FIRSTPMPBLIDX));
+
+ hdr = LS_64(IRDMA_CQP_OP_ALLOC_STAG, IRDMA_CQPSQ_OPCODE) |
+ LS_64(1, IRDMA_CQPSQ_STAG_MR) |
+ LS_64(info->access_rights, IRDMA_CQPSQ_STAG_ARIGHTS) |
+ LS_64(info->chunk_size, IRDMA_CQPSQ_STAG_LPBLSIZE) |
+ LS_64(page_size, IRDMA_CQPSQ_STAG_HPAGESIZE) |
+ LS_64(info->remote_access, IRDMA_CQPSQ_STAG_REMACCENABLED) |
+ LS_64(info->use_hmc_fcn_index, IRDMA_CQPSQ_STAG_USEHMCFNIDX) |
+ LS_64(info->use_pf_rid, IRDMA_CQPSQ_STAG_USEPFRID) |
+ LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ irdma_insert_wqe_hdr(wqe, hdr);
+
+ irdma_debug_buf(dev, IRDMA_DEBUG_WQE, "ALLOC_STAG WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+ if (post_sq)
+ irdma_sc_cqp_post_sq(cqp);
+
+ return 0;
+}
+
+/**
+ * irdma_sc_mr_reg_non_shared - non-shared mr registration
+ * @dev: sc device struct
+ * @info: mr info
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code
+irdma_sc_mr_reg_non_shared(struct irdma_sc_dev *dev,
+ struct irdma_reg_ns_stag_info *info,
+ u64 scratch,
+ bool post_sq)
+{
+ __le64 *wqe;
+ u64 temp;
+ struct irdma_sc_cqp *cqp;
+ u64 hdr;
+ u32 pble_obj_cnt;
+ bool remote_access;
+ u8 addr_type;
+ enum irdma_page_size page_size;
+
+ if (info->page_size == 0x40000000)
+ page_size = IRDMA_PAGE_SIZE_1G;
+ else if (info->page_size == 0x200000)
+ page_size = IRDMA_PAGE_SIZE_2M;
+ else
+ page_size = IRDMA_PAGE_SIZE_4K;
+
+ if (info->access_rights & (IRDMA_ACCESS_FLAGS_REMOTEREAD_ONLY |
+ IRDMA_ACCESS_FLAGS_REMOTEWRITE_ONLY))
+ remote_access = true;
+ else
+ remote_access = false;
+
+ pble_obj_cnt = dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;
+ if (info->chunk_size && info->first_pm_pbl_index >= pble_obj_cnt)
+ return IRDMA_ERR_INVALID_PBLE_INDEX;
+
+ cqp = dev->cqp;
+ wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+
+ temp = (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED) ?
+ (uintptr_t)info->va : info->fbo;
+ set_64bit_val(wqe, 0, temp);
+ set_64bit_val(wqe,
+ 8,
+ LS_64(info->total_len, IRDMA_CQPSQ_STAG_STAGLEN) |
+ FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID));
+ set_64bit_val(wqe,
+ 16,
+ LS_64(info->stag_key, IRDMA_CQPSQ_STAG_KEY) |
+ LS_64(info->stag_idx, IRDMA_CQPSQ_STAG_IDX));
+ if (!info->chunk_size) {
+ set_64bit_val(wqe, 32, info->reg_addr_pa);
+ set_64bit_val(wqe, 48, 0);
+ } else {
+ set_64bit_val(wqe, 32, 0);
+ set_64bit_val(wqe, 48, info->first_pm_pbl_index);
+ }
+ set_64bit_val(wqe, 40, info->hmc_fcn_index);
+ set_64bit_val(wqe, 56, 0);
+
+ addr_type = (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED) ? 1 : 0;
+ hdr = LS_64(IRDMA_CQP_OP_REG_MR, IRDMA_CQPSQ_OPCODE) |
+ LS_64(1, IRDMA_CQPSQ_STAG_MR) |
+ LS_64(info->chunk_size, IRDMA_CQPSQ_STAG_LPBLSIZE) |
+ LS_64(page_size, IRDMA_CQPSQ_STAG_HPAGESIZE) |
+ LS_64(info->access_rights, IRDMA_CQPSQ_STAG_ARIGHTS) |
+ LS_64(remote_access, IRDMA_CQPSQ_STAG_REMACCENABLED) |
+ LS_64(addr_type, IRDMA_CQPSQ_STAG_VABASEDTO) |
+ LS_64(info->use_hmc_fcn_index, IRDMA_CQPSQ_STAG_USEHMCFNIDX) |
+ LS_64(info->use_pf_rid, IRDMA_CQPSQ_STAG_USEPFRID) |
+ LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ irdma_insert_wqe_hdr(wqe, hdr);
+
+ irdma_debug_buf(dev, IRDMA_DEBUG_WQE, "MR_REG_NS WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+ if (post_sq)
+ irdma_sc_cqp_post_sq(cqp);
+
+ return 0;
+}
+
+/**
+ * irdma_sc_mr_reg_shared - registered shared memory region
+ * @dev: sc device struct
+ * @info: info for shared memory registration
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code
+irdma_sc_mr_reg_shared(struct irdma_sc_dev *dev,
+ struct irdma_register_shared_stag *info,
+ u64 scratch,
+ bool post_sq)
+{
+ __le64 *wqe;
+ struct irdma_sc_cqp *cqp;
+ u64 temp, va64, fbo, hdr;
+ u32 va32;
+ bool remote_access;
+ u8 addr_type;
+
+ if (info->access_rights & (IRDMA_ACCESS_FLAGS_REMOTEREAD_ONLY |
+ IRDMA_ACCESS_FLAGS_REMOTEWRITE_ONLY))
+ remote_access = true;
+ else
+ remote_access = false;
+ cqp = dev->cqp;
+ wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+ va64 = (uintptr_t)(info->va);
+ va32 = (u32)(va64 & 0x00000000FFFFFFFF);
+ fbo = (u64)(va32 & (4096 - 1));
+
+ set_64bit_val(wqe,
+ 0,
+ (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED ?
+ (uintptr_t)info->va : fbo));
+ set_64bit_val(wqe,
+ 8,
+ FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID));
+ temp = LS_64(info->new_stag_key, IRDMA_CQPSQ_STAG_KEY) |
+ LS_64(info->new_stag_idx, IRDMA_CQPSQ_STAG_IDX) |
+ LS_64(info->parent_stag_idx, IRDMA_CQPSQ_STAG_PARENTSTAGIDX);
+ set_64bit_val(wqe, 16, temp);
+
+ addr_type = (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED) ? 1 : 0;
+ hdr = LS_64(IRDMA_CQP_OP_REG_SMR, IRDMA_CQPSQ_OPCODE) |
+ LS_64(1, IRDMA_CQPSQ_STAG_MR) |
+ LS_64(info->access_rights, IRDMA_CQPSQ_STAG_ARIGHTS) |
+ LS_64(remote_access, IRDMA_CQPSQ_STAG_REMACCENABLED) |
+ LS_64(addr_type, IRDMA_CQPSQ_STAG_VABASEDTO) |
+ LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ irdma_insert_wqe_hdr(wqe, hdr);
+
+ irdma_debug_buf(dev, IRDMA_DEBUG_WQE, "MR_REG_SHARED WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+ if (post_sq)
+ irdma_sc_cqp_post_sq(cqp);
+
+ return 0;
+}
+
+/**
+ * irdma_sc_dealloc_stag - deallocate stag
+ * @dev: sc device struct
+ * @info: dealloc stag info
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code
+irdma_sc_dealloc_stag(struct irdma_sc_dev *dev,
+ struct irdma_dealloc_stag_info *info,
+ u64 scratch,
+ bool post_sq)
+{
+ u64 hdr;
+ __le64 *wqe;
+ struct irdma_sc_cqp *cqp;
+
+ cqp = dev->cqp;
+ wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+
+ set_64bit_val(wqe,
+ 8,
+ FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID));
+ set_64bit_val(wqe,
+ 16,
+ LS_64(info->stag_idx, IRDMA_CQPSQ_STAG_IDX));
+
+ hdr = LS_64(IRDMA_CQP_OP_DEALLOC_STAG, IRDMA_CQPSQ_OPCODE) |
+ LS_64(info->mr, IRDMA_CQPSQ_STAG_MR) |
+ LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ irdma_insert_wqe_hdr(wqe, hdr);
+
+ irdma_debug_buf(dev, IRDMA_DEBUG_WQE, "DEALLOC_STAG WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+ if (post_sq)
+ irdma_sc_cqp_post_sq(cqp);
+
+ return 0;
+}
+
+/**
+ * irdma_sc_query_stag - query hardware for stag
+ * @dev: sc device struct
+ * @scratch: u64 saved to be used during cqp completion
+ * @stag_index: stag index for query
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code irdma_sc_query_stag(struct irdma_sc_dev *dev,
+ u64 scratch,
+ u32 stag_index,
+ bool post_sq)
+{
+ u64 hdr;
+ __le64 *wqe;
+ struct irdma_sc_cqp *cqp;
+
+ cqp = dev->cqp;
+ wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+
+ set_64bit_val(wqe,
+ 16,
+ LS_64(stag_index, IRDMA_CQPSQ_QUERYSTAG_IDX));
+
+ hdr = LS_64(IRDMA_CQP_OP_QUERY_STAG, IRDMA_CQPSQ_OPCODE) |
+ LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ irdma_insert_wqe_hdr(wqe, hdr);
+
+ irdma_debug_buf(dev, IRDMA_DEBUG_WQE, "QUERY_STAG WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+ if (post_sq)
+ irdma_sc_cqp_post_sq(cqp);
+
+ return 0;
+}
+
+/**
+ * irdma_sc_mw_alloc - mw allocate
+ * @dev: sc device struct
+ * @scratch: u64 saved to be used during cqp completion
+ * @mw_stag_index:stag index
+ * @pd_id: pd is for this mw
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code
+irdma_sc_mw_alloc(struct irdma_sc_dev *dev,
+ struct irdma_mw_alloc_info *info,
+ u64 scratch,
+ bool post_sq)
+{
+ u64 hdr;
+ struct irdma_sc_cqp *cqp;
+ __le64 *wqe;
+
+ cqp = dev->cqp;
+ wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+
+ set_64bit_val(wqe,
+ 8,
+ FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID));
+ set_64bit_val(wqe,
+ 16,
+ LS_64(info->mw_stag_index, IRDMA_CQPSQ_STAG_IDX));
+
+ hdr = LS_64(IRDMA_CQP_OP_ALLOC_STAG, IRDMA_CQPSQ_OPCODE) |
+ LS_64(info->mw_wide, IRDMA_CQPSQ_STAG_MWTYPE) |
+ LS_64(info->mw1_bind_dont_vldt_key, IRDMA_CQPSQ_STAG_MW1_BIND_DONT_VLDT_KEY) |
+ LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ irdma_insert_wqe_hdr(wqe, hdr);
+
+ irdma_debug_buf(dev, IRDMA_DEBUG_WQE, "MW_ALLOC WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+ if (post_sq)
+ irdma_sc_cqp_post_sq(cqp);
+
+ return 0;
+}
+
+/**
+ * irdma_sc_mr_fast_register - Posts RDMA fast register mr WR to iwarp qp
+ * @qp: sc qp struct
+ * @info: fast mr info
+ * @post_sq: flag for cqp db to ring
+ */
+enum irdma_status_code
+irdma_sc_mr_fast_register(struct irdma_sc_qp *qp,
+ struct irdma_fast_reg_stag_info *info,
+ bool post_sq)
+{
+ u64 temp, hdr;
+ __le64 *wqe;
+ u32 wqe_idx;
+ enum irdma_page_size page_size;
+ struct irdma_post_sq_info sq_info = {};
+
+ if (info->page_size == 0x40000000)
+ page_size = IRDMA_PAGE_SIZE_1G;
+ else if (info->page_size == 0x200000)
+ page_size = IRDMA_PAGE_SIZE_2M;
+ else
+ page_size = IRDMA_PAGE_SIZE_4K;
+
+ sq_info.wr_id = info->wr_id;
+ sq_info.push_wqe = info->push_wqe;
+
+ wqe = irdma_qp_get_next_send_wqe(&qp->qp_uk, &wqe_idx,
+ IRDMA_QP_WQE_MIN_QUANTA, 0, &sq_info);
+ if (!wqe)
+ return IRDMA_ERR_QP_TOOMANY_WRS_POSTED;
+
+ irdma_debug(qp->dev, IRDMA_DEBUG_MR, "wr_id[%llxh] wqe_idx[%04d] location[%p]\n",
+ info->wr_id, wqe_idx,
+ &qp->qp_uk.sq_wrtrk_array[wqe_idx].wrid);
+ temp = (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED) ?
+ (uintptr_t)info->va : info->fbo;
+ set_64bit_val(wqe, 0, temp);
+
+ temp = RS_64(info->first_pm_pbl_index >> 16, IRDMAQPSQ_FIRSTPMPBLIDXHI);
+ set_64bit_val(wqe,
+ 8,
+ LS_64(temp, IRDMAQPSQ_FIRSTPMPBLIDXHI) |
+ LS_64(info->reg_addr_pa >> IRDMAQPSQ_PBLADDR_S, IRDMAQPSQ_PBLADDR));
+ set_64bit_val(wqe,
+ 16,
+ info->total_len |
+ LS_64(info->first_pm_pbl_index, IRDMAQPSQ_FIRSTPMPBLIDXLO));
+
+ hdr = LS_64(info->stag_key, IRDMAQPSQ_STAGKEY) |
+ LS_64(info->stag_idx, IRDMAQPSQ_STAGINDEX) |
+ LS_64(IRDMAQP_OP_FAST_REGISTER, IRDMAQPSQ_OPCODE) |
+ LS_64(info->chunk_size, IRDMAQPSQ_LPBLSIZE) |
+ LS_64(page_size, IRDMAQPSQ_HPAGESIZE) |
+ LS_64(info->access_rights, IRDMAQPSQ_STAGRIGHTS) |
+ LS_64(info->addr_type, IRDMAQPSQ_VABASEDTO) |
+ LS_64((sq_info.push_wqe ? 1 : 0), IRDMAQPSQ_PUSHWQE) |
+ LS_64(info->read_fence, IRDMAQPSQ_READFENCE) |
+ LS_64(info->local_fence, IRDMAQPSQ_LOCALFENCE) |
+ LS_64(info->signaled, IRDMAQPSQ_SIGCOMPL) |
+ LS_64(qp->qp_uk.swqe_polarity, IRDMAQPSQ_VALID);
+ irdma_insert_wqe_hdr(wqe, hdr);
+
+ irdma_debug_buf(qp->dev, IRDMA_DEBUG_WQE, "FAST_REG WQE",
+ wqe, IRDMA_QP_WQE_MIN_SIZE);
+ if (sq_info.push_wqe) {
+ irdma_qp_push_wqe(&qp->qp_uk, wqe,
+ IRDMA_QP_WQE_MIN_QUANTA, wqe_idx, post_sq);
+ } else {
+ if (post_sq)
+ irdma_qp_post_wr(&qp->qp_uk);
+ }
+
+ return 0;
+}
+
+/**
+ * irdma_sc_gen_rts_ae - request AE generated after RTS
+ * @qp: sc qp struct
+ */
+static void irdma_sc_gen_rts_ae(struct irdma_sc_qp *qp)
+{
+ __le64 *wqe;
+ u64 hdr;
+ struct irdma_qp_uk *qp_uk;
+
+ qp_uk = &qp->qp_uk;
+
+ wqe = qp_uk->sq_base[1].elem;
+
+ hdr = LS_64(IRDMAQP_OP_NOP, IRDMAQPSQ_OPCODE) |
+ LS_64(1, IRDMAQPSQ_LOCALFENCE) |
+ LS_64(qp->qp_uk.swqe_polarity, IRDMAQPSQ_VALID);
+ irdma_insert_wqe_hdr(wqe, hdr);
+ irdma_debug_buf(qp->dev, IRDMA_DEBUG_QP, "NOP W/LOCAL FENCE WQE",
+ wqe, IRDMA_QP_WQE_MIN_SIZE);
+
+ wqe = qp_uk->sq_base[2].elem;
+ hdr = LS_64(IRDMAQP_OP_GEN_RTS_AE, IRDMAQPSQ_OPCODE) |
+ LS_64(qp->qp_uk.swqe_polarity, IRDMAQPSQ_VALID);
+ irdma_insert_wqe_hdr(wqe, hdr);
+ irdma_debug_buf(qp->dev, IRDMA_DEBUG_QP, "CONN EST WQE",
+ wqe, IRDMA_QP_WQE_MIN_SIZE);
+}
+
+/**
+ * irdma_sc_send_lsmm - send last streaming mode message
+ * @qp: sc qp struct
+ * @lsmm_buf: buffer with lsmm message
+ * @size: size of lsmm buffer
+ * @stag: stag of lsmm buffer
+ */
+static void irdma_sc_send_lsmm(struct irdma_sc_qp *qp,
+ void *lsmm_buf,
+ u32 size,
+ irdma_stag stag)
+{
+ __le64 *wqe;
+ u64 hdr;
+ struct irdma_qp_uk *qp_uk;
+
+ qp_uk = &qp->qp_uk;
+ wqe = qp_uk->sq_base->elem;
+
+ set_64bit_val(wqe, 0, (uintptr_t)lsmm_buf);
+ if (qp->qp_uk.hw_attrs->hw_rev == IRDMA_GEN_1) {
+ set_64bit_val(wqe, 8,
+ LS_64(size, IRDMAQPSQ_GEN1_FRAG_LEN) |
+ LS_64(stag, IRDMAQPSQ_GEN1_FRAG_STAG));
+ } else {
+ set_64bit_val(wqe, 8,
+ LS_64(size, IRDMAQPSQ_FRAG_LEN) |
+ LS_64(stag, IRDMAQPSQ_FRAG_STAG) |
+ LS_64(qp->qp_uk.swqe_polarity, IRDMAQPSQ_VALID));
+ }
+ set_64bit_val(wqe, 16, 0);
+
+ hdr = LS_64(IRDMAQP_OP_RDMA_SEND, IRDMAQPSQ_OPCODE) |
+ LS_64(1, IRDMAQPSQ_STREAMMODE) |
+ LS_64(1, IRDMAQPSQ_WAITFORRCVPDU) |
+ LS_64(qp->qp_uk.swqe_polarity, IRDMAQPSQ_VALID);
+ irdma_insert_wqe_hdr(wqe, hdr);
+
+ irdma_debug_buf(qp->dev, IRDMA_DEBUG_WQE, "SEND_LSMM WQE",
+ wqe, IRDMA_QP_WQE_MIN_SIZE);
+
+ if (qp->dev->hw_attrs.feature_flags & IRDMA_FEATURE_RTS_AE)
+ irdma_sc_gen_rts_ae(qp);
+}
+
+/**
+ * irdma_sc_send_lsmm_nostag - for privilege qp
+ * @qp: sc qp struct
+ * @lsmm_buf: buffer with lsmm message
+ * @size: size of lsmm buffer
+ */
+static void irdma_sc_send_lsmm_nostag(struct irdma_sc_qp *qp,
+ void *lsmm_buf,
+ u32 size)
+{
+ __le64 *wqe;
+ u64 hdr;
+ struct irdma_qp_uk *qp_uk;
+
+ qp_uk = &qp->qp_uk;
+ wqe = qp_uk->sq_base->elem;
+
+ set_64bit_val(wqe, 0, (uintptr_t)lsmm_buf);
+
+ if (qp->qp_uk.hw_attrs->hw_rev == IRDMA_GEN_1)
+ set_64bit_val(wqe, 8,
+ LS_64(size, IRDMAQPSQ_GEN1_FRAG_LEN));
+ else
+ set_64bit_val(wqe, 8,
+ LS_64(size, IRDMAQPSQ_FRAG_LEN) |
+ LS_64(qp->qp_uk.swqe_polarity, IRDMAQPSQ_VALID));
+ set_64bit_val(wqe, 16, 0);
+
+ hdr = LS_64(IRDMAQP_OP_RDMA_SEND, IRDMAQPSQ_OPCODE) |
+ LS_64(1, IRDMAQPSQ_STREAMMODE) |
+ LS_64(1, IRDMAQPSQ_WAITFORRCVPDU) |
+ LS_64(qp->qp_uk.swqe_polarity, IRDMAQPSQ_VALID);
+ irdma_insert_wqe_hdr(wqe, hdr);
+
+ irdma_debug_buf(qp->dev, IRDMA_DEBUG_WQE, "SEND_LSMM_NOSTAG WQE",
+ wqe, IRDMA_QP_WQE_MIN_SIZE);
+}
+
+/**
+ * irdma_sc_send_rtt - send last read0 or write0
+ * @qp: sc qp struct
+ * @read: Do read0 or write0
+ */
+static void irdma_sc_send_rtt(struct irdma_sc_qp *qp, bool read)
+{
+ __le64 *wqe;
+ u64 hdr;
+ struct irdma_qp_uk *qp_uk;
+
+ qp_uk = &qp->qp_uk;
+ wqe = qp_uk->sq_base->elem;
+
+ set_64bit_val(wqe, 0, 0);
+ set_64bit_val(wqe, 16, 0);
+ if (read) {
+ if (qp->qp_uk.hw_attrs->hw_rev == IRDMA_GEN_1) {
+ set_64bit_val(wqe,
+ 8,
+ LS_64(0xabcd, IRDMAQPSQ_GEN1_FRAG_STAG));
+ } else {
+ set_64bit_val(wqe,
+ 8,
+ (u64)0xabcd | LS_64(qp->qp_uk.swqe_polarity,
+ IRDMAQPSQ_VALID));
+ }
+ hdr = LS_64(0x1234, IRDMAQPSQ_REMSTAG) |
+ LS_64(IRDMAQP_OP_RDMA_READ, IRDMAQPSQ_OPCODE) |
+ LS_64(qp->qp_uk.swqe_polarity, IRDMAQPSQ_VALID);
+
+ } else {
+ if (qp->qp_uk.hw_attrs->hw_rev == IRDMA_GEN_1) {
+ set_64bit_val(wqe, 8, 0);
+ } else {
+ set_64bit_val(wqe, 8,
+ LS_64(qp->qp_uk.swqe_polarity,
+ IRDMAQPSQ_VALID));
+ }
+ hdr = LS_64(IRDMAQP_OP_RDMA_WRITE, IRDMAQPSQ_OPCODE) |
+ LS_64(qp->qp_uk.swqe_polarity, IRDMAQPSQ_VALID);
+ }
+
+ irdma_insert_wqe_hdr(wqe, hdr);
+
+ irdma_debug_buf(qp->dev, IRDMA_DEBUG_WQE, "RTR WQE",
+ wqe, IRDMA_QP_WQE_MIN_SIZE);
+
+ if (qp->dev->hw_attrs.feature_flags & IRDMA_FEATURE_RTS_AE)
+ irdma_sc_gen_rts_ae(qp);
+}
+
+/**
+ * irdma_sc_gather_stats - collect the statistics
+ * @cqp: struct for cqp hw
+ * @info: gather stats info structure
+ * @scratch: u64 saved to be used during cqp completion
+ */
+static enum irdma_status_code
+irdma_sc_gather_stats(struct irdma_sc_cqp *cqp,
+ struct irdma_stats_gather_info *info,
+ u64 scratch)
+{
+ __le64 *wqe;
+ u64 temp;
+
+ if (info->stats_buff_mem.size < IRDMA_GATHER_STATS_BUF_SIZE)
+ return IRDMA_ERR_BUF_TOO_SHORT;
+
+ wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+
+ set_64bit_val(wqe, 40,
+ LS_64(info->hmc_fcn_index, IRDMA_CQPSQ_STATS_HMC_FCN_INDEX));
+ set_64bit_val(wqe, 32, info->stats_buff_mem.pa);
+
+ temp = LS_64(cqp->polarity, IRDMA_CQPSQ_STATS_WQEVALID) |
+ LS_64(info->use_stats_inst, IRDMA_CQPSQ_STATS_USE_INST) |
+ LS_64(info->stats_inst_index, IRDMA_CQPSQ_STATS_INST_INDEX) |
+ LS_64(info->use_hmc_fcn_index, IRDMA_CQPSQ_STATS_USE_HMC_FCN_INDEX) |
+ LS_64(IRDMA_CQP_OP_GATHER_STATS, IRDMA_CQPSQ_STATS_OP);
+ irdma_insert_wqe_hdr(wqe, temp);
+
+ irdma_debug_buf(cqp->dev, IRDMA_DEBUG_STATS, "GATHER_STATS WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+ wr32(cqp->dev->hw, cqp->dev->hw_regs[IRDMA_CQPDB],
+ IRDMA_RING_CURRENT_HEAD(cqp->sq_ring));
+ irdma_debug(cqp->dev,
+ IRDMA_DEBUG_STATS,
+ "CQP SQ head 0x%x tail 0x%x size 0x%x\n",
+ cqp->sq_ring.head,
+ cqp->sq_ring.tail,
+ cqp->sq_ring.size);
+
+ return 0;
+}
+
+/**
+ * irdma_sc_manage_stats_inst - allocate or free stats instance
+ * @cqp: struct for cqp hw
+ * @info: stats info structure
+ * @scratch: u64 saved to be used during cqp completion
+ */
+static enum irdma_status_code
+irdma_sc_manage_stats_inst(struct irdma_sc_cqp *cqp,
+ struct irdma_stats_inst_info *info,
+ bool alloc,
+ u64 scratch)
+{
+ __le64 *wqe;
+ u64 temp;
+
+ wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+
+ set_64bit_val(wqe, 40,
+ LS_64(info->hmc_fn_id, IRDMA_CQPSQ_STATS_HMC_FCN_INDEX));
+ temp = LS_64(cqp->polarity, IRDMA_CQPSQ_STATS_WQEVALID) |
+ LS_64(alloc, IRDMA_CQPSQ_STATS_ALLOC_INST) |
+ LS_64(info->use_hmc_fcn_index, IRDMA_CQPSQ_STATS_USE_HMC_FCN_INDEX) |
+ LS_64(info->stats_idx, IRDMA_CQPSQ_STATS_INST_INDEX) |
+ LS_64(IRDMA_CQP_OP_MANAGE_STATS, IRDMA_CQPSQ_STATS_OP);
+
+ irdma_insert_wqe_hdr(wqe, temp);
+
+ irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "MANAGE_STATS WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+
+ irdma_sc_cqp_post_sq(cqp);
+ return 0;
+}
+
+/**
+ * irdma_sc_set_up_mapping - set the up map table
+ * @cqp: struct for cqp hw
+ * @info: User priority map info
+ * @scratch: u64 saved to be used during cqp completion
+ */
+static enum irdma_status_code irdma_sc_set_up_map(struct irdma_sc_cqp *cqp,
+ struct irdma_up_info *info,
+ u64 scratch)
+{
+ __le64 *wqe;
+ u64 temp;
+
+ wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+
+ temp = info->map[0] |
+ LS_64_1(info->map[1], 8) | LS_64_1(info->map[2], 16) |
+ LS_64_1(info->map[3], 24) | LS_64_1(info->map[4], 32) |
+ LS_64_1(info->map[5], 40) | LS_64_1(info->map[6], 48) |
+ LS_64_1(info->map[7], 56);
+
+ set_64bit_val(wqe, 0, temp);
+ set_64bit_val(wqe, 40,
+ LS_64(info->cnp_up_override, IRDMA_CQPSQ_UP_CNPOVERRIDE) |
+ LS_64(info->hmc_fcn_idx, IRDMA_CQPSQ_UP_HMCFCNIDX));
+
+ temp = LS_64(cqp->polarity, IRDMA_CQPSQ_UP_WQEVALID) |
+ LS_64(info->use_vlan, IRDMA_CQPSQ_UP_USEVLAN) |
+ LS_64(info->use_cnp_up_override, IRDMA_CQPSQ_UP_USEOVERRIDE) |
+ LS_64(IRDMA_CQP_OP_UP_MAP, IRDMA_CQPSQ_UP_OP);
+ irdma_insert_wqe_hdr(wqe, temp);
+
+ irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "UPMAP WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+ irdma_sc_cqp_post_sq(cqp);
+
+ return 0;
+}
+
+/**
+ * irdma_sc_manage_ws_node - create/modify/destroy WS node
+ * @cqp: struct for cqp hw
+ * @info: node info structure
+ * @node_op: 0 for add 1 for modify, 2 for delete
+ * @scratch: u64 saved to be used during cqp completion
+ */
+static enum irdma_status_code
+irdma_sc_manage_ws_node(struct irdma_sc_cqp *cqp,
+ struct irdma_ws_node_info *info,
+ enum irdma_ws_node_op node_op,
+ u64 scratch)
+{
+ __le64 *wqe;
+ u64 temp = 0;
+
+ wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+
+ set_64bit_val(wqe, 32,
+ LS_64(info->vsi, IRDMA_CQPSQ_WS_VSI) |
+ LS_64(info->weight, IRDMA_CQPSQ_WS_WEIGHT));
+
+ temp = LS_64(cqp->polarity, IRDMA_CQPSQ_WS_WQEVALID) |
+ LS_64(node_op, IRDMA_CQPSQ_WS_NODEOP) |
+ LS_64(info->enable, IRDMA_CQPSQ_WS_ENABLENODE) |
+ LS_64(info->type_leaf, IRDMA_CQPSQ_WS_NODETYPE) |
+ LS_64(info->prio_type, IRDMA_CQPSQ_WS_PRIOTYPE) |
+ LS_64(info->tc, IRDMA_CQPSQ_WS_TC) |
+ LS_64(IRDMA_CQP_OP_WORK_SCHED_NODE, IRDMA_CQPSQ_WS_OP) |
+ LS_64(info->parent_id, IRDMA_CQPSQ_WS_PARENTID) |
+ LS_64(info->id, IRDMA_CQPSQ_WS_NODEID);
+ irdma_insert_wqe_hdr(wqe, temp);
+
+ irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "MANAGE_WS WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+ irdma_sc_cqp_post_sq(cqp);
+
+ return 0;
+}
+
+/**
+ * irdma_iwarp_opcode - determine if incoming is rdma layer
+ * @info: aeq info for the packet
+ * @pkt: packet for error
+ */
+static u32 irdma_iwarp_opcode(struct irdma_aeqe_info *info, u8 *pkt)
+{
+ __be16 *mpa;
+ u32 opcode = 0xffffffff;
+
+ if (info->q2_data_written) {
+ mpa = (__be16 *)pkt;
+ opcode = ntohs(mpa[1]) & 0xf;
+ }
+
+ return opcode;
+}
+
+/**
+ * irdma_locate_mpa - return pointer to mpa in the pkt
+ * @pkt: packet with data
+ */
+static u8 *irdma_locate_mpa(u8 *pkt)
+{
+ /* skip over ethernet header */
+ pkt += IRDMA_MAC_HLEN;
+
+ /* Skip over IP and TCP headers */
+ pkt += 4 * (pkt[0] & 0x0f);
+ pkt += 4 * ((pkt[12] >> 4) & 0x0f);
+
+ return pkt;
+}
+
+/**
+ * irdma_bld_termhdr_ctrl - setup terminate hdr control fields
+ * @qp: sc qp ptr for pkt
+ * @hdr: term hdr
+ * @opcode: flush opcode for termhdr
+ * @layer_etype: error layer + error type
+ * @err: error cod ein the header
+ */
+static void irdma_bld_termhdr_ctrl(struct irdma_sc_qp *qp,
+ struct irdma_terminate_hdr *hdr,
+ enum irdma_flush_opcode opcode,
+ u8 layer_etype,
+ u8 err)
+{
+ qp->flush_code = opcode;
+ hdr->layer_etype = layer_etype;
+ hdr->error_code = err;
+}
+
+/**
+ * irdma_bld_termhdr_ddp_rdma - setup ddp and rdma hdrs in terminate hdr
+ * @pkt: ptr to mpa in offending pkt
+ * @hdr: term hdr
+ * @copy_len: offending pkt length to be copied to term hdr
+ * @is_tagged: DDP tagged or untagged
+ */
+static void irdma_bld_termhdr_ddp_rdma(u8 *pkt,
+ struct irdma_terminate_hdr *hdr,
+ int *copy_len,
+ u8 *is_tagged)
+{
+ u16 ddp_seg_len;
+
+ ddp_seg_len = ntohs(*(__be16 *)pkt);
+ if (ddp_seg_len) {
+ *copy_len = 2;
+ hdr->hdrct = DDP_LEN_FLAG;
+ if (pkt[2] & 0x80) {
+ *is_tagged = 1;
+ if (ddp_seg_len >= TERM_DDP_LEN_TAGGED) {
+ *copy_len += TERM_DDP_LEN_TAGGED;
+ hdr->hdrct |= DDP_HDR_FLAG;
+ }
+ } else {
+ if (ddp_seg_len >= TERM_DDP_LEN_UNTAGGED) {
+ *copy_len += TERM_DDP_LEN_UNTAGGED;
+ hdr->hdrct |= DDP_HDR_FLAG;
+ }
+ if (ddp_seg_len >= (TERM_DDP_LEN_UNTAGGED + TERM_RDMA_LEN) &&
+ ((pkt[3] & RDMA_OPCODE_M) == RDMA_READ_REQ_OPCODE)) {
+ *copy_len += TERM_RDMA_LEN;
+ hdr->hdrct |= RDMA_HDR_FLAG;
+ }
+ }
+ }
+}
+
+/**
+ * irdma_bld_terminate_hdr - build terminate message header
+ * @qp: qp associated with received terminate AE
+ * @info: the struct contiaing AE information
+ */
+static int irdma_bld_terminate_hdr(struct irdma_sc_qp *qp,
+ struct irdma_aeqe_info *info)
+{
+ u8 *pkt = qp->q2_buf + Q2_BAD_FRAME_OFFSET;
+ int copy_len = 0;
+ u8 is_tagged = 0;
+ u32 opcode;
+ struct irdma_terminate_hdr *termhdr;
+
+ termhdr = (struct irdma_terminate_hdr *)qp->q2_buf;
+ memset(termhdr, 0, Q2_BAD_FRAME_OFFSET);
+
+ if (info->q2_data_written) {
+ pkt = irdma_locate_mpa(pkt);
+ irdma_bld_termhdr_ddp_rdma(pkt, termhdr, ©_len, &is_tagged);
+ }
+
+ opcode = irdma_iwarp_opcode(info, pkt);
+ qp->eventtype = TERM_EVENT_QP_FATAL;
+
+ switch (info->ae_id) {
+ case IRDMA_AE_AMP_UNALLOCATED_STAG:
+ qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
+ if (opcode == IRDMA_OP_TYPE_RDMA_WRITE)
+ irdma_bld_termhdr_ctrl(qp,
+ termhdr,
+ FLUSH_PROT_ERR,
+ (LAYER_DDP << 4) | DDP_TAGGED_BUF,
+ DDP_TAGGED_INV_STAG);
+ else
+ irdma_bld_termhdr_ctrl(qp,
+ termhdr,
+ FLUSH_REM_ACCESS_ERR,
+ (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,
+ RDMAP_INV_STAG);
+ break;
+ case IRDMA_AE_AMP_BOUNDS_VIOLATION:
+ qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
+ if (info->q2_data_written)
+ irdma_bld_termhdr_ctrl(qp,
+ termhdr,
+ FLUSH_PROT_ERR,
+ (LAYER_DDP << 4) | DDP_TAGGED_BUF,
+ DDP_TAGGED_BOUNDS);
+ else
+ irdma_bld_termhdr_ctrl(qp,
+ termhdr,
+ FLUSH_REM_ACCESS_ERR,
+ (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,
+ RDMAP_INV_BOUNDS);
+ break;
+ case IRDMA_AE_AMP_BAD_PD:
+ switch (opcode) {
+ case IRDMA_OP_TYPE_RDMA_WRITE:
+ irdma_bld_termhdr_ctrl(qp,
+ termhdr,
+ FLUSH_PROT_ERR,
+ (LAYER_DDP << 4) | DDP_TAGGED_BUF,
+ DDP_TAGGED_UNASSOC_STAG);
+ break;
+ case IRDMA_OP_TYPE_SEND_INV:
+ case IRDMA_OP_TYPE_SEND_SOL_INV:
+ irdma_bld_termhdr_ctrl(qp,
+ termhdr,
+ FLUSH_REM_ACCESS_ERR,
+ (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,
+ RDMAP_CANT_INV_STAG);
+ break;
+ default:
+ irdma_bld_termhdr_ctrl(qp,
+ termhdr,
+ FLUSH_REM_ACCESS_ERR,
+ (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,
+ RDMAP_UNASSOC_STAG);
+ }
+ break;
+ case IRDMA_AE_AMP_INVALID_STAG:
+ qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
+ irdma_bld_termhdr_ctrl(qp,
+ termhdr,
+ FLUSH_REM_ACCESS_ERR,
+ (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,
+ RDMAP_INV_STAG);
+ break;
+ case IRDMA_AE_AMP_BAD_QP:
+ irdma_bld_termhdr_ctrl(qp,
+ termhdr,
+ FLUSH_LOC_QP_OP_ERR,
+ (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,
+ DDP_UNTAGGED_INV_QN);
+ break;
+ case IRDMA_AE_AMP_BAD_STAG_KEY:
+ case IRDMA_AE_AMP_BAD_STAG_INDEX:
+ qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
+ switch (opcode) {
+ case IRDMA_OP_TYPE_SEND_INV:
+ case IRDMA_OP_TYPE_SEND_SOL_INV:
+ irdma_bld_termhdr_ctrl(qp,
+ termhdr,
+ FLUSH_REM_OP_ERR,
+ (LAYER_RDMA << 4) | RDMAP_REMOTE_OP,
+ RDMAP_CANT_INV_STAG);
+ break;
+ default:
+ irdma_bld_termhdr_ctrl(qp,
+ termhdr,
+ FLUSH_REM_ACCESS_ERR,
+ (LAYER_RDMA << 4) | RDMAP_REMOTE_OP,
+ RDMAP_INV_STAG);
+ }
+ break;
+ case IRDMA_AE_AMP_RIGHTS_VIOLATION:
+ case IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
+ case IRDMA_AE_PRIV_OPERATION_DENIED:
+ qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
+ irdma_bld_termhdr_ctrl(qp,
+ termhdr,
+ FLUSH_REM_ACCESS_ERR,
+ (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,
+ RDMAP_ACCESS);
+ break;
+ case IRDMA_AE_AMP_TO_WRAP:
+ qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
+ irdma_bld_termhdr_ctrl(qp,
+ termhdr,
+ FLUSH_REM_ACCESS_ERR,
+ (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,
+ RDMAP_TO_WRAP);
+ break;
+ case IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR:
+ irdma_bld_termhdr_ctrl(qp,
+ termhdr,
+ FLUSH_GENERAL_ERR,
+ (LAYER_MPA << 4) | DDP_LLP,
+ MPA_CRC);
+ break;
+ case IRDMA_AE_LLP_SEGMENT_TOO_SMALL:
+ irdma_bld_termhdr_ctrl(qp,
+ termhdr,
+ FLUSH_LOC_LEN_ERR,
+ (LAYER_DDP << 4) | DDP_CATASTROPHIC,
+ DDP_CATASTROPHIC_LOCAL);
+ break;
+ case IRDMA_AE_LCE_QP_CATASTROPHIC:
+ case IRDMA_AE_DDP_NO_L_BIT:
+ irdma_bld_termhdr_ctrl(qp,
+ termhdr,
+ FLUSH_FATAL_ERR,
+ (LAYER_DDP << 4) | DDP_CATASTROPHIC,
+ DDP_CATASTROPHIC_LOCAL);
+ break;
+ case IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN:
+ irdma_bld_termhdr_ctrl(qp,
+ termhdr,
+ FLUSH_GENERAL_ERR,
+ (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,
+ DDP_UNTAGGED_INV_MSN_RANGE);
+ break;
+ case IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
+ qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
+ irdma_bld_termhdr_ctrl(qp,
+ termhdr,
+ FLUSH_LOC_LEN_ERR,
+ (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,
+ DDP_UNTAGGED_INV_TOO_LONG);
+ break;
+ case IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION:
+ if (is_tagged)
+ irdma_bld_termhdr_ctrl(qp,
+ termhdr,
+ FLUSH_GENERAL_ERR,
+ (LAYER_DDP << 4) | DDP_TAGGED_BUF,
+ DDP_TAGGED_INV_DDP_VER);
+ else
+ irdma_bld_termhdr_ctrl(qp,
+ termhdr,
+ FLUSH_GENERAL_ERR,
+ (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,
+ DDP_UNTAGGED_INV_DDP_VER);
+ break;
+ case IRDMA_AE_DDP_UBE_INVALID_MO:
+ irdma_bld_termhdr_ctrl(qp,
+ termhdr,
+ FLUSH_GENERAL_ERR,
+ (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,
+ DDP_UNTAGGED_INV_MO);
+ break;
+ case IRDMA_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
+ irdma_bld_termhdr_ctrl(qp,
+ termhdr,
+ FLUSH_REM_OP_ERR,
+ (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,
+ DDP_UNTAGGED_INV_MSN_NO_BUF);
+ break;
+ case IRDMA_AE_DDP_UBE_INVALID_QN:
+ irdma_bld_termhdr_ctrl(qp,
+ termhdr,
+ FLUSH_GENERAL_ERR,
+ (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,
+ DDP_UNTAGGED_INV_QN);
+ break;
+ case IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION:
+ irdma_bld_termhdr_ctrl(qp,
+ termhdr,
+ FLUSH_GENERAL_ERR,
+ (LAYER_RDMA << 4) | RDMAP_REMOTE_OP,
+ RDMAP_INV_RDMAP_VER);
+ break;
+ default:
+ irdma_bld_termhdr_ctrl(qp,
+ termhdr,
+ FLUSH_FATAL_ERR,
+ (LAYER_RDMA << 4) | RDMAP_REMOTE_OP,
+ RDMAP_UNSPECIFIED);
+ break;
+ }
+
+ if (copy_len)
+ memcpy(termhdr + 1, pkt, copy_len);
+
+ return sizeof(struct irdma_terminate_hdr) + copy_len;
+}
+
+/**
+ * irdma_terminate_send_fin() - Send fin for terminate message
+ * @qp: qp associated with received terminate AE
+ */
+void irdma_terminate_send_fin(struct irdma_sc_qp *qp)
+{
+ irdma_term_modify_qp(qp,
+ IRDMA_QP_STATE_TERMINATE,
+ IRDMAQP_TERM_SEND_FIN_ONLY,
+ 0);
+}
+
+/**
+ * irdma_terminate_connection() - Bad AE and send terminate to remote QP
+ * @qp: qp associated with received terminate AE
+ * @info: the struct contiaing AE information
+ */
+void irdma_terminate_connection(struct irdma_sc_qp *qp,
+ struct irdma_aeqe_info *info)
+{
+ u8 termlen = 0;
+
+ if (qp->term_flags & IRDMA_TERM_SENT)
+ return;
+
+ termlen = irdma_bld_terminate_hdr(qp, info);
+ irdma_terminate_start_timer(qp);
+ qp->term_flags |= IRDMA_TERM_SENT;
+ irdma_term_modify_qp(qp, IRDMA_QP_STATE_TERMINATE,
+ IRDMAQP_TERM_SEND_TERM_ONLY, termlen);
+}
+
+/**
+ * irdma_terminate_received - handle terminate received AE
+ * @qp: qp associated with received terminate AE
+ * @info: the struct contiaing AE information
+ */
+void irdma_terminate_received(struct irdma_sc_qp *qp,
+ struct irdma_aeqe_info *info)
+{
+ u8 *pkt = qp->q2_buf + Q2_BAD_FRAME_OFFSET;
+ __be32 *mpa;
+ u8 ddp_ctl;
+ u8 rdma_ctl;
+ u16 aeq_id = 0;
+ struct irdma_terminate_hdr *termhdr;
+
+ mpa = (__be32 *)irdma_locate_mpa(pkt);
+ if (info->q2_data_written) {
+ /* did not validate the frame - do it now */
+ ddp_ctl = (ntohl(mpa[0]) >> 8) & 0xff;
+ rdma_ctl = ntohl(mpa[0]) & 0xff;
+ if ((ddp_ctl & 0xc0) != 0x40)
+ aeq_id = IRDMA_AE_LCE_QP_CATASTROPHIC;
+ else if ((ddp_ctl & 0x03) != 1)
+ aeq_id = IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION;
+ else if (ntohl(mpa[2]) != 2)
+ aeq_id = IRDMA_AE_DDP_UBE_INVALID_QN;
+ else if (ntohl(mpa[3]) != 1)
+ aeq_id = IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN;
+ else if (ntohl(mpa[4]) != 0)
+ aeq_id = IRDMA_AE_DDP_UBE_INVALID_MO;
+ else if ((rdma_ctl & 0xc0) != 0x40)
+ aeq_id = IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION;
+
+ info->ae_id = aeq_id;
+ if (info->ae_id) {
+ /* Bad terminate recvd - send back a terminate */
+ irdma_terminate_connection(qp, info);
+ return;
+ }
+ }
+
+ qp->term_flags |= IRDMA_TERM_RCVD;
+ qp->eventtype = TERM_EVENT_QP_FATAL;
+ termhdr = (struct irdma_terminate_hdr *)&mpa[5];
+ if (termhdr->layer_etype == RDMAP_REMOTE_PROT ||
+ termhdr->layer_etype == RDMAP_REMOTE_OP) {
+ irdma_terminate_done(qp, 0);
+ } else {
+ irdma_terminate_start_timer(qp);
+ irdma_terminate_send_fin(qp);
+ }
+}
+
+static enum irdma_status_code irdma_null_ws_add(struct irdma_sc_vsi *vsi, u8 user_pri)
+{
+ return 0;
+}
+
+static void irdma_null_ws_remove(struct irdma_sc_vsi *vsi, u8 user_pri)
+{
+ /* do nothing */
+}
+
+ /**
+ * irdma_sc_vsi_init - Init the vsi structure
+ * @vsi: pointer to vsi structure to initialize
+ * @info: the info used to initialize the vsi struct
+ */
+void irdma_sc_vsi_init(struct irdma_sc_vsi *vsi,
+ struct irdma_vsi_init_info *info)
+{
+ int i;
+ u32 reg_data = 0;
+ u32 reg_offset = 0;
+ struct irdma_l2params *l2p;
+
+ vsi->dev = info->dev;
+ vsi->back_vsi = info->back_vsi;
+ vsi->mtu = info->params->mtu;
+ vsi->exception_lan_q = info->exception_lan_q;
+ vsi->vsi_idx = info->pf_data_vsi_num;
+ vsi->vm_vf_type = info->vm_vf_type;
+ vsi->vm_id = info->vm_id;
+ if (vsi->dev->hw_attrs.hw_rev == IRDMA_GEN_1)
+ vsi->fcn_id = info->dev->hmc_fn_id;
+
+ l2p = info->params;
+ vsi->qos_rel_bw = l2p->vsi_rel_bw;
+ vsi->qos_prio_type = l2p->vsi_prio_type;
+ for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) {
+ if (vsi->dev->hw_attrs.hw_rev == IRDMA_GEN_1)
+ vsi->qos[i].qs_handle = l2p->qs_handle_list[i];
+ vsi->qos[i].traffic_class = info->params->up2tc[i];
+ vsi->qos[i].rel_bw =
+ l2p->tc_info[vsi->qos[i].traffic_class].rel_bw;
+ vsi->qos[i].prio_type =
+ l2p->tc_info[vsi->qos[i].traffic_class].prio_type;
+ spin_lock_init(&vsi->qos[i].lock);
+ INIT_LIST_HEAD(&vsi->qos[i].qplist);
+ }
+ if (vsi->dev->hw_attrs.hw_rev == IRDMA_GEN_1) {
+ vsi->dev->ws_add = irdma_null_ws_add;
+ vsi->dev->ws_remove = irdma_null_ws_remove;
+ } else {
+ vsi->dev->ws_add = irdma_ws_add;
+ vsi->dev->ws_remove = irdma_ws_remove;
+ }
+ if (info->dev->is_pf) {
+ reg_offset = info->dev->hw_regs[IRDMA_VSIQF_PE_CTL1] + 4 * (vsi->vsi_idx);
+ if (vsi->dev->hw_attrs.hw_rev != IRDMA_GEN_1) {
+ wr32(info->dev->hw,
+ info->dev->hw_regs[IRDMA_VSIQF_PE_CTL1] + 4 * (vsi->vsi_idx), 0x1);
+ wr32(info->dev->hw, reg_offset, 0x1);
+ } else {
+ reg_data = rd32(info->dev->hw, reg_offset);
+ reg_data |= 0x2;
+ wr32(info->dev->hw, reg_offset, reg_data);
+ }
+ }
+}
+
+/**
+ * irdma_get_fcn_id - Return the function id
+ * @vsi: pointer to the vsi
+ */
+static u8 irdma_get_fcn_id(struct irdma_sc_vsi *vsi)
+{
+ struct irdma_stats_inst_info stats_info = {};
+ struct irdma_sc_dev *dev = vsi->dev;
+ u8 fcn_id = IRDMA_INVALID_FCN_ID;
+ u8 start_idx, max_stats, i;
+
+ if (dev->hw_attrs.hw_rev != IRDMA_GEN_1) {
+ if (!irdma_cqp_stats_inst_cmd(vsi, IRDMA_OP_STATS_ALLOCATE, &stats_info))
+ return stats_info.stats_idx;
+ }
+
+ start_idx = 1;
+ max_stats = 16;
+ for (i = start_idx; i < max_stats; i++)
+ if (!dev->fcn_id_array[i]) {
+ fcn_id = i;
+ dev->fcn_id_array[i] = true;
+ break;
+ }
+
+ return fcn_id;
+}
+
+/**
+ * irdma_vsi_stats_init - Initialize the vsi statistics
+ * @vsi: pointer to the vsi structure
+ * @info: The info structure used for initialization
+ */
+enum irdma_status_code irdma_vsi_stats_init(struct irdma_sc_vsi *vsi,
+ struct irdma_vsi_stats_info *info)
+{
+ u8 fcn_id = info->fcn_id;
+
+ vsi->pestat = info->pestat;
+ vsi->pestat->hw = vsi->dev->hw;
+ vsi->pestat->vsi = vsi;
+ if (irdma_allocate_dma_mem(vsi->pestat->hw,
+ &vsi->pestat->gather_info.stats_buff_mem,
+ IRDMA_GATHER_STATS_BUF_SIZE * 2, 1))
+ return IRDMA_ERR_NO_MEMORY;
+
+ vsi->pestat->gather_info.gather_stats =
+ vsi->pestat->gather_info.stats_buff_mem.va;
+
+ vsi->pestat->gather_info.last_gather_stats =
+ (void *)((uintptr_t)vsi->pestat->gather_info.stats_buff_mem.va +
+ IRDMA_GATHER_STATS_BUF_SIZE);
+
+ if (vsi->dev->is_pf)
+ irdma_hw_stats_start_timer(vsi);
+
+ if (info->alloc_fcn_id)
+ fcn_id = irdma_get_fcn_id(vsi);
+ if (fcn_id == IRDMA_INVALID_FCN_ID)
+ goto stats_error;
+
+ vsi->stats_fcn_id_alloc = info->alloc_fcn_id;
+ vsi->fcn_id = fcn_id;
+ if (info->alloc_fcn_id) {
+ vsi->pestat->gather_info.use_stats_inst = true;
+ vsi->pestat->gather_info.stats_inst_index = fcn_id;
+ }
+
+ return 0;
+
+stats_error:
+ irdma_free_dma_mem(vsi->pestat->hw,
+ &vsi->pestat->gather_info.stats_buff_mem);
+
+ return IRDMA_ERR_CQP_COMPL_ERROR;
+}
+
+/**
+ * irdma_vsi_stats_free - Free the vsi stats
+ * @vsi: pointer to the vsi structure
+ */
+void irdma_vsi_stats_free(struct irdma_sc_vsi *vsi)
+{
+ struct irdma_stats_inst_info stats_info = {};
+ u8 fcn_id = vsi->fcn_id;
+ struct irdma_sc_dev *dev = vsi->dev;
+
+ if (dev->hw_attrs.hw_rev != IRDMA_GEN_1) {
+ if (vsi->stats_fcn_id_alloc) {
+ stats_info.stats_idx = vsi->fcn_id;
+ irdma_cqp_stats_inst_cmd(vsi, IRDMA_OP_STATS_FREE, &stats_info);
+ }
+ } else {
+ if (vsi->stats_fcn_id_alloc && fcn_id < vsi->dev->hw_attrs.max_stat_inst)
+ vsi->dev->fcn_id_array[fcn_id] = false;
+ }
+
+ if (!vsi->pestat)
+ return;
+ if (vsi->dev->is_pf)
+ irdma_hw_stats_stop_timer(vsi);
+ irdma_free_dma_mem(vsi->pestat->hw,
+ &vsi->pestat->gather_info.stats_buff_mem);
+}
+
+/**
+ * irdma_sc_cq_ack - acknowledge completion q
+ * @cq: cq struct
+ */
+static void irdma_sc_cq_ack(struct irdma_sc_cq *cq)
+{
+ writel(cq->cq_uk.cq_id, cq->cq_uk.cq_ack_db);
+}
+
+/**
+ * irdma_sc_cq_init - initialize completion q
+ * @cq: cq struct
+ * @info: cq initialization info
+ */
+static enum irdma_status_code
+irdma_sc_cq_init(struct irdma_sc_cq *cq, struct irdma_cq_init_info *info)
+{
+ enum irdma_status_code ret_code;
+ u32 pble_obj_cnt;
+
+ if (info->cq_uk_init_info.cq_size < info->dev->hw_attrs.min_hw_cq_size ||
+ info->cq_uk_init_info.cq_size > info->dev->hw_attrs.max_hw_cq_size)
+ return IRDMA_ERR_INVALID_SIZE;
+
+ pble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;
+ if (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt)
+ return IRDMA_ERR_INVALID_PBLE_INDEX;
+
+ cq->cq_pa = info->cq_base_pa;
+ cq->dev = info->dev;
+ cq->ceq_id = info->ceq_id;
+ info->cq_uk_init_info.cqe_alloc_db = cq->dev->cq_arm_db;
+ info->cq_uk_init_info.cq_ack_db = cq->dev->cq_ack_db;
+ ret_code = irdma_cq_uk_init(&cq->cq_uk, &info->cq_uk_init_info);
+ if (ret_code)
+ return ret_code;
+
+ cq->virtual_map = info->virtual_map;
+ cq->pbl_chunk_size = info->pbl_chunk_size;
+ cq->ceqe_mask = info->ceqe_mask;
+ cq->cq_type = (info->type) ? info->type : IRDMA_CQ_TYPE_IWARP;
+ cq->shadow_area_pa = info->shadow_area_pa;
+ cq->shadow_read_threshold = info->shadow_read_threshold;
+ cq->ceq_id_valid = info->ceq_id_valid;
+ cq->tph_en = info->tph_en;
+ cq->tph_val = info->tph_val;
+ cq->first_pm_pbl_idx = info->first_pm_pbl_idx;
+ cq->vsi = info->vsi;
+
+ return 0;
+}
+
+/**
+ * irdma_sc_cq_create - create completion q
+ * @cq: cq struct
+ * @scratch: u64 saved to be used during cqp completion
+ * @check_overflow: flag for overflow check
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code irdma_sc_cq_create(struct irdma_sc_cq *cq,
+ u64 scratch,
+ bool check_overflow,
+ bool post_sq)
+{
+ __le64 *wqe;
+ struct irdma_sc_cqp *cqp;
+ u64 hdr;
+ struct irdma_sc_ceq *ceq;
+ enum irdma_status_code ret_code = 0;
+
+ cqp = cq->dev->cqp;
+ if (cq->cq_uk.cq_id > (cqp->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].max_cnt - 1))
+ return IRDMA_ERR_INVALID_CQ_ID;
+
+ if (cq->ceq_id > (cq->dev->hmc_fpm_misc.max_ceqs - 1))
+ return IRDMA_ERR_INVALID_CEQ_ID;
+
+ wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+
+ ceq = cq->dev->ceq[cq->ceq_id];
+ if (ceq && ceq->reg_cq)
+ ret_code = irdma_sc_add_cq_ctx(ceq, cq);
+
+ if (ret_code)
+ return ret_code;
+
+ set_64bit_val(wqe, 0, cq->cq_uk.cq_size);
+ set_64bit_val(wqe, 8, RS_64_1(cq, 1));
+ set_64bit_val(wqe,
+ 16,
+ LS_64(cq->shadow_read_threshold,
+ IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD));
+ set_64bit_val(wqe, 32, (cq->virtual_map ? 0 : cq->cq_pa));
+ set_64bit_val(wqe, 40, cq->shadow_area_pa);
+ set_64bit_val(wqe,
+ 48,
+ (cq->virtual_map ? cq->first_pm_pbl_idx : 0));
+ set_64bit_val(wqe, 56,
+ LS_64(cq->tph_val, IRDMA_CQPSQ_TPHVAL) |
+ LS_64(cq->vsi->vsi_idx, IRDMA_CQPSQ_VSIIDX));
+
+ hdr = cq->cq_uk.cq_id |
+ FLD_LS_64(cq->dev,
+ (cq->ceq_id_valid ? cq->ceq_id : 0),
+ IRDMA_CQPSQ_CQ_CEQID) |
+ LS_64(IRDMA_CQP_OP_CREATE_CQ, IRDMA_CQPSQ_OPCODE) |
+ LS_64(cq->pbl_chunk_size, IRDMA_CQPSQ_CQ_LPBLSIZE) |
+ LS_64(check_overflow, IRDMA_CQPSQ_CQ_CHKOVERFLOW) |
+ LS_64(cq->virtual_map, IRDMA_CQPSQ_CQ_VIRTMAP) |
+ LS_64(cq->ceqe_mask, IRDMA_CQPSQ_CQ_ENCEQEMASK) |
+ LS_64(cq->ceq_id_valid, IRDMA_CQPSQ_CQ_CEQIDVALID) |
+ LS_64(cq->tph_en, IRDMA_CQPSQ_TPHEN) |
+ LS_64(cq->cq_uk.avoid_mem_cflct, IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT) |
+ LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ irdma_insert_wqe_hdr(wqe, hdr);
+
+ irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "CQ_CREATE WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+ if (post_sq)
+ irdma_sc_cqp_post_sq(cqp);
+
+ return 0;
+}
+
+/**
+ * irdma_sc_cq_destroy - destroy completion q
+ * @cq: cq struct
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code irdma_sc_cq_destroy(struct irdma_sc_cq *cq,
+ u64 scratch,
+ bool post_sq)
+{
+ struct irdma_sc_cqp *cqp;
+ __le64 *wqe;
+ u64 hdr;
+ struct irdma_sc_ceq *ceq;
+
+ cqp = cq->dev->cqp;
+ wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+
+ ceq = cq->dev->ceq[cq->ceq_id];
+ if (ceq->reg_cq)
+ irdma_sc_remove_cq_ctx(ceq, cq);
+
+ set_64bit_val(wqe, 0, cq->cq_uk.cq_size);
+ set_64bit_val(wqe, 8, RS_64_1(cq, 1));
+ set_64bit_val(wqe, 40, cq->shadow_area_pa);
+ set_64bit_val(wqe,
+ 48,
+ (cq->virtual_map ? cq->first_pm_pbl_idx : 0));
+
+ hdr = cq->cq_uk.cq_id |
+ FLD_LS_64(cq->dev,
+ (cq->ceq_id_valid ? cq->ceq_id : 0),
+ IRDMA_CQPSQ_CQ_CEQID) |
+ LS_64(IRDMA_CQP_OP_DESTROY_CQ, IRDMA_CQPSQ_OPCODE) |
+ LS_64(cq->pbl_chunk_size, IRDMA_CQPSQ_CQ_LPBLSIZE) |
+ LS_64(cq->virtual_map, IRDMA_CQPSQ_CQ_VIRTMAP) |
+ LS_64(cq->ceqe_mask, IRDMA_CQPSQ_CQ_ENCEQEMASK) |
+ LS_64(cq->ceq_id_valid, IRDMA_CQPSQ_CQ_CEQIDVALID) |
+ LS_64(cq->tph_en, IRDMA_CQPSQ_TPHEN) |
+ LS_64(cq->cq_uk.avoid_mem_cflct, IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT) |
+ LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ irdma_insert_wqe_hdr(wqe, hdr);
+
+ irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "CQ_DESTROY WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+ if (post_sq)
+ irdma_sc_cqp_post_sq(cqp);
+
+ return 0;
+}
+
+/**
+ * irdma_sc_cq_modify - modify a Completion Queue
+ * @cq: cq struct
+ * @info: modification info struct
+ * @scratch:
+ * @post_sq: flag to post to sq
+ */
+static enum irdma_status_code
+irdma_sc_cq_modify(struct irdma_sc_cq *cq,
+ struct irdma_modify_cq_info *info,
+ u64 scratch,
+ bool post_sq)
+{
+ struct irdma_sc_cqp *cqp;
+ __le64 *wqe;
+ u64 hdr;
+ u32 cq_size, ceq_id, first_pm_pbl_idx;
+ u8 pbl_chunk_size;
+ bool virtual_map, ceq_id_valid, check_overflow;
+ u32 pble_obj_cnt;
+
+ if (info->ceq_valid && info->ceq_id > (cq->dev->hmc_fpm_misc.max_ceqs - 1))
+ return IRDMA_ERR_INVALID_CEQ_ID;
+
+ pble_obj_cnt = cq->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;
+ if (info->cq_resize && info->virtual_map &&
+ info->first_pm_pbl_idx >= pble_obj_cnt)
+ return IRDMA_ERR_INVALID_PBLE_INDEX;
+
+ cqp = cq->dev->cqp;
+ wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+
+ cq->pbl_list = info->pbl_list;
+ cq->cq_pa = info->cq_pa;
+ cq->first_pm_pbl_idx = info->first_pm_pbl_idx;
+ cq_size = info->cq_resize ? info->cq_size : cq->cq_uk.cq_size;
+ if (info->ceq_change) {
+ ceq_id_valid = true;
+ ceq_id = info->ceq_id;
+ } else {
+ ceq_id_valid = cq->ceq_id_valid;
+ ceq_id = ceq_id_valid ? cq->ceq_id : 0;
+ }
+ virtual_map = info->cq_resize ? info->virtual_map : cq->virtual_map;
+ first_pm_pbl_idx = (info->cq_resize ?
+ (info->virtual_map ? info->first_pm_pbl_idx : 0) :
+ (cq->virtual_map ? cq->first_pm_pbl_idx : 0));
+ pbl_chunk_size = (info->cq_resize ?
+ (info->virtual_map ? info->pbl_chunk_size : 0) :
+ (cq->virtual_map ? cq->pbl_chunk_size : 0));
+ check_overflow = info->check_overflow_change ? info->check_overflow :
+ cq->check_overflow;
+
+ cq->cq_uk.cq_size = cq_size;
+ cq->ceq_id_valid = ceq_id_valid;
+ cq->ceq_id = ceq_id;
+ cq->virtual_map = virtual_map;
+ cq->first_pm_pbl_idx = first_pm_pbl_idx;
+ cq->pbl_chunk_size = pbl_chunk_size;
+ cq->check_overflow = check_overflow;
+
+ set_64bit_val(wqe, 0, cq_size);
+ set_64bit_val(wqe, 8, RS_64_1(cq, 1));
+ set_64bit_val(wqe, 16,
+ LS_64(info->shadow_read_threshold,
+ IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD));
+ set_64bit_val(wqe, 32, (cq->virtual_map ? 0 : cq->cq_pa));
+ set_64bit_val(wqe, 40, cq->shadow_area_pa);
+ set_64bit_val(wqe,
+ 48,
+ (cq->virtual_map ? first_pm_pbl_idx : 0));
+ set_64bit_val(wqe, 56,
+ LS_64(cq->tph_val, IRDMA_CQPSQ_TPHVAL) |
+ LS_64(cq->vsi->vsi_idx, IRDMA_CQPSQ_VSIIDX));
+
+ hdr = cq->cq_uk.cq_id |
+ FLD_LS_64(cq->dev,
+ (cq->ceq_id_valid ? cq->ceq_id : 0),
+ IRDMA_CQPSQ_CQ_CEQID) |
+ LS_64(IRDMA_CQP_OP_MODIFY_CQ, IRDMA_CQPSQ_OPCODE) |
+ LS_64(info->cq_resize, IRDMA_CQPSQ_CQ_CQRESIZE) |
+ LS_64(pbl_chunk_size, IRDMA_CQPSQ_CQ_LPBLSIZE) |
+ LS_64(check_overflow, IRDMA_CQPSQ_CQ_CHKOVERFLOW) |
+ LS_64(virtual_map, IRDMA_CQPSQ_CQ_VIRTMAP) |
+ LS_64(cq->ceqe_mask, IRDMA_CQPSQ_CQ_ENCEQEMASK) |
+ LS_64(ceq_id_valid, IRDMA_CQPSQ_CQ_CEQIDVALID) |
+ LS_64(cq->tph_en, IRDMA_CQPSQ_TPHEN) |
+ LS_64(cq->cq_uk.avoid_mem_cflct, IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT) |
+ LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ irdma_insert_wqe_hdr(wqe, hdr);
+
+ irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "CQ_MODIFY WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+ if (post_sq)
+ irdma_sc_cqp_post_sq(cqp);
+
+ return 0;
+}
+
+/**
+ * irdma_check_cqp_progress - check cqp processing progress
+ * @cqp_timeout: timeout info struct
+ * @dev: sc device struct
+ */
+static void irdma_check_cqp_progress(struct irdma_cqp_timeout *timeout,
+ struct irdma_sc_dev *dev)
+{
+ if (timeout->compl_cqp_cmds != dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS]) {
+ timeout->compl_cqp_cmds = dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS];
+ timeout->count = 0;
+ } else {
+ if (dev->cqp_cmd_stats[IRDMA_OP_REQ_CMDS] != timeout->compl_cqp_cmds)
+ timeout->count++;
+ }
+}
+
+/**
+ * irdma_get_cqp_reg_info - get head and tail for cqp using registers
+ * @cqp: struct for cqp hw
+ * @val: cqp tail register value
+ * @tail: wqtail register value
+ * @error: cqp processing err
+ */
+static inline void irdma_get_cqp_reg_info(struct irdma_sc_cqp *cqp,
+ u32 *val,
+ u32 *tail,
+ u32 *error)
+{
+ *val = rd32(cqp->dev->hw, cqp->dev->hw_regs[IRDMA_CQPTAIL]);
+ *tail = RS_32(*val, IRDMA_CQPTAIL_WQTAIL);
+ *error = RS_32(*val, IRDMA_CQPTAIL_CQP_OP_ERR);
+}
+
+/**
+ * irdma_cqp_poll_registers - poll cqp registers
+ * @cqp: struct for cqp hw
+ * @tail: wqtail register value
+ * @count: how many times to try for completion
+ */
+static enum irdma_status_code irdma_cqp_poll_registers(struct irdma_sc_cqp *cqp,
+ u32 tail,
+ u32 count)
+{
+ u32 i = 0;
+ u32 newtail, error, val;
+
+ while (i++ < count) {
+ irdma_get_cqp_reg_info(cqp, &val, &newtail, &error);
+ if (error) {
+ error = rd32(cqp->dev->hw, cqp->dev->hw_regs[IRDMA_CQPERRCODES]);
+ irdma_debug(cqp->dev, IRDMA_DEBUG_CQP, "CQPERRCODES error_code[x%08X]\n",
+ error);
+ return IRDMA_ERR_CQP_COMPL_ERROR;
+ }
+ if (newtail != tail) {
+ /* SUCCESS */
+ IRDMA_RING_MOVE_TAIL(cqp->sq_ring);
+ cqp->dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS]++;
+ return 0;
+ }
+ udelay(cqp->dev->hw_attrs.max_sleep_count);
+ }
+
+ return IRDMA_ERR_TIMEOUT;
+}
+
+/**
+ * irdma_sc_decode_fpm_commit - decode a 64 bit value into count and base
+ * @buf: pointer to commit buffer
+ * @buf_idx: buffer index
+ * @obj_info: object info pointer
+ * @rsrc_idx: indexs of memory resource
+ */
+static u64 irdma_sc_decode_fpm_commit(__le64 *buf,
+ u32 buf_idx,
+ struct irdma_hmc_obj_info *obj_info,
+ u32 rsrc_idx)
+{
+ u64 temp;
+
+ get_64bit_val(buf, buf_idx, &temp);
+
+ switch (rsrc_idx) {
+ case IRDMA_HMC_IW_QP:
+ obj_info[rsrc_idx].cnt = (u32)RS_64(temp, IRDMA_COMMIT_FPM_QPCNT);
+ break;
+ case IRDMA_HMC_IW_CQ:
+ obj_info[rsrc_idx].cnt = (u32)RS_64(temp, IRDMA_COMMIT_FPM_CQCNT);
+ break;
+ case IRDMA_HMC_IW_APBVT_ENTRY:
+ obj_info[rsrc_idx].cnt = 1;
+ break;
+ default:
+ obj_info[rsrc_idx].cnt = (u32)temp;
+ break;
+ }
+
+ obj_info[rsrc_idx].base = (u64)RS_64_1(temp, IRDMA_COMMIT_FPM_BASE_S) * 512;
+
+ return temp;
+}
+
+/**
+ * irdma_sc_parse_fpm_commit_buf - parse fpm commit buffer
+ * @dev: pointer to dev struct
+ * @buf: ptr to fpm commit buffer
+ * @info: ptr to irdma_hmc_obj_info struct
+ * @sd: number of SDs for HMC objects
+ *
+ * parses fpm commit info and copy base value
+ * of hmc objects in hmc_info
+ */
+static enum irdma_status_code
+irdma_sc_parse_fpm_commit_buf(struct irdma_sc_dev *dev,
+ __le64 *buf,
+ struct irdma_hmc_obj_info *info,
+ u32 *sd)
+{
+ u64 size;
+ u32 i;
+ u64 max_base = 0;
+ u32 last_hmc_obj = 0;
+
+ irdma_sc_decode_fpm_commit(buf, 0, info, IRDMA_HMC_IW_QP);
+ irdma_sc_decode_fpm_commit(buf, 8, info, IRDMA_HMC_IW_CQ);
+ /* skiping RSRVD */
+ irdma_sc_decode_fpm_commit(buf, 24, info, IRDMA_HMC_IW_HTE);
+ irdma_sc_decode_fpm_commit(buf, 32, info, IRDMA_HMC_IW_ARP);
+ irdma_sc_decode_fpm_commit(buf, 40, info, IRDMA_HMC_IW_APBVT_ENTRY);
+ irdma_sc_decode_fpm_commit(buf, 48, info, IRDMA_HMC_IW_MR);
+ irdma_sc_decode_fpm_commit(buf, 56, info, IRDMA_HMC_IW_XF);
+ irdma_sc_decode_fpm_commit(buf, 64, info, IRDMA_HMC_IW_XFFL);
+ irdma_sc_decode_fpm_commit(buf, 72, info, IRDMA_HMC_IW_Q1);
+ irdma_sc_decode_fpm_commit(buf, 80, info, IRDMA_HMC_IW_Q1FL);
+ irdma_sc_decode_fpm_commit(buf, 88, info, IRDMA_HMC_IW_TIMER);
+ irdma_sc_decode_fpm_commit(buf, 112, info, IRDMA_HMC_IW_PBLE);
+ /* skipping RSVD. */
+ if (dev->hw_attrs.hw_rev != IRDMA_GEN_1) {
+ irdma_sc_decode_fpm_commit(buf, 96, info, IRDMA_HMC_IW_FSIMC);
+ irdma_sc_decode_fpm_commit(buf, 104, info, IRDMA_HMC_IW_FSIAV);
+ irdma_sc_decode_fpm_commit(buf, 128, info, IRDMA_HMC_IW_RRF);
+ irdma_sc_decode_fpm_commit(buf, 136, info, IRDMA_HMC_IW_RRFFL);
+ irdma_sc_decode_fpm_commit(buf, 144, info, IRDMA_HMC_IW_HDR);
+ irdma_sc_decode_fpm_commit(buf, 152, info, IRDMA_HMC_IW_MD);
+ irdma_sc_decode_fpm_commit(buf, 160, info, IRDMA_HMC_IW_OOISC);
+ irdma_sc_decode_fpm_commit(buf, 168, info, IRDMA_HMC_IW_OOISCFFL);
+ }
+
+ /* searching for the last object in HMC to find the size of the HMC area. */
+ for (i = IRDMA_HMC_IW_QP; i < IRDMA_HMC_IW_MAX; i++) {
+ if (info[i].base > max_base) {
+ max_base = info[i].base;
+ last_hmc_obj = i;
+ }
+ }
+
+ size = info[last_hmc_obj].cnt * info[last_hmc_obj].size +
+ info[last_hmc_obj].base;
+
+ if (size & 0x1FFFFF)
+ *sd = (u32)((size >> 21) + 1); /* add 1 for remainder */
+ else
+ *sd = (u32)(size >> 21);
+
+ return 0;
+}
+
+/**
+ * irdma_sc_decode_fpm_query() - Decode a 64 bit value into max count and size
+ * @buf: ptr to fpm query buffer
+ * @buf_idx: index into buf
+ * @info: ptr to irdma_hmc_obj_info struct
+ * @rsrc_idx: resource index into info
+ *
+ * Decode a 64 bit value from fpm query buffer into max count and size
+ */
+static u64 irdma_sc_decode_fpm_query(__le64 *buf,
+ u32 buf_idx,
+ struct irdma_hmc_obj_info *obj_info,
+ u32 rsrc_idx)
+{
+ u64 temp;
+ u32 size;
+
+ get_64bit_val(buf, buf_idx, &temp);
+ obj_info[rsrc_idx].max_cnt = (u32)temp;
+ size = (u32)RS_64_1(temp, 32);
+ obj_info[rsrc_idx].size = LS_64_1(1, size);
+
+ return temp;
+}
+
+/**
+ * irdma_sc_parse_fpm_query_buf() - parses fpm query buffer
+ * @dev: ptr to shared code device
+ * @buf: ptr to fpm query buffer
+ * @info: ptr to irdma_hmc_obj_info struct
+ * @hmc_fpm_misc: ptr to fpm data
+ *
+ * parses fpm query buffer and copy max_cnt and
+ * size value of hmc objects in hmc_info
+ */
+static enum irdma_status_code
+irdma_sc_parse_fpm_query_buf(struct irdma_sc_dev *dev,
+ __le64 *buf,
+ struct irdma_hmc_info *hmc_info,
+ struct irdma_hmc_fpm_misc *hmc_fpm_misc)
+{
+ struct irdma_hmc_obj_info *obj_info;
+ u64 temp;
+ u32 size;
+ u16 max_pe_sds;
+
+ obj_info = hmc_info->hmc_obj;
+
+ get_64bit_val(buf, 0, &temp);
+ hmc_info->first_sd_index = (u16)RS_64(temp, IRDMA_QUERY_FPM_FIRST_PE_SD_INDEX);
+ max_pe_sds = (u16)RS_64(temp, IRDMA_QUERY_FPM_MAX_PE_SDS);
+
+ /* Reduce SD count for VFs by 1 to account
+ * for PBLE backing page rounding
+ */
+ if (hmc_info->hmc_fn_id >= dev->hw_attrs.first_hw_vf_fpm_id)
+ max_pe_sds--;
+ hmc_fpm_misc->max_sds = max_pe_sds;
+ hmc_info->sd_table.sd_cnt = max_pe_sds + hmc_info->first_sd_index;
+ get_64bit_val(buf, 8, &temp);
+ obj_info[IRDMA_HMC_IW_QP].max_cnt = (u32)RS_64(temp, IRDMA_QUERY_FPM_MAX_QPS);
+ size = (u32)RS_64_1(temp, 32);
+ obj_info[IRDMA_HMC_IW_QP].size = LS_64_1(1, size);
+
+ get_64bit_val(buf, 16, &temp);
+ obj_info[IRDMA_HMC_IW_CQ].max_cnt = (u32)RS_64(temp, IRDMA_QUERY_FPM_MAX_CQS);
+ size = (u32)RS_64_1(temp, 32);
+ obj_info[IRDMA_HMC_IW_CQ].size = LS_64_1(1, size);
+
+ irdma_sc_decode_fpm_query(buf, 32, obj_info, IRDMA_HMC_IW_HTE);
+ irdma_sc_decode_fpm_query(buf, 40, obj_info, IRDMA_HMC_IW_ARP);
+
+ obj_info[IRDMA_HMC_IW_APBVT_ENTRY].size = 8192;
+ obj_info[IRDMA_HMC_IW_APBVT_ENTRY].max_cnt = 1;
+
+ irdma_sc_decode_fpm_query(buf, 48, obj_info, IRDMA_HMC_IW_MR);
+ irdma_sc_decode_fpm_query(buf, 56, obj_info, IRDMA_HMC_IW_XF);
+
+ get_64bit_val(buf, 64, &temp);
+ obj_info[IRDMA_HMC_IW_XFFL].max_cnt = (u32)temp;
+ obj_info[IRDMA_HMC_IW_XFFL].size = 4;
+ hmc_fpm_misc->xf_block_size = RS_64(temp, IRDMA_QUERY_FPM_XFBLOCKSIZE);
+ if (!hmc_fpm_misc->xf_block_size)
+ return IRDMA_ERR_INVALID_SIZE;
+
+ irdma_sc_decode_fpm_query(buf, 72, obj_info, IRDMA_HMC_IW_Q1);
+ get_64bit_val(buf, 80, &temp);
+ obj_info[IRDMA_HMC_IW_Q1FL].max_cnt = (u32)temp;
+ obj_info[IRDMA_HMC_IW_Q1FL].size = 4;
+
+ hmc_fpm_misc->q1_block_size = RS_64(temp, IRDMA_QUERY_FPM_Q1BLOCKSIZE);
+ if (!hmc_fpm_misc->q1_block_size)
+ return IRDMA_ERR_INVALID_SIZE;
+
+ irdma_sc_decode_fpm_query(buf, 88, obj_info, IRDMA_HMC_IW_TIMER);
+
+ get_64bit_val(buf, 112, &temp);
+ obj_info[IRDMA_HMC_IW_PBLE].max_cnt = (u32)temp;
+ obj_info[IRDMA_HMC_IW_PBLE].size = 8;
+
+ get_64bit_val(buf, 120, &temp);
+ hmc_fpm_misc->max_ceqs = RS_64(temp, IRDMA_QUERY_FPM_MAX_CEQS);
+ hmc_fpm_misc->ht_multiplier = RS_64(temp, IRDMA_QUERY_FPM_HTMULTIPLIER);
+ hmc_fpm_misc->timer_bucket = RS_64(temp, IRDMA_QUERY_FPM_TIMERBUCKET);
+ if (dev->hw_attrs.hw_rev == IRDMA_GEN_1)
+ return 0;
+ irdma_sc_decode_fpm_query(buf, 96, obj_info, IRDMA_HMC_IW_FSIMC);
+ irdma_sc_decode_fpm_query(buf, 104, obj_info, IRDMA_HMC_IW_FSIAV);
+ irdma_sc_decode_fpm_query(buf, 128, obj_info, IRDMA_HMC_IW_RRF);
+
+ get_64bit_val(buf, 136, &temp);
+ obj_info[IRDMA_HMC_IW_RRFFL].max_cnt = (u32)temp;
+ obj_info[IRDMA_HMC_IW_RRFFL].size = 4;
+ hmc_fpm_misc->rrf_block_size = RS_64(temp, IRDMA_QUERY_FPM_RRFBLOCKSIZE);
+ if (!hmc_fpm_misc->rrf_block_size &&
+ obj_info[IRDMA_HMC_IW_RRFFL].max_cnt)
+ return IRDMA_ERR_INVALID_SIZE;
+
+ irdma_sc_decode_fpm_query(buf, 144, obj_info, IRDMA_HMC_IW_HDR);
+ irdma_sc_decode_fpm_query(buf, 152, obj_info, IRDMA_HMC_IW_MD);
+ irdma_sc_decode_fpm_query(buf, 160, obj_info, IRDMA_HMC_IW_OOISC);
+
+ get_64bit_val(buf, 168, &temp);
+ obj_info[IRDMA_HMC_IW_OOISCFFL].max_cnt = (u32)temp;
+ obj_info[IRDMA_HMC_IW_OOISCFFL].size = 4;
+ hmc_fpm_misc->ooiscf_block_size = RS_64(temp, IRDMA_QUERY_FPM_OOISCFBLOCKSIZE);
+ if (!hmc_fpm_misc->ooiscf_block_size &&
+ obj_info[IRDMA_HMC_IW_OOISCFFL].max_cnt)
+ return IRDMA_ERR_INVALID_SIZE;
+
+ return 0;
+}
+
+/**
+ * irdma_sc_find_reg_cq - find cq ctx index
+ * @ceq: ceq sc structure
+ * @cq: cq sc structure
+ */
+static u32 irdma_sc_find_reg_cq(struct irdma_sc_ceq *ceq,
+ struct irdma_sc_cq *cq)
+{
+ u32 i;
+
+ for (i = 0; i < ceq->reg_cq_size; i++) {
+ if (cq == ceq->reg_cq[i])
+ return i;
+ }
+
+ return IRDMA_INVALID_CQ_IDX;
+}
+
+/**
+ * irdma_sc_add_cq_ctx - add cq ctx tracking for ceq
+ * @ceq: ceq sc structure
+ * @cq: cq sc structure
+ */
+enum irdma_status_code irdma_sc_add_cq_ctx(struct irdma_sc_ceq *ceq,
+ struct irdma_sc_cq *cq)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&ceq->req_cq_lock, flags);
+
+ if (ceq->reg_cq_size == ceq->elem_cnt) {
+ spin_unlock_irqrestore(&ceq->req_cq_lock, flags);
+ return IRDMA_ERR_REG_CQ_FULL;
+ }
+
+ ceq->reg_cq[ceq->reg_cq_size++] = cq;
+
+ spin_unlock_irqrestore(&ceq->req_cq_lock, flags);
+
+ return 0;
+}
+
+/**
+ * irdma_sc_remove_cq_ctx - remove cq ctx tracking for ceq
+ * @ceq: ceq sc structure
+ * @cq: cq sc structure
+ */
+void irdma_sc_remove_cq_ctx(struct irdma_sc_ceq *ceq, struct irdma_sc_cq *cq)
+{
+ unsigned long flags;
+ u32 cq_ctx_idx;
+
+ spin_lock_irqsave(&ceq->req_cq_lock, flags);
+ cq_ctx_idx = irdma_sc_find_reg_cq(ceq, cq);
+ if (cq_ctx_idx == IRDMA_INVALID_CQ_IDX)
+ goto exit;
+
+ ceq->reg_cq_size--;
+ if (cq_ctx_idx != ceq->reg_cq_size)
+ ceq->reg_cq[cq_ctx_idx] = ceq->reg_cq[ceq->reg_cq_size];
+ ceq->reg_cq[ceq->reg_cq_size] = NULL;
+
+exit:
+ spin_unlock_irqrestore(&ceq->req_cq_lock, flags);
+}
+
+/**
+ * irdma_sc_cqp_init - Initialize buffers for a control Queue Pair
+ * @cqp: IWARP control queue pair pointer
+ * @info: IWARP control queue pair init info pointer
+ *
+ * Initializes the object and context buffers for a control Queue Pair.
+ */
+static enum irdma_status_code
+irdma_sc_cqp_init(struct irdma_sc_cqp *cqp, struct irdma_cqp_init_info *info)
+{
+ u8 hw_sq_size;
+
+ if (info->sq_size > IRDMA_CQP_SW_SQSIZE_2048 ||
+ info->sq_size < IRDMA_CQP_SW_SQSIZE_4 ||
+ ((info->sq_size & (info->sq_size - 1))))
+ return IRDMA_ERR_INVALID_SIZE;
+
+ hw_sq_size = irdma_get_encoded_wqe_size(info->sq_size, true);
+ cqp->size = sizeof(*cqp);
+ cqp->sq_size = info->sq_size;
+ cqp->hw_sq_size = hw_sq_size;
+ cqp->sq_base = info->sq;
+ cqp->host_ctx = info->host_ctx;
+ cqp->sq_pa = info->sq_pa;
+ cqp->host_ctx_pa = info->host_ctx_pa;
+ cqp->dev = info->dev;
+ cqp->struct_ver = info->struct_ver;
+ cqp->hw_maj_ver = info->hw_maj_ver;
+ cqp->hw_min_ver = info->hw_min_ver;
+ cqp->scratch_array = info->scratch_array;
+ cqp->polarity = 0;
+ cqp->en_datacenter_tcp = info->en_datacenter_tcp;
+ cqp->ena_vf_count = info->ena_vf_count;
+ cqp->hmc_profile = info->hmc_profile;
+ cqp->ceqs_per_vf = info->ceqs_per_vf;
+ cqp->disable_packed = info->disable_packed;
+ cqp->rocev2_rto_policy = info->rocev2_rto_policy;
+ cqp->protocol_used = info->protocol_used;
+ info->dev->cqp = cqp;
+
+ IRDMA_RING_INIT(cqp->sq_ring, cqp->sq_size);
+ cqp->dev->cqp_cmd_stats[IRDMA_OP_REQ_CMDS] = 0;
+ cqp->dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS] = 0;
+ /* for the cqp commands backlog. */
+ INIT_LIST_HEAD(&cqp->dev->cqp_cmd_head);
+
+ wr32(cqp->dev->hw, cqp->dev->hw_regs[IRDMA_CQPTAIL], 0);
+ wr32(cqp->dev->hw, cqp->dev->hw_regs[IRDMA_CQPDB], 0);
+ wr32(cqp->dev->hw, cqp->dev->hw_regs[IRDMA_CCQPSTATUS], 0);
+
+ irdma_debug(cqp->dev, IRDMA_DEBUG_WQE,
+ "sq_size[%04d] hw_sq_size[%04d] sq_base[%p] sq_pa[%pK] cqp[%p] polarity[x%04x]\n",
+ cqp->sq_size, cqp->hw_sq_size,
+ cqp->sq_base, (void *)cqp->sq_pa, cqp, cqp->polarity);
+
+ return 0;
+}
+
+/**
+ * irdma_sc_cqp_create - create cqp during bringup
+ * @cqp: struct for cqp hw
+ * @disable_pfpdus: if pfpdu to be disabled
+ * @maj_err: If error, major err number
+ * @min_err: If error, minor err number
+ */
+static enum irdma_status_code irdma_sc_cqp_create(struct irdma_sc_cqp *cqp,
+ u16 *maj_err,
+ u16 *min_err)
+{
+ u64 temp;
+ u32 cnt = 0, p1, p2, val = 0, err_code;
+ enum irdma_status_code ret_code;
+
+ ret_code = irdma_allocate_dma_mem(cqp->dev->hw,
+ &cqp->sdbuf,
+ IRDMA_UPDATE_SD_BUFF_SIZE * cqp->sq_size,
+ IRDMA_SD_BUF_ALIGNMENT);
+ if (ret_code)
+ return ret_code;
+
+ temp = LS_64(cqp->hw_sq_size, IRDMA_CQPHC_SQSIZE) |
+ LS_64(cqp->struct_ver, IRDMA_CQPHC_SVER) |
+ LS_64(cqp->rocev2_rto_policy, IRDMA_CQPHC_ROCEV2_RTO_POLICY) |
+ LS_64(cqp->protocol_used, IRDMA_CQPHC_PROTOCOL_USED) |
+ LS_64(cqp->disable_packed, IRDMA_CQPHC_DISABLE_PFPDUS) |
+ LS_64(cqp->ceqs_per_vf, IRDMA_CQPHC_CEQPERVF);
+ set_64bit_val(cqp->host_ctx, 0, temp);
+ set_64bit_val(cqp->host_ctx, 8, cqp->sq_pa);
+
+ temp = LS_64(cqp->ena_vf_count, IRDMA_CQPHC_ENABLED_VFS) |
+ LS_64(cqp->hmc_profile, IRDMA_CQPHC_HMC_PROFILE);
+ set_64bit_val(cqp->host_ctx, 16, temp);
+ set_64bit_val(cqp->host_ctx, 24, (uintptr_t)cqp);
+ set_64bit_val(cqp->host_ctx, 32, 0);
+ temp = LS_64(cqp->hw_maj_ver, IRDMA_CQPHC_HW_MAJVER) |
+ LS_64(cqp->hw_maj_ver, IRDMA_CQPHC_HW_MINVER);
+ set_64bit_val(cqp->host_ctx, 32, temp);
+ set_64bit_val(cqp->host_ctx, 40, 0);
+ set_64bit_val(cqp->host_ctx, 48, 0);
+ set_64bit_val(cqp->host_ctx, 56, 0);
+ irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "CQP_HOST_CTX WQE",
+ cqp->host_ctx, IRDMA_CQP_CTX_SIZE * 8);
+ p1 = RS_32_1(cqp->host_ctx_pa, 32);
+ p2 = (u32)cqp->host_ctx_pa;
+
+ wr32(cqp->dev->hw, cqp->dev->hw_regs[IRDMA_CCQPHIGH], p1);
+ wr32(cqp->dev->hw, cqp->dev->hw_regs[IRDMA_CCQPLOW], p2);
+
+ do {
+ if (cnt++ > cqp->dev->hw_attrs.max_done_count) {
+ ret_code = IRDMA_ERR_TIMEOUT;
+ goto err;
+ }
+ udelay(cqp->dev->hw_attrs.max_sleep_count);
+ val = rd32(cqp->dev->hw, cqp->dev->hw_regs[IRDMA_CCQPSTATUS]);
+ } while (!val);
+
+ if (FLD_RS_32(cqp->dev, val, IRDMA_CCQPSTATUS_CCQP_ERR)) {
+ ret_code = IRDMA_ERR_DEVICE_NOT_SUPPORTED;
+ goto err;
+ }
+
+ cqp->process_cqp_sds = irdma_update_sds_noccq;
+ if (cqp->dev->is_pf && cqp->dev->hw_attrs.hw_rev == IRDMA_GEN_2) {
+ val = rd32(cqp->dev->hw, 0x000AE124);
+ val |= 0x80000000;
+ wr32(cqp->dev->hw, 0x000AE124, val);
+ }
+ return 0;
+
+err:
+ irdma_free_dma_mem(cqp->dev->hw, &cqp->sdbuf);
+ err_code = rd32(cqp->dev->hw, cqp->dev->hw_regs[IRDMA_CQPERRCODES]);
+ *min_err = RS_32(err_code, IRDMA_CQPERRCODES_CQP_MINOR_CODE);
+ *maj_err = RS_32(err_code, IRDMA_CQPERRCODES_CQP_MAJOR_CODE);
+ return ret_code;
+}
+
+/**
+ * irdma_sc_cqp_post_sq - post of cqp's sq
+ * @cqp: struct for cqp hw
+ */
+void irdma_sc_cqp_post_sq(struct irdma_sc_cqp *cqp)
+{
+ writel(IRDMA_RING_CURRENT_HEAD(cqp->sq_ring), cqp->dev->cqp_db);
+
+ irdma_debug(cqp->dev,
+ IRDMA_DEBUG_WQE,
+ "CQP SQ head 0x%x tail 0x%x size 0x%x\n",
+ cqp->sq_ring.head,
+ cqp->sq_ring.tail,
+ cqp->sq_ring.size);
+}
+
+/**
+ * irdma_sc_cqp_get_next_send_wqe_idx - get next wqe on cqp sq
+ * and pass back index
+ * @cqp: CQP HW structure
+ * @scratch: private data for CQP WQE
+ * @wqe_idx: WQE index of CQP SQ
+ */
+static __le64 *irdma_sc_cqp_get_next_send_wqe_idx(struct irdma_sc_cqp *cqp,
+ u64 scratch,
+ u32 *wqe_idx)
+{
+ __le64 *wqe = NULL;
+ enum irdma_status_code ret_code;
+
+ if (IRDMA_RING_FULL_ERR(cqp->sq_ring)) {
+ irdma_debug(cqp->dev,
+ IRDMA_DEBUG_WQE,
+ "CQP SQ is full, head 0x%x tail 0x%x size 0x%x\n",
+ cqp->sq_ring.head,
+ cqp->sq_ring.tail,
+ cqp->sq_ring.size);
+ return NULL;
+ }
+ IRDMA_ATOMIC_RING_MOVE_HEAD(cqp->sq_ring, *wqe_idx, ret_code);
+ if (ret_code)
+ return NULL;
+
+ cqp->dev->cqp_cmd_stats[IRDMA_OP_REQ_CMDS]++;
+ if (!*wqe_idx)
+ cqp->polarity = !cqp->polarity;
+ wqe = cqp->sq_base[*wqe_idx].elem;
+ cqp->scratch_array[*wqe_idx] = scratch;
+ IRDMA_CQP_INIT_WQE(wqe);
+
+ return wqe;
+}
+
+/**
+ * irdma_sc_cqp_get_next_send_wqe - get next wqe on cqp sq
+ * @cqp: struct for cqp hw
+ * @scratch: private data for CQP WQE
+ */
+__le64 *irdma_sc_cqp_get_next_send_wqe(struct irdma_sc_cqp *cqp, u64 scratch)
+{
+ u32 wqe_idx;
+
+ return irdma_sc_cqp_get_next_send_wqe_idx(cqp, scratch, &wqe_idx);
+}
+
+/**
+ * irdma_sc_cqp_destroy - destroy cqp during close
+ * @cqp: struct for cqp hw
+ */
+static enum irdma_status_code irdma_sc_cqp_destroy(struct irdma_sc_cqp *cqp)
+{
+ u32 cnt = 0, val = 1;
+ enum irdma_status_code ret_code = 0;
+
+ wr32(cqp->dev->hw, cqp->dev->hw_regs[IRDMA_CCQPHIGH], 0);
+ wr32(cqp->dev->hw, cqp->dev->hw_regs[IRDMA_CCQPLOW], 0);
+ do {
+ if (cnt++ > cqp->dev->hw_attrs.max_done_count) {
+ ret_code = IRDMA_ERR_TIMEOUT;
+ break;
+ }
+ udelay(cqp->dev->hw_attrs.max_sleep_count);
+ val = rd32(cqp->dev->hw, cqp->dev->hw_regs[IRDMA_CCQPSTATUS]);
+ } while (FLD_RS_32(cqp->dev, val, IRDMA_CCQPSTATUS_CCQP_DONE));
+
+ irdma_free_dma_mem(cqp->dev->hw, &cqp->sdbuf);
+
+ return ret_code;
+}
+
+/**
+ * irdma_sc_ccq_arm - enable intr for control cq
+ * @ccq: ccq sc struct
+ */
+static void irdma_sc_ccq_arm(struct irdma_sc_cq *ccq)
+{
+ u64 temp_val;
+ u16 sw_cq_sel;
+ u8 arm_next_se;
+ u8 arm_seq_num;
+
+ get_64bit_val(ccq->cq_uk.shadow_area, 32, &temp_val);
+ sw_cq_sel = (u16)RS_64(temp_val, IRDMA_CQ_DBSA_SW_CQ_SELECT);
+ arm_next_se = (u8)RS_64(temp_val, IRDMA_CQ_DBSA_ARM_NEXT_SE);
+ arm_seq_num = (u8)RS_64(temp_val, IRDMA_CQ_DBSA_ARM_SEQ_NUM);
+ arm_seq_num++;
+ temp_val = LS_64(arm_seq_num, IRDMA_CQ_DBSA_ARM_SEQ_NUM) |
+ LS_64(sw_cq_sel, IRDMA_CQ_DBSA_SW_CQ_SELECT) |
+ LS_64(arm_next_se, IRDMA_CQ_DBSA_ARM_NEXT_SE) |
+ LS_64(1, IRDMA_CQ_DBSA_ARM_NEXT);
+ set_64bit_val(ccq->cq_uk.shadow_area, 32, temp_val);
+
+ wmb(); /* make sure shadow area is updated before arming */
+
+ wr32(ccq->dev->hw, ccq->dev->hw_regs[IRDMA_CQARM], ccq->cq_uk.cq_id);
+}
+
+/**
+ * irdma_sc_ccq_get_cqe_info - get ccq's cq entry
+ * @ccq: ccq sc struct
+ * @info: completion q entry to return
+ */
+static enum irdma_status_code
+irdma_sc_ccq_get_cqe_info(struct irdma_sc_cq *ccq,
+ struct irdma_ccq_cqe_info *info)
+{
+ u64 qp_ctx, temp, temp1;
+ __le64 *cqe;
+ struct irdma_sc_cqp *cqp;
+ u32 wqe_idx;
+ u32 error;
+ u8 polarity;
+ enum irdma_status_code ret_code = 0;
+
+ if (ccq->cq_uk.avoid_mem_cflct)
+ cqe = IRDMA_GET_CURRENT_EXTENDED_CQ_ELEM(&ccq->cq_uk);
+ else
+ cqe = IRDMA_GET_CURRENT_CQ_ELEM(&ccq->cq_uk);
+
+ get_64bit_val(cqe, 24, &temp);
+ polarity = (u8)RS_64(temp, IRDMA_CQ_VALID);
+ if (polarity != ccq->cq_uk.polarity)
+ return IRDMA_ERR_Q_EMPTY;
+
+ get_64bit_val(cqe, 8, &qp_ctx);
+ cqp = (struct irdma_sc_cqp *)(unsigned long)qp_ctx;
+ info->error = (bool)RS_64(temp, IRDMA_CQ_ERROR);
+ info->min_err_code = (u16)RS_64(temp, IRDMA_CQ_MINERR);
+ if (info->error) {
+ info->maj_err_code = (u16)RS_64(temp, IRDMA_CQ_MAJERR);
+ info->min_err_code = (u16)RS_64(temp, IRDMA_CQ_MINERR);
+ error = rd32(cqp->dev->hw, cqp->dev->hw_regs[IRDMA_CQPERRCODES]);
+ irdma_debug(cqp->dev, IRDMA_DEBUG_CQP,
+ "CQPERRCODES error_code[x%08X]\n",
+ error);
+ }
+ wqe_idx = (u32)RS_64(temp, IRDMA_CQ_WQEIDX);
+ info->scratch = cqp->scratch_array[wqe_idx];
+
+ get_64bit_val(cqe, 16, &temp1);
+ info->op_ret_val = (u32)RS_64(temp1, IRDMA_CCQ_OPRETVAL);
+ get_64bit_val(cqp->sq_base[wqe_idx].elem, 24, &temp1);
+ info->op_code = (u8)RS_64(temp1, IRDMA_CQPSQ_OPCODE);
+ info->cqp = cqp;
+
+ /* move the head for cq */
+ IRDMA_RING_MOVE_HEAD(ccq->cq_uk.cq_ring, ret_code);
+ if (IRDMA_RING_CURRENT_HEAD(ccq->cq_uk.cq_ring) == 0)
+ ccq->cq_uk.polarity ^= 1;
+
+ /* update cq tail in cq shadow memory also */
+ IRDMA_RING_MOVE_TAIL(ccq->cq_uk.cq_ring);
+ set_64bit_val(ccq->cq_uk.shadow_area,
+ 0,
+ IRDMA_RING_CURRENT_HEAD(ccq->cq_uk.cq_ring));
+
+ wmb(); /* make sure shadow area is updated before moving tail */
+
+ IRDMA_RING_MOVE_TAIL(cqp->sq_ring);
+ ccq->dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS]++;
+
+ return ret_code;
+}
+
+/**
+ * irdma_sc_poll_for_cqp_op_done - Waits for last write to complete in CQP SQ
+ * @cqp: struct for cqp hw
+ * @op_code: cqp opcode for completion
+ * @info: completion q entry to return
+ */
+static enum irdma_status_code
+irdma_sc_poll_for_cqp_op_done(struct irdma_sc_cqp *cqp,
+ u8 op_code,
+ struct irdma_ccq_cqe_info *compl_info)
+{
+ struct irdma_ccq_cqe_info info = {};
+ struct irdma_sc_cq *ccq;
+ enum irdma_status_code ret_code = 0;
+ u32 cnt = 0;
+
+ ccq = cqp->dev->ccq;
+ while (1) {
+ if (cnt++ > 100 * cqp->dev->hw_attrs.max_done_count)
+ return IRDMA_ERR_TIMEOUT;
+
+ if (irdma_sc_ccq_get_cqe_info(ccq, &info)) {
+ udelay(cqp->dev->hw_attrs.max_sleep_count);
+ continue;
+ }
+ if (info.error) {
+ ret_code = IRDMA_ERR_CQP_COMPL_ERROR;
+ break;
+ }
+ /* make sure op code matches*/
+ if (op_code == info.op_code)
+ break;
+ irdma_debug(cqp->dev, IRDMA_DEBUG_WQE,
+ "opcode mismatch for my op code 0x%x, returned opcode %x\n",
+ op_code, info.op_code);
+ }
+
+ if (compl_info)
+ memcpy(compl_info, &info, sizeof(*compl_info));
+
+ return ret_code;
+}
+
+/**
+ * irdma_sc_manage_hmc_pm_func_table - manage of function table
+ * @cqp: struct for cqp hw
+ * @scratch: u64 saved to be used during cqp completion
+ * @vf_index: vf index for cqp
+ * @free_pm_fcn: function number
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code
+irdma_sc_manage_hmc_pm_func_table(struct irdma_sc_cqp *cqp,
+ u64 scratch,
+ u8 vf_index,
+ bool free_pm_fcn,
+ bool post_sq)
+{
+ __le64 *wqe;
+ u64 hdr;
+
+ wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+
+ hdr = LS_64(vf_index, IRDMA_CQPSQ_MHMC_VFIDX) |
+ LS_64(IRDMA_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE, IRDMA_CQPSQ_OPCODE) |
+ LS_64(free_pm_fcn, IRDMA_CQPSQ_MHMC_FREEPMFN) |
+ LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ irdma_insert_wqe_hdr(wqe, hdr);
+
+ irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "MANAGE_HMC_PM_FUNC_TABLE WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+ if (post_sq)
+ irdma_sc_cqp_post_sq(cqp);
+
+ return 0;
+}
+
+/**
+ * irdma_sc_manage_hmc_pm_func_table_done - wait for cqp wqe completion for function table
+ * @cqp: struct for cqp hw
+ */
+static enum irdma_status_code
+irdma_sc_manage_hmc_pm_func_table_done(struct irdma_sc_cqp *cqp)
+{
+ return irdma_sc_poll_for_cqp_op_done(cqp,
+ IRDMA_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE,
+ NULL);
+}
+
+/**
+ * irdma_sc_commit_fpm_values_done - wait for cqp eqe completion for fpm commit
+ * @cqp: struct for cqp hw
+ */
+static enum irdma_status_code irdma_sc_commit_fpm_val_done(struct irdma_sc_cqp *cqp)
+{
+ return irdma_sc_poll_for_cqp_op_done(cqp,
+ IRDMA_CQP_OP_COMMIT_FPM_VAL,
+ NULL);
+}
+
+/**
+ * irdma_sc_commit_fpm_values - cqp wqe for commit fpm values
+ * @cqp: struct for cqp hw
+ * @scratch: u64 saved to be used during cqp completion
+ * @hmc_fn_id: hmc function id
+ * @commit_fpm_mem; Memory for fpm values
+ * @post_sq: flag for cqp db to ring
+ * @wait_type: poll ccq or cqp registers for cqp completion
+ */
+static enum irdma_status_code
+irdma_sc_commit_fpm_val(struct irdma_sc_cqp *cqp,
+ u64 scratch,
+ u8 hmc_fn_id,
+ struct irdma_dma_mem *commit_fpm_mem,
+ bool post_sq,
+ u8 wait_type)
+{
+ __le64 *wqe;
+ u64 hdr;
+ u32 tail, val, error;
+ enum irdma_status_code ret_code = 0;
+
+ wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+
+ set_64bit_val(wqe, 16, hmc_fn_id);
+ set_64bit_val(wqe, 32, commit_fpm_mem->pa);
+
+ hdr = LS_64(IRDMA_COMMIT_FPM_BUF_SIZE, IRDMA_CQPSQ_BUFSIZE) |
+ LS_64(IRDMA_CQP_OP_COMMIT_FPM_VAL, IRDMA_CQPSQ_OPCODE) |
+ LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+
+ irdma_insert_wqe_hdr(wqe, hdr);
+
+ irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "COMMIT_FPM_VAL WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+
+ irdma_get_cqp_reg_info(cqp, &val, &tail, &error);
+
+ if (post_sq) {
+ irdma_sc_cqp_post_sq(cqp);
+
+ if (wait_type == IRDMA_CQP_WAIT_POLL_REGS)
+ ret_code = irdma_cqp_poll_registers(cqp,
+ tail,
+ cqp->dev->hw_attrs.max_done_count);
+ else if (wait_type == IRDMA_CQP_WAIT_POLL_CQ)
+ ret_code = irdma_sc_commit_fpm_val_done(cqp);
+ }
+
+ return ret_code;
+}
+
+/**
+ * irdma_sc_query_fpm_values_done - poll for cqp wqe completion for query fpm
+ * @cqp: struct for cqp hw
+ */
+static enum irdma_status_code irdma_sc_query_fpm_val_done(struct irdma_sc_cqp *cqp)
+{
+ return irdma_sc_poll_for_cqp_op_done(cqp,
+ IRDMA_CQP_OP_QUERY_FPM_VAL,
+ NULL);
+}
+
+/**
+ * irdma_sc_query_fpm_val - cqp wqe query fpm values
+ * @cqp: struct for cqp hw
+ * @scratch: u64 saved to be used during cqp completion
+ * @hmc_fn_id: hmc function id
+ * @query_fpm_mem: memory for return fpm values
+ * @post_sq: flag for cqp db to ring
+ * @wait_type: poll ccq or cqp registers for cqp completion
+ */
+static enum irdma_status_code
+irdma_sc_query_fpm_val(struct irdma_sc_cqp *cqp,
+ u64 scratch,
+ u8 hmc_fn_id,
+ struct irdma_dma_mem *query_fpm_mem,
+ bool post_sq,
+ u8 wait_type)
+{
+ __le64 *wqe;
+ u64 hdr;
+ u32 tail, val, error;
+ enum irdma_status_code ret_code = 0;
+
+ wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+
+ set_64bit_val(wqe, 16, hmc_fn_id);
+ set_64bit_val(wqe, 32, query_fpm_mem->pa);
+
+ hdr = LS_64(IRDMA_CQP_OP_QUERY_FPM_VAL, IRDMA_CQPSQ_OPCODE) |
+ LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ irdma_insert_wqe_hdr(wqe, hdr);
+
+ irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "QUERY_FPM WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+ irdma_get_cqp_reg_info(cqp, &val, &tail, &error);
+ if (post_sq) {
+ irdma_sc_cqp_post_sq(cqp);
+ if (wait_type == IRDMA_CQP_WAIT_POLL_REGS)
+ ret_code = irdma_cqp_poll_registers(cqp,
+ tail,
+ cqp->dev->hw_attrs.max_done_count);
+ else if (wait_type == IRDMA_CQP_WAIT_POLL_CQ)
+ ret_code = irdma_sc_query_fpm_val_done(cqp);
+ }
+
+ return ret_code;
+}
+
+/**
+ * irdma_sc_ceq_init - initialize ceq
+ * @ceq: ceq sc structure
+ * @info: ceq initialization info
+ */
+static enum irdma_status_code
+irdma_sc_ceq_init(struct irdma_sc_ceq *ceq, struct irdma_ceq_init_info *info)
+{
+ u32 pble_obj_cnt;
+
+ if (info->elem_cnt < info->dev->hw_attrs.min_hw_ceq_size ||
+ info->elem_cnt > info->dev->hw_attrs.max_hw_ceq_size)
+ return IRDMA_ERR_INVALID_SIZE;
+
+ if (info->ceq_id > (info->dev->hmc_fpm_misc.max_ceqs - 1))
+ return IRDMA_ERR_INVALID_CEQ_ID;
+ pble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;
+
+ if (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt)
+ return IRDMA_ERR_INVALID_PBLE_INDEX;
+
+ ceq->size = sizeof(*ceq);
+ ceq->ceqe_base = (struct irdma_ceqe *)info->ceqe_base;
+ ceq->ceq_id = info->ceq_id;
+ ceq->dev = info->dev;
+ ceq->elem_cnt = info->elem_cnt;
+ ceq->ceq_elem_pa = info->ceqe_pa;
+ ceq->virtual_map = info->virtual_map;
+ ceq->itr_no_expire = info->itr_no_expire;
+ ceq->reg_cq = info->reg_cq;
+ ceq->reg_cq_size = 0;
+ spin_lock_init(&ceq->req_cq_lock);
+ ceq->pbl_chunk_size = (ceq->virtual_map ? info->pbl_chunk_size : 0);
+ ceq->first_pm_pbl_idx = (ceq->virtual_map ? info->first_pm_pbl_idx : 0);
+ ceq->pbl_list = (ceq->virtual_map ? info->pbl_list : NULL);
+ ceq->tph_en = info->tph_en;
+ ceq->tph_val = info->tph_val;
+ ceq->vsi = info->vsi;
+ ceq->polarity = 1;
+ IRDMA_RING_INIT(ceq->ceq_ring, ceq->elem_cnt);
+ ceq->dev->ceq[info->ceq_id] = ceq;
+
+ return 0;
+}
+
+/**
+ * irdma_sc_ceq_create - create ceq wqe
+ * @ceq: ceq sc structure
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag for cqp db to ring
+ */
+
+static enum irdma_status_code irdma_sc_ceq_create(struct irdma_sc_ceq *ceq,
+ u64 scratch,
+ bool post_sq)
+{
+ struct irdma_sc_cqp *cqp;
+ __le64 *wqe;
+ u64 hdr;
+
+ cqp = ceq->dev->cqp;
+ wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+ set_64bit_val(wqe, 16, ceq->elem_cnt);
+ set_64bit_val(wqe,
+ 32,
+ (ceq->virtual_map ? 0 : ceq->ceq_elem_pa));
+ set_64bit_val(wqe,
+ 48,
+ (ceq->virtual_map ? ceq->first_pm_pbl_idx : 0));
+ set_64bit_val(wqe, 56,
+ LS_64(ceq->tph_val, IRDMA_CQPSQ_TPHVAL) |
+ LS_64(ceq->vsi->vsi_idx, IRDMA_CQPSQ_VSIIDX));
+ hdr = ceq->ceq_id |
+ LS_64(IRDMA_CQP_OP_CREATE_CEQ, IRDMA_CQPSQ_OPCODE) |
+ LS_64(ceq->pbl_chunk_size, IRDMA_CQPSQ_CEQ_LPBLSIZE) |
+ LS_64(ceq->virtual_map, IRDMA_CQPSQ_CEQ_VMAP) |
+ LS_64(ceq->itr_no_expire, IRDMA_CQPSQ_CEQ_ITRNOEXPIRE) |
+ LS_64(ceq->tph_en, IRDMA_CQPSQ_TPHEN) |
+ LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ irdma_insert_wqe_hdr(wqe, hdr);
+
+ irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "CEQ_CREATE WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+ if (post_sq)
+ irdma_sc_cqp_post_sq(cqp);
+
+ return 0;
+}
+
+/**
+ * irdma_sc_cceq_create_done - poll for control ceq wqe to complete
+ * @ceq: ceq sc structure
+ */
+static enum irdma_status_code irdma_sc_cceq_create_done(struct irdma_sc_ceq *ceq)
+{
+ struct irdma_sc_cqp *cqp;
+
+ cqp = ceq->dev->cqp;
+ return irdma_sc_poll_for_cqp_op_done(cqp,
+ IRDMA_CQP_OP_CREATE_CEQ,
+ NULL);
+}
+
+/**
+ * irdma_sc_cceq_destroy_done - poll for destroy cceq to complete
+ * @ceq: ceq sc structure
+ */
+static enum irdma_status_code irdma_sc_cceq_destroy_done(struct irdma_sc_ceq *ceq)
+{
+ struct irdma_sc_cqp *cqp;
+
+ if (ceq->reg_cq)
+ irdma_sc_remove_cq_ctx(ceq, ceq->dev->ccq);
+
+ cqp = ceq->dev->cqp;
+ cqp->process_cqp_sds = irdma_update_sds_noccq;
+
+ return irdma_sc_poll_for_cqp_op_done(cqp,
+ IRDMA_CQP_OP_DESTROY_CEQ,
+ NULL);
+}
+
+/**
+ * irdma_sc_cceq_create - create cceq
+ * @ceq: ceq sc structure
+ * @scratch: u64 saved to be used during cqp completion
+ */
+static enum irdma_status_code irdma_sc_cceq_create(struct irdma_sc_ceq *ceq,
+ u64 scratch)
+{
+ enum irdma_status_code ret_code;
+
+ ceq->dev->ccq->vsi = ceq->vsi;
+ if (ceq->reg_cq) {
+ ret_code = irdma_sc_add_cq_ctx(ceq, ceq->dev->ccq);
+ if (ret_code)
+ return ret_code;
+ }
+
+ ret_code = irdma_sc_ceq_create(ceq, scratch, true);
+ if (!ret_code)
+ return irdma_sc_cceq_create_done(ceq);
+
+ return ret_code;
+}
+
+/**
+ * irdma_sc_ceq_destroy - destroy ceq
+ * @ceq: ceq sc structure
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code irdma_sc_ceq_destroy(struct irdma_sc_ceq *ceq,
+ u64 scratch,
+ bool post_sq)
+{
+ struct irdma_sc_cqp *cqp;
+ __le64 *wqe;
+ u64 hdr;
+
+ cqp = ceq->dev->cqp;
+ wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+
+ set_64bit_val(wqe, 16, ceq->elem_cnt);
+ set_64bit_val(wqe, 48, ceq->first_pm_pbl_idx);
+ hdr = ceq->ceq_id |
+ LS_64(IRDMA_CQP_OP_DESTROY_CEQ, IRDMA_CQPSQ_OPCODE) |
+ LS_64(ceq->pbl_chunk_size, IRDMA_CQPSQ_CEQ_LPBLSIZE) |
+ LS_64(ceq->virtual_map, IRDMA_CQPSQ_CEQ_VMAP) |
+ LS_64(ceq->tph_en, IRDMA_CQPSQ_TPHEN) |
+ LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ irdma_insert_wqe_hdr(wqe, hdr);
+
+ irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "CEQ_DESTROY WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+ if (post_sq)
+ irdma_sc_cqp_post_sq(cqp);
+
+ return 0;
+}
+
+/**
+ * irdma_sc_process_ceq - process ceq
+ * @dev: sc device struct
+ * @ceq: ceq sc structure
+ */
+static void *irdma_sc_process_ceq(struct irdma_sc_dev *dev,
+ struct irdma_sc_ceq *ceq)
+{
+ u64 temp;
+ __le64 *ceqe;
+ struct irdma_sc_cq *cq;
+ u8 polarity;
+ u32 cq_idx = 0;
+ unsigned long flags;
+
+ do {
+ ceqe = IRDMA_GET_CURRENT_CEQ_ELEM(ceq);
+ get_64bit_val(ceqe, 0, &temp);
+ polarity = (u8)RS_64(temp, IRDMA_CEQE_VALID);
+ if (polarity != ceq->polarity)
+ return NULL;
+
+ cq = (struct irdma_sc_cq *)(unsigned long)LS_64_1(temp, 1);
+ if (!cq)
+ return NULL;
+
+ if (ceq->reg_cq) {
+ spin_lock_irqsave(&ceq->req_cq_lock, flags);
+ cq_idx = irdma_sc_find_reg_cq(ceq, cq);
+ spin_unlock_irqrestore(&ceq->req_cq_lock, flags);
+ }
+
+ IRDMA_RING_MOVE_TAIL(ceq->ceq_ring);
+ if (IRDMA_RING_CURRENT_TAIL(ceq->ceq_ring) == 0)
+ ceq->polarity ^= 1;
+ } while (cq_idx == IRDMA_INVALID_CQ_IDX);
+
+ irdma_sc_cq_ack(cq);
+
+ return cq;
+}
+
+/**
+ * irdma_sc_aeq_init - initialize aeq
+ * @aeq: aeq structure ptr
+ * @info: aeq initialization info
+ */
+static enum irdma_status_code
+irdma_sc_aeq_init(struct irdma_sc_aeq *aeq, struct irdma_aeq_init_info *info)
+{
+ u32 pble_obj_cnt;
+
+ if (info->elem_cnt < info->dev->hw_attrs.min_hw_aeq_size ||
+ info->elem_cnt > info->dev->hw_attrs.max_hw_aeq_size)
+ return IRDMA_ERR_INVALID_SIZE;
+
+ pble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;
+
+ if (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt)
+ return IRDMA_ERR_INVALID_PBLE_INDEX;
+
+ aeq->size = sizeof(*aeq);
+ aeq->polarity = 1;
+ aeq->aeqe_base = (struct irdma_sc_aeqe *)info->aeqe_base;
+ aeq->dev = info->dev;
+ aeq->elem_cnt = info->elem_cnt;
+ aeq->aeq_elem_pa = info->aeq_elem_pa;
+ IRDMA_RING_INIT(aeq->aeq_ring, aeq->elem_cnt);
+ aeq->virtual_map = info->virtual_map;
+ aeq->pbl_list = (aeq->virtual_map ? info->pbl_list : NULL);
+ aeq->pbl_chunk_size = (aeq->virtual_map ? info->pbl_chunk_size : 0);
+ aeq->first_pm_pbl_idx = (aeq->virtual_map ? info->first_pm_pbl_idx : 0);
+ info->dev->aeq = aeq;
+
+ return 0;
+}
+
+/**
+ * irdma_sc_aeq_create - create aeq
+ * @aeq: aeq structure ptr
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code irdma_sc_aeq_create(struct irdma_sc_aeq *aeq,
+ u64 scratch,
+ bool post_sq)
+{
+ __le64 *wqe;
+ struct irdma_sc_cqp *cqp;
+ u64 hdr;
+
+ cqp = aeq->dev->cqp;
+ wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+ set_64bit_val(wqe, 16, aeq->elem_cnt);
+ set_64bit_val(wqe, 32,
+ (aeq->virtual_map ? 0 : aeq->aeq_elem_pa));
+ set_64bit_val(wqe, 48,
+ (aeq->virtual_map ? aeq->first_pm_pbl_idx : 0));
+
+ hdr = LS_64(IRDMA_CQP_OP_CREATE_AEQ, IRDMA_CQPSQ_OPCODE) |
+ LS_64(aeq->pbl_chunk_size, IRDMA_CQPSQ_AEQ_LPBLSIZE) |
+ LS_64(aeq->virtual_map, IRDMA_CQPSQ_AEQ_VMAP) |
+ LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ irdma_insert_wqe_hdr(wqe, hdr);
+
+ irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "AEQ_CREATE WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+ if (post_sq)
+ irdma_sc_cqp_post_sq(cqp);
+
+ return 0;
+}
+
+/**
+ * irdma_sc_aeq_destroy - destroy aeq during close
+ * @aeq: aeq structure ptr
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code irdma_sc_aeq_destroy(struct irdma_sc_aeq *aeq,
+ u64 scratch,
+ bool post_sq)
+{
+ __le64 *wqe;
+ struct irdma_sc_cqp *cqp;
+ struct irdma_sc_dev *dev;
+ u64 hdr;
+
+ dev = aeq->dev;
+ if (dev->is_pf)
+ wr32(dev->hw, dev->hw_regs[IRDMA_PFINT_AEQCTL], 0);
+
+ cqp = dev->cqp;
+ wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+ set_64bit_val(wqe, 16, aeq->elem_cnt);
+ set_64bit_val(wqe, 48, aeq->first_pm_pbl_idx);
+ hdr = LS_64(IRDMA_CQP_OP_DESTROY_AEQ, IRDMA_CQPSQ_OPCODE) |
+ LS_64(aeq->pbl_chunk_size, IRDMA_CQPSQ_AEQ_LPBLSIZE) |
+ LS_64(aeq->virtual_map, IRDMA_CQPSQ_AEQ_VMAP) |
+ LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ irdma_insert_wqe_hdr(wqe, hdr);
+
+ irdma_debug_buf(dev, IRDMA_DEBUG_WQE, "AEQ_DESTROY WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+ if (post_sq)
+ irdma_sc_cqp_post_sq(cqp);
+ return 0;
+}
+
+/**
+ * irdma_sc_get_next_aeqe - get next aeq entry
+ * @aeq: aeq structure ptr
+ * @info: aeqe info to be returned
+ */
+static enum irdma_status_code
+irdma_sc_get_next_aeqe(struct irdma_sc_aeq *aeq, struct irdma_aeqe_info *info)
+{
+ u64 temp, compl_ctx;
+ __le64 *aeqe;
+ u16 wqe_idx;
+ u8 ae_src;
+ u8 polarity;
+
+ aeqe = IRDMA_GET_CURRENT_AEQ_ELEM(aeq);
+ get_64bit_val(aeqe, 0, &compl_ctx);
+ get_64bit_val(aeqe, 8, &temp);
+ polarity = (u8)RS_64(temp, IRDMA_AEQE_VALID);
+
+ if (aeq->polarity != polarity)
+ return IRDMA_ERR_Q_EMPTY;
+
+ irdma_debug_buf(aeq->dev, IRDMA_DEBUG_WQE, "AEQ_ENTRY WQE", aeqe, 16);
+
+ ae_src = (u8)RS_64(temp, IRDMA_AEQE_AESRC);
+ wqe_idx = (u16)RS_64(temp, IRDMA_AEQE_WQDESCIDX);
+ info->qp_cq_id = (u32)RS_64(temp, IRDMA_AEQE_QPCQID_LOW) |
+ ((u32)RS_64(temp, IRDMA_AEQE_QPCQID_HI) << 18);
+ info->ae_id = (u16)RS_64(temp, IRDMA_AEQE_AECODE);
+ info->tcp_state = (u8)RS_64(temp, IRDMA_AEQE_TCPSTATE);
+ info->iwarp_state = (u8)RS_64(temp, IRDMA_AEQE_IWSTATE);
+ info->q2_data_written = (u8)RS_64(temp, IRDMA_AEQE_Q2DATA);
+ info->aeqe_overflow = (bool)RS_64(temp, IRDMA_AEQE_OVERFLOW);
+
+ switch (info->ae_id) {
+ case IRDMA_AE_PRIV_OPERATION_DENIED:
+ case IRDMA_AE_AMP_INVALIDATE_TYPE1_MW:
+ case IRDMA_AE_AMP_MWBIND_ZERO_BASED_TYPE1_MW:
+ case IRDMA_AE_AMP_FASTREG_INVALID_PBL_HPS_CFG:
+ case IRDMA_AE_AMP_FASTREG_PBLE_MISMATCH:
+ case IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG:
+ case IRDMA_AE_UDA_XMIT_BAD_PD:
+ case IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT:
+ case IRDMA_AE_BAD_CLOSE:
+ case IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE:
+ case IRDMA_AE_RDMA_READ_WHILE_ORD_ZERO:
+ case IRDMA_AE_STAG_ZERO_INVALID:
+ case IRDMA_AE_IB_RREQ_AND_Q1_FULL:
+ case IRDMA_AE_IB_INVALID_REQUEST:
+ case IRDMA_AE_WQE_UNEXPECTED_OPCODE:
+ case IRDMA_AE_IB_REMOTE_ACCESS_ERROR:
+ case IRDMA_AE_IB_REMOTE_OP_ERROR:
+ case IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION:
+ case IRDMA_AE_DDP_UBE_INVALID_MO:
+ case IRDMA_AE_DDP_UBE_INVALID_QN:
+ case IRDMA_AE_DDP_NO_L_BIT:
+ case IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION:
+ case IRDMA_AE_RDMAP_ROE_UNEXPECTED_OPCODE:
+ case IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST:
+ case IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP:
+ case IRDMA_AE_ROCE_RSP_LENGTH_ERROR:
+ case IRDMA_AE_INVALID_ARP_ENTRY:
+ case IRDMA_AE_INVALID_TCP_OPTION_RCVD:
+ case IRDMA_AE_STALE_ARP_ENTRY:
+ case IRDMA_AE_INVALID_AH_ENTRY:
+ case IRDMA_AE_LLP_CLOSE_COMPLETE:
+ case IRDMA_AE_LLP_CONNECTION_RESET:
+ case IRDMA_AE_LLP_FIN_RECEIVED:
+ case IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR:
+ case IRDMA_AE_LLP_SEGMENT_TOO_SMALL:
+ case IRDMA_AE_LLP_SYN_RECEIVED:
+ case IRDMA_AE_LLP_TERMINATE_RECEIVED:
+ case IRDMA_AE_LLP_TOO_MANY_RETRIES:
+ case IRDMA_AE_LLP_DOUBT_REACHABILITY:
+ case IRDMA_AE_LLP_CONNECTION_ESTABLISHED:
+ case IRDMA_AE_RESET_SENT:
+ case IRDMA_AE_TERMINATE_SENT:
+ case IRDMA_AE_RESET_NOT_SENT:
+ case IRDMA_AE_LCE_QP_CATASTROPHIC:
+ case IRDMA_AE_QP_SUSPEND_COMPLETE:
+ case IRDMA_AE_UDA_L4LEN_INVALID:
+ info->qp = true;
+ info->compl_ctx = compl_ctx;
+ ae_src = IRDMA_AE_SOURCE_RSVD;
+ break;
+ case IRDMA_AE_LCE_CQ_CATASTROPHIC:
+ info->cq = true;
+ info->compl_ctx = LS_64_1(compl_ctx, 1);
+ ae_src = IRDMA_AE_SOURCE_RSVD;
+ break;
+ case IRDMA_AE_ROCE_EMPTY_MCG:
+ case IRDMA_AE_ROCE_BAD_MC_IP_ADDR:
+ case IRDMA_AE_ROCE_BAD_MC_QPID:
+ case IRDMA_AE_MCG_QP_PROTOCOL_MISMATCH:
+ ae_src = IRDMA_AE_SOURCE_RSVD;
+ break;
+ default:
+ break;
+ }
+
+ switch (ae_src) {
+ case IRDMA_AE_SOURCE_RQ:
+ case IRDMA_AE_SOURCE_RQ_0011:
+ info->qp = true;
+ info->wqe_idx = wqe_idx;
+ info->compl_ctx = compl_ctx;
+ break;
+ case IRDMA_AE_SOURCE_CQ:
+ case IRDMA_AE_SOURCE_CQ_0110:
+ case IRDMA_AE_SOURCE_CQ_1010:
+ case IRDMA_AE_SOURCE_CQ_1110:
+ info->cq = true;
+ info->compl_ctx = LS_64_1(compl_ctx, 1);
+ break;
+ case IRDMA_AE_SOURCE_SQ:
+ case IRDMA_AE_SOURCE_SQ_0111:
+ info->qp = true;
+ info->sq = true;
+ info->wqe_idx = wqe_idx;
+ info->compl_ctx = compl_ctx;
+ break;
+ case IRDMA_AE_SOURCE_IN_RR_WR:
+ case IRDMA_AE_SOURCE_IN_RR_WR_1011:
+ info->qp = true;
+ info->compl_ctx = compl_ctx;
+ info->in_rdrsp_wr = true;
+ break;
+ case IRDMA_AE_SOURCE_OUT_RR:
+ case IRDMA_AE_SOURCE_OUT_RR_1111:
+ info->qp = true;
+ info->compl_ctx = compl_ctx;
+ info->out_rdrsp = true;
+ break;
+ case IRDMA_AE_SOURCE_RSVD:
+ /* fallthrough */
+ default:
+ break;
+ }
+
+ IRDMA_RING_MOVE_TAIL(aeq->aeq_ring);
+ if (IRDMA_RING_CURRENT_TAIL(aeq->aeq_ring) == 0)
+ aeq->polarity ^= 1;
+
+ return 0;
+}
+
+/**
+ * irdma_sc_repost_aeq_entries - repost completed aeq entries
+ * @dev: sc device struct
+ * @count: allocate count
+ */
+static enum irdma_status_code
+irdma_sc_repost_aeq_entries(struct irdma_sc_dev *dev, u32 count)
+{
+ wr32(dev->hw, dev->hw_regs[IRDMA_AEQALLOC], count);
+
+ return 0;
+}
+
+/**
+ * irdma_sc_aeq_create_done - create aeq
+ * @aeq: aeq structure ptr
+ */
+static enum irdma_status_code irdma_sc_aeq_create_done(struct irdma_sc_aeq *aeq)
+{
+ struct irdma_sc_cqp *cqp;
+
+ cqp = aeq->dev->cqp;
+
+ return irdma_sc_poll_for_cqp_op_done(cqp,
+ IRDMA_CQP_OP_CREATE_AEQ,
+ NULL);
+}
+
+/**
+ * irdma_sc_aeq_destroy_done - destroy of aeq during close
+ * @aeq: aeq structure ptr
+ */
+static enum irdma_status_code irdma_sc_aeq_destroy_done(struct irdma_sc_aeq *aeq)
+{
+ struct irdma_sc_cqp *cqp;
+
+ cqp = aeq->dev->cqp;
+
+ return irdma_sc_poll_for_cqp_op_done(cqp,
+ IRDMA_CQP_OP_DESTROY_AEQ,
+ NULL);
+}
+
+/**
+ * irdma_sc_ccq_init - initialize control cq
+ * @cq: sc's cq ctruct
+ * @info: info for control cq initialization
+ */
+static enum irdma_status_code
+irdma_sc_ccq_init(struct irdma_sc_cq *cq, struct irdma_ccq_init_info *info)
+{
+ u32 pble_obj_cnt;
+
+ if (info->num_elem < info->dev->hw_attrs.min_hw_cq_size ||
+ info->num_elem > info->dev->hw_attrs.max_hw_cq_size)
+ return IRDMA_ERR_INVALID_SIZE;
+
+ if (info->ceq_id > (info->dev->hmc_fpm_misc.max_ceqs - 1))
+ return IRDMA_ERR_INVALID_CEQ_ID;
+
+ pble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;
+
+ if (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt)
+ return IRDMA_ERR_INVALID_PBLE_INDEX;
+
+ cq->cq_pa = info->cq_pa;
+ cq->cq_uk.cq_base = info->cq_base;
+ cq->shadow_area_pa = info->shadow_area_pa;
+ cq->cq_uk.shadow_area = info->shadow_area;
+ cq->shadow_read_threshold = info->shadow_read_threshold;
+ cq->dev = info->dev;
+ cq->ceq_id = info->ceq_id;
+ cq->cq_uk.cq_size = info->num_elem;
+ cq->cq_type = IRDMA_CQ_TYPE_CQP;
+ cq->ceqe_mask = info->ceqe_mask;
+ IRDMA_RING_INIT(cq->cq_uk.cq_ring, info->num_elem);
+ cq->cq_uk.cq_id = 0; /* control cq is id 0 always */
+ cq->ceq_id_valid = info->ceq_id_valid;
+ cq->tph_en = info->tph_en;
+ cq->tph_val = info->tph_val;
+ cq->cq_uk.avoid_mem_cflct = info->avoid_mem_cflct;
+ cq->pbl_list = info->pbl_list;
+ cq->virtual_map = info->virtual_map;
+ cq->pbl_chunk_size = info->pbl_chunk_size;
+ cq->first_pm_pbl_idx = info->first_pm_pbl_idx;
+ cq->cq_uk.polarity = true;
+ cq->vsi = info->vsi;
+ cq->cq_uk.cq_ack_db = cq->dev->cq_ack_db;
+
+ /* Only applicable to CQs other than CCQ so initialize to zero */
+ cq->cq_uk.cqe_alloc_db = NULL;
+
+ info->dev->ccq = cq;
+ return 0;
+}
+
+/**
+ * irdma_sc_ccq_create_done - poll cqp for ccq create
+ * @ccq: ccq sc struct
+ */
+static enum irdma_status_code irdma_sc_ccq_create_done(struct irdma_sc_cq *ccq)
+{
+ struct irdma_sc_cqp *cqp;
+
+ cqp = ccq->dev->cqp;
+ return irdma_sc_poll_for_cqp_op_done(cqp,
+ IRDMA_CQP_OP_CREATE_CQ,
+ NULL);
+}
+
+/**
+ * irdma_sc_ccq_create - create control cq
+ * @ccq: ccq sc struct
+ * @scratch: u64 saved to be used during cqp completion
+ * @check_overflow: overlow flag for ccq
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code irdma_sc_ccq_create(struct irdma_sc_cq *ccq,
+ u64 scratch,
+ bool check_overflow,
+ bool post_sq)
+{
+ enum irdma_status_code ret_code;
+
+ ret_code = irdma_sc_cq_create(ccq, scratch, check_overflow, post_sq);
+ if (ret_code)
+ return ret_code;
+
+ if (post_sq) {
+ ret_code = irdma_sc_ccq_create_done(ccq);
+ if (ret_code)
+ return ret_code;
+ }
+ ccq->dev->cqp->process_cqp_sds = irdma_cqp_sds_cmd;
+
+ return 0;
+}
+
+/**
+ * irdma_sc_ccq_destroy - destroy ccq during close
+ * @ccq: ccq sc struct
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code irdma_sc_ccq_destroy(struct irdma_sc_cq *ccq,
+ u64 scratch,
+ bool post_sq)
+{
+ struct irdma_sc_cqp *cqp;
+ __le64 *wqe;
+ u64 hdr;
+ enum irdma_status_code ret_code = 0;
+ u32 tail, val, error;
+
+ cqp = ccq->dev->cqp;
+ wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+
+ set_64bit_val(wqe, 0, ccq->cq_uk.cq_size);
+ set_64bit_val(wqe, 8, RS_64_1(ccq, 1));
+ set_64bit_val(wqe, 40, ccq->shadow_area_pa);
+
+ hdr = ccq->cq_uk.cq_id |
+ LS_64((ccq->ceq_id_valid ? ccq->ceq_id : 0), IRDMA_CQPSQ_CQ_CEQID) |
+ LS_64(IRDMA_CQP_OP_DESTROY_CQ, IRDMA_CQPSQ_OPCODE) |
+ LS_64(ccq->ceqe_mask, IRDMA_CQPSQ_CQ_ENCEQEMASK) |
+ LS_64(ccq->ceq_id_valid, IRDMA_CQPSQ_CQ_CEQIDVALID) |
+ LS_64(ccq->tph_en, IRDMA_CQPSQ_TPHEN) |
+ LS_64(ccq->cq_uk.avoid_mem_cflct, IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT) |
+ LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ irdma_insert_wqe_hdr(wqe, hdr);
+
+ irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "CCQ_DESTROY WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+ irdma_get_cqp_reg_info(cqp, &val, &tail, &error);
+ if (error)
+ return IRDMA_ERR_CQP_COMPL_ERROR;
+
+ if (post_sq) {
+ irdma_sc_cqp_post_sq(cqp);
+ ret_code = irdma_cqp_poll_registers(cqp, tail, 1000);
+ }
+
+ cqp->process_cqp_sds = irdma_update_sds_noccq;
+
+ return ret_code;
+}
+
+/**
+ * irdma_sc_init_iw_hmc() - queries fpm values using cqp and populates hmc_info
+ * @dev : ptr to irdma_dev struct
+ * @hmc_fn_id: hmc function id
+ */
+enum irdma_status_code irdma_sc_init_iw_hmc(struct irdma_sc_dev *dev,
+ u8 hmc_fn_id)
+{
+ struct irdma_hmc_info *hmc_info;
+ struct irdma_dma_mem query_fpm_mem;
+ enum irdma_status_code ret_code = 0;
+ bool poll_registers = true;
+ u8 wait_type;
+
+ if (hmc_fn_id > dev->hw_attrs.max_hw_vf_fpm_id ||
+ (dev->hmc_fn_id != hmc_fn_id && hmc_fn_id < dev->hw_attrs.first_hw_vf_fpm_id))
+ return IRDMA_ERR_INVALID_HMCFN_ID;
+
+ irdma_debug(dev, IRDMA_DEBUG_HMC, "hmc_fn_id %u, dev->hmc_fn_id %u\n",
+ hmc_fn_id, dev->hmc_fn_id);
+ if (hmc_fn_id == dev->hmc_fn_id) {
+ hmc_info = dev->hmc_info;
+ query_fpm_mem.pa = dev->fpm_query_buf_pa;
+ query_fpm_mem.va = dev->fpm_query_buf;
+ } else {
+ irdma_debug(dev, IRDMA_DEBUG_HMC, "Bad hmc function id: hmc_fn_id %u, dev->hmc_fn_id %u\n",
+ hmc_fn_id, dev->hmc_fn_id);
+ return IRDMA_ERR_INVALID_HMCFN_ID;
+ }
+ hmc_info->hmc_fn_id = hmc_fn_id;
+ wait_type = poll_registers ? (u8)IRDMA_CQP_WAIT_POLL_REGS :
+ (u8)IRDMA_CQP_WAIT_POLL_CQ;
+
+ ret_code = irdma_sc_query_fpm_val(dev->cqp,
+ 0,
+ hmc_info->hmc_fn_id,
+ &query_fpm_mem,
+ true,
+ wait_type);
+ if (ret_code)
+ return ret_code;
+
+ /* parse the fpm_query_buf and fill hmc obj info */
+ ret_code = irdma_sc_parse_fpm_query_buf(dev,
+ query_fpm_mem.va,
+ hmc_info,
+ &dev->hmc_fpm_misc);
+
+ irdma_debug_buf(dev, IRDMA_DEBUG_HMC, "QUERY FPM BUFFER",
+ query_fpm_mem.va, IRDMA_QUERY_FPM_BUF_SIZE);
+ return ret_code;
+}
+
+/**
+ * irdma_sc_configure_iw_fpm() - commits hmc obj cnt values using cqp command and
+ * populates fpm base address in hmc_info
+ * @dev : ptr to irdma_dev struct
+ * @hmc_fn_id: hmc function id
+ */
+static enum irdma_status_code irdma_sc_cfg_iw_fpm(struct irdma_sc_dev *dev,
+ u8 hmc_fn_id)
+{
+ struct irdma_hmc_info *hmc_info;
+ struct irdma_hmc_obj_info *obj_info;
+ __le64 *buf;
+ struct irdma_dma_mem commit_fpm_mem;
+ enum irdma_status_code ret_code = 0;
+ bool poll_registers = true;
+ u8 wait_type;
+
+ if (hmc_fn_id > dev->hw_attrs.max_hw_vf_fpm_id ||
+ (dev->hmc_fn_id != hmc_fn_id && hmc_fn_id < dev->hw_attrs.first_hw_vf_fpm_id))
+ return IRDMA_ERR_INVALID_HMCFN_ID;
+
+ if (hmc_fn_id != dev->hmc_fn_id)
+ return IRDMA_ERR_INVALID_FPM_FUNC_ID;
+
+ hmc_info = dev->hmc_info;
+
+ if (!hmc_info)
+ return IRDMA_ERR_BAD_PTR;
+
+ obj_info = hmc_info->hmc_obj;
+ buf = dev->fpm_commit_buf;
+
+ set_64bit_val(buf, 0, (u64)obj_info[IRDMA_HMC_IW_QP].cnt);
+ set_64bit_val(buf, 8, (u64)obj_info[IRDMA_HMC_IW_CQ].cnt);
+ set_64bit_val(buf, 16, (u64)0); /* RSRVD */
+ set_64bit_val(buf, 24, (u64)obj_info[IRDMA_HMC_IW_HTE].cnt);
+ set_64bit_val(buf, 32, (u64)obj_info[IRDMA_HMC_IW_ARP].cnt);
+ set_64bit_val(buf, 40, (u64)0); /* RSVD */
+ set_64bit_val(buf, 48, (u64)obj_info[IRDMA_HMC_IW_MR].cnt);
+ set_64bit_val(buf, 56, (u64)obj_info[IRDMA_HMC_IW_XF].cnt);
+ set_64bit_val(buf, 64, (u64)obj_info[IRDMA_HMC_IW_XFFL].cnt);
+ set_64bit_val(buf, 72, (u64)obj_info[IRDMA_HMC_IW_Q1].cnt);
+ set_64bit_val(buf, 80, (u64)obj_info[IRDMA_HMC_IW_Q1FL].cnt);
+ set_64bit_val(buf, 88, (u64)obj_info[IRDMA_HMC_IW_TIMER].cnt);
+ set_64bit_val(buf, 96, (u64)obj_info[IRDMA_HMC_IW_FSIMC].cnt);
+ set_64bit_val(buf, 104, (u64)obj_info[IRDMA_HMC_IW_FSIAV].cnt);
+ set_64bit_val(buf, 112, (u64)obj_info[IRDMA_HMC_IW_PBLE].cnt);
+ set_64bit_val(buf, 120, (u64)0); /* RSVD */
+ set_64bit_val(buf, 128, (u64)obj_info[IRDMA_HMC_IW_RRF].cnt);
+ set_64bit_val(buf, 136, (u64)obj_info[IRDMA_HMC_IW_RRFFL].cnt);
+ set_64bit_val(buf, 144, (u64)obj_info[IRDMA_HMC_IW_HDR].cnt);
+ set_64bit_val(buf, 152, (u64)obj_info[IRDMA_HMC_IW_MD].cnt);
+ set_64bit_val(buf, 160, (u64)obj_info[IRDMA_HMC_IW_OOISC].cnt);
+ set_64bit_val(buf, 168, (u64)obj_info[IRDMA_HMC_IW_OOISCFFL].cnt);
+
+ commit_fpm_mem.pa = dev->fpm_commit_buf_pa;
+ commit_fpm_mem.va = dev->fpm_commit_buf;
+
+ wait_type = poll_registers ? (u8)IRDMA_CQP_WAIT_POLL_REGS :
+ (u8)IRDMA_CQP_WAIT_POLL_CQ;
+ irdma_debug_buf(dev, IRDMA_DEBUG_HMC, "COMMIT FPM BUFFER",
+ commit_fpm_mem.va, IRDMA_COMMIT_FPM_BUF_SIZE);
+ ret_code = irdma_sc_commit_fpm_val(dev->cqp,
+ 0,
+ hmc_info->hmc_fn_id,
+ &commit_fpm_mem,
+ true,
+ wait_type);
+ if (!ret_code)
+ ret_code = irdma_sc_parse_fpm_commit_buf(dev, dev->fpm_commit_buf,
+ hmc_info->hmc_obj,
+ &hmc_info->sd_table.sd_cnt);
+ irdma_debug_buf(dev, IRDMA_DEBUG_HMC, "COMMIT FPM BUFFER",
+ commit_fpm_mem.va, IRDMA_COMMIT_FPM_BUF_SIZE);
+
+ return ret_code;
+}
+
+/**
+ * cqp_sds_wqe_fill - fill cqp wqe doe sd
+ * @cqp: struct for cqp hw
+ * @info; sd info for wqe
+ * @scratch: u64 saved to be used during cqp completion
+ */
+static enum irdma_status_code
+cqp_sds_wqe_fill(struct irdma_sc_cqp *cqp,
+ struct irdma_update_sds_info *info,
+ u64 scratch)
+{
+ u64 data;
+ u64 hdr;
+ __le64 *wqe;
+ int mem_entries, wqe_entries;
+ struct irdma_dma_mem *sdbuf = &cqp->sdbuf;
+ u64 offset;
+ u32 wqe_idx;
+
+ wqe = irdma_sc_cqp_get_next_send_wqe_idx(cqp, scratch, &wqe_idx);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+
+ IRDMA_CQP_INIT_WQE(wqe);
+ wqe_entries = (info->cnt > 3) ? 3 : info->cnt;
+ mem_entries = info->cnt - wqe_entries;
+
+ if (mem_entries) {
+ offset = wqe_idx * IRDMA_UPDATE_SD_BUFF_SIZE;
+ memcpy(((char *)sdbuf->va + offset), &info->entry[3], mem_entries << 4);
+
+ data = (u64)sdbuf->pa + offset;
+ } else {
+ data = 0;
+ }
+ data |= LS_64(info->hmc_fn_id, IRDMA_CQPSQ_UPESD_HMCFNID);
+ set_64bit_val(wqe, 16, data);
+
+ switch (wqe_entries) {
+ case 3:
+ set_64bit_val(wqe, 48,
+ (LS_64(info->entry[2].cmd, IRDMA_CQPSQ_UPESD_SDCMD) |
+ LS_64(1, IRDMA_CQPSQ_UPESD_ENTRY_VALID)));
+
+ set_64bit_val(wqe, 56, info->entry[2].data);
+ /* fallthrough */
+ case 2:
+ set_64bit_val(wqe, 32,
+ (LS_64(info->entry[1].cmd, IRDMA_CQPSQ_UPESD_SDCMD) |
+ LS_64(1, IRDMA_CQPSQ_UPESD_ENTRY_VALID)));
+
+ set_64bit_val(wqe, 40, info->entry[1].data);
+ /* fallthrough */
+ case 1:
+ set_64bit_val(wqe,
+ 0,
+ LS_64(info->entry[0].cmd, IRDMA_CQPSQ_UPESD_SDCMD));
+
+ set_64bit_val(wqe, 8, info->entry[0].data);
+ break;
+ default:
+ break;
+ }
+
+ hdr = LS_64(IRDMA_CQP_OP_UPDATE_PE_SDS, IRDMA_CQPSQ_OPCODE) |
+ LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID) |
+ LS_64(mem_entries, IRDMA_CQPSQ_UPESD_ENTRY_COUNT);
+ irdma_insert_wqe_hdr(wqe, hdr);
+
+ irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "UPDATE_PE_SDS WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+
+ return 0;
+}
+
+/**
+ * irdma_update_pe_sds - cqp wqe for sd
+ * @dev: ptr to irdma_dev struct
+ * @info: sd info for sd's
+ * @scratch: u64 saved to be used during cqp completion
+ */
+static enum irdma_status_code
+irdma_update_pe_sds(struct irdma_sc_dev *dev,
+ struct irdma_update_sds_info *info,
+ u64 scratch)
+{
+ struct irdma_sc_cqp *cqp = dev->cqp;
+ enum irdma_status_code ret_code;
+
+ ret_code = cqp_sds_wqe_fill(cqp, info, scratch);
+ if (!ret_code)
+ irdma_sc_cqp_post_sq(cqp);
+
+ return ret_code;
+}
+
+/**
+ * irdma_update_sds_noccq - update sd before ccq created
+ * @dev: sc device struct
+ * @info: sd info for sd's
+ */
+enum irdma_status_code
+irdma_update_sds_noccq(struct irdma_sc_dev *dev,
+ struct irdma_update_sds_info *info)
+{
+ u32 error, val, tail;
+ struct irdma_sc_cqp *cqp = dev->cqp;
+ enum irdma_status_code ret_code;
+
+ ret_code = cqp_sds_wqe_fill(cqp, info, 0);
+ if (ret_code)
+ return ret_code;
+
+ irdma_get_cqp_reg_info(cqp, &val, &tail, &error);
+ if (error)
+ return IRDMA_ERR_CQP_COMPL_ERROR;
+ irdma_sc_cqp_post_sq(cqp);
+ return irdma_cqp_poll_registers(cqp, tail, cqp->dev->hw_attrs.max_done_count);
+}
+
+/**
+ * irdma_sc_static_hmc_pages_allocated - cqp wqe to allocate hmc pages
+ * @cqp: struct for cqp hw
+ * @scratch: u64 saved to be used during cqp completion
+ * @hmc_fn_id: hmc function id
+ * @post_sq: flag for cqp db to ring
+ * @poll_registers: flag to poll register for cqp completion
+ */
+enum irdma_status_code
+irdma_sc_static_hmc_pages_allocated(struct irdma_sc_cqp *cqp,
+ u64 scratch,
+ u8 hmc_fn_id,
+ bool post_sq,
+ bool poll_registers)
+{
+ u64 hdr;
+ __le64 *wqe;
+ u32 tail, val, error;
+
+ wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+
+ set_64bit_val(wqe,
+ 16,
+ LS_64(hmc_fn_id, IRDMA_SHMC_PAGE_ALLOCATED_HMC_FN_ID));
+
+ hdr = LS_64(IRDMA_CQP_OP_SHMC_PAGES_ALLOCATED, IRDMA_CQPSQ_OPCODE) |
+ LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+ irdma_insert_wqe_hdr(wqe, hdr);
+
+ irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "SHMC_PAGES_ALLOCATED WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+ irdma_get_cqp_reg_info(cqp, &val, &tail, &error);
+ if (error)
+ return IRDMA_ERR_CQP_COMPL_ERROR;
+
+ if (post_sq) {
+ irdma_sc_cqp_post_sq(cqp);
+ if (poll_registers)
+ /* check for cqp sq tail update */
+ return irdma_cqp_poll_registers(cqp, tail, 1000);
+ else
+ return irdma_sc_poll_for_cqp_op_done(cqp,
+ IRDMA_CQP_OP_SHMC_PAGES_ALLOCATED,
+ NULL);
+ }
+
+ return 0;
+}
+
+/**
+ * irdma_cqp_ring_full - check if cqp ring is full
+ * @cqp: struct for cqp hw
+ */
+static bool irdma_cqp_ring_full(struct irdma_sc_cqp *cqp)
+{
+ return IRDMA_RING_FULL_ERR(cqp->sq_ring);
+}
+
+/**
+ * irdma_est_sd - returns approximate number of SDs for HMC
+ * @dev: sc device struct
+ * @hmc_info: hmc structure, size and count for HMC objects
+ */
+static u32 irdma_est_sd(struct irdma_sc_dev *dev,
+ struct irdma_hmc_info *hmc_info)
+{
+ int i;
+ u64 size = 0;
+ u64 sd;
+
+ for (i = IRDMA_HMC_IW_QP; i < IRDMA_HMC_IW_MAX; i++)
+ if (i != IRDMA_HMC_IW_PBLE)
+ size += round_up(hmc_info->hmc_obj[i].cnt *
+ hmc_info->hmc_obj[i].size, 512);
+ if (dev->is_pf)
+ size += round_up(hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt *
+ hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].size, 512);
+ if (size & 0x1FFFFF)
+ sd = (size >> 21) + 1; /* add 1 for remainder */
+ else
+ sd = size >> 21;
+ if (!dev->is_pf) {
+ /* 2MB alignment for VF PBLE HMC */
+ size = hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt *
+ hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].size;
+ if (size & 0x1FFFFF)
+ sd += (size >> 21) + 1; /* add 1 for remainder */
+ else
+ sd += size >> 21;
+ }
+ if (sd > 0xFFFFFFFF) {
+ irdma_debug(dev, IRDMA_DEBUG_HMC, "sd overflow[%lld]\n", sd);
+ sd = 0xFFFFFFFF - 1;
+ }
+
+ return (u32)sd;
+}
+
+/**
+ * irdma_sc_query_rdma_features_done - poll cqp for query features done
+ * @cqp: struct for cqp hw
+ */
+static enum irdma_status_code
+irdma_sc_query_rdma_features_done(struct irdma_sc_cqp *cqp)
+{
+ return irdma_sc_poll_for_cqp_op_done(cqp,
+ IRDMA_CQP_OP_QUERY_RDMA_FEATURES,
+ NULL);
+}
+
+/**
+ * irdma_sc_query_rdma_features - query RDMA features and FW ver
+ * @cqp: struct for cqp hw
+ * @buffer: buffer to hold query info
+ * @scratch: u64 saved to be used during cqp completion
+ */
+static enum irdma_status_code
+irdma_sc_query_rdma_features(struct irdma_sc_cqp *cqp,
+ struct irdma_dma_mem *buf,
+ u64 scratch)
+{
+ __le64 *wqe;
+ u64 temp;
+
+ wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+
+ temp = buf->pa;
+ set_64bit_val(wqe, 32, temp);
+
+ temp = LS_64(cqp->polarity, IRDMA_CQPSQ_QUERY_RDMA_FEATURES_WQEVALID) |
+ LS_64(buf->size, IRDMA_CQPSQ_QUERY_RDMA_FEATURES_BUF_LEN) |
+ LS_64(IRDMA_CQP_OP_QUERY_RDMA_FEATURES, IRDMA_CQPSQ_UP_OP);
+ irdma_insert_wqe_hdr(wqe, temp);
+
+ irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "QUERY RDMA FEATURES",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+ irdma_sc_cqp_post_sq(cqp);
+
+ return 0;
+}
+
+/**
+ * irdma_sc_alloc_local_mac_entry - allocate a mac entry
+ * @cqp: struct for cqp hw
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code
+irdma_sc_alloc_local_mac_entry(struct irdma_sc_cqp *cqp,
+ u64 scratch,
+ bool post_sq)
+{
+ u64 *wqe;
+ u64 hdr;
+
+ wqe = cqp->dev->cqp_ops->cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+
+ hdr = LS_64(IRDMA_CQP_OP_ALLOCATE_LOC_MAC_TABLE_ENTRY,
+ IRDMA_CQPSQ_OPCODE) |
+ LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+
+ irdma_insert_wqe_hdr(wqe, hdr);
+
+ irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "ALLOCATE_LOCAL_MAC WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+
+ if (post_sq)
+ cqp->dev->cqp_ops->cqp_post_sq(cqp);
+ return 0;
+}
+
+/**
+ * irdma_sc_add_local_mac_entry - add mac enry
+ * @cqp: struct for cqp hw
+ * @info:mac addr info
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code
+irdma_sc_add_local_mac_entry(struct irdma_sc_cqp *cqp,
+ struct irdma_local_mac_entry_info *info,
+ u64 scratch,
+ bool post_sq)
+{
+ u64 *wqe;
+ u64 temp, header;
+
+ wqe = cqp->dev->cqp_ops->cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+ temp = info->mac_addr[5] |
+ LS_64_1(info->mac_addr[4], 8) |
+ LS_64_1(info->mac_addr[3], 16) |
+ LS_64_1(info->mac_addr[2], 24) |
+ LS_64_1(info->mac_addr[1], 32) |
+ LS_64_1(info->mac_addr[0], 40);
+
+ set_64bit_val(wqe, 32, temp);
+
+ header = LS_64(info->entry_idx, IRDMA_CQPSQ_MLM_TABLEIDX) |
+ LS_64(IRDMA_CQP_OP_MANAGE_LOC_MAC_TABLE, IRDMA_CQPSQ_OPCODE) |
+ LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+
+ irdma_insert_wqe_hdr(wqe, header);
+
+ irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "ADD_LOCAL_MAC WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+
+ if (post_sq)
+ cqp->dev->cqp_ops->cqp_post_sq(cqp);
+ return 0;
+}
+
+/**
+ * irdma_sc_del_local_mac_entry - cqp wqe to dele local mac
+ * @cqp: struct for cqp hw
+ * @scratch: u64 saved to be used during cqp completion
+ * @entry_idx: index of mac entry
+ * @ ignore_ref_count: to force mac adde delete
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code
+irdma_sc_del_local_mac_entry(struct irdma_sc_cqp *cqp,
+ u64 scratch,
+ u16 entry_idx,
+ u8 ignore_ref_count,
+ bool post_sq)
+{
+ u64 *wqe;
+ u64 header;
+
+ wqe = cqp->dev->cqp_ops->cqp_get_next_send_wqe(cqp, scratch);
+ if (!wqe)
+ return IRDMA_ERR_RING_FULL;
+ header = LS_64(entry_idx, IRDMA_CQPSQ_MLM_TABLEIDX) |
+ LS_64(IRDMA_CQP_OP_MANAGE_LOC_MAC_TABLE, IRDMA_CQPSQ_OPCODE) |
+ LS_64(1, IRDMA_CQPSQ_MLM_FREEENTRY) |
+ LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID) |
+ LS_64(ignore_ref_count, IRDMA_CQPSQ_MLM_IGNORE_REF_CNT);
+
+ irdma_insert_wqe_hdr(wqe, header);
+
+ irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "DEL_LOCAL_MAC_IPADDR WQE",
+ wqe, IRDMA_CQP_WQE_SIZE * 8);
+
+ if (post_sq)
+ cqp->dev->cqp_ops->cqp_post_sq(cqp);
+ return 0;
+}
+
+/**
+ * irdma_get_rdma_features - get RDMA features
+ * @dev: sc device struct
+ */
+enum irdma_status_code irdma_get_rdma_features(struct irdma_sc_dev *dev)
+{
+ enum irdma_status_code ret_code;
+ struct irdma_dma_mem feat_buf;
+ u64 temp;
+ u16 byte_idx, feat_type, feat_cnt;
+
+ ret_code = irdma_allocate_dma_mem(dev->hw,
+ &feat_buf,
+ IRDMA_FEATURE_BUF_SIZE,
+ IRDMA_FEATURE_BUF_ALIGNMENT);
+ if (ret_code)
+ return IRDMA_ERR_NO_MEMORY;
+
+ ret_code = irdma_sc_query_rdma_features(dev->cqp, &feat_buf, 0);
+ if (!ret_code)
+ ret_code = irdma_sc_query_rdma_features_done(dev->cqp);
+ if (ret_code)
+ goto exit;
+
+ get_64bit_val(feat_buf.va, 0, &temp);
+ feat_cnt = (u16)RS_64(temp, IRDMA_FEATURE_CNT);
+ if (feat_cnt < IRDMA_MAX_FEATURES) {
+ ret_code = IRDMA_ERR_INVALID_FEAT_CNT;
+ goto exit;
+ } else if (feat_cnt > IRDMA_MAX_FEATURES) {
+ irdma_debug(dev, IRDMA_DEBUG_DEV,
+ "feature buf size insufficient\n");
+ }
+
+ for (byte_idx = 0, feat_type = 0;
+ feat_type < IRDMA_MAX_FEATURES;
+ feat_type++, byte_idx += 8) {
+ get_64bit_val(feat_buf.va, byte_idx, &temp);
+ dev->feature_info[feat_type] = RS_64(temp, IRDMA_FEATURE_INFO);
+ }
+exit:
+ irdma_free_dma_mem(dev->hw, &feat_buf);
+ return ret_code;
+}
+
+static void cfg_fpm_value_gen_1(struct irdma_sc_dev *dev, struct irdma_hmc_info *hmc_info, u32 qpwanted)
+{
+ u32 powerof2 = 1;
+
+ while (powerof2 < dev->hw_attrs.max_hw_wqes)
+ powerof2 *= 2;
+ hmc_info->hmc_obj[IRDMA_HMC_IW_XF].cnt = powerof2 * qpwanted;
+
+ powerof2 = 1;
+ while (powerof2 < dev->hw_attrs.max_hw_ird)
+ powerof2 *= 2;
+ hmc_info->hmc_obj[IRDMA_HMC_IW_Q1].cnt = powerof2 * qpwanted * 2;
+}
+
+static void cfg_fpm_value_gen_2(struct irdma_sc_dev *dev, struct irdma_hmc_info *hmc_info, u32 qpwanted)
+{
+ struct irdma_hmc_fpm_misc *hmc_fpm_misc = &dev->hmc_fpm_misc;
+
+ hmc_info->hmc_obj[IRDMA_HMC_IW_XF].cnt =
+ 2 * hmc_fpm_misc->xf_block_size * qpwanted;
+ hmc_info->hmc_obj[IRDMA_HMC_IW_Q1].cnt = 2 * 16 * qpwanted;
+ hmc_info->hmc_obj[IRDMA_HMC_IW_HDR].cnt = qpwanted;
+
+ if (hmc_info->hmc_obj[IRDMA_HMC_IW_RRF].max_cnt)
+ hmc_info->hmc_obj[IRDMA_HMC_IW_RRF].cnt = 32 * qpwanted;
+ if (hmc_info->hmc_obj[IRDMA_HMC_IW_RRFFL].max_cnt)
+ hmc_info->hmc_obj[IRDMA_HMC_IW_RRFFL].cnt =
+ hmc_info->hmc_obj[IRDMA_HMC_IW_RRF].cnt /
+ hmc_fpm_misc->rrf_block_size;
+ if (hmc_info->hmc_obj[IRDMA_HMC_IW_OOISC].max_cnt)
+ hmc_info->hmc_obj[IRDMA_HMC_IW_OOISC].cnt = 32 * qpwanted;
+ if (hmc_info->hmc_obj[IRDMA_HMC_IW_OOISCFFL].max_cnt)
+ hmc_info->hmc_obj[IRDMA_HMC_IW_OOISCFFL].cnt =
+ hmc_info->hmc_obj[IRDMA_HMC_IW_OOISC].cnt /
+ hmc_fpm_misc->ooiscf_block_size;
+}
+
+/**
+ * irdma_config_fpm_values - configure HMC objects
+ * @dev: sc device struct
+ * @qp_count: desired qp count
+ */
+enum irdma_status_code irdma_cfg_fpm_val(struct irdma_sc_dev *dev, u32 qp_count)
+{
+ struct irdma_virt_mem virt_mem;
+ u32 i, mem_size;
+ u32 qpwantedoriginal, qpwanted, mrwanted, pblewanted;
+ u32 powerof2, hte;
+ u32 sd_needed;
+ u32 sd_diff;
+ u32 loop_count = 0;
+ struct irdma_hmc_info *hmc_info;
+ struct irdma_hmc_fpm_misc *hmc_fpm_misc;
+ enum irdma_status_code ret_code = 0;
+
+ hmc_info = dev->hmc_info;
+ hmc_fpm_misc = &dev->hmc_fpm_misc;
+
+ ret_code = irdma_sc_init_iw_hmc(dev, dev->hmc_fn_id);
+ if (ret_code) {
+ irdma_debug(dev, IRDMA_DEBUG_HMC,
+ "irdma_sc_init_iw_hmc returned error_code = %d\n",
+ ret_code);
+ return ret_code;
+ }
+
+ for (i = IRDMA_HMC_IW_QP; i < IRDMA_HMC_IW_MAX; i++)
+ hmc_info->hmc_obj[i].cnt = hmc_info->hmc_obj[i].max_cnt;
+ sd_needed = irdma_est_sd(dev, hmc_info);
+ irdma_debug(dev, IRDMA_DEBUG_HMC,
+ "FW max resources sd_needed[%08d] first_sd_index[%04d]\n",
+ sd_needed, hmc_info->first_sd_index);
+ irdma_debug(dev, IRDMA_DEBUG_HMC,
+ "sd count %d where max sd is %d\n",
+ hmc_info->sd_table.sd_cnt,
+ hmc_fpm_misc->max_sds);
+
+ qpwanted = min(qp_count, hmc_info->hmc_obj[IRDMA_HMC_IW_QP].max_cnt);
+
+ powerof2 = 1;
+ while (powerof2 <= qpwanted)
+ powerof2 *= 2;
+ powerof2 /= 2;
+ qpwanted = powerof2;
+
+ qpwantedoriginal = qpwanted;
+
+ mrwanted = hmc_info->hmc_obj[IRDMA_HMC_IW_MR].max_cnt;
+ pblewanted = hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].max_cnt;
+
+ irdma_debug(dev, IRDMA_DEBUG_HMC,
+ "req_qp=%d max_sd=%d, max_qp = %d, max_cq=%d, max_mr=%d, max_pble=%d, mc=%d, av=%d\n",
+ qp_count, hmc_fpm_misc->max_sds,
+ hmc_info->hmc_obj[IRDMA_HMC_IW_QP].max_cnt,
+ hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].max_cnt,
+ hmc_info->hmc_obj[IRDMA_HMC_IW_MR].max_cnt,
+ hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].max_cnt,
+ hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].max_cnt,
+ hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].max_cnt);
+ hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt =
+ hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].max_cnt;
+ hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt =
+ hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].max_cnt;
+ hmc_info->hmc_obj[IRDMA_HMC_IW_ARP].cnt =
+ hmc_info->hmc_obj[IRDMA_HMC_IW_ARP].max_cnt;
+
+ hmc_info->hmc_obj[IRDMA_HMC_IW_APBVT_ENTRY].cnt = 1;
+
+ do {
+ ++loop_count;
+ hmc_info->hmc_obj[IRDMA_HMC_IW_QP].cnt = qpwanted;
+ hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt =
+ min(2 * qpwanted, hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt);
+ hmc_info->hmc_obj[IRDMA_HMC_IW_RESERVED].cnt = 0; /* Reserved */
+ hmc_info->hmc_obj[IRDMA_HMC_IW_MR].cnt = mrwanted;
+
+ hte = round_up(qpwanted + hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt, 512);
+ powerof2 = 1;
+ while (powerof2 < hte)
+ powerof2 *= 2;
+ hmc_info->hmc_obj[IRDMA_HMC_IW_HTE].cnt =
+ powerof2 * hmc_fpm_misc->ht_multiplier;
+ if (dev->hw_attrs.hw_rev == IRDMA_GEN_1)
+ cfg_fpm_value_gen_1(dev, hmc_info, qpwanted);
+ else
+ cfg_fpm_value_gen_2(dev, hmc_info, qpwanted);
+
+ hmc_info->hmc_obj[IRDMA_HMC_IW_XFFL].cnt =
+ hmc_info->hmc_obj[IRDMA_HMC_IW_XF].cnt / hmc_fpm_misc->xf_block_size;
+ hmc_info->hmc_obj[IRDMA_HMC_IW_Q1FL].cnt =
+ hmc_info->hmc_obj[IRDMA_HMC_IW_Q1].cnt / hmc_fpm_misc->q1_block_size;
+ hmc_info->hmc_obj[IRDMA_HMC_IW_TIMER].cnt =
+ (round_up(qpwanted, 512) / 512 + 1) * hmc_fpm_misc->timer_bucket;
+
+ hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt = pblewanted;
+ sd_needed = irdma_est_sd(dev, hmc_info);
+ irdma_debug(dev, IRDMA_DEBUG_HMC,
+ "sd_needed = %d, hmc_fpm_misc->max_sds=%d, mrwanted=%d, pblewanted=%d qpwanted=%d\n",
+ sd_needed, hmc_fpm_misc->max_sds, mrwanted,
+ pblewanted, qpwanted);
+
+ /* Do not reduce resources further. All objects fit with max SDs */
+ if (sd_needed <= hmc_fpm_misc->max_sds)
+ break;
+
+ sd_diff = sd_needed - hmc_fpm_misc->max_sds;
+ if (sd_diff > 128) {
+ if (qpwanted > 128)
+ qpwanted /= 2;
+ mrwanted /= 2;
+ pblewanted /= 2;
+ continue;
+ }
+ if (dev->cqp->hmc_profile != IRDMA_HMC_PROFILE_FAVOR_VF &&
+ pblewanted > (512 * FPM_MULTIPLIER * sd_diff)) {
+ pblewanted -= 256 * FPM_MULTIPLIER * sd_diff;
+ continue;
+ } else if (pblewanted > (100 * FPM_MULTIPLIER)) {
+ pblewanted -= 10 * FPM_MULTIPLIER;
+ } else if (pblewanted > FPM_MULTIPLIER) {
+ pblewanted -= FPM_MULTIPLIER;
+ } else if (qpwanted <= 128) {
+ if (hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt > 256)
+ hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt /= 2;
+ if (hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt > 256)
+ hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt /= 2;
+ }
+ if (mrwanted > FPM_MULTIPLIER)
+ mrwanted -= FPM_MULTIPLIER;
+ if (!(loop_count % 10) && qpwanted > 128) {
+ qpwanted /= 2;
+ if (hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt > 256)
+ hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt /= 2;
+ }
+ } while (loop_count < 2000);
+
+ if (sd_needed > hmc_fpm_misc->max_sds) {
+ irdma_debug(dev, IRDMA_DEBUG_HMC,
+ "cfg_fpm failed loop_cnt=%d, sd_needed=%d, max sd count %d\n",
+ loop_count, sd_needed, hmc_info->sd_table.sd_cnt);
+ return IRDMA_ERR_CFG;
+ }
+
+ if (loop_count > 1 && sd_needed < hmc_fpm_misc->max_sds) {
+ pblewanted += (hmc_fpm_misc->max_sds - sd_needed) *
+ 256 * FPM_MULTIPLIER;
+ hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt = pblewanted;
+ sd_needed = irdma_est_sd(dev, hmc_info);
+ }
+
+ irdma_debug(dev, IRDMA_DEBUG_HMC,
+ "loop_cnt=%d, sd_needed=%d, qpcnt = %d, cqcnt=%d, mrcnt=%d, pblecnt=%d, mc=%d, ah=%d, max sd count %d, first sd index %d\n",
+ loop_count, sd_needed,
+ hmc_info->hmc_obj[IRDMA_HMC_IW_QP].cnt,
+ hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt,
+ hmc_info->hmc_obj[IRDMA_HMC_IW_MR].cnt,
+ hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt,
+ hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt,
+ hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt,
+ hmc_info->sd_table.sd_cnt,
+ hmc_info->first_sd_index);
+
+ ret_code = irdma_sc_cfg_iw_fpm(dev, dev->hmc_fn_id);
+ if (ret_code) {
+ irdma_debug(dev, IRDMA_DEBUG_HMC,
+ "cfg_iw_fpm returned error_code[x%08X]\n",
+ rd32(dev->hw, dev->hw_regs[IRDMA_CQPERRCODES]));
+ return ret_code;
+ }
+
+ mem_size = sizeof(struct irdma_hmc_sd_entry) *
+ (hmc_info->sd_table.sd_cnt + hmc_info->first_sd_index + 1);
+ ret_code = irdma_allocate_virt_mem(dev->hw, &virt_mem, mem_size);
+ if (ret_code) {
+ irdma_debug(dev, IRDMA_DEBUG_HMC,
+ "failed to allocate memory for sd_entry buffer\n");
+ return ret_code;
+ }
+ hmc_info->sd_table.sd_entry = virt_mem.va;
+
+ return ret_code;
+}
+
+/**
+ * irdma_exec_cqp_cmd - execute cqp cmd when wqe are available
+ * @dev: rdma device
+ * @pcmdinfo: cqp command info
+ */
+static enum irdma_status_code irdma_exec_cqp_cmd(struct irdma_sc_dev *dev,
+ struct cqp_cmds_info *pcmdinfo)
+{
+ enum irdma_status_code status;
+ struct irdma_dma_mem val_mem;
+ bool alloc = false;
+
+ dev->cqp_cmd_stats[pcmdinfo->cqp_cmd]++;
+ switch (pcmdinfo->cqp_cmd) {
+ case IRDMA_OP_CEQ_DESTROY:
+ status = irdma_sc_ceq_destroy(pcmdinfo->in.u.ceq_destroy.ceq,
+ pcmdinfo->in.u.ceq_destroy.scratch,
+ pcmdinfo->post_sq);
+ break;
+ case IRDMA_OP_AEQ_DESTROY:
+ status = irdma_sc_aeq_destroy(pcmdinfo->in.u.aeq_destroy.aeq,
+ pcmdinfo->in.u.aeq_destroy.scratch,
+ pcmdinfo->post_sq);
+
+ break;
+ case IRDMA_OP_DELETE_ARP_CACHE_ENTRY:
+ status = irdma_sc_del_arp_cache_entry(pcmdinfo->in.u.del_arp_cache_entry.cqp,
+ pcmdinfo->in.u.del_arp_cache_entry.scratch,
+ pcmdinfo->in.u.del_arp_cache_entry.arp_index,
+ pcmdinfo->post_sq);
+ break;
+ case IRDMA_OP_MANAGE_APBVT_ENTRY:
+ status = irdma_sc_manage_apbvt_entry(pcmdinfo->in.u.manage_apbvt_entry.cqp,
+ &pcmdinfo->in.u.manage_apbvt_entry.info,
+ pcmdinfo->in.u.manage_apbvt_entry.scratch,
+ pcmdinfo->post_sq);
+ break;
+ case IRDMA_OP_CEQ_CREATE:
+ status = irdma_sc_ceq_create(pcmdinfo->in.u.ceq_create.ceq,
+ pcmdinfo->in.u.ceq_create.scratch,
+ pcmdinfo->post_sq);
+ break;
+ case IRDMA_OP_AEQ_CREATE:
+ status = irdma_sc_aeq_create(pcmdinfo->in.u.aeq_create.aeq,
+ pcmdinfo->in.u.aeq_create.scratch,
+ pcmdinfo->post_sq);
+ break;
+ case IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY:
+ status = irdma_sc_manage_qhash_table_entry(pcmdinfo->in.u.manage_qhash_table_entry.cqp,
+ &pcmdinfo->in.u.manage_qhash_table_entry.info,
+ pcmdinfo->in.u.manage_qhash_table_entry.scratch,
+ pcmdinfo->post_sq);
+ break;
+ case IRDMA_OP_QP_MODIFY:
+ status = irdma_sc_qp_modify(pcmdinfo->in.u.qp_modify.qp,
+ &pcmdinfo->in.u.qp_modify.info,
+ pcmdinfo->in.u.qp_modify.scratch,
+ pcmdinfo->post_sq);
+ break;
+ case IRDMA_OP_QP_UPLOAD_CONTEXT:
+ status = irdma_sc_qp_upload_context(pcmdinfo->in.u.qp_upload_context.dev,
+ &pcmdinfo->in.u.qp_upload_context.info,
+ pcmdinfo->in.u.qp_upload_context.scratch,
+ pcmdinfo->post_sq);
+ break;
+ case IRDMA_OP_CQ_CREATE:
+ status = irdma_sc_cq_create(pcmdinfo->in.u.cq_create.cq,
+ pcmdinfo->in.u.cq_create.scratch,
+ pcmdinfo->in.u.cq_create.check_overflow,
+ pcmdinfo->post_sq);
+ break;
+ case IRDMA_OP_CQ_DESTROY:
+ status = irdma_sc_cq_destroy(pcmdinfo->in.u.cq_destroy.cq,
+ pcmdinfo->in.u.cq_destroy.scratch,
+ pcmdinfo->post_sq);
+ break;
+ case IRDMA_OP_QP_CREATE:
+ status = irdma_sc_qp_create(pcmdinfo->in.u.qp_create.qp,
+ &pcmdinfo->in.u.qp_create.info,
+ pcmdinfo->in.u.qp_create.scratch,
+ pcmdinfo->post_sq);
+ break;
+ case IRDMA_OP_QP_DESTROY:
+ status = irdma_sc_qp_destroy(pcmdinfo->in.u.qp_destroy.qp,
+ pcmdinfo->in.u.qp_destroy.scratch,
+ pcmdinfo->in.u.qp_destroy.remove_hash_idx,
+ pcmdinfo->in.u.qp_destroy.ignore_mw_bnd,
+ pcmdinfo->post_sq);
+ break;
+ case IRDMA_OP_ALLOC_STAG:
+ status = irdma_sc_alloc_stag(pcmdinfo->in.u.alloc_stag.dev,
+ &pcmdinfo->in.u.alloc_stag.info,
+ pcmdinfo->in.u.alloc_stag.scratch,
+ pcmdinfo->post_sq);
+ break;
+ case IRDMA_OP_MR_REG_NON_SHARED:
+ status = irdma_sc_mr_reg_non_shared(pcmdinfo->in.u.mr_reg_non_shared.dev,
+ &pcmdinfo->in.u.mr_reg_non_shared.info,
+ pcmdinfo->in.u.mr_reg_non_shared.scratch,
+ pcmdinfo->post_sq);
+ break;
+ case IRDMA_OP_DEALLOC_STAG:
+ status =
+ irdma_sc_dealloc_stag(pcmdinfo->in.u.dealloc_stag.dev,
+ &pcmdinfo->in.u.dealloc_stag.info,
+ pcmdinfo->in.u.dealloc_stag.scratch,
+ pcmdinfo->post_sq);
+ break;
+ case IRDMA_OP_MW_ALLOC:
+ status = irdma_sc_mw_alloc(pcmdinfo->in.u.mw_alloc.dev,
+ &pcmdinfo->in.u.mw_alloc.info,
+ pcmdinfo->in.u.mw_alloc.scratch,
+ pcmdinfo->post_sq);
+ break;
+ case IRDMA_OP_QP_FLUSH_WQES:
+ status = irdma_sc_qp_flush_wqes(pcmdinfo->in.u.qp_flush_wqes.qp,
+ &pcmdinfo->in.u.qp_flush_wqes.info,
+ pcmdinfo->in.u.qp_flush_wqes.scratch,
+ pcmdinfo->post_sq);
+ break;
+ case IRDMA_OP_GEN_AE:
+ status = irdma_sc_gen_ae(pcmdinfo->in.u.gen_ae.qp,
+ &pcmdinfo->in.u.gen_ae.info,
+ pcmdinfo->in.u.gen_ae.scratch,
+ pcmdinfo->post_sq);
+ break;
+ case IRDMA_OP_ADD_ARP_CACHE_ENTRY:
+ status = irdma_sc_add_arp_cache_entry(pcmdinfo->in.u.add_arp_cache_entry.cqp,
+ &pcmdinfo->in.u.add_arp_cache_entry.info,
+ pcmdinfo->in.u.add_arp_cache_entry.scratch,
+ pcmdinfo->post_sq);
+ break;
+ case IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY:
+ status = dev->cqp_misc_ops->alloc_local_mac_entry(pcmdinfo->in.u.alloc_local_mac_entry.cqp,
+ pcmdinfo->in.u.alloc_local_mac_entry.scratch,
+ pcmdinfo->post_sq);
+ break;
+ case IRDMA_OP_ADD_LOCAL_MAC_ENTRY:
+ status = dev->cqp_misc_ops->add_local_mac_entry(pcmdinfo->in.u.add_local_mac_entry.cqp,
+ &pcmdinfo->in.u.add_local_mac_entry.info,
+ pcmdinfo->in.u.add_local_mac_entry.scratch,
+ pcmdinfo->post_sq);
+ break;
+ case IRDMA_OP_DELETE_LOCAL_MAC_ENTRY:
+ status = dev->cqp_misc_ops->del_local_mac_entry(pcmdinfo->in.u.del_local_mac_entry.cqp,
+ pcmdinfo->in.u.del_local_mac_entry.scratch,
+ pcmdinfo->in.u.del_local_mac_entry.entry_idx,
+ pcmdinfo->in.u.del_local_mac_entry.ignore_ref_count,
+ pcmdinfo->post_sq);
+ break;
+ case IRDMA_OP_MANAGE_PUSH_PAGE:
+ status = irdma_sc_manage_push_page(pcmdinfo->in.u.manage_push_page.cqp,
+ &pcmdinfo->in.u.manage_push_page.info,
+ pcmdinfo->in.u.manage_push_page.scratch,
+ pcmdinfo->post_sq);
+ break;
+ case IRDMA_OP_UPDATE_PE_SDS:
+ status = irdma_update_pe_sds(pcmdinfo->in.u.update_pe_sds.dev,
+ &pcmdinfo->in.u.update_pe_sds.info,
+ pcmdinfo->in.u.update_pe_sds.scratch);
+ break;
+ case IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE:
+ status =
+ irdma_sc_manage_hmc_pm_func_table(pcmdinfo->in.u.manage_hmc_pm.dev->cqp,
+ pcmdinfo->in.u.manage_hmc_pm.scratch,
+ (u8)pcmdinfo->in.u.manage_hmc_pm.info.vf_id,
+ pcmdinfo->in.u.manage_hmc_pm.info.free_fcn,
+ true);
+ break;
+ case IRDMA_OP_SUSPEND:
+ status = irdma_sc_suspend_qp(pcmdinfo->in.u.suspend_resume.cqp,
+ pcmdinfo->in.u.suspend_resume.qp,
+ pcmdinfo->in.u.suspend_resume.scratch);
+ break;
+ case IRDMA_OP_RESUME:
+ status = irdma_sc_resume_qp(pcmdinfo->in.u.suspend_resume.cqp,
+ pcmdinfo->in.u.suspend_resume.qp,
+ pcmdinfo->in.u.suspend_resume.scratch);
+ break;
+ case IRDMA_OP_QUERY_FPM_VAL:
+ val_mem.pa = pcmdinfo->in.u.query_fpm_val.fpm_val_pa;
+ val_mem.va = pcmdinfo->in.u.query_fpm_val.fpm_val_va;
+ status = irdma_sc_query_fpm_val(pcmdinfo->in.u.query_fpm_val.cqp,
+ pcmdinfo->in.u.query_fpm_val.scratch,
+ pcmdinfo->in.u.query_fpm_val.hmc_fn_id,
+ &val_mem, true, IRDMA_CQP_WAIT_EVENT);
+ break;
+ case IRDMA_OP_COMMIT_FPM_VAL:
+ val_mem.pa = pcmdinfo->in.u.commit_fpm_val.fpm_val_pa;
+ val_mem.va = pcmdinfo->in.u.commit_fpm_val.fpm_val_va;
+ status = irdma_sc_commit_fpm_val(pcmdinfo->in.u.commit_fpm_val.cqp,
+ pcmdinfo->in.u.commit_fpm_val.scratch,
+ pcmdinfo->in.u.commit_fpm_val.hmc_fn_id,
+ &val_mem,
+ true,
+ IRDMA_CQP_WAIT_EVENT);
+ break;
+ case IRDMA_OP_AH_CREATE:
+ status = dev->iw_uda_ops->create_ah(pcmdinfo->in.u.ah_create.cqp,
+ &pcmdinfo->in.u.ah_create.info,
+ pcmdinfo->in.u.ah_create.scratch);
+ break;
+ case IRDMA_OP_AH_DESTROY:
+ status = dev->iw_uda_ops->destroy_ah(pcmdinfo->in.u.ah_destroy.cqp,
+ &pcmdinfo->in.u.ah_destroy.info,
+ pcmdinfo->in.u.ah_destroy.scratch);
+ break;
+ case IRDMA_OP_MC_CREATE:
+ status = dev->iw_uda_ops->mcast_grp_create(pcmdinfo->in.u.mc_create.cqp,
+ &pcmdinfo->in.u.mc_create.info,
+ pcmdinfo->in.u.mc_create.scratch);
+ break;
+ case IRDMA_OP_MC_DESTROY:
+ status = dev->iw_uda_ops->mcast_grp_destroy(pcmdinfo->in.u.mc_destroy.cqp,
+ &pcmdinfo->in.u.mc_destroy.info,
+ pcmdinfo->in.u.mc_destroy.scratch);
+ break;
+ case IRDMA_OP_MC_MODIFY:
+ status = dev->iw_uda_ops->mcast_grp_modify(pcmdinfo->in.u.mc_modify.cqp,
+ &pcmdinfo->in.u.mc_modify.info,
+ pcmdinfo->in.u.mc_modify.scratch);
+ break;
+ case IRDMA_OP_STATS_ALLOCATE:
+ alloc = true;
+ /* fall-through */
+ case IRDMA_OP_STATS_FREE:
+ status = irdma_sc_manage_stats_inst(pcmdinfo->in.u.stats_manage.cqp,
+ &pcmdinfo->in.u.stats_manage.info,
+ alloc,
+ pcmdinfo->in.u.stats_manage.scratch);
+ break;
+ case IRDMA_OP_STATS_GATHER:
+ status = irdma_sc_gather_stats(pcmdinfo->in.u.stats_gather.cqp,
+ &pcmdinfo->in.u.stats_gather.info,
+ pcmdinfo->in.u.stats_gather.scratch);
+ break;
+ case IRDMA_OP_WS_MODIFY_NODE:
+ status = irdma_sc_manage_ws_node(pcmdinfo->in.u.ws_node.cqp,
+ &pcmdinfo->in.u.ws_node.info,
+ IRDMA_MODIFY_NODE,
+ pcmdinfo->in.u.ws_node.scratch);
+ break;
+ case IRDMA_OP_WS_DELETE_NODE:
+ status = irdma_sc_manage_ws_node(pcmdinfo->in.u.ws_node.cqp,
+ &pcmdinfo->in.u.ws_node.info,
+ IRDMA_DEL_NODE,
+ pcmdinfo->in.u.ws_node.scratch);
+ break;
+ case IRDMA_OP_WS_ADD_NODE:
+ status = irdma_sc_manage_ws_node(pcmdinfo->in.u.ws_node.cqp,
+ &pcmdinfo->in.u.ws_node.info,
+ IRDMA_ADD_NODE,
+ pcmdinfo->in.u.ws_node.scratch);
+ break;
+ case IRDMA_OP_SET_UP_MAP:
+ status = irdma_sc_set_up_map(pcmdinfo->in.u.up_map.cqp,
+ &pcmdinfo->in.u.up_map.info,
+ pcmdinfo->in.u.up_map.scratch);
+ break;
+ case IRDMA_OP_QUERY_RDMA_FEATURES:
+ status = irdma_sc_query_rdma_features(pcmdinfo->in.u.query_rdma.cqp,
+ &pcmdinfo->in.u.query_rdma.query_buff_mem,
+ pcmdinfo->in.u.query_rdma.scratch);
+ break;
+ default:
+ status = IRDMA_NOT_SUPPORTED;
+ break;
+ }
+
+ return status;
+}
+
+/**
+ * irdma_process_cqp_cmd - process all cqp commands
+ * @dev: sc device struct
+ * @pcmdinfo: cqp command info
+ */
+enum irdma_status_code irdma_process_cqp_cmd(struct irdma_sc_dev *dev,
+ struct cqp_cmds_info *pcmdinfo)
+{
+ enum irdma_status_code status = 0;
+ unsigned long flags;
+
+ spin_lock_irqsave(&dev->cqp_lock, flags);
+ if (list_empty(&dev->cqp_cmd_head) && !irdma_cqp_ring_full(dev->cqp))
+ status = irdma_exec_cqp_cmd(dev, pcmdinfo);
+ else
+ list_add_tail(&pcmdinfo->cqp_cmd_entry, &dev->cqp_cmd_head);
+ spin_unlock_irqrestore(&dev->cqp_lock, flags);
+ return status;
+}
+
+/**
+ * irdma_process_bh - called from tasklet for cqp list
+ * @dev: sc device struct
+ */
+enum irdma_status_code irdma_process_bh(struct irdma_sc_dev *dev)
+{
+ enum irdma_status_code status = 0;
+ struct cqp_cmds_info *pcmdinfo;
+ unsigned long flags;
+
+ spin_lock_irqsave(&dev->cqp_lock, flags);
+ while (!list_empty(&dev->cqp_cmd_head) &&
+ !irdma_cqp_ring_full(dev->cqp)) {
+ pcmdinfo = (struct cqp_cmds_info *)
+ irdma_remove_head(&dev->cqp_cmd_head);
+
+ status = irdma_exec_cqp_cmd(dev, pcmdinfo);
+ if (status)
+ break;
+ }
+ spin_unlock_irqrestore(&dev->cqp_lock, flags);
+ return status;
+}
+
+/**
+ * irdma_enable_irq - Enable interrupt
+ * @dev: pointer to the device structure
+ * @idx: vector index
+ */
+static void irdma_ena_irq(struct irdma_sc_dev *dev, u32 idx)
+{
+ u32 val;
+
+ val = IRDMA_GLINT_DYN_CTL_INTENA_M |
+ IRDMA_GLINT_DYN_CTL_CLEARPBA_M |
+ IRDMA_GLINT_DYN_CTL_ITR_INDX_M;
+ if (dev->hw_attrs.hw_rev != IRDMA_GEN_1)
+ wr32(dev->hw, dev->hw_regs[IRDMA_GLINT_DYN_CTL] + 4 * idx, val);
+ else
+ wr32(dev->hw, dev->hw_regs[IRDMA_GLINT_DYN_CTL] + 4 * (idx - 1), val);
+}
+
+/**
+ * irdma_disable_irq - Disable interrupt
+ * @dev: pointer to the device structure
+ * @idx: vector index
+ */
+static void irdma_disable_irq(struct irdma_sc_dev *dev, u32 idx)
+{
+ if (dev->hw_attrs.hw_rev != IRDMA_GEN_1)
+ wr32(dev->hw, dev->hw_regs[IRDMA_GLINT_DYN_CTL] + 4 * idx, 0);
+ else
+ wr32(dev->hw, dev->hw_regs[IRDMA_GLINT_DYN_CTL] + 4 * (idx - 1), 0);
+}
+
+/**
+ * irdma_config_ceq- Configure CEQ interrupt
+ * @dev: pointer to the device structure
+ * @ceq_id: Completion Event Queue ID
+ * @idx: vector index
+ */
+static void irdma_cfg_ceq(struct irdma_sc_dev *dev, u32 ceq_id, u32 idx)
+{
+ u32 reg_val;
+
+ reg_val = (IRDMA_GLINT_CEQCTL_CAUSE_ENA_M |
+ (idx << IRDMA_GLINT_CEQCTL_MSIX_INDX_S) |
+ IRDMA_GLINT_CEQCTL_ITR_INDX_M);
+
+ wr32(dev->hw, dev->hw_regs[IRDMA_GLINT_CEQCTL] + 4 * ceq_id, reg_val);
+}
+
+/**
+ * irdma_config_aeq- Configure AEQ interrupt
+ * @dev: pointer to the device structure
+ * @idx: vector index
+ */
+static void irdma_cfg_aeq(struct irdma_sc_dev *dev, u32 idx)
+{
+ u32 reg_val;
+
+ reg_val = (IRDMA_PFINT_AEQCTL_CAUSE_ENA_M |
+ (idx << IRDMA_PFINT_AEQCTL_MSIX_INDX_S) |
+ IRDMA_PFINT_AEQCTL_ITR_INDX_M);
+
+ wr32(dev->hw, dev->hw_regs[IRDMA_PFINT_AEQCTL], reg_val);
+}
+
+/* iwarp pd ops */
+static struct irdma_pd_ops iw_pd_ops = {
+ .pd_init = irdma_sc_pd_init
+};
+
+static struct irdma_priv_qp_ops iw_priv_qp_ops = {
+ .qp_init = irdma_sc_qp_init,
+ .qp_create = irdma_sc_qp_create,
+ .qp_modify = irdma_sc_qp_modify,
+ .qp_destroy = irdma_sc_qp_destroy,
+ .qp_flush_wqes = irdma_sc_qp_flush_wqes,
+ .qp_upload_context = irdma_sc_qp_upload_context,
+ .qp_setctx = irdma_sc_qp_setctx,
+ .qp_setctx_roce = irdma_sc_qp_setctx_roce,
+ .qp_send_lsmm = irdma_sc_send_lsmm,
+ .qp_send_lsmm_nostag = irdma_sc_send_lsmm_nostag,
+ .qp_send_rtt = irdma_sc_send_rtt,
+ .iw_mr_fast_register = irdma_sc_mr_fast_register,
+ .update_suspend_qp = irdma_sc_suspend_qp,
+ .update_resume_qp = irdma_sc_resume_qp,
+};
+
+static struct irdma_mr_ops iw_mr_ops = {
+ .alloc_stag = irdma_sc_alloc_stag,
+ .mr_reg_non_shared = irdma_sc_mr_reg_non_shared,
+ .mr_reg_shared = irdma_sc_mr_reg_shared,
+ .dealloc_stag = irdma_sc_dealloc_stag,
+ .query_stag = irdma_sc_query_stag,
+ .mw_alloc = irdma_sc_mw_alloc
+};
+
+static struct irdma_cqp_misc_ops iw_cqp_misc_ops = {
+ .manage_push_page = irdma_sc_manage_push_page,
+ .add_arp_cache_entry = irdma_sc_add_arp_cache_entry,
+ .del_arp_cache_entry = irdma_sc_del_arp_cache_entry,
+ .query_arp_cache_entry = irdma_sc_query_arp_cache_entry,
+ .manage_apbvt_entry = irdma_sc_manage_apbvt_entry,
+ .manage_qhash_table_entry = irdma_sc_manage_qhash_table_entry,
+ .cqp_nop = irdma_sc_cqp_nop,
+ .manage_stats_instance = irdma_sc_manage_stats_inst,
+ .gather_stats = irdma_sc_gather_stats,
+ .manage_ws_node = irdma_sc_manage_ws_node,
+ .set_up_map = irdma_sc_set_up_map,
+ .query_rdma_features = irdma_sc_query_rdma_features,
+ .alloc_local_mac_entry = irdma_sc_alloc_local_mac_entry,
+ .add_local_mac_entry = irdma_sc_add_local_mac_entry,
+ .del_local_mac_entry = irdma_sc_del_local_mac_entry,
+};
+
+static struct irdma_irq_ops iw_irq_ops = {
+ .irdma_en_irq = irdma_ena_irq,
+ .irdma_dis_irq = irdma_disable_irq,
+ .irdma_cfg_aeq = irdma_cfg_aeq,
+ .irdma_cfg_ceq = irdma_cfg_ceq
+};
+
+static struct irdma_cqp_ops iw_cqp_ops = {
+ .cqp_init = irdma_sc_cqp_init,
+ .cqp_create = irdma_sc_cqp_create,
+ .cqp_post_sq = irdma_sc_cqp_post_sq,
+ .cqp_get_next_send_wqe = irdma_sc_cqp_get_next_send_wqe,
+ .cqp_destroy = irdma_sc_cqp_destroy,
+ .poll_for_cqp_op_done = irdma_sc_poll_for_cqp_op_done,
+ .check_cqp_progress = irdma_check_cqp_progress,
+};
+
+static struct irdma_priv_cq_ops iw_priv_cq_ops = {
+ .cq_init = irdma_sc_cq_init,
+ .cq_create = irdma_sc_cq_create,
+ .cq_destroy = irdma_sc_cq_destroy,
+ .cq_modify = irdma_sc_cq_modify,
+ .cq_ack = irdma_sc_cq_ack,
+};
+
+static struct irdma_ccq_ops iw_ccq_ops = {
+ .ccq_init = irdma_sc_ccq_init,
+ .ccq_create = irdma_sc_ccq_create,
+ .ccq_destroy = irdma_sc_ccq_destroy,
+ .ccq_create_done = irdma_sc_ccq_create_done,
+ .ccq_get_cqe_info = irdma_sc_ccq_get_cqe_info,
+ .ccq_arm = irdma_sc_ccq_arm
+};
+
+static struct irdma_ceq_ops iw_ceq_ops = {
+ .ceq_init = irdma_sc_ceq_init,
+ .ceq_create = irdma_sc_ceq_create,
+ .cceq_create_done = irdma_sc_cceq_create_done,
+ .cceq_destroy_done = irdma_sc_cceq_destroy_done,
+ .cceq_create = irdma_sc_cceq_create,
+ .ceq_destroy = irdma_sc_ceq_destroy,
+ .process_ceq = irdma_sc_process_ceq
+};
+
+static struct irdma_aeq_ops iw_aeq_ops = {
+ .aeq_init = irdma_sc_aeq_init,
+ .aeq_create = irdma_sc_aeq_create,
+ .aeq_destroy = irdma_sc_aeq_destroy,
+ .get_next_aeqe = irdma_sc_get_next_aeqe,
+ .repost_aeq_entries = irdma_sc_repost_aeq_entries,
+ .aeq_create_done = irdma_sc_aeq_create_done,
+ .aeq_destroy_done = irdma_sc_aeq_destroy_done
+};
+
+static struct irdma_hmc_ops iw_hmc_ops = {
+ .init_iw_hmc = irdma_sc_init_iw_hmc,
+ .manage_hmc_pm_func_table = irdma_sc_manage_hmc_pm_func_table,
+ .manage_hmc_pm_func_table_done = irdma_sc_manage_hmc_pm_func_table_done,
+ .static_hmc_pages_allocated = irdma_sc_static_hmc_pages_allocated,
+ .query_fpm_val = irdma_sc_query_fpm_val,
+ .parse_fpm_query_buf = irdma_sc_parse_fpm_query_buf,
+ .query_fpm_val_done = irdma_sc_query_fpm_val_done,
+ .cfg_iw_fpm = irdma_sc_cfg_iw_fpm,
+ .commit_fpm_val = irdma_sc_commit_fpm_val,
+ .parse_fpm_commit_buf = irdma_sc_parse_fpm_commit_buf,
+ .commit_fpm_val_done = irdma_sc_commit_fpm_val_done,
+ .create_hmc_object = irdma_sc_create_hmc_obj,
+ .del_hmc_object = irdma_sc_del_hmc_obj,
+ .pf_init_vfhmc = NULL,
+ .vf_cfg_vffpm = NULL
+};
+
+/**
+ * irdma_wait_pe_ready - Check if firmware is ready
+ * @hw: provides access to registers
+ */
+static int irdma_wait_pe_ready(struct irdma_sc_dev *dev)
+{
+ u32 statuscpu0;
+ u32 statuscpu1;
+ u32 statuscpu2;
+ u32 retrycount = 0;
+
+ do {
+ statuscpu0 = rd32(dev->hw, dev->hw_regs[IRDMA_GLPE_CPUSTATUS0]);
+ statuscpu1 = rd32(dev->hw, dev->hw_regs[IRDMA_GLPE_CPUSTATUS1]);
+ statuscpu2 = rd32(dev->hw, dev->hw_regs[IRDMA_GLPE_CPUSTATUS2]);
+ if (statuscpu0 == 0x80 && statuscpu1 == 0x80 &&
+ statuscpu2 == 0x80)
+ return 0;
+ mdelay(1000);
+ } while (retrycount < dev->hw_attrs.max_pe_ready_count);
+ return -1;
+}
+
+/**
+ * irdma_sc_ctrl_init - Initialize control part of device
+ * @ver: version
+ * @dev: Device pointer
+ * @info: Device init info
+ */
+enum irdma_status_code irdma_sc_ctrl_init(enum irdma_vers ver,
+ struct irdma_sc_dev *dev,
+ struct irdma_device_init_info *info)
+{
+ u32 val;
+ u16 hmc_fcn = 0;
+ enum irdma_status_code ret_code = 0;
+ u8 db_size;
+
+ spin_lock_init(&dev->cqp_lock);
+ INIT_LIST_HEAD(&dev->cqp_cmd_head); /* for CQP command backlog */
+#ifndef CONFIG_DYNAMIC_DEBUG
+ dev->debug_mask = info->debug_mask;
+#endif
+ dev->hmc_fn_id = info->hmc_fn_id;
+ dev->is_pf = info->is_pf;
+ dev->fpm_query_buf_pa = info->fpm_query_buf_pa;
+ dev->fpm_query_buf = info->fpm_query_buf;
+ dev->fpm_commit_buf_pa = info->fpm_commit_buf_pa;
+ dev->fpm_commit_buf = info->fpm_commit_buf;
+ dev->hw = info->hw;
+ dev->hw->hw_addr = info->bar0;
+ dev->irq_ops = &iw_irq_ops;
+ dev->cqp_ops = &iw_cqp_ops;
+ dev->ccq_ops = &iw_ccq_ops;
+ dev->ceq_ops = &iw_ceq_ops;
+ dev->aeq_ops = &iw_aeq_ops;
+ dev->hmc_ops = &iw_hmc_ops;
+ dev->iw_priv_cq_ops = &iw_priv_cq_ops;
+
+ /* Setup the hardware limits, hmc may limit further */
+ dev->hw_attrs.min_hw_qp_id = IRDMA_MIN_IW_QP_ID;
+ dev->hw_attrs.min_hw_aeq_size = IRDMA_MIN_AEQ_ENTRIES;
+ dev->hw_attrs.max_hw_aeq_size = IRDMA_MAX_AEQ_ENTRIES;
+ dev->hw_attrs.min_hw_ceq_size = IRDMA_MIN_CEQ_ENTRIES;
+ dev->hw_attrs.max_hw_ceq_size = IRDMA_MAX_CEQ_ENTRIES;
+ dev->hw_attrs.min_hw_cq_size = IRDMA_MIN_CQ_SIZE;
+ dev->hw_attrs.max_hw_cq_size = IRDMA_MAX_CQ_SIZE;
+ dev->hw_attrs.max_hw_wq_frags = IRDMA_MAX_WQ_FRAGMENT_COUNT;
+ dev->hw_attrs.max_hw_read_sges = IRDMA_MAX_SGE_RD;
+ dev->hw_attrs.max_hw_outbound_msg_size = IRDMA_MAX_OUTBOUND_MSG_SIZE;
+ dev->hw_attrs.max_hw_inbound_msg_size = IRDMA_MAX_INBOUND_MSG_SIZE;
+ dev->hw_attrs.max_hw_device_pages = IRDMA_MAX_PUSH_PAGE_COUNT;
+ dev->hw_attrs.max_hw_vf_fpm_id = IRDMA_MAX_VF_FPM_ID;
+ dev->hw_attrs.first_hw_vf_fpm_id = IRDMA_FIRST_VF_FPM_ID;
+ dev->hw_attrs.max_hw_inline = IRDMA_MAX_INLINE_DATA_SIZE;
+ dev->hw_attrs.max_hw_push_inline = IRDMA_MAX_PUSHMODE_INLINE_DATA_SIZE;
+ dev->hw_attrs.max_hw_ird = IRDMA_MAX_IRD_SIZE;
+ dev->hw_attrs.max_hw_ord = IRDMA_MAX_ORD_SIZE;
+ dev->hw_attrs.max_hw_wqes = IRDMA_MAX_WQ_ENTRIES;
+ dev->hw_attrs.max_qp_wr = IRDMA_MAX_QP_WRS;
+
+ dev->hw_attrs.max_hw_sq_quanta = IRDMA_QP_SW_MAX_SQ_QUANTA;
+ dev->hw_attrs.max_hw_rq_quanta = IRDMA_QP_SW_MAX_RQ_QUANTA;
+ dev->hw_attrs.max_hw_wq_quanta = IRDMA_QP_SW_MAX_WQ_QUANTA;
+ dev->hw_attrs.max_hw_pds = IRDMA_MAX_PDS;
+ dev->hw_attrs.max_hw_ena_vf_count = IRDMA_MAX_PE_ENA_VF_COUNT;
+
+ dev->hw_attrs.max_pe_ready_count = 14;
+ dev->hw_attrs.max_done_count = IRDMA_DONE_COUNT;
+ dev->hw_attrs.max_sleep_count = IRDMA_SLEEP_COUNT;
+ dev->hw_attrs.max_cqp_compl_wait_time_ms = CQP_COMPL_WAIT_TIME_MS;
+
+ dev->hw_attrs.hw_rev = ver;
+
+ info->init_hw(dev);
+ if (dev->is_pf) {
+ if (irdma_wait_pe_ready(dev))
+ return IRDMA_ERR_TIMEOUT;
+
+ val = rd32(dev->hw, dev->hw_regs[IRDMA_GLPCI_LBARCTRL]);
+ db_size = (u8)RS_32(val, IRDMA_GLPCI_LBARCTRL_PE_DB_SIZE);
+ if (db_size != IRDMA_PE_DB_SIZE_4M &&
+ db_size != IRDMA_PE_DB_SIZE_8M) {
+ pr_err("RDMA feature not enabled! db_size=%d\n", db_size);
+ return IRDMA_ERR_PE_DOORBELL_NOT_ENA;
+ }
+ }
+ dev->db_addr = dev->hw->hw_addr + dev->hw_regs[IRDMA_DB_ADDR_OFFSET];
+ dev->hw->hmc.hmc_fn_id = (u8)hmc_fcn;
+
+ return ret_code;
+}
+
+/**
+ * irdma_sc_rt_init - Runtime initialize device
+ * @dev: IWARP device pointer
+ */
+void irdma_sc_rt_init(struct irdma_sc_dev *dev)
+{
+ mutex_init(&dev->ws_mutex);
+ irdma_device_init_uk(&dev->dev_uk);
+ dev->cqp_misc_ops = &iw_cqp_misc_ops;
+ dev->iw_pd_ops = &iw_pd_ops;
+ dev->iw_priv_qp_ops = &iw_priv_qp_ops;
+ dev->mr_ops = &iw_mr_ops;
+ dev->iw_uda_ops = &irdma_uda_ops;
+}
+
+/**
+ * irdma_update_stats - Update statistics
+ * @hw_stats: hw_stats instance to update
+ * @gather_stats: updated stat counters
+ * @last_gather_stats: last stat counters
+ */
+void irdma_update_stats(struct irdma_dev_hw_stats *hw_stats,
+ struct irdma_gather_stats *gather_stats,
+ struct irdma_gather_stats *last_gather_stats)
+{
+ u64 *stats_val = hw_stats->stats_val_32;
+
+ stats_val[IRDMA_HW_STAT_INDEX_RXVLANERR] +=
+ IRDMA_STATS_DELTA(gather_stats->rxvlanerr,
+ last_gather_stats->rxvlanerr,
+ IRDMA_MAX_STATS_32);
+ stats_val[IRDMA_HW_STAT_INDEX_IP4RXDISCARD] +=
+ IRDMA_STATS_DELTA(gather_stats->ip4rxdiscard,
+ last_gather_stats->ip4rxdiscard,
+ IRDMA_MAX_STATS_32);
+ stats_val[IRDMA_HW_STAT_INDEX_IP4RXTRUNC] +=
+ IRDMA_STATS_DELTA(gather_stats->ip4rxtrunc,
+ last_gather_stats->ip4rxtrunc,
+ IRDMA_MAX_STATS_32);
+ stats_val[IRDMA_HW_STAT_INDEX_IP4TXNOROUTE] +=
+ IRDMA_STATS_DELTA(gather_stats->ip4txnoroute,
+ last_gather_stats->ip4txnoroute,
+ IRDMA_MAX_STATS_32);
+ stats_val[IRDMA_HW_STAT_INDEX_IP6RXDISCARD] +=
+ IRDMA_STATS_DELTA(gather_stats->ip6rxdiscard,
+ last_gather_stats->ip6rxdiscard,
+ IRDMA_MAX_STATS_32);
+ stats_val[IRDMA_HW_STAT_INDEX_IP6RXTRUNC] +=
+ IRDMA_STATS_DELTA(gather_stats->ip6rxtrunc,
+ last_gather_stats->ip6rxtrunc,
+ IRDMA_MAX_STATS_32);
+ stats_val[IRDMA_HW_STAT_INDEX_IP6TXNOROUTE] +=
+ IRDMA_STATS_DELTA(gather_stats->ip6txnoroute,
+ last_gather_stats->ip6txnoroute,
+ IRDMA_MAX_STATS_32);
+ stats_val[IRDMA_HW_STAT_INDEX_TCPRTXSEG] +=
+ IRDMA_STATS_DELTA(gather_stats->tcprtxseg,
+ last_gather_stats->tcprtxseg,
+ IRDMA_MAX_STATS_32);
+ stats_val[IRDMA_HW_STAT_INDEX_TCPRXOPTERR] +=
+ IRDMA_STATS_DELTA(gather_stats->tcprxopterr,
+ last_gather_stats->tcprxopterr,
+ IRDMA_MAX_STATS_32);
+ stats_val[IRDMA_HW_STAT_INDEX_TCPRXPROTOERR] +=
+ IRDMA_STATS_DELTA(gather_stats->tcprxprotoerr,
+ last_gather_stats->tcprxprotoerr,
+ IRDMA_MAX_STATS_32);
+ stats_val[IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED] +=
+ IRDMA_STATS_DELTA(gather_stats->rxrpcnphandled,
+ last_gather_stats->rxrpcnphandled,
+ IRDMA_MAX_STATS_32);
+ stats_val[IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED] +=
+ IRDMA_STATS_DELTA(gather_stats->rxrpcnpignored,
+ last_gather_stats->rxrpcnpignored,
+ IRDMA_MAX_STATS_32);
+ stats_val[IRDMA_HW_STAT_INDEX_TXNPCNPSENT] +=
+ IRDMA_STATS_DELTA(gather_stats->txnpcnpsent,
+ last_gather_stats->txnpcnpsent,
+ IRDMA_MAX_STATS_32);
+ stats_val = hw_stats->stats_val_64;
+ stats_val[IRDMA_HW_STAT_INDEX_IP4RXOCTS] +=
+ IRDMA_STATS_DELTA(gather_stats->ip4rxocts,
+ last_gather_stats->ip4rxocts,
+ IRDMA_MAX_STATS_48);
+ stats_val[IRDMA_HW_STAT_INDEX_IP4RXPKTS] +=
+ IRDMA_STATS_DELTA(gather_stats->ip4rxpkts,
+ last_gather_stats->ip4rxpkts,
+ IRDMA_MAX_STATS_48);
+ stats_val[IRDMA_HW_STAT_INDEX_IP4RXFRAGS] +=
+ IRDMA_STATS_DELTA(gather_stats->ip4txfrag,
+ last_gather_stats->ip4txfrag,
+ IRDMA_MAX_STATS_48);
+ stats_val[IRDMA_HW_STAT_INDEX_IP4RXMCPKTS] +=
+ IRDMA_STATS_DELTA(gather_stats->ip4rxmcpkts,
+ last_gather_stats->ip4rxmcpkts,
+ IRDMA_MAX_STATS_48);
+ stats_val[IRDMA_HW_STAT_INDEX_IP4TXOCTS] +=
+ IRDMA_STATS_DELTA(gather_stats->ip4txocts,
+ last_gather_stats->ip4txocts,
+ IRDMA_MAX_STATS_48);
+ stats_val[IRDMA_HW_STAT_INDEX_IP4TXPKTS] +=
+ IRDMA_STATS_DELTA(gather_stats->ip4txpkts,
+ last_gather_stats->ip4txpkts,
+ IRDMA_MAX_STATS_48);
+ stats_val[IRDMA_HW_STAT_INDEX_IP4TXFRAGS] +=
+ IRDMA_STATS_DELTA(gather_stats->ip4txfrag,
+ last_gather_stats->ip4txfrag,
+ IRDMA_MAX_STATS_48);
+ stats_val[IRDMA_HW_STAT_INDEX_IP4TXMCPKTS] +=
+ IRDMA_STATS_DELTA(gather_stats->ip4txmcpkts,
+ last_gather_stats->ip4txmcpkts,
+ IRDMA_MAX_STATS_48);
+ stats_val[IRDMA_HW_STAT_INDEX_IP6RXOCTS] +=
+ IRDMA_STATS_DELTA(gather_stats->ip6rxocts,
+ last_gather_stats->ip6rxocts,
+ IRDMA_MAX_STATS_48);
+ stats_val[IRDMA_HW_STAT_INDEX_IP6RXPKTS] +=
+ IRDMA_STATS_DELTA(gather_stats->ip6rxpkts,
+ last_gather_stats->ip6rxpkts,
+ IRDMA_MAX_STATS_48);
+ stats_val[IRDMA_HW_STAT_INDEX_IP6RXFRAGS] +=
+ IRDMA_STATS_DELTA(gather_stats->ip6txfrags,
+ last_gather_stats->ip6txfrags,
+ IRDMA_MAX_STATS_48);
+ stats_val[IRDMA_HW_STAT_INDEX_IP6RXMCPKTS] +=
+ IRDMA_STATS_DELTA(gather_stats->ip6rxmcpkts,
+ last_gather_stats->ip6rxmcpkts,
+ IRDMA_MAX_STATS_48);
+ stats_val[IRDMA_HW_STAT_INDEX_IP6TXOCTS] +=
+ IRDMA_STATS_DELTA(gather_stats->ip6txocts,
+ last_gather_stats->ip6txocts,
+ IRDMA_MAX_STATS_48);
+ stats_val[IRDMA_HW_STAT_INDEX_IP6TXPKTS] +=
+ IRDMA_STATS_DELTA(gather_stats->ip6txpkts,
+ last_gather_stats->ip6txpkts,
+ IRDMA_MAX_STATS_48);
+ stats_val[IRDMA_HW_STAT_INDEX_IP6TXFRAGS] +=
+ IRDMA_STATS_DELTA(gather_stats->ip6txfrags,
+ last_gather_stats->ip6txfrags,
+ IRDMA_MAX_STATS_48);
+ stats_val[IRDMA_HW_STAT_INDEX_IP6TXMCPKTS] +=
+ IRDMA_STATS_DELTA(gather_stats->ip6txmcpkts,
+ last_gather_stats->ip6txmcpkts,
+ IRDMA_MAX_STATS_48);
+ stats_val[IRDMA_HW_STAT_INDEX_TCPRXSEGS] +=
+ IRDMA_STATS_DELTA(gather_stats->tcprxsegs,
+ last_gather_stats->tcprxsegs,
+ IRDMA_MAX_STATS_48);
+ stats_val[IRDMA_HW_STAT_INDEX_TCPTXSEG] +=
+ IRDMA_STATS_DELTA(gather_stats->tcptxsegs,
+ last_gather_stats->tcptxsegs,
+ IRDMA_MAX_STATS_48);
+ stats_val[IRDMA_HW_STAT_INDEX_RDMARXRDS] +=
+ IRDMA_STATS_DELTA(gather_stats->rdmarxrds,
+ last_gather_stats->rdmarxrds,
+ IRDMA_MAX_STATS_48);
+ stats_val[IRDMA_HW_STAT_INDEX_RDMARXSNDS] +=
+ IRDMA_STATS_DELTA(gather_stats->rdmarxsnds,
+ last_gather_stats->rdmarxsnds,
+ IRDMA_MAX_STATS_48);
+ stats_val[IRDMA_HW_STAT_INDEX_RDMARXWRS] +=
+ IRDMA_STATS_DELTA(gather_stats->rdmarxwrs,
+ last_gather_stats->rdmarxwrs,
+ IRDMA_MAX_STATS_48);
+ stats_val[IRDMA_HW_STAT_INDEX_RDMATXRDS] +=
+ IRDMA_STATS_DELTA(gather_stats->rdmatxrds,
+ last_gather_stats->rdmatxrds,
+ IRDMA_MAX_STATS_48);
+ stats_val[IRDMA_HW_STAT_INDEX_RDMATXSNDS] +=
+ IRDMA_STATS_DELTA(gather_stats->rdmatxsnds,
+ last_gather_stats->rdmatxsnds,
+ IRDMA_MAX_STATS_48);
+ stats_val[IRDMA_HW_STAT_INDEX_RDMATXWRS] +=
+ IRDMA_STATS_DELTA(gather_stats->rdmatxwrs,
+ last_gather_stats->rdmatxwrs,
+ IRDMA_MAX_STATS_48);
+ stats_val[IRDMA_HW_STAT_INDEX_RDMAVBND] +=
+ IRDMA_STATS_DELTA(gather_stats->rdmavbn,
+ last_gather_stats->rdmavbn,
+ IRDMA_MAX_STATS_48);
+ stats_val[IRDMA_HW_STAT_INDEX_RDMAVINV] +=
+ IRDMA_STATS_DELTA(gather_stats->rdmavinv,
+ last_gather_stats->rdmavinv,
+ IRDMA_MAX_STATS_48);
+ stats_val[IRDMA_HW_STAT_INDEX_UDPRXPKTS] +=
+ IRDMA_STATS_DELTA(gather_stats->udprxpkts,
+ last_gather_stats->udprxpkts,
+ IRDMA_MAX_STATS_48);
+ stats_val[IRDMA_HW_STAT_INDEX_UDPTXPKTS] +=
+ IRDMA_STATS_DELTA(gather_stats->udptxpkts,
+ last_gather_stats->udptxpkts,
+ IRDMA_MAX_STATS_48);
+ stats_val[IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS] +=
+ IRDMA_STATS_DELTA(gather_stats->rxnpecnmrkpkts,
+ last_gather_stats->rxnpecnmrkpkts,
+ IRDMA_MAX_STATS_48);
+ memcpy(last_gather_stats, gather_stats, sizeof(*last_gather_stats));
+}
+
+/**
+ * irdma_hw_stats_read_32 - Read 32-bit HW stats counters and accommodates
+ * for roll-overs.
+ * @stat: stat instance contains data and offsets
+ * @hw_stats_regs: hw stat reg array for the pf or vf
+ * @index: index in HW stats table which contains offset reg-addr
+ * @value: hw stats value
+ */
+static void
+irdma_hw_stats_read_32(struct irdma_vsi_pestat *stats,
+ u64 *hw_stats_regs,
+ enum irdma_hw_stats_index_32b index,
+ u64 *value)
+{
+ struct irdma_dev_hw_stats *hw_stats = &stats->hw_stats;
+ struct irdma_dev_hw_stats *last_hw_stats = &stats->last_hw_stats;
+ u64 new_stats_value;
+
+ new_stats_value = rd32(stats->hw, hw_stats_regs[index]);
+ /*roll-over case */
+ if (new_stats_value < last_hw_stats->stats_val_32[index])
+ hw_stats->stats_val_32[index] += new_stats_value;
+ else
+ hw_stats->stats_val_32[index] +=
+ new_stats_value -
+ last_hw_stats->stats_val_32[index];
+
+ last_hw_stats->stats_val_32[index] = new_stats_value;
+ *value = hw_stats->stats_val_32[index];
+}
+
+/**
+ * irdma_hw_stats_read_64 - Read 64-bit HW stats counters and accommodates
+ * for roll-overs.
+ * @stat: stat instance contains data and offsets
+ * @hw_stats_regs: hw stat reg array for the pf or vf
+ * @index: index in HW stats table which contains offset reg-addr
+ * @value: hw stats value
+ */
+static void
+irdma_hw_stats_read_64(struct irdma_vsi_pestat *stats,
+ u64 *hw_stats_regs,
+ enum irdma_hw_stats_index_64b index,
+ u64 *value)
+{
+ struct irdma_dev_hw_stats *last_hw_stats = &stats->last_hw_stats;
+ struct irdma_dev_hw_stats *hw_stats = &stats->hw_stats;
+ u64 new_stats_value;
+
+ new_stats_value = rd64(stats->hw, hw_stats_regs[index]);
+ /*roll-over case */
+ if (new_stats_value < last_hw_stats->stats_val_64[index])
+ hw_stats->stats_val_64[index] += new_stats_value;
+ else
+ hw_stats->stats_val_64[index] +=
+ new_stats_value -
+ last_hw_stats->stats_val_64[index];
+
+ last_hw_stats->stats_val_64[index] = new_stats_value;
+ *value = hw_stats->stats_val_64[index];
+}
+
+/**
+ * irdma_hw_stats_read_all - read all HW stat counters
+ * @stats: pestat struct
+ * @stats_values: hw stats structure
+ * @hw_stats_regs_32: 32-bit stat reg array from pf or vf
+ * @hw_stats_regs_64: 64-bit stat reg array from pf or vf
+ * @hw_rev: hardware revision
+ *
+ * Read all the HW stat counters and populates hw_stats structure
+ * of passed-in dev's pestat as well as copy created in stat_values.
+ */
+void
+irdma_hw_stats_read_all(struct irdma_vsi_pestat *stats,
+ struct irdma_dev_hw_stats *stats_values,
+ u64 *hw_stats_regs_32,
+ u64 *hw_stats_regs_64,
+ u8 hw_rev)
+{
+ u8 max_stats_idx_32, max_stats_idx_64;
+ unsigned long flags;
+ u32 stats_index;
+
+ if (hw_rev == IRDMA_GEN_1) {
+ max_stats_idx_32 = IRDMA_HW_STAT_INDEX_MAX_32_GEN_1;
+ max_stats_idx_64 = IRDMA_HW_STAT_INDEX_MAX_64_GEN_1;
+ } else {
+ max_stats_idx_32 = IRDMA_HW_STAT_INDEX_MAX_32;
+ max_stats_idx_64 = IRDMA_HW_STAT_INDEX_MAX_64;
+ }
+
+ spin_lock_irqsave(&stats->lock, flags);
+
+ for (stats_index = 0; stats_index < max_stats_idx_32; stats_index++)
+ irdma_hw_stats_read_32(stats,
+ hw_stats_regs_32,
+ stats_index,
+ &stats_values->stats_val_32[stats_index]);
+ for (stats_index = 0; stats_index < max_stats_idx_64; stats_index++)
+ irdma_hw_stats_read_64(stats,
+ hw_stats_regs_64,
+ stats_index,
+ &stats_values->stats_val_64[stats_index]);
+
+ spin_unlock_irqrestore(&stats->lock, flags);
+}
diff --git a/drivers/infiniband/hw/irdma/defs.h b/drivers/infiniband/hw/irdma/defs.h
new file mode 100644
index 0000000..9f9cd90
--- /dev/null
+++ b/drivers/infiniband/hw/irdma/defs.h
@@ -0,0 +1,2053 @@
+/* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB */
+/* Copyright (c) 2019, Intel Corporation. */
+
+#ifndef IRDMA_DEFS_H
+#define IRDMA_DEFS_H
+#define IRDMA_FIRST_USER_QP_ID 3
+
+#define IRDMA_PUSH_OFFSET (8 * 1024 * 1024)
+#define IRDMA_PF_FIRST_PUSH_PAGE_INDEX 16
+#define IRDMA_VF_PUSH_OFFSET ((8 + 64) * 1024)
+#define IRDMA_VF_FIRST_PUSH_PAGE_INDEX 2
+#define IRDMA_VF_STATS_SIZE_V0 280
+
+#define IRDMA_PE_DB_SIZE_4M 1
+#define IRDMA_PE_DB_SIZE_8M 2
+
+enum irdma_protocol_used {
+ IRDMA_ANY_PROTOCOL = 0,
+ IRDMA_IWARP_PROTOCOL_ONLY = 1,
+ IRDMA_ROCE_PROTOCOL_ONLY = 2,
+};
+
+#define IRDMA_QP_STATE_INVALID 0
+#define IRDMA_QP_STATE_IDLE 1
+#define IRDMA_QP_STATE_RTS 2
+#define IRDMA_QP_STATE_CLOSING 3
+#define IRDMA_QP_STATE_RTR 4
+#define IRDMA_QP_STATE_TERMINATE 5
+#define IRDMA_QP_STATE_ERROR 6
+
+#define IRDMA_MAX_USER_PRIORITY 8
+#define IRDMA_MAX_APPS 8
+#define IRDMA_MAX_STATS_COUNT 128
+#define IRDMA_FIRST_NON_PF_STAT 4
+
+#define IRDMA_FEATURE_RTS_AE 1
+
+#define IRDMA_MIN_MTU_IPV4 576
+#define IRDMA_MIN_MTU_IPV6 1280
+#define IRDMA_MTU_TO_MSS_IPV4 40
+#define IRDMA_MTU_TO_MSS_IPV6 60
+#define IRDMA_DEFAULT_MTU 1500
+
+#define IRDMA_MAX_ENCODED_IRD_SIZE 4
+
+#define Q2_FPSN_OFFSET 64
+#define TERM_DDP_LEN_TAGGED 14
+#define TERM_DDP_LEN_UNTAGGED 18
+#define TERM_RDMA_LEN 28
+#define RDMA_OPCODE_M 0x0f
+#define RDMA_READ_REQ_OPCODE 1
+#define Q2_BAD_FRAME_OFFSET 72
+#define CQE_MAJOR_DRV 0x8000
+
+#define IRDMA_TERM_SENT 1
+#define IRDMA_TERM_RCVD 2
+#define IRDMA_TERM_DONE 4
+#define IRDMA_MAC_HLEN 14
+
+#define IRDMA_CQP_WAIT_POLL_REGS 1
+#define IRDMA_CQP_WAIT_POLL_CQ 2
+#define IRDMA_CQP_WAIT_EVENT 3
+
+#define IRDMA_AE_SOURCE_RSVD 0x0
+#define IRDMA_AE_SOURCE_RQ 0x1
+#define IRDMA_AE_SOURCE_RQ_0011 0x3
+
+#define IRDMA_AE_SOURCE_CQ 0x2
+#define IRDMA_AE_SOURCE_CQ_0110 0x6
+#define IRDMA_AE_SOURCE_CQ_1010 0xa
+#define IRDMA_AE_SOURCE_CQ_1110 0xe
+
+#define IRDMA_AE_SOURCE_SQ 0x5
+#define IRDMA_AE_SOURCE_SQ_0111 0x7
+
+#define IRDMA_AE_SOURCE_IN_RR_WR 0x9
+#define IRDMA_AE_SOURCE_IN_RR_WR_1011 0xb
+#define IRDMA_AE_SOURCE_OUT_RR 0xd
+#define IRDMA_AE_SOURCE_OUT_RR_1111 0xf
+
+#define IRDMA_TCP_STATE_NON_EXISTENT 0
+#define IRDMA_TCP_STATE_CLOSED 1
+#define IRDMA_TCP_STATE_LISTEN 2
+#define IRDMA_STATE_SYN_SEND 3
+#define IRDMA_TCP_STATE_SYN_RECEIVED 4
+#define IRDMA_TCP_STATE_ESTABLISHED 5
+#define IRDMA_TCP_STATE_CLOSE_WAIT 6
+#define IRDMA_TCP_STATE_FIN_WAIT_1 7
+#define IRDMA_TCP_STATE_CLOSING 8
+#define IRDMA_TCP_STATE_LAST_ACK 9
+#define IRDMA_TCP_STATE_FIN_WAIT_2 10
+#define IRDMA_TCP_STATE_TIME_WAIT 11
+#define IRDMA_TCP_STATE_RESERVED_1 12
+#define IRDMA_TCP_STATE_RESERVED_2 13
+#define IRDMA_TCP_STATE_RESERVED_3 14
+#define IRDMA_TCP_STATE_RESERVED_4 15
+
+#define IRDMA_CQP_SW_SQSIZE_4 4
+#define IRDMA_CQP_SW_SQSIZE_2048 2048
+
+#define IRDMA_CQ_TYPE_IWARP 1
+#define IRDMA_CQ_TYPE_ILQ 2
+#define IRDMA_CQ_TYPE_IEQ 3
+#define IRDMA_CQ_TYPE_CQP 4
+/* CQP SQ WQES */
+#define IRDMA_QP_TYPE_IWARP 1
+#define IRDMA_QP_TYPE_UDA 2
+#define IRDMA_QP_TYPE_ROCE_RC 3
+#define IRDMA_QP_TYPE_ROCE_UD 4
+
+#define IRDMA_HW_PAGE_SIZE 4096
+#define IRDMA_DONE_COUNT 1000
+#define IRDMA_SLEEP_COUNT 10
+
+#define IRDMA_QP_WQE_MIN_SIZE 32
+#define IRDMA_QP_WQE_MAX_SIZE 256
+#define IRDMA_QP_WQE_MIN_QUANTA 1
+
+#define IRDMA_UPDATE_SD_BUFF_SIZE 128
+#define IRDMA_FEATURE_BUF_SIZE (8 * IRDMA_MAX_FEATURES)
+
+#define IRDMA_CQE_QTYPE_RQ 0
+#define IRDMA_CQE_QTYPE_SQ 1
+
+#define IRDMA_SQ_RSVD 2
+#define IRDMA_RQ_RSVD 1
+
+#define IRDMA_QP_SW_MIN_WQSIZE 8 /* in WRs*/
+#define IRDMA_MAX_QUANTA_PER_WR 8
+
+#define IRDMA_QP_SW_MAX_WQ_QUANTA 32768
+#define IRDMA_QP_SW_MAX_SQ_QUANTA 32768
+#define IRDMA_QP_SW_MAX_RQ_QUANTA 32768
+#define IRDMA_MAX_QP_WRS (((IRDMA_QP_SW_MAX_WQ_QUANTA - IRDMA_SQ_RSVD) / IRDMA_MAX_QUANTA_PER_WR))
+
+#define IRDMAQP_TERM_SEND_TERM_AND_FIN 0
+#define IRDMAQP_TERM_SEND_TERM_ONLY 1
+#define IRDMAQP_TERM_SEND_FIN_ONLY 2
+#define IRDMAQP_TERM_DONOT_SEND_TERM_OR_FIN 3
+
+#define IRDMAQP_OP_RDMA_WRITE 0x00
+#define IRDMAQP_OP_RDMA_READ 0x01
+#define IRDMAQP_OP_RDMA_SEND 0x03
+#define IRDMAQP_OP_RDMA_SEND_INV 0x04
+#define IRDMAQP_OP_RDMA_SEND_SOL_EVENT 0x05
+#define IRDMAQP_OP_RDMA_SEND_SOL_EVENT_INV 0x06
+#define IRDMAQP_OP_BIND_MW 0x08
+#define IRDMAQP_OP_FAST_REGISTER 0x09
+#define IRDMAQP_OP_LOCAL_INVALIDATE 0x0a
+#define IRDMAQP_OP_RDMA_READ_LOC_INV 0x0b
+#define IRDMAQP_OP_NOP 0x0c
+#define IRDMAQP_OP_RDMA_WRITE_SOL 0x0d
+#define IRDMAQP_OP_GEN_RTS_AE 0x30
+
+#define IRDMA_OP_CEQ_DESTROY 1
+#define IRDMA_OP_AEQ_DESTROY 2
+#define IRDMA_OP_DELETE_ARP_CACHE_ENTRY 3
+#define IRDMA_OP_MANAGE_APBVT_ENTRY 4
+#define IRDMA_OP_CEQ_CREATE 5
+#define IRDMA_OP_AEQ_CREATE 6
+#define IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY 7
+#define IRDMA_OP_QP_MODIFY 8
+#define IRDMA_OP_QP_UPLOAD_CONTEXT 9
+#define IRDMA_OP_CQ_CREATE 10
+#define IRDMA_OP_CQ_DESTROY 11
+#define IRDMA_OP_QP_CREATE 12
+#define IRDMA_OP_QP_DESTROY 13
+#define IRDMA_OP_ALLOC_STAG 14
+#define IRDMA_OP_MR_REG_NON_SHARED 15
+#define IRDMA_OP_DEALLOC_STAG 16
+#define IRDMA_OP_MW_ALLOC 17
+#define IRDMA_OP_QP_FLUSH_WQES 18
+#define IRDMA_OP_ADD_ARP_CACHE_ENTRY 19
+#define IRDMA_OP_MANAGE_PUSH_PAGE 20
+#define IRDMA_OP_UPDATE_PE_SDS 21
+#define IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE 22
+#define IRDMA_OP_SUSPEND 23
+#define IRDMA_OP_RESUME 24
+#define IRDMA_OP_MANAGE_VF_PBLE_BP 25
+#define IRDMA_OP_QUERY_FPM_VAL 26
+#define IRDMA_OP_COMMIT_FPM_VAL 27
+#define IRDMA_OP_REQ_CMDS 28
+#define IRDMA_OP_CMPL_CMDS 29
+#define IRDMA_OP_AH_CREATE 30
+#define IRDMA_OP_AH_MODIFY 31
+#define IRDMA_OP_AH_DESTROY 32
+#define IRDMA_OP_MC_CREATE 33
+#define IRDMA_OP_MC_DESTROY 34
+#define IRDMA_OP_MC_MODIFY 35
+#define IRDMA_OP_STATS_ALLOCATE 36
+#define IRDMA_OP_STATS_FREE 37
+#define IRDMA_OP_STATS_GATHER 38
+#define IRDMA_OP_WS_ADD_NODE 39
+#define IRDMA_OP_WS_MODIFY_NODE 40
+#define IRDMA_OP_WS_DELETE_NODE 41
+#define IRDMA_OP_SET_UP_MAP 42
+#define IRDMA_OP_GEN_AE 43
+#define IRDMA_OP_QUERY_RDMA_FEATURES 44
+#define IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY 45
+#define IRDMA_OP_ADD_LOCAL_MAC_ENTRY 46
+#define IRDMA_OP_DELETE_LOCAL_MAC_ENTRY 47
+#define IRDMA_OP_SIZE_CQP_STAT_ARRAY 48
+
+#define IRDMA_CQP_OP_CREATE_QP 0
+#define IRDMA_CQP_OP_MODIFY_QP 0x1
+#define IRDMA_CQP_OP_DESTROY_QP 0x02
+#define IRDMA_CQP_OP_CREATE_CQ 0x03
+#define IRDMA_CQP_OP_MODIFY_CQ 0x04
+#define IRDMA_CQP_OP_DESTROY_CQ 0x05
+#define IRDMA_CQP_OP_ALLOC_STAG 0x09
+#define IRDMA_CQP_OP_REG_MR 0x0a
+#define IRDMA_CQP_OP_QUERY_STAG 0x0b
+#define IRDMA_CQP_OP_REG_SMR 0x0c
+#define IRDMA_CQP_OP_DEALLOC_STAG 0x0d
+#define IRDMA_CQP_OP_MANAGE_LOC_MAC_TABLE 0x0e
+#define IRDMA_CQP_OP_MANAGE_ARP 0x0f
+#define IRDMA_CQP_OP_MANAGE_VF_PBLE_BP 0x10
+#define IRDMA_CQP_OP_MANAGE_PUSH_PAGES 0x11
+#define IRDMA_CQP_OP_QUERY_RDMA_FEATURES 0x12
+#define IRDMA_CQP_OP_UPLOAD_CONTEXT 0x13
+#define IRDMA_CQP_OP_ALLOCATE_LOC_MAC_TABLE_ENTRY 0x14
+#define IRDMA_CQP_OP_UPLOAD_CONTEXT 0x13
+#define IRDMA_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE 0x15
+#define IRDMA_CQP_OP_CREATE_CEQ 0x16
+#define IRDMA_CQP_OP_DESTROY_CEQ 0x18
+#define IRDMA_CQP_OP_CREATE_AEQ 0x19
+#define IRDMA_CQP_OP_DESTROY_AEQ 0x1b
+#define IRDMA_CQP_OP_CREATE_ADDR_HANDLE 0x1c
+#define IRDMA_CQP_OP_MODIFY_ADDR_HANDLE 0x1d
+#define IRDMA_CQP_OP_DESTROY_ADDR_HANDLE 0x1e
+#define IRDMA_CQP_OP_UPDATE_PE_SDS 0x1f
+#define IRDMA_CQP_OP_QUERY_FPM_VAL 0x20
+#define IRDMA_CQP_OP_COMMIT_FPM_VAL 0x21
+#define IRDMA_CQP_OP_FLUSH_WQES 0x22
+/* IRDMA_CQP_OP_GEN_AE is the same value as IRDMA_CQP_OP_FLUSH_WQES */
+#define IRDMA_CQP_OP_GEN_AE 0x22
+#define IRDMA_CQP_OP_MANAGE_APBVT 0x23
+#define IRDMA_CQP_OP_NOP 0x24
+#define IRDMA_CQP_OP_MANAGE_QUAD_HASH_TABLE_ENTRY 0x25
+#define IRDMA_CQP_OP_CREATE_MCAST_GRP 0x26
+#define IRDMA_CQP_OP_MODIFY_MCAST_GRP 0x27
+#define IRDMA_CQP_OP_DESTROY_MCAST_GRP 0x28
+#define IRDMA_CQP_OP_SUSPEND_QP 0x29
+#define IRDMA_CQP_OP_RESUME_QP 0x2a
+#define IRDMA_CQP_OP_SHMC_PAGES_ALLOCATED 0x2b
+#define IRDMA_CQP_OP_WORK_SCHED_NODE 0x2c
+#define IRDMA_CQP_OP_MANAGE_STATS 0x2d
+#define IRDMA_CQP_OP_GATHER_STATS 0x2e
+#define IRDMA_CQP_OP_UP_MAP 0x2f
+
+/* Async Events codes */
+#define IRDMA_AE_AMP_UNALLOCATED_STAG 0x0102
+#define IRDMA_AE_AMP_INVALID_STAG 0x0103
+#define IRDMA_AE_AMP_BAD_QP 0x0104
+#define IRDMA_AE_AMP_BAD_PD 0x0105
+#define IRDMA_AE_AMP_BAD_STAG_KEY 0x0106
+#define IRDMA_AE_AMP_BAD_STAG_INDEX 0x0107
+#define IRDMA_AE_AMP_BOUNDS_VIOLATION 0x0108
+#define IRDMA_AE_AMP_RIGHTS_VIOLATION 0x0109
+#define IRDMA_AE_AMP_TO_WRAP 0x010a
+#define IRDMA_AE_AMP_FASTREG_VALID_STAG 0x010c
+#define IRDMA_AE_AMP_FASTREG_MW_STAG 0x010d
+#define IRDMA_AE_AMP_FASTREG_INVALID_RIGHTS 0x010e
+#define IRDMA_AE_AMP_FASTREG_INVALID_LENGTH 0x0110
+#define IRDMA_AE_AMP_INVALIDATE_SHARED 0x0111
+#define IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS 0x0112
+#define IRDMA_AE_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS 0x0113
+#define IRDMA_AE_AMP_MWBIND_VALID_STAG 0x0114
+#define IRDMA_AE_AMP_MWBIND_OF_MR_STAG 0x0115
+#define IRDMA_AE_AMP_MWBIND_TO_ZERO_BASED_STAG 0x0116
+#define IRDMA_AE_AMP_MWBIND_TO_MW_STAG 0x0117
+#define IRDMA_AE_AMP_MWBIND_INVALID_RIGHTS 0x0118
+#define IRDMA_AE_AMP_MWBIND_INVALID_BOUNDS 0x0119
+#define IRDMA_AE_AMP_MWBIND_TO_INVALID_PARENT 0x011a
+#define IRDMA_AE_AMP_MWBIND_BIND_DISABLED 0x011b
+#define IRDMA_AE_PRIV_OPERATION_DENIED 0x011c
+#define IRDMA_AE_AMP_INVALIDATE_TYPE1_MW 0x011d
+#define IRDMA_AE_AMP_MWBIND_ZERO_BASED_TYPE1_MW 0x011e
+#define IRDMA_AE_AMP_FASTREG_INVALID_PBL_HPS_CFG 0x011f
+#define IRDMA_AE_AMP_FASTREG_PBLE_MISMATCH 0x0121
+#define IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG 0x0132
+#define IRDMA_AE_UDA_XMIT_BAD_PD 0x0133
+#define IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT 0x0134
+#define IRDMA_AE_UDA_L4LEN_INVALID 0x0135
+#define IRDMA_AE_BAD_CLOSE 0x0201
+#define IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE 0x0202
+#define IRDMA_AE_CQ_OPERATION_ERROR 0x0203
+#define IRDMA_AE_RDMA_READ_WHILE_ORD_ZERO 0x0205
+#define IRDMA_AE_STAG_ZERO_INVALID 0x0206
+#define IRDMA_AE_IB_RREQ_AND_Q1_FULL 0x0207
+#define IRDMA_AE_IB_INVALID_REQUEST 0x0208
+#define IRDMA_AE_WQE_UNEXPECTED_OPCODE 0x020a
+#define IRDMA_AE_WQE_INVALID_PARAMETER 0x020b
+#define IRDMA_AE_WQE_INVALID_FRAG_DATA 0x020c
+#define IRDMA_AE_IB_REMOTE_ACCESS_ERROR 0x020d
+#define IRDMA_AE_IB_REMOTE_OP_ERROR 0x020e
+#define IRDMA_AE_WQE_LSMM_TOO_LONG 0x0220
+#define IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN 0x0301
+#define IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER 0x0303
+#define IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION 0x0304
+#define IRDMA_AE_DDP_UBE_INVALID_MO 0x0305
+#define IRDMA_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE 0x0306
+#define IRDMA_AE_DDP_UBE_INVALID_QN 0x0307
+#define IRDMA_AE_DDP_NO_L_BIT 0x0308
+#define IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION 0x0311
+#define IRDMA_AE_RDMAP_ROE_UNEXPECTED_OPCODE 0x0312
+#define IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST 0x0313
+#define IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP 0x0314
+#define IRDMA_AE_ROCE_RSP_LENGTH_ERROR 0x0316
+#define IRDMA_AE_ROCE_EMPTY_MCG 0x0380
+#define IRDMA_AE_ROCE_BAD_MC_IP_ADDR 0x0381
+#define IRDMA_AE_ROCE_BAD_MC_QPID 0x0382
+#define IRDMA_AE_MCG_QP_PROTOCOL_MISMATCH 0x0383
+#define IRDMA_AE_INVALID_ARP_ENTRY 0x0401
+#define IRDMA_AE_INVALID_TCP_OPTION_RCVD 0x0402
+#define IRDMA_AE_STALE_ARP_ENTRY 0x0403
+#define IRDMA_AE_INVALID_AH_ENTRY 0x0406
+#define IRDMA_AE_LLP_CLOSE_COMPLETE 0x0501
+#define IRDMA_AE_LLP_CONNECTION_RESET 0x0502
+#define IRDMA_AE_LLP_FIN_RECEIVED 0x0503
+#define IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR 0x0505
+#define IRDMA_AE_LLP_SEGMENT_TOO_SMALL 0x0507
+#define IRDMA_AE_LLP_SYN_RECEIVED 0x0508
+#define IRDMA_AE_LLP_TERMINATE_RECEIVED 0x0509
+#define IRDMA_AE_LLP_TOO_MANY_RETRIES 0x050a
+#define IRDMA_AE_LLP_TOO_MANY_KEEPALIVE_RETRIES 0x050b
+#define IRDMA_AE_LLP_DOUBT_REACHABILITY 0x050c
+#define IRDMA_AE_LLP_CONNECTION_ESTABLISHED 0x050e
+#define IRDMA_AE_RESOURCE_EXHAUSTION 0x0520
+#define IRDMA_AE_RESET_SENT 0x0601
+#define IRDMA_AE_TERMINATE_SENT 0x0602
+#define IRDMA_AE_RESET_NOT_SENT 0x0603
+#define IRDMA_AE_LCE_QP_CATASTROPHIC 0x0700
+#define IRDMA_AE_LCE_FUNCTION_CATASTROPHIC 0x0701
+#define IRDMA_AE_LCE_CQ_CATASTROPHIC 0x0702
+#define IRDMA_AE_QP_SUSPEND_COMPLETE 0x0900
+
+#define LS_64_1(val, bits) ((u64)(uintptr_t)(val) << (bits))
+#define RS_64_1(val, bits) ((u64)(uintptr_t)(val) >> (bits))
+#define LS_32_1(val, bits) (u32)((val) << (bits))
+#define RS_32_1(val, bits) (u32)((val) >> (bits))
+#define LS_64(val, field) (((u64)(val) << field ## _S) & (field ## _M))
+#define RS_64(val, field) ((u64)((val) & field ## _M) >> field ## _S)
+#define LS_32(val, field) (((val) << field ## _S) & (field ## _M))
+#define RS_32(val, field) (((val) & field ## _M) >> field ## _S)
+
+#define FLD_LS_64(dev, val, field) \
+ (((u64)(val) << (dev)->hw_shifts[field ## _S]) & (dev)->hw_masks[field ## _M])
+#define FLD_RS_64(dev, val, field) \
+ ((u64)((val) & (dev)->hw_masks[field ## _M]) >> (dev)->hw_shifts[field ## _S])
+#define FLD_LS_32(dev, val, field) \
+ (((val) << (dev)->hw_shifts[field ## _S]) & (dev)->hw_masks[field ## _M])
+#define FLD_RS_32(dev, val, field) \
+ ((u64)((val) & (dev)->hw_masks[field ## _M]) >> (dev)->hw_shifts[field ## _S])
+
+#define FW_MAJOR_VER(dev) \
+ ((u16)RS_64((dev)->feature_info[IRDMA_FEATURE_FW_INFO], IRDMA_FW_VER_MAJOR))
+#define FW_MINOR_VER(dev) \
+ ((u16)RS_64((dev)->feature_info[IRDMA_FEATURE_FW_INFO], IRDMA_FW_VER_MINOR))
+
+#define IRDMA_STATS_DELTA(a, b, c) ((a) >= (b) ? (a) - (b) : (a) + (c) - (b))
+#define IRDMA_MAX_STATS_32 0xFFFFFFFFULL
+#define IRDMA_MAX_STATS_48 0xFFFFFFFFFFFFULL
+#define IRDMA_MAX_RQ_WQE_SHIFT_GEN1 2
+
+/* ILQ CQP hash table fields */
+#define IRDMA_CQPSQ_QHASH_VLANID_S 32
+#define IRDMA_CQPSQ_QHASH_VLANID_M \
+ ((u64)0xfff << IRDMA_CQPSQ_QHASH_VLANID_S)
+
+#define IRDMA_CQPSQ_QHASH_QPN_S 32
+#define IRDMA_CQPSQ_QHASH_QPN_M \
+ ((u64)0x3ffff << IRDMA_CQPSQ_QHASH_QPN_S)
+
+#define IRDMA_CQPSQ_QHASH_QS_HANDLE_S 0
+#define IRDMA_CQPSQ_QHASH_QS_HANDLE_M ((u64)0x3ff << IRDMA_CQPSQ_QHASH_QS_HANDLE_S)
+
+#define IRDMA_CQPSQ_QHASH_SRC_PORT_S 16
+#define IRDMA_CQPSQ_QHASH_SRC_PORT_M \
+ ((u64)0xffff << IRDMA_CQPSQ_QHASH_SRC_PORT_S)
+
+#define IRDMA_CQPSQ_QHASH_DEST_PORT_S 0
+#define IRDMA_CQPSQ_QHASH_DEST_PORT_M \
+ ((u64)0xffff << IRDMA_CQPSQ_QHASH_DEST_PORT_S)
+
+#define IRDMA_CQPSQ_QHASH_ADDR0_S 32
+#define IRDMA_CQPSQ_QHASH_ADDR0_M \
+ ((u64)0xffffffff << IRDMA_CQPSQ_QHASH_ADDR0_S)
+
+#define IRDMA_CQPSQ_QHASH_ADDR1_S 0
+#define IRDMA_CQPSQ_QHASH_ADDR1_M \
+ ((u64)0xffffffff << IRDMA_CQPSQ_QHASH_ADDR1_S)
+
+#define IRDMA_CQPSQ_QHASH_ADDR2_S 32
+#define IRDMA_CQPSQ_QHASH_ADDR2_M \
+ ((u64)0xffffffff << IRDMA_CQPSQ_QHASH_ADDR2_S)
+
+#define IRDMA_CQPSQ_QHASH_ADDR3_S 0
+#define IRDMA_CQPSQ_QHASH_ADDR3_M \
+ ((u64)0xffffffff << IRDMA_CQPSQ_QHASH_ADDR3_S)
+
+#define IRDMA_CQPSQ_QHASH_WQEVALID_S 63
+#define IRDMA_CQPSQ_QHASH_WQEVALID_M \
+ BIT_ULL(IRDMA_CQPSQ_QHASH_WQEVALID_S)
+#define IRDMA_CQPSQ_QHASH_OPCODE_S 32
+#define IRDMA_CQPSQ_QHASH_OPCODE_M \
+ ((u64)0x3f << IRDMA_CQPSQ_QHASH_OPCODE_S)
+
+#define IRDMA_CQPSQ_QHASH_MANAGE_S 61
+#define IRDMA_CQPSQ_QHASH_MANAGE_M \
+ ((u64)0x3 << IRDMA_CQPSQ_QHASH_MANAGE_S)
+
+#define IRDMA_CQPSQ_QHASH_IPV4VALID_S 60
+#define IRDMA_CQPSQ_QHASH_IPV4VALID_M \
+ BIT_ULL(IRDMA_CQPSQ_QHASH_IPV4VALID_S)
+
+#define IRDMA_CQPSQ_QHASH_VLANVALID_S 59
+#define IRDMA_CQPSQ_QHASH_VLANVALID_M \
+ BIT_ULL(IRDMA_CQPSQ_QHASH_VLANVALID_S)
+
+#define IRDMA_CQPSQ_QHASH_ENTRYTYPE_S 42
+#define IRDMA_CQPSQ_QHASH_ENTRYTYPE_M \
+ ((u64)0x7 << IRDMA_CQPSQ_QHASH_ENTRYTYPE_S)
+
+/* Stats */
+#define IRDMA_CQPSQ_STATS_WQEVALID_S 63
+#define IRDMA_CQPSQ_STATS_WQEVALID_M \
+ BIT_ULL(IRDMA_CQPSQ_STATS_WQEVALID_S)
+
+#define IRDMA_CQPSQ_STATS_ALLOC_INST_S 62
+#define IRDMA_CQPSQ_STATS_ALLOC_INST_M \
+ BIT_ULL(IRDMA_CQPSQ_STATS_ALLOC_INST_S)
+
+#define IRDMA_CQPSQ_STATS_USE_HMC_FCN_INDEX_S 60
+#define IRDMA_CQPSQ_STATS_USE_HMC_FCN_INDEX_M \
+ BIT_ULL(IRDMA_CQPSQ_STATS_USE_HMC_FCN_INDEX_S)
+
+#define IRDMA_CQPSQ_STATS_USE_INST_S 61
+#define IRDMA_CQPSQ_STATS_USE_INST_M \
+ BIT_ULL(IRDMA_CQPSQ_STATS_USE_INST_S)
+
+#define IRDMA_CQPSQ_STATS_OP_S 32
+#define IRDMA_CQPSQ_STATS_OP_M \
+ ((u64)0x3f << IRDMA_CQPSQ_STATS_OP_S)
+
+#define IRDMA_CQPSQ_STATS_INST_INDEX_S 0
+#define IRDMA_CQPSQ_STATS_INST_INDEX_M \
+ ((u64)0x7f << IRDMA_CQPSQ_STATS_INST_INDEX_S)
+
+#define IRDMA_CQPSQ_STATS_HMC_FCN_INDEX_S 0
+#define IRDMA_CQPSQ_STATS_HMC_FCN_INDEX_M \
+ ((u64)0x3f << IRDMA_CQPSQ_STATS_HMC_FCN_INDEX_S)
+
+/* WS - Work Scheduler */
+#define IRDMA_CQPSQ_WS_WQEVALID_S 63
+#define IRDMA_CQPSQ_WS_WQEVALID_M \
+ BIT_ULL(IRDMA_CQPSQ_WS_WQEVALID_S)
+
+#define IRDMA_CQPSQ_WS_NODEOP_S 52
+#define IRDMA_CQPSQ_WS_NODEOP_M \
+ ((u64)0x3 << IRDMA_CQPSQ_WS_NODEOP_S)
+
+#define IRDMA_CQPSQ_WS_ENABLENODE_S 62
+#define IRDMA_CQPSQ_WS_ENABLENODE_M \
+ BIT_ULL(IRDMA_CQPSQ_WS_ENABLENODE_S)
+
+#define IRDMA_CQPSQ_WS_NODETYPE_S 61
+#define IRDMA_CQPSQ_WS_NODETYPE_M \
+ BIT_ULL(IRDMA_CQPSQ_WS_NODETYPE_S)
+
+#define IRDMA_CQPSQ_WS_PRIOTYPE_S 59
+#define IRDMA_CQPSQ_WS_PRIOTYPE_M \
+ ((u64)0x3 << IRDMA_CQPSQ_WS_PRIOTYPE_S)
+
+#define IRDMA_CQPSQ_WS_TC_S 56
+#define IRDMA_CQPSQ_WS_TC_M \
+ ((u64)0x7 << IRDMA_CQPSQ_WS_TC_S)
+
+#define IRDMA_CQPSQ_WS_VMVFTYPE_S 54
+#define IRDMA_CQPSQ_WS_VMVFTYPE_M \
+ ((u64)0x3 << IRDMA_CQPSQ_WS_VMVFTYPE_S)
+
+#define IRDMA_CQPSQ_WS_VMVFNUM_S 42
+#define IRDMA_CQPSQ_WS_VMVFNUM_M \
+ ((u64)0x3ff << IRDMA_CQPSQ_WS_VMVFNUM_S)
+
+#define IRDMA_CQPSQ_WS_OP_S 32
+#define IRDMA_CQPSQ_WS_OP_M \
+ ((u64)0x3f << IRDMA_CQPSQ_WS_OP_S)
+
+#define IRDMA_CQPSQ_WS_PARENTID_S 16
+#define IRDMA_CQPSQ_WS_PARENTID_M \
+ ((u64)0x3ff << IRDMA_CQPSQ_WS_PARENTID_S)
+
+#define IRDMA_CQPSQ_WS_NODEID_S 0
+#define IRDMA_CQPSQ_WS_NODEID_M \
+ ((u64)0x3ff << IRDMA_CQPSQ_WS_NODEID_S)
+
+#define IRDMA_CQPSQ_WS_VSI_S 48
+#define IRDMA_CQPSQ_WS_VSI_M \
+ ((u64)0x3ff << IRDMA_CQPSQ_WS_VSI_S)
+
+#define IRDMA_CQPSQ_WS_WEIGHT_S 32
+#define IRDMA_CQPSQ_WS_WEIGHT_M \
+ ((u64)0x7f << IRDMA_CQPSQ_WS_WEIGHT_S)
+
+/* UP to UP mapping */
+#define IRDMA_CQPSQ_UP_WQEVALID_S 63
+#define IRDMA_CQPSQ_UP_WQEVALID_M \
+ BIT_ULL(IRDMA_CQPSQ_UP_WQEVALID_S)
+
+#define IRDMA_CQPSQ_UP_USEVLAN_S 62
+#define IRDMA_CQPSQ_UP_USEVLAN_M \
+ BIT_ULL(IRDMA_CQPSQ_UP_USEVLAN_S)
+
+#define IRDMA_CQPSQ_UP_USEOVERRIDE_S 61
+#define IRDMA_CQPSQ_UP_USEOVERRIDE_M \
+ BIT_ULL(IRDMA_CQPSQ_UP_USEOVERRIDE_S)
+
+#define IRDMA_CQPSQ_UP_OP_S 32
+#define IRDMA_CQPSQ_UP_OP_M \
+ ((u64)0x3f << IRDMA_CQPSQ_UP_OP_S)
+
+#define IRDMA_CQPSQ_UP_HMCFCNIDX_S 0
+#define IRDMA_CQPSQ_UP_HMCFCNIDX_M \
+ ((u64)0x3f << IRDMA_CQPSQ_UP_HMCFCNIDX_S)
+
+#define IRDMA_CQPSQ_UP_CNPOVERRIDE_S 32
+#define IRDMA_CQPSQ_UP_CNPOVERRIDE_M \
+ ((u64)0x3f << IRDMA_CQPSQ_UP_CNPOVERRIDE_S)
+
+/* Query RDMA features*/
+#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_WQEVALID_S 63
+#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_WQEVALID_M \
+ BIT_ULL(IRDMA_CQPSQ_QUERY_RDMA_FEATURES_WQEVALID_S)
+
+#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_BUF_LEN_S 0
+#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_BUF_LEN_M \
+ ((u64)0xffffffff << IRDMA_CQPSQ_QUERY_RDMA_FEATURES_BUF_LEN_S)
+
+#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_OP_S 32
+#define IRDMA_CQPSQ_QUERY_RDMA_FEATURES_OP_M \
+ ((u64)0x3f << IRDMA_CQPSQ_QUERY_RDMA_FEATURES_OP_S)
+
+/* CQP Host Context */
+#define IRDMA_CQPHC_EN_DC_TCP_S 25
+#define IRDMA_CQPHC_EN_DC_TCP_M BIT_ULL(IRDMA_CQPHC_EN_DC_TCP_S)
+
+#define IRDMA_CQPHC_SQSIZE_S 8
+#define IRDMA_CQPHC_SQSIZE_M (0xfULL << IRDMA_CQPHC_SQSIZE_S)
+
+#define IRDMA_CQPHC_DISABLE_PFPDUS_S 1
+#define IRDMA_CQPHC_DISABLE_PFPDUS_M BIT_ULL(IRDMA_CQPHC_DISABLE_PFPDUS_S)
+
+#define IRDMA_CQPHC_ROCEV2_RTO_POLICY_S 2
+#define IRDMA_CQPHC_ROCEV2_RTO_POLICY_M BIT_ULL(IRDMA_CQPHC_ROCEV2_RTO_POLICY_S)
+#define IRDMA_CQPHC_PROTOCOL_USED_S 3
+#define IRDMA_CQPHC_PROTOCOL_USED_M (0x3ULL << IRDMA_CQPHC_PROTOCOL_USED_S)
+
+#define IRDMA_CQPHC_HW_MINVER_S 0
+#define IRDMA_CQPHC_HW_MINVER_M (0xffffULL << IRDMA_CQPHC_HW_MINVER_S)
+
+#define IRDMA_CQPHC_HW_MAJVER_S 16
+#define IRDMA_CQPHC_HW_MAJVER_M (0xffffULL << IRDMA_CQPHC_HW_MAJVER_S)
+
+#define IRDMA_CQPHC_STRUCTVER_S 24
+#define IRDMA_CQPHC_STRUCTVER_M (0xffULL << IRDMA_CQPHC_STRUCTVER_S)
+
+#define IRDMA_CQPHC_CEQPERVF_S 32
+#define IRDMA_CQPHC_CEQPERVF_M (0xffULL << IRDMA_CQPHC_CEQPERVF_S)
+
+#define IRDMA_CQPHC_ENABLED_VFS_S 32
+#define IRDMA_CQPHC_ENABLED_VFS_M (0x3fULL << IRDMA_CQPHC_ENABLED_VFS_S)
+
+#define IRDMA_CQPHC_HMC_PROFILE_S 0
+#define IRDMA_CQPHC_HMC_PROFILE_M (0x7ULL << IRDMA_CQPHC_HMC_PROFILE_S)
+
+#define IRDMA_CQPHC_SVER_S 24
+#define IRDMA_CQPHC_SVER_M (0xffULL << IRDMA_CQPHC_SVER_S)
+
+#define IRDMA_CQPHC_SQBASE_S 9
+#define IRDMA_CQPHC_SQBASE_M \
+ (0xfffffffffffffeULL << IRDMA_CQPHC_SQBASE_S)
+
+#define IRDMA_CQPHC_QPCTX_S 0
+#define IRDMA_CQPHC_QPCTX_M \
+ (0xffffffffffffffffULL << IRDMA_CQPHC_QPCTX_S)
+
+/* iWARP QP Doorbell shadow area */
+#define IRDMA_QP_DBSA_HW_SQ_TAIL_S 0
+#define IRDMA_QP_DBSA_HW_SQ_TAIL_M \
+ (0x7fffULL << IRDMA_QP_DBSA_HW_SQ_TAIL_S)
+
+/* Completion Queue Doorbell shadow area */
+#define IRDMA_CQ_DBSA_CQEIDX_S 0
+#define IRDMA_CQ_DBSA_CQEIDX_M (0xfffffULL << IRDMA_CQ_DBSA_CQEIDX_S)
+
+#define IRDMA_CQ_DBSA_SW_CQ_SELECT_S 0
+#define IRDMA_CQ_DBSA_SW_CQ_SELECT_M \
+ (0x3fffULL << IRDMA_CQ_DBSA_SW_CQ_SELECT_S)
+
+#define IRDMA_CQ_DBSA_ARM_NEXT_S 14
+#define IRDMA_CQ_DBSA_ARM_NEXT_M BIT_ULL(IRDMA_CQ_DBSA_ARM_NEXT_S)
+
+#define IRDMA_CQ_DBSA_ARM_NEXT_SE_S 15
+#define IRDMA_CQ_DBSA_ARM_NEXT_SE_M BIT_ULL(IRDMA_CQ_DBSA_ARM_NEXT_SE_S)
+
+#define IRDMA_CQ_DBSA_ARM_SEQ_NUM_S 16
+#define IRDMA_CQ_DBSA_ARM_SEQ_NUM_M \
+ (0x3ULL << IRDMA_CQ_DBSA_ARM_SEQ_NUM_S)
+
+/* CQP and iWARP Completion Queue */
+#define IRDMA_CQ_QPCTX_S IRDMA_CQPHC_QPCTX_S
+#define IRDMA_CQ_QPCTX_M IRDMA_CQPHC_QPCTX_M
+
+#define IRDMA_CCQ_OPRETVAL_S 0
+#define IRDMA_CCQ_OPRETVAL_M (0xffffffffULL << IRDMA_CCQ_OPRETVAL_S)
+
+#define IRDMA_CQ_MINERR_S 0
+#define IRDMA_CQ_MINERR_M (0xffffULL << IRDMA_CQ_MINERR_S)
+
+#define IRDMA_CQ_MAJERR_S 16
+#define IRDMA_CQ_MAJERR_M (0xffffULL << IRDMA_CQ_MAJERR_S)
+
+#define IRDMA_CQ_WQEIDX_S 32
+#define IRDMA_CQ_WQEIDX_M (0x7fffULL << IRDMA_CQ_WQEIDX_S)
+
+#define IRDMA_CQ_EXTCQE_S 50
+#define IRDMA_CQ_EXTCQE_M BIT_ULL(IRDMA_CQ_EXTCQE_S)
+
+#define IRDMA_CQ_ERROR_S 55
+#define IRDMA_CQ_ERROR_M BIT_ULL(IRDMA_CQ_ERROR_S)
+
+#define IRDMA_CQ_SQ_S 62
+#define IRDMA_CQ_SQ_M BIT_ULL(IRDMA_CQ_SQ_S)
+
+#define IRDMA_CQ_VALID_S 63
+#define IRDMA_CQ_VALID_M BIT_ULL(IRDMA_CQ_VALID_S)
+
+#define IRDMA_CQ_IMMVALID_S 62
+#define IRDMA_CQ_IMMVALID_M BIT_ULL(IRDMA_CQ_IMMVALID_S)
+
+#define IRDMA_CQ_UDSMACVALID_S 61
+#define IRDMA_CQ_UDSMACVALID_M BIT_ULL(IRDMA_CQ_UDSMACVALID_S)
+
+#define IRDMA_CQ_UDVLANVALID_S 60
+#define IRDMA_CQ_UDVLANVALID_M BIT_ULL(IRDMA_CQ_UDVLANVALID_S)
+
+#define IRDMA_CQ_UDSMAC_S 0
+#define IRDMA_CQ_UDSMAC_M (0xffffffffffffULL << IRDMA_CQ_UDSMAC_S)
+
+#define IRDMA_CQ_UDVLAN_S 48
+#define IRDMA_CQ_UDVLAN_M (0xffffULL << IRDMA_CQ_UDVLAN_S)
+
+#define IRDMA_CQ_IMMDATA_S 0
+#define IRDMA_CQ_IMMDATA_M (0xffffffffffffffffULL << IRDMA_CQ_IMMVALID_S)
+
+#define IRDMA_CQ_IMMDATALOW32_S 0
+#define IRDMA_CQ_IMMDATALOW32_M (0xffffffffULL << IRDMA_CQ_IMMDATALOW32_S)
+
+#define IRDMA_CQ_IMMDATAUP32_S 32
+#define IRDMA_CQ_IMMDATAUP32_M (0xffffffffULL << IRDMA_CQ_IMMDATAUP32_S)
+
+#define IRDMACQ_PAYLDLEN_S 0
+#define IRDMACQ_PAYLDLEN_M (0xffffffffULL << IRDMACQ_PAYLDLEN_S)
+
+#define IRDMACQ_TCPSEQNUMRTT_S 32
+#define IRDMACQ_TCPSEQNUMRTT_M (0xffffffffULL << IRDMACQ_TCPSEQNUMRTT_S)
+
+#define IRDMACQ_INVSTAG_S 0
+#define IRDMACQ_INVSTAG_M (0xffffffffULL << IRDMACQ_INVSTAG_S)
+
+#define IRDMACQ_QPID_S 32
+#define IRDMACQ_QPID_M (0x3ffffULL << IRDMACQ_QPID_S)
+
+#define IRDMACQ_UDSRCQPN_S 0
+#define IRDMACQ_UDSRCQPN_M (0xffffffffULL << IRDMACQ_UDSRCQPN_S)
+
+#define IRDMACQ_PSHDROP_S 51
+#define IRDMACQ_PSHDROP_M BIT_ULL(IRDMACQ_PSHDROP_S)
+
+#define IRDMACQ_STAG_S 53
+#define IRDMACQ_STAG_M BIT_ULL(IRDMACQ_STAG_S)
+
+#define IRDMACQ_IPV4_S 53
+#define IRDMACQ_IPV4_M BIT_ULL(IRDMACQ_IPV4_S)
+
+#define IRDMACQ_SOEVENT_S 54
+#define IRDMACQ_SOEVENT_M BIT_ULL(IRDMACQ_SOEVENT_S)
+
+#define IRDMACQ_OP_S 56
+#define IRDMACQ_OP_M (0x3fULL << IRDMACQ_OP_S)
+
+/* CEQE format */
+#define IRDMA_CEQE_CQCTX_S 0
+#define IRDMA_CEQE_CQCTX_M \
+ (0x7fffffffffffffffULL << IRDMA_CEQE_CQCTX_S)
+
+#define IRDMA_CEQE_VALID_S 63
+#define IRDMA_CEQE_VALID_M BIT_ULL(IRDMA_CEQE_VALID_S)
+
+/* AEQE format */
+#define IRDMA_AEQE_COMPCTX_S IRDMA_CQPHC_QPCTX_S
+#define IRDMA_AEQE_COMPCTX_M IRDMA_CQPHC_QPCTX_M
+
+#define IRDMA_AEQE_QPCQID_LOW_S 0
+#define IRDMA_AEQE_QPCQID_LOW_M (0x3ffffULL << IRDMA_AEQE_QPCQID_LOW_S)
+
+#define IRDMA_AEQE_QPCQID_HI_S 46
+#define IRDMA_AEQE_QPCQID_HI_M BIT_ULL(IRDMA_AEQE_QPCQID_HI_S)
+
+#define IRDMA_AEQE_WQDESCIDX_S 18
+#define IRDMA_AEQE_WQDESCIDX_M (0x7fffULL << IRDMA_AEQE_WQDESCIDX_S)
+
+#define IRDMA_AEQE_OVERFLOW_S 33
+#define IRDMA_AEQE_OVERFLOW_M BIT_ULL(IRDMA_AEQE_OVERFLOW_S)
+
+#define IRDMA_AEQE_AECODE_S 34
+#define IRDMA_AEQE_AECODE_M (0xfffULL << IRDMA_AEQE_AECODE_S)
+
+#define IRDMA_AEQE_AESRC_S 50
+#define IRDMA_AEQE_AESRC_M (0xfULL << IRDMA_AEQE_AESRC_S)
+
+#define IRDMA_AEQE_IWSTATE_S 54
+#define IRDMA_AEQE_IWSTATE_M (0x7ULL << IRDMA_AEQE_IWSTATE_S)
+
+#define IRDMA_AEQE_TCPSTATE_S 57
+#define IRDMA_AEQE_TCPSTATE_M (0xfULL << IRDMA_AEQE_TCPSTATE_S)
+
+#define IRDMA_AEQE_Q2DATA_S 61
+#define IRDMA_AEQE_Q2DATA_M (0x3ULL << IRDMA_AEQE_Q2DATA_S)
+
+#define IRDMA_AEQE_VALID_S 63
+#define IRDMA_AEQE_VALID_M BIT_ULL(IRDMA_AEQE_VALID_S)
+
+#define IRDMA_UDA_QPSQ_NEXT_HDR_S 16
+#define IRDMA_UDA_QPSQ_NEXT_HDR_M ((u64)0xff << IRDMA_UDA_QPSQ_NEXT_HDR_S)
+
+#define IRDMA_UDA_QPSQ_OPCODE_S 32
+#define IRDMA_UDA_QPSQ_OPCODE_M ((u64)0x3f << IRDMA_UDA_QPSQ_OPCODE_S)
+
+#define IRDMA_UDA_QPSQ_L4LEN_S 42
+#define IRDMA_UDA_QPSQ_L4LEN_M ((u64)0xf << IRDMA_UDA_QPSQ_L4LEN_S)
+
+#define IRDMA_GEN1_UDA_QPSQ_L4LEN_S 24
+#define IRDMA_GEN1_UDA_QPSQ_L4LEN_M ((u64)0xf << IRDMA_GEN1_UDA_QPSQ_L4LEN_S)
+
+#define IRDMA_UDA_QPSQ_AHIDX_S 0
+#define IRDMA_UDA_QPSQ_AHIDX_M ((u64)0x1ffff << IRDMA_UDA_QPSQ_AHIDX_S)
+
+#define IRDMA_UDA_QPSQ_VALID_S 63
+#define IRDMA_UDA_QPSQ_VALID_M \
+ BIT_ULL(IRDMA_UDA_QPSQ_VALID_S)
+
+#define IRDMA_UDA_QPSQ_SIGCOMPL_S 62
+#define IRDMA_UDA_QPSQ_SIGCOMPL_M BIT_ULL(IRDMA_UDA_QPSQ_SIGCOMPL_S)
+
+#define IRDMA_UDA_QPSQ_MACLEN_S 56
+#define IRDMA_UDA_QPSQ_MACLEN_M \
+ ((u64)0x7f << IRDMA_UDA_QPSQ_MACLEN_S)
+
+#define IRDMA_UDA_QPSQ_IPLEN_S 48
+#define IRDMA_UDA_QPSQ_IPLEN_M \
+ ((u64)0x7f << IRDMA_UDA_QPSQ_IPLEN_S)
+
+#define IRDMA_UDA_QPSQ_L4T_S 30
+#define IRDMA_UDA_QPSQ_L4T_M \
+ ((u64)0x3 << IRDMA_UDA_QPSQ_L4T_S)
+
+#define IRDMA_UDA_QPSQ_IIPT_S 28
+#define IRDMA_UDA_QPSQ_IIPT_M \
+ ((u64)0x3 << IRDMA_UDA_QPSQ_IIPT_S)
+
+#define IRDMA_UDA_PAYLOADLEN_S 0
+#define IRDMA_UDA_PAYLOADLEN_M ((u64)0x3fff << IRDMA_UDA_PAYLOADLEN_S)
+
+#define IRDMA_UDA_HDRLEN_S 16
+#define IRDMA_UDA_HDRLEN_M ((u64)0x1ff << IRDMA_UDA_HDRLEN_S)
+
+#define IRDMA_VLAN_TAG_VALID_S 50
+#define IRDMA_VLAN_TAG_VALID_M BIT_ULL(IRDMA_VLAN_TAG_VALID_S)
+
+#define IRDMA_UDA_L3PROTO_S 0
+#define IRDMA_UDA_L3PROTO_M ((u64)0x3 << IRDMA_UDA_L3PROTO_S)
+
+#define IRDMA_UDA_L4PROTO_S 16
+#define IRDMA_UDA_L4PROTO_M ((u64)0x3 << IRDMA_UDA_L4PROTO_S)
+
+#define IRDMA_UDA_QPSQ_DOLOOPBACK_S 44
+#define IRDMA_UDA_QPSQ_DOLOOPBACK_M \
+ BIT_ULL(IRDMA_UDA_QPSQ_DOLOOPBACK_S)
+
+/* CQP SQ WQE common fields */
+#define IRDMA_CQPSQ_BUFSIZE_S 0
+#define IRDMA_CQPSQ_BUFSIZE_M (0xffffffffULL << IRDMA_CQPSQ_BUFSIZE_S)
+
+#define IRDMA_CQPSQ_OPCODE_S 32
+#define IRDMA_CQPSQ_OPCODE_M (0x3fULL << IRDMA_CQPSQ_OPCODE_S)
+
+#define IRDMA_CQPSQ_WQEVALID_S 63
+#define IRDMA_CQPSQ_WQEVALID_M BIT_ULL(IRDMA_CQPSQ_WQEVALID_S)
+
+#define IRDMA_CQPSQ_TPHVAL_S 0
+#define IRDMA_CQPSQ_TPHVAL_M (0xffULL << IRDMA_CQPSQ_TPHVAL_S)
+
+#define IRDMA_CQPSQ_VSIIDX_S 8
+#define IRDMA_CQPSQ_VSIIDX_M (0x3ffULL << IRDMA_CQPSQ_VSIIDX_S)
+
+#define IRDMA_CQPSQ_TPHEN_S 60
+#define IRDMA_CQPSQ_TPHEN_M BIT_ULL(IRDMA_CQPSQ_TPHEN_S)
+
+#define IRDMA_CQPSQ_PBUFADDR_S IRDMA_CQPHC_QPCTX_S
+#define IRDMA_CQPSQ_PBUFADDR_M IRDMA_CQPHC_QPCTX_M
+
+/* Create/Modify/Destroy QP */
+
+#define IRDMA_CQPSQ_QP_NEWMSS_S 32
+#define IRDMA_CQPSQ_QP_NEWMSS_M (0x3fffULL << IRDMA_CQPSQ_QP_NEWMSS_S)
+
+#define IRDMA_CQPSQ_QP_TERMLEN_S 48
+#define IRDMA_CQPSQ_QP_TERMLEN_M (0xfULL << IRDMA_CQPSQ_QP_TERMLEN_S)
+
+#define IRDMA_CQPSQ_QP_QPCTX_S IRDMA_CQPHC_QPCTX_S
+#define IRDMA_CQPSQ_QP_QPCTX_M IRDMA_CQPHC_QPCTX_M
+
+#define IRDMA_CQPSQ_QP_QPID_S 0
+#define IRDMA_CQPSQ_QP_QPID_M (0x3FFFFUL)
+
+#define IRDMA_CQPSQ_QP_OP_S 32
+#define IRDMA_CQPSQ_QP_OP_M IRDMACQ_OP_M
+
+#define IRDMA_CQPSQ_QP_ORDVALID_S 42
+#define IRDMA_CQPSQ_QP_ORDVALID_M BIT_ULL(IRDMA_CQPSQ_QP_ORDVALID_S)
+
+#define IRDMA_CQPSQ_QP_TOECTXVALID_S 43
+#define IRDMA_CQPSQ_QP_TOECTXVALID_M \
+ BIT_ULL(IRDMA_CQPSQ_QP_TOECTXVALID_S)
+
+#define IRDMA_CQPSQ_QP_CACHEDVARVALID_S 44
+#define IRDMA_CQPSQ_QP_CACHEDVARVALID_M \
+ BIT_ULL(IRDMA_CQPSQ_QP_CACHEDVARVALID_S)
+
+#define IRDMA_CQPSQ_QP_VQ_S 45
+#define IRDMA_CQPSQ_QP_VQ_M BIT_ULL(IRDMA_CQPSQ_QP_VQ_S)
+
+#define IRDMA_CQPSQ_QP_FORCELOOPBACK_S 46
+#define IRDMA_CQPSQ_QP_FORCELOOPBACK_M \
+ BIT_ULL(IRDMA_CQPSQ_QP_FORCELOOPBACK_S)
+
+#define IRDMA_CQPSQ_QP_CQNUMVALID_S 47
+#define IRDMA_CQPSQ_QP_CQNUMVALID_M \
+ BIT_ULL(IRDMA_CQPSQ_QP_CQNUMVALID_S)
+
+#define IRDMA_CQPSQ_QP_QPTYPE_S 48
+#define IRDMA_CQPSQ_QP_QPTYPE_M (0x7ULL << IRDMA_CQPSQ_QP_QPTYPE_S)
+
+#define IRDMA_CQPSQ_QP_MACVALID_S 51
+#define IRDMA_CQPSQ_QP_MACVALID_M BIT_ULL(IRDMA_CQPSQ_QP_MACVALID_S)
+
+#define IRDMA_CQPSQ_QP_MSSCHANGE_S 52
+#define IRDMA_CQPSQ_QP_MSSCHANGE_M BIT_ULL(IRDMA_CQPSQ_QP_MSSCHANGE_S)
+
+#define IRDMA_CQPSQ_QP_IGNOREMWBOUND_S 54
+#define IRDMA_CQPSQ_QP_IGNOREMWBOUND_M \
+ BIT_ULL(IRDMA_CQPSQ_QP_IGNOREMWBOUND_S)
+
+#define IRDMA_CQPSQ_QP_REMOVEHASHENTRY_S 55
+#define IRDMA_CQPSQ_QP_REMOVEHASHENTRY_M \
+ BIT_ULL(IRDMA_CQPSQ_QP_REMOVEHASHENTRY_S)
+
+#define IRDMA_CQPSQ_QP_TERMACT_S 56
+#define IRDMA_CQPSQ_QP_TERMACT_M (0x3ULL << IRDMA_CQPSQ_QP_TERMACT_S)
+
+#define IRDMA_CQPSQ_QP_RESETCON_S 58
+#define IRDMA_CQPSQ_QP_RESETCON_M BIT_ULL(IRDMA_CQPSQ_QP_RESETCON_S)
+
+#define IRDMA_CQPSQ_QP_ARPTABIDXVALID_S 59
+#define IRDMA_CQPSQ_QP_ARPTABIDXVALID_M \
+ BIT_ULL(IRDMA_CQPSQ_QP_ARPTABIDXVALID_S)
+
+#define IRDMA_CQPSQ_QP_NEXTIWSTATE_S 60
+#define IRDMA_CQPSQ_QP_NEXTIWSTATE_M \
+ (0x7ULL << IRDMA_CQPSQ_QP_NEXTIWSTATE_S)
+
+#define IRDMA_CQPSQ_QP_DBSHADOWADDR_S IRDMA_CQPHC_QPCTX_S
+#define IRDMA_CQPSQ_QP_DBSHADOWADDR_M IRDMA_CQPHC_QPCTX_M
+
+/* Create/Modify/Destroy CQ */
+#define IRDMA_CQPSQ_CQ_CQSIZE_S 0
+#define IRDMA_CQPSQ_CQ_CQSIZE_M (0x1fffffULL << IRDMA_CQPSQ_CQ_CQSIZE_S)
+
+#define IRDMA_CQPSQ_CQ_CQCTX_S 0
+#define IRDMA_CQPSQ_CQ_CQCTX_M \
+ (0x7fffffffffffffffULL << IRDMA_CQPSQ_CQ_CQCTX_S)
+
+#define IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD_S 0
+#define IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD_M \
+ (0x3ffff << IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD_S)
+
+#define IRDMA_CQPSQ_CQ_OP_S 32
+#define IRDMA_CQPSQ_CQ_OP_M (0x3fULL << IRDMA_CQPSQ_CQ_OP_S)
+
+#define IRDMA_CQPSQ_CQ_CQRESIZE_S 43
+#define IRDMA_CQPSQ_CQ_CQRESIZE_M BIT_ULL(IRDMA_CQPSQ_CQ_CQRESIZE_S)
+
+#define IRDMA_CQPSQ_CQ_LPBLSIZE_S 44
+#define IRDMA_CQPSQ_CQ_LPBLSIZE_M (3ULL << IRDMA_CQPSQ_CQ_LPBLSIZE_S)
+
+#define IRDMA_CQPSQ_CQ_CHKOVERFLOW_S 46
+#define IRDMA_CQPSQ_CQ_CHKOVERFLOW_M \
+ BIT_ULL(IRDMA_CQPSQ_CQ_CHKOVERFLOW_S)
+
+#define IRDMA_CQPSQ_CQ_VIRTMAP_S 47
+#define IRDMA_CQPSQ_CQ_VIRTMAP_M BIT_ULL(IRDMA_CQPSQ_CQ_VIRTMAP_S)
+
+#define IRDMA_CQPSQ_CQ_ENCEQEMASK_S 48
+#define IRDMA_CQPSQ_CQ_ENCEQEMASK_M \
+ BIT_ULL(IRDMA_CQPSQ_CQ_ENCEQEMASK_S)
+
+#define IRDMA_CQPSQ_CQ_CEQIDVALID_S 49
+#define IRDMA_CQPSQ_CQ_CEQIDVALID_M \
+ BIT_ULL(IRDMA_CQPSQ_CQ_CEQIDVALID_S)
+
+#define IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT_S 61
+#define IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT_M \
+ BIT_ULL(IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT_S)
+
+/* Allocate/Register/Register Shared/Deallocate Stag */
+#define IRDMA_CQPSQ_STAG_VA_FBO_S IRDMA_CQPHC_QPCTX_S
+#define IRDMA_CQPSQ_STAG_VA_FBO_M IRDMA_CQPHC_QPCTX_M
+
+#define IRDMA_CQPSQ_STAG_STAGLEN_S 0
+#define IRDMA_CQPSQ_STAG_STAGLEN_M \
+ (0x3fffffffffffULL << IRDMA_CQPSQ_STAG_STAGLEN_S)
+
+#define IRDMA_CQPSQ_STAG_KEY_S 0
+#define IRDMA_CQPSQ_STAG_KEY_M (0xffULL << IRDMA_CQPSQ_STAG_KEY_S)
+
+#define IRDMA_CQPSQ_STAG_IDX_S 8
+#define IRDMA_CQPSQ_STAG_IDX_M (0xffffffULL << IRDMA_CQPSQ_STAG_IDX_S)
+
+#define IRDMA_CQPSQ_STAG_PARENTSTAGIDX_S 32
+#define IRDMA_CQPSQ_STAG_PARENTSTAGIDX_M \
+ (0xffffffULL << IRDMA_CQPSQ_STAG_PARENTSTAGIDX_S)
+
+#define IRDMA_CQPSQ_STAG_MR_S 43
+#define IRDMA_CQPSQ_STAG_MR_M BIT_ULL(IRDMA_CQPSQ_STAG_MR_S)
+
+#define IRDMA_CQPSQ_STAG_MWTYPE_S 42
+#define IRDMA_CQPSQ_STAG_MWTYPE_M BIT_ULL(IRDMA_CQPSQ_STAG_MWTYPE_S)
+
+#define IRDMA_CQPSQ_STAG_MW1_BIND_DONT_VLDT_KEY_S 58
+#define IRDMA_CQPSQ_STAG_MW1_BIND_DONT_VLDT_KEY_M \
+ BIT_ULL(IRDMA_CQPSQ_STAG_MW1_BIND_DONT_VLDT_KEY_S)
+
+#define IRDMA_CQPSQ_STAG_LPBLSIZE_S IRDMA_CQPSQ_CQ_LPBLSIZE_S
+#define IRDMA_CQPSQ_STAG_LPBLSIZE_M IRDMA_CQPSQ_CQ_LPBLSIZE_M
+
+#define IRDMA_CQPSQ_STAG_HPAGESIZE_S 46
+#define IRDMA_CQPSQ_STAG_HPAGESIZE_M \
+ ((u64)3 << IRDMA_CQPSQ_STAG_HPAGESIZE_S)
+
+#define IRDMA_CQPSQ_STAG_ARIGHTS_S 48
+#define IRDMA_CQPSQ_STAG_ARIGHTS_M \
+ (0x1fULL << IRDMA_CQPSQ_STAG_ARIGHTS_S)
+
+#define IRDMA_CQPSQ_STAG_REMACCENABLED_S 53
+#define IRDMA_CQPSQ_STAG_REMACCENABLED_M \
+ BIT_ULL(IRDMA_CQPSQ_STAG_REMACCENABLED_S)
+
+#define IRDMA_CQPSQ_STAG_VABASEDTO_S 59
+#define IRDMA_CQPSQ_STAG_VABASEDTO_M \
+ BIT_ULL(IRDMA_CQPSQ_STAG_VABASEDTO_S)
+
+#define IRDMA_CQPSQ_STAG_USEHMCFNIDX_S 60
+#define IRDMA_CQPSQ_STAG_USEHMCFNIDX_M \
+ BIT_ULL(IRDMA_CQPSQ_STAG_USEHMCFNIDX_S)
+
+#define IRDMA_CQPSQ_STAG_USEPFRID_S 61
+#define IRDMA_CQPSQ_STAG_USEPFRID_M \
+ BIT_ULL(IRDMA_CQPSQ_STAG_USEPFRID_S)
+
+#define IRDMA_CQPSQ_STAG_PBA_S IRDMA_CQPHC_QPCTX_S
+#define IRDMA_CQPSQ_STAG_PBA_M IRDMA_CQPHC_QPCTX_M
+
+#define IRDMA_CQPSQ_STAG_HMCFNIDX_S 0
+#define IRDMA_CQPSQ_STAG_HMCFNIDX_M \
+ (0x3fULL << IRDMA_CQPSQ_STAG_HMCFNIDX_S)
+
+#define IRDMA_CQPSQ_STAG_FIRSTPMPBLIDX_S 0
+#define IRDMA_CQPSQ_STAG_FIRSTPMPBLIDX_M \
+ (0xfffffffULL << IRDMA_CQPSQ_STAG_FIRSTPMPBLIDX_S)
+
+#define IRDMA_CQPSQ_QUERYSTAG_IDX_S IRDMA_CQPSQ_STAG_IDX_S
+#define IRDMA_CQPSQ_QUERYSTAG_IDX_M IRDMA_CQPSQ_STAG_IDX_M
+
+/* Manage Local MAC Table - MLM */
+#define IRDMA_CQPSQ_MLM_TABLEIDX_S 0
+#define IRDMA_CQPSQ_MLM_TABLEIDX_M \
+ (0x3fULL << IRDMA_CQPSQ_MLM_TABLEIDX_S)
+
+#define IRDMA_CQPSQ_MLM_FREEENTRY_S 62
+#define IRDMA_CQPSQ_MLM_FREEENTRY_M \
+ BIT_ULL(IRDMA_CQPSQ_MLM_FREEENTRY_S)
+
+#define IRDMA_CQPSQ_MLM_IGNORE_REF_CNT_S 61
+#define IRDMA_CQPSQ_MLM_IGNORE_REF_CNT_M \
+ BIT_ULL(IRDMA_CQPSQ_MLM_IGNORE_REF_CNT_S)
+
+#define IRDMA_CQPSQ_MLM_MAC0_S 0
+#define IRDMA_CQPSQ_MLM_MAC0_M (0xffULL << IRDMA_CQPSQ_MLM_MAC0_S)
+
+#define IRDMA_CQPSQ_MLM_MAC1_S 8
+#define IRDMA_CQPSQ_MLM_MAC1_M (0xffULL << IRDMA_CQPSQ_MLM_MAC1_S)
+
+#define IRDMA_CQPSQ_MLM_MAC2_S 16
+#define IRDMA_CQPSQ_MLM_MAC2_M (0xffULL << IRDMA_CQPSQ_MLM_MAC2_S)
+
+#define IRDMA_CQPSQ_MLM_MAC3_S 24
+#define IRDMA_CQPSQ_MLM_MAC3_M (0xffULL << IRDMA_CQPSQ_MLM_MAC3_S)
+
+#define IRDMA_CQPSQ_MLM_MAC4_S 32
+#define IRDMA_CQPSQ_MLM_MAC4_M (0xffULL << IRDMA_CQPSQ_MLM_MAC4_S)
+
+#define IRDMA_CQPSQ_MLM_MAC5_S 40
+#define IRDMA_CQPSQ_MLM_MAC5_M (0xffULL << IRDMA_CQPSQ_MLM_MAC5_S)
+
+/* Manage ARP Table - MAT */
+#define IRDMA_CQPSQ_MAT_REACHMAX_S 0
+#define IRDMA_CQPSQ_MAT_REACHMAX_M \
+ (0xffffffffULL << IRDMA_CQPSQ_MAT_REACHMAX_S)
+
+#define IRDMA_CQPSQ_MAT_MACADDR_S 0
+#define IRDMA_CQPSQ_MAT_MACADDR_M \
+ (0xffffffffffffULL << IRDMA_CQPSQ_MAT_MACADDR_S)
+
+#define IRDMA_CQPSQ_MAT_ARPENTRYIDX_S 0
+#define IRDMA_CQPSQ_MAT_ARPENTRYIDX_M \
+ (0xfffULL << IRDMA_CQPSQ_MAT_ARPENTRYIDX_S)
+
+#define IRDMA_CQPSQ_MAT_ENTRYVALID_S 42
+#define IRDMA_CQPSQ_MAT_ENTRYVALID_M \
+ BIT_ULL(IRDMA_CQPSQ_MAT_ENTRYVALID_S)
+
+#define IRDMA_CQPSQ_MAT_PERMANENT_S 43
+#define IRDMA_CQPSQ_MAT_PERMANENT_M \
+ BIT_ULL(IRDMA_CQPSQ_MAT_PERMANENT_S)
+
+#define IRDMA_CQPSQ_MAT_QUERY_S 44
+#define IRDMA_CQPSQ_MAT_QUERY_M BIT_ULL(IRDMA_CQPSQ_MAT_QUERY_S)
+
+/* Manage VF PBLE Backing Pages - MVPBP*/
+#define IRDMA_CQPSQ_MVPBP_PD_ENTRY_CNT_S 0
+#define IRDMA_CQPSQ_MVPBP_PD_ENTRY_CNT_M \
+ (0x3ffULL << IRDMA_CQPSQ_MVPBP_PD_ENTRY_CNT_S)
+
+#define IRDMA_CQPSQ_MVPBP_FIRST_PD_INX_S 16
+#define IRDMA_CQPSQ_MVPBP_FIRST_PD_INX_M \
+ (0x1ffULL << IRDMA_CQPSQ_MVPBP_FIRST_PD_INX_S)
+
+#define IRDMA_CQPSQ_MVPBP_SD_INX_S 32
+#define IRDMA_CQPSQ_MVPBP_SD_INX_M \
+ (0xfffULL << IRDMA_CQPSQ_MVPBP_SD_INX_S)
+
+#define IRDMA_CQPSQ_MVPBP_INV_PD_ENT_S 62
+#define IRDMA_CQPSQ_MVPBP_INV_PD_ENT_M \
+ BIT_ULL(IRDMA_CQPSQ_MVPBP_INV_PD_ENT_S)
+
+#define IRDMA_CQPSQ_MVPBP_PD_PLPBA_S 3
+#define IRDMA_CQPSQ_MVPBP_PD_PLPBA_M \
+ (0x1fffffffffffffffULL << IRDMA_CQPSQ_MVPBP_PD_PLPBA_S)
+
+/* Manage Push Page - MPP */
+#define IRDMA_INVALID_PUSH_PAGE_INDEX 0xffff
+
+#define IRDMA_CQPSQ_MPP_QS_HANDLE_S 0
+#define IRDMA_CQPSQ_MPP_QS_HANDLE_M \
+ (0x3ffULL << IRDMA_CQPSQ_MPP_QS_HANDLE_S)
+
+#define IRDMA_CQPSQ_MPP_PPIDX_S 0
+#define IRDMA_CQPSQ_MPP_PPIDX_M (0x3ffULL << IRDMA_CQPSQ_MPP_PPIDX_S)
+
+#define IRDMA_CQPSQ_MPP_PPTYPE_S 60
+#define IRDMA_CQPSQ_MPP_PPTYPE_M (0x3ULL << IRDMA_CQPSQ_MPP_PPTYPE_S)
+
+#define IRDMA_CQPSQ_MPP_FREE_PAGE_S 62
+#define IRDMA_CQPSQ_MPP_FREE_PAGE_M BIT_ULL(IRDMA_CQPSQ_MPP_FREE_PAGE_S)
+
+/* Upload Context - UCTX */
+#define IRDMA_CQPSQ_UCTX_QPCTXADDR_S IRDMA_CQPHC_QPCTX_S
+#define IRDMA_CQPSQ_UCTX_QPCTXADDR_M IRDMA_CQPHC_QPCTX_M
+
+#define IRDMA_CQPSQ_UCTX_QPID_S 0
+#define IRDMA_CQPSQ_UCTX_QPID_M (0x3ffffULL << IRDMA_CQPSQ_UCTX_QPID_S)
+
+#define IRDMA_CQPSQ_UCTX_QPTYPE_S 48
+#define IRDMA_CQPSQ_UCTX_QPTYPE_M (0xfULL << IRDMA_CQPSQ_UCTX_QPTYPE_S)
+
+#define IRDMA_CQPSQ_UCTX_RAWFORMAT_S 61
+#define IRDMA_CQPSQ_UCTX_RAWFORMAT_M \
+ BIT_ULL(IRDMA_CQPSQ_UCTX_RAWFORMAT_S)
+
+#define IRDMA_CQPSQ_UCTX_FREEZEQP_S 62
+#define IRDMA_CQPSQ_UCTX_FREEZEQP_M \
+ BIT_ULL(IRDMA_CQPSQ_UCTX_FREEZEQP_S)
+
+/* Manage HMC PM Function Table - MHMC */
+#define IRDMA_CQPSQ_MHMC_VFIDX_S 0
+#define IRDMA_CQPSQ_MHMC_VFIDX_M (0xffULL << IRDMA_CQPSQ_MHMC_VFIDX_S)
+
+#define IRDMA_CQPSQ_MHMC_FREEPMFN_S 62
+#define IRDMA_CQPSQ_MHMC_FREEPMFN_M \
+ BIT_ULL(IRDMA_CQPSQ_MHMC_FREEPMFN_S)
+
+/* Set HMC Resource Profile - SHMCRP */
+#define IRDMA_CQPSQ_SHMCRP_HMC_PROFILE_S 0
+#define IRDMA_CQPSQ_SHMCRP_HMC_PROFILE_M \
+ (0x7ULL << IRDMA_CQPSQ_SHMCRP_HMC_PROFILE_S)
+#define IRDMA_CQPSQ_SHMCRP_VFNUM_S 32
+#define IRDMA_CQPSQ_SHMCRP_VFNUM_M (0x3fULL << IRDMA_CQPSQ_SHMCRP_VFNUM_S)
+
+/* Create/Destroy CEQ */
+#define IRDMA_CQPSQ_CEQ_CEQSIZE_S 0
+#define IRDMA_CQPSQ_CEQ_CEQSIZE_M \
+ (0x1ffffULL << IRDMA_CQPSQ_CEQ_CEQSIZE_S)
+
+#define IRDMA_CQPSQ_CEQ_CEQID_S 0
+#define IRDMA_CQPSQ_CEQ_CEQID_M (0x3ffULL << IRDMA_CQPSQ_CEQ_CEQID_S)
+
+#define IRDMA_CQPSQ_CEQ_LPBLSIZE_S IRDMA_CQPSQ_CQ_LPBLSIZE_S
+#define IRDMA_CQPSQ_CEQ_LPBLSIZE_M IRDMA_CQPSQ_CQ_LPBLSIZE_M
+
+#define IRDMA_CQPSQ_CEQ_VMAP_S 47
+#define IRDMA_CQPSQ_CEQ_VMAP_M BIT_ULL(IRDMA_CQPSQ_CEQ_VMAP_S)
+
+#define IRDMA_CQPSQ_CEQ_ITRNOEXPIRE_S 46
+#define IRDMA_CQPSQ_CEQ_ITRNOEXPIRE_M BIT_ULL(IRDMA_CQPSQ_CEQ_ITRNOEXPIRE_S)
+
+#define IRDMA_CQPSQ_CEQ_FIRSTPMPBLIDX_S 0
+#define IRDMA_CQPSQ_CEQ_FIRSTPMPBLIDX_M \
+ (0xfffffffULL << IRDMA_CQPSQ_CEQ_FIRSTPMPBLIDX_S)
+
+/* Create/Destroy AEQ */
+#define IRDMA_CQPSQ_AEQ_AEQECNT_S 0
+#define IRDMA_CQPSQ_AEQ_AEQECNT_M \
+ (0x7ffffULL << IRDMA_CQPSQ_AEQ_AEQECNT_S)
+
+#define IRDMA_CQPSQ_AEQ_LPBLSIZE_S IRDMA_CQPSQ_CQ_LPBLSIZE_S
+#define IRDMA_CQPSQ_AEQ_LPBLSIZE_M IRDMA_CQPSQ_CQ_LPBLSIZE_M
+
+#define IRDMA_CQPSQ_AEQ_VMAP_S 47
+#define IRDMA_CQPSQ_AEQ_VMAP_M BIT_ULL(IRDMA_CQPSQ_AEQ_VMAP_S)
+
+#define IRDMA_CQPSQ_AEQ_FIRSTPMPBLIDX_S 0
+#define IRDMA_CQPSQ_AEQ_FIRSTPMPBLIDX_M \
+ (0xfffffffULL << IRDMA_CQPSQ_AEQ_FIRSTPMPBLIDX_S)
+
+/* Commit FPM Values - CFPM */
+#define IRDMA_COMMIT_FPM_QPCNT_S 0
+#define IRDMA_COMMIT_FPM_QPCNT_M (0x7ffffULL << IRDMA_COMMIT_FPM_QPCNT_S)
+
+#define IRDMA_COMMIT_FPM_CQCNT_S 0
+#define IRDMA_COMMIT_FPM_CQCNT_M (0xfffffULL << IRDMA_COMMIT_FPM_CQCNT_S)
+
+#define IRDMA_COMMIT_FPM_BASE_S 32
+
+#define IRDMA_CQPSQ_CFPM_HMCFNID_S 0
+#define IRDMA_CQPSQ_CFPM_HMCFNID_M (0x3fULL << IRDMA_CQPSQ_CFPM_HMCFNID_S)
+
+/* Flush WQEs - FWQE */
+#define IRDMA_CQPSQ_FWQE_AECODE_S 0
+#define IRDMA_CQPSQ_FWQE_AECODE_M (0xffffULL << IRDMA_CQPSQ_FWQE_AECODE_S)
+
+#define IRDMA_CQPSQ_FWQE_AESOURCE_S 16
+#define IRDMA_CQPSQ_FWQE_AESOURCE_M \
+ (0xfULL << IRDMA_CQPSQ_FWQE_AESOURCE_S)
+
+#define IRDMA_CQPSQ_FWQE_RQMNERR_S 0
+#define IRDMA_CQPSQ_FWQE_RQMNERR_M \
+ (0xffffULL << IRDMA_CQPSQ_FWQE_RQMNERR_S)
+
+#define IRDMA_CQPSQ_FWQE_RQMJERR_S 16
+#define IRDMA_CQPSQ_FWQE_RQMJERR_M \
+ (0xffffULL << IRDMA_CQPSQ_FWQE_RQMJERR_S)
+
+#define IRDMA_CQPSQ_FWQE_SQMNERR_S 32
+#define IRDMA_CQPSQ_FWQE_SQMNERR_M \
+ (0xffffULL << IRDMA_CQPSQ_FWQE_SQMNERR_S)
+
+#define IRDMA_CQPSQ_FWQE_SQMJERR_S 48
+#define IRDMA_CQPSQ_FWQE_SQMJERR_M \
+ (0xffffULL << IRDMA_CQPSQ_FWQE_SQMJERR_S)
+
+#define IRDMA_CQPSQ_FWQE_QPID_S 0
+#define IRDMA_CQPSQ_FWQE_QPID_M (0x3ffffULL << IRDMA_CQPSQ_FWQE_QPID_S)
+
+#define IRDMA_CQPSQ_FWQE_GENERATE_AE_S 59
+#define IRDMA_CQPSQ_FWQE_GENERATE_AE_M \
+ BIT_ULL(IRDMA_CQPSQ_FWQE_GENERATE_AE_S)
+
+#define IRDMA_CQPSQ_FWQE_USERFLCODE_S 60
+#define IRDMA_CQPSQ_FWQE_USERFLCODE_M \
+ BIT_ULL(IRDMA_CQPSQ_FWQE_USERFLCODE_S)
+
+#define IRDMA_CQPSQ_FWQE_FLUSHSQ_S 61
+#define IRDMA_CQPSQ_FWQE_FLUSHSQ_M BIT_ULL(IRDMA_CQPSQ_FWQE_FLUSHSQ_S)
+
+#define IRDMA_CQPSQ_FWQE_FLUSHRQ_S 62
+#define IRDMA_CQPSQ_FWQE_FLUSHRQ_M BIT_ULL(IRDMA_CQPSQ_FWQE_FLUSHRQ_S)
+
+/* Manage Accelerated Port Table - MAPT */
+#define IRDMA_CQPSQ_MAPT_PORT_S 0
+#define IRDMA_CQPSQ_MAPT_PORT_M (0xffffULL << IRDMA_CQPSQ_MAPT_PORT_S)
+
+#define IRDMA_CQPSQ_MAPT_ADDPORT_S 62
+#define IRDMA_CQPSQ_MAPT_ADDPORT_M BIT_ULL(IRDMA_CQPSQ_MAPT_ADDPORT_S)
+
+/* Update Protocol Engine SDs */
+#define IRDMA_CQPSQ_UPESD_SDCMD_S 0
+#define IRDMA_CQPSQ_UPESD_SDCMD_M (0xffffffffULL << IRDMA_CQPSQ_UPESD_SDCMD_S)
+
+#define IRDMA_CQPSQ_UPESD_SDDATALOW_S 0
+#define IRDMA_CQPSQ_UPESD_SDDATALOW_M \
+ (0xffffffffULL << IRDMA_CQPSQ_UPESD_SDDATALOW_S)
+
+#define IRDMA_CQPSQ_UPESD_SDDATAHI_S 32
+#define IRDMA_CQPSQ_UPESD_SDDATAHI_M \
+ (0xffffffffULL << IRDMA_CQPSQ_UPESD_SDDATAHI_S)
+#define IRDMA_CQPSQ_UPESD_HMCFNID_S 0
+#define IRDMA_CQPSQ_UPESD_HMCFNID_M \
+ (0x3fULL << IRDMA_CQPSQ_UPESD_HMCFNID_S)
+
+#define IRDMA_CQPSQ_UPESD_ENTRY_VALID_S 63
+#define IRDMA_CQPSQ_UPESD_ENTRY_VALID_M \
+ BIT_ULL(IRDMA_CQPSQ_UPESD_ENTRY_VALID_S)
+
+#define IRDMA_CQPSQ_UPESD_ENTRY_COUNT_S 0
+#define IRDMA_CQPSQ_UPESD_ENTRY_COUNT_M \
+ (0xfULL << IRDMA_CQPSQ_UPESD_ENTRY_COUNT_S)
+
+#define IRDMA_CQPSQ_UPESD_SKIP_ENTRY_S 7
+#define IRDMA_CQPSQ_UPESD_SKIP_ENTRY_M \
+ BIT_ULL(IRDMA_CQPSQ_UPESD_SKIP_ENTRY_S)
+
+/* Suspend QP */
+#define IRDMA_CQPSQ_SUSPENDQP_QPID_S 0
+#define IRDMA_CQPSQ_SUSPENDQP_QPID_M (0x3FFFFULL)
+
+/* Resume QP */
+#define IRDMA_CQPSQ_RESUMEQP_QSHANDLE_S 0
+#define IRDMA_CQPSQ_RESUMEQP_QSHANDLE_M \
+ (0xffffffffULL << IRDMA_CQPSQ_RESUMEQP_QSHANDLE_S)
+
+#define IRDMA_CQPSQ_RESUMEQP_QPID_S 0
+#define IRDMA_CQPSQ_RESUMEQP_QPID_M (0x3FFFFUL)
+
+/* IW QP Context */
+#define IRDMAQPC_DDP_VER_S 0
+#define IRDMAQPC_DDP_VER_M (3ULL << IRDMAQPC_DDP_VER_S)
+
+#define IRDMAQPC_IBRDENABLE_S 2
+#define IRDMAQPC_IBRDENABLE_M BIT_ULL(IRDMAQPC_IBRDENABLE_S)
+
+#define IRDMAQPC_IPV4_S 3
+#define IRDMAQPC_IPV4_M BIT_ULL(IRDMAQPC_IPV4_S)
+
+#define IRDMAQPC_NONAGLE_S 4
+#define IRDMAQPC_NONAGLE_M BIT_ULL(IRDMAQPC_NONAGLE_S)
+
+#define IRDMAQPC_INSERTVLANTAG_S 5
+#define IRDMAQPC_INSERTVLANTAG_M BIT_ULL(IRDMAQPC_INSERTVLANTAG_S)
+
+#define IRDMAQPC_ISQP1_S 6
+#define IRDMAQPC_ISQP1_M BIT_ULL(IRDMAQPC_ISQP1_S)
+
+#define IRDMAQPC_TIMESTAMP_S 7
+#define IRDMAQPC_TIMESTAMP_M BIT_ULL(IRDMAQPC_TIMESTAMP_S)
+
+#define IRDMAQPC_RQWQESIZE_S 8
+#define IRDMAQPC_RQWQESIZE_M (3ULL << IRDMAQPC_RQWQESIZE_S)
+
+#define IRDMAQPC_INSERTL2TAG2_S 11
+#define IRDMAQPC_INSERTL2TAG2_M BIT_ULL(IRDMAQPC_INSERTL2TAG2_S)
+
+#define IRDMAQPC_LIMIT_S 12
+#define IRDMAQPC_LIMIT_M (3ULL << IRDMAQPC_LIMIT_S)
+
+#define IRDMAQPC_ECN_EN_S 14
+#define IRDMAQPC_ECN_EN_M BIT_ULL(IRDMAQPC_ECN_EN_S)
+
+#define IRDMAQPC_DROPOOOSEG_S 15
+#define IRDMAQPC_DROPOOOSEG_M BIT_ULL(IRDMAQPC_DROPOOOSEG_S)
+
+#define IRDMAQPC_DUPACK_THRESH_S 16
+#define IRDMAQPC_DUPACK_THRESH_M (7ULL << IRDMAQPC_DUPACK_THRESH_S)
+
+#define IRDMAQPC_ERR_RQ_IDX_VALID_S 19
+#define IRDMAQPC_ERR_RQ_IDX_VALID_M BIT_ULL(IRDMAQPC_ERR_RQ_IDX_VALID_S)
+
+#define IRDMAQPC_DIS_VLAN_CHECKS_S 19
+#define IRDMAQPC_DIS_VLAN_CHECKS_M (7ULL << IRDMAQPC_DIS_VLAN_CHECKS_S)
+
+#define IRDMAQPC_DC_TCP_EN_S 25
+#define IRDMAQPC_DC_TCP_EN_M BIT_ULL(IRDMAQPC_DC_TCP_EN_S)
+
+#define IRDMAQPC_RCVTPHEN_S 28
+#define IRDMAQPC_RCVTPHEN_M BIT_ULL(IRDMAQPC_RCVTPHEN_S)
+
+#define IRDMAQPC_XMITTPHEN_S 29
+#define IRDMAQPC_XMITTPHEN_M BIT_ULL(IRDMAQPC_XMITTPHEN_S)
+
+#define IRDMAQPC_RQTPHEN_S 30
+#define IRDMAQPC_RQTPHEN_M BIT_ULL(IRDMAQPC_RQTPHEN_S)
+
+#define IRDMAQPC_SQTPHEN_S 31
+#define IRDMAQPC_SQTPHEN_M BIT_ULL(IRDMAQPC_SQTPHEN_S)
+
+#define IRDMAQPC_PPIDX_S 32
+#define IRDMAQPC_PPIDX_M (0x3ffULL << IRDMAQPC_PPIDX_S)
+
+#define IRDMAQPC_PMENA_S 47
+#define IRDMAQPC_PMENA_M BIT_ULL(IRDMAQPC_PMENA_S)
+
+#define IRDMAQPC_RDMAP_VER_S 62
+#define IRDMAQPC_RDMAP_VER_M (3ULL << IRDMAQPC_RDMAP_VER_S)
+
+#define IRDMAQPC_ROCE_TVER_S 60
+#define IRDMAQPC_ROCE_TVER_M (0x0fULL << IRDMAQPC_ROCE_TVER_S)
+#define IRDMAQPC_SQADDR_S IRDMA_CQPHC_QPCTX_S
+#define IRDMAQPC_SQADDR_M IRDMA_CQPHC_QPCTX_M
+
+#define IRDMAQPC_RQADDR_S IRDMA_CQPHC_QPCTX_S
+#define IRDMAQPC_RQADDR_M IRDMA_CQPHC_QPCTX_M
+
+#define IRDMAQPC_TTL_S 0
+#define IRDMAQPC_TTL_M (0xffULL << IRDMAQPC_TTL_S)
+
+#define IRDMAQPC_RQSIZE_S 8
+#define IRDMAQPC_RQSIZE_M (0xfULL << IRDMAQPC_RQSIZE_S)
+
+#define IRDMAQPC_SQSIZE_S 12
+#define IRDMAQPC_SQSIZE_M (0xfULL << IRDMAQPC_SQSIZE_S)
+
+#define IRDMAQPC_GEN1_SRCMACADDRIDX_S 16
+#define IRDMAQPC_GEN1_SRCMACADDRIDX_M (0x3fUL << IRDMAQPC_GEN1_SRCMACADDRIDX_S)
+
+#define IRDMAQPC_AVOIDSTRETCHACK_S 23
+#define IRDMAQPC_AVOIDSTRETCHACK_M BIT_ULL(IRDMAQPC_AVOIDSTRETCHACK_S)
+
+#define IRDMAQPC_TOS_S 24
+#define IRDMAQPC_TOS_M (0xffULL << IRDMAQPC_TOS_S)
+
+#define IRDMAQPC_SRCPORTNUM_S 32
+#define IRDMAQPC_SRCPORTNUM_M (0xffffULL << IRDMAQPC_SRCPORTNUM_S)
+
+#define IRDMAQPC_DESTPORTNUM_S 48
+#define IRDMAQPC_DESTPORTNUM_M (0xffffULL << IRDMAQPC_DESTPORTNUM_S)
+
+#define IRDMAQPC_DESTIPADDR0_S 32
+#define IRDMAQPC_DESTIPADDR0_M \
+ (0xffffffffULL << IRDMAQPC_DESTIPADDR0_S)
+
+#define IRDMAQPC_DESTIPADDR1_S 0
+#define IRDMAQPC_DESTIPADDR1_M \
+ (0xffffffffULL << IRDMAQPC_DESTIPADDR1_S)
+
+#define IRDMAQPC_DESTIPADDR2_S 32
+#define IRDMAQPC_DESTIPADDR2_M \
+ (0xffffffffULL << IRDMAQPC_DESTIPADDR2_S)
+
+#define IRDMAQPC_DESTIPADDR3_S 0
+#define IRDMAQPC_DESTIPADDR3_M \
+ (0xffffffffULL << IRDMAQPC_DESTIPADDR3_S)
+
+#define IRDMAQPC_SNDMSS_S 16
+#define IRDMAQPC_SNDMSS_M (0x3fffULL << IRDMAQPC_SNDMSS_S)
+
+#define IRDMAQPC_VLANTAG_S 32
+#define IRDMAQPC_VLANTAG_M (0xffffULL << IRDMAQPC_VLANTAG_S)
+
+#define IRDMAQPC_ARPIDX_S 48
+#define IRDMAQPC_ARPIDX_M (0xffffULL << IRDMAQPC_ARPIDX_S)
+
+#define IRDMAQPC_FLOWLABEL_S 0
+#define IRDMAQPC_FLOWLABEL_M (0xfffffULL << IRDMAQPC_FLOWLABEL_S)
+
+#define IRDMAQPC_WSCALE_S 20
+#define IRDMAQPC_WSCALE_M BIT_ULL(IRDMAQPC_WSCALE_S)
+
+#define IRDMAQPC_KEEPALIVE_S 21
+#define IRDMAQPC_KEEPALIVE_M BIT_ULL(IRDMAQPC_KEEPALIVE_S)
+
+#define IRDMAQPC_IGNORE_TCP_OPT_S 22
+#define IRDMAQPC_IGNORE_TCP_OPT_M BIT_ULL(IRDMAQPC_IGNORE_TCP_OPT_S)
+
+#define IRDMAQPC_IGNORE_TCP_UNS_OPT_S 23
+#define IRDMAQPC_IGNORE_TCP_UNS_OPT_M \
+ BIT_ULL(IRDMAQPC_IGNORE_TCP_UNS_OPT_S)
+
+#define IRDMAQPC_TCPSTATE_S 28
+#define IRDMAQPC_TCPSTATE_M (0xfULL << IRDMAQPC_TCPSTATE_S)
+
+#define IRDMAQPC_RCVSCALE_S 32
+#define IRDMAQPC_RCVSCALE_M (0xfULL << IRDMAQPC_RCVSCALE_S)
+
+#define IRDMAQPC_SNDSCALE_S 40
+#define IRDMAQPC_SNDSCALE_M (0xfULL << IRDMAQPC_SNDSCALE_S)
+
+#define IRDMAQPC_PDIDX_S 48
+#define IRDMAQPC_PDIDX_M (0xffffULL << IRDMAQPC_PDIDX_S)
+
+#define IRDMAQPC_PDIDXHI_S 20
+#define IRDMAQPC_PDIDXHI_M (0x3ULL << IRDMAQPC_PDIDXHI_S)
+
+#define IRDMAQPC_PKEY_S 32
+#define IRDMAQPC_PKEY_M (0xffffULL << IRDMAQPC_PKEY_S)
+
+#define IRDMAQPC_ACKCREDITS_S 20
+#define IRDMAQPC_ACKCREDITS_M (0x1fULL << IRDMAQPC_ACKCREDITS_S)
+
+#define IRDMAQPC_QKEY_S 32
+#define IRDMAQPC_QKEY_M (0xffffffffULL << IRDMAQPC_QKEY_S)
+
+#define IRDMAQPC_DESTQP_S 0
+#define IRDMAQPC_DESTQP_M (0xffffffULL << IRDMAQPC_DESTQP_S)
+
+#define IRDMAQPC_KALIVE_TIMER_MAX_PROBES_S 16
+#define IRDMAQPC_KALIVE_TIMER_MAX_PROBES_M \
+ (0xffULL << IRDMAQPC_KALIVE_TIMER_MAX_PROBES_S)
+
+#define IRDMAQPC_KEEPALIVE_INTERVAL_S 24
+#define IRDMAQPC_KEEPALIVE_INTERVAL_M \
+ (0xffULL << IRDMAQPC_KEEPALIVE_INTERVAL_S)
+
+#define IRDMAQPC_TIMESTAMP_RECENT_S 0
+#define IRDMAQPC_TIMESTAMP_RECENT_M \
+ (0xffffffffULL << IRDMAQPC_TIMESTAMP_RECENT_S)
+
+#define IRDMAQPC_TIMESTAMP_AGE_S 32
+#define IRDMAQPC_TIMESTAMP_AGE_M \
+ (0xffffffffULL << IRDMAQPC_TIMESTAMP_AGE_S)
+
+#define IRDMAQPC_SNDNXT_S 0
+#define IRDMAQPC_SNDNXT_M (0xffffffffULL << IRDMAQPC_SNDNXT_S)
+
+#define IRDMAQPC_ISN_S 32
+#define IRDMAQPC_ISN_M (0x00ffffffULL << IRDMAQPC_ISN_S)
+
+#define IRDMAQPC_PSNNXT_S 0
+#define IRDMAQPC_PSNNXT_M (0x00ffffffULL << IRDMAQPC_PSNNXT_S)
+
+#define IRDMAQPC_LSN_S 32
+#define IRDMAQPC_LSN_M (0x00ffffffULL << IRDMAQPC_LSN_S)
+
+#define IRDMAQPC_SNDWND_S 32
+#define IRDMAQPC_SNDWND_M (0xffffffffULL << IRDMAQPC_SNDWND_S)
+
+#define IRDMAQPC_RCVNXT_S 0
+#define IRDMAQPC_RCVNXT_M (0xffffffffULL << IRDMAQPC_RCVNXT_S)
+
+#define IRDMAQPC_EPSN_S 0
+#define IRDMAQPC_EPSN_M (0x00ffffffULL << IRDMAQPC_EPSN_S)
+
+#define IRDMAQPC_RCVWND_S 32
+#define IRDMAQPC_RCVWND_M (0xffffffffULL << IRDMAQPC_RCVWND_S)
+
+#define IRDMAQPC_SNDMAX_S 0
+#define IRDMAQPC_SNDMAX_M (0xffffffffULL << IRDMAQPC_SNDMAX_S)
+
+#define IRDMAQPC_SNDUNA_S 32
+#define IRDMAQPC_SNDUNA_M (0xffffffffULL << IRDMAQPC_SNDUNA_S)
+
+#define IRDMAQPC_PSNMAX_S 0
+#define IRDMAQPC_PSNMAX_M (0x00ffffffULL << IRDMAQPC_PSNMAX_S)
+#define IRDMAQPC_PSNUNA_S 32
+#define IRDMAQPC_PSNUNA_M (0xffffffULL << IRDMAQPC_PSNUNA_S)
+
+#define IRDMAQPC_SRTT_S 0
+#define IRDMAQPC_SRTT_M (0xffffffffULL << IRDMAQPC_SRTT_S)
+
+#define IRDMAQPC_RTTVAR_S 32
+#define IRDMAQPC_RTTVAR_M (0xffffffffULL << IRDMAQPC_RTTVAR_S)
+
+#define IRDMAQPC_SSTHRESH_S 0
+#define IRDMAQPC_SSTHRESH_M (0xffffffffULL << IRDMAQPC_SSTHRESH_S)
+
+#define IRDMAQPC_CWND_S 32
+#define IRDMAQPC_CWND_M (0xffffffffULL << IRDMAQPC_CWND_S)
+
+#define IRDMAQPC_CWNDROCE_S 32
+#define IRDMAQPC_CWNDROCE_M (0xffffffULL << IRDMAQPC_CWNDROCE_S)
+#define IRDMAQPC_SNDWL1_S 0
+#define IRDMAQPC_SNDWL1_M (0xffffffffULL << IRDMAQPC_SNDWL1_S)
+
+#define IRDMAQPC_SNDWL2_S 32
+#define IRDMAQPC_SNDWL2_M (0xffffffffULL << IRDMAQPC_SNDWL2_S)
+
+#define IRDMAQPC_ERR_RQ_IDX_S 32
+#define IRDMAQPC_ERR_RQ_IDX_M (0x3fffULL << IRDMAQPC_ERR_RQ_IDX_S)
+
+#define IRDMAQPC_MAXSNDWND_S 0
+#define IRDMAQPC_MAXSNDWND_M (0xffffffffULL << IRDMAQPC_MAXSNDWND_S)
+
+#define IRDMAQPC_REXMIT_THRESH_S 48
+#define IRDMAQPC_REXMIT_THRESH_M (0x3fULL << IRDMAQPC_REXMIT_THRESH_S)
+
+#define IRDMAQPC_RNRNAK_THRESH_S 54
+#define IRDMAQPC_RNRNAK_THRESH_M (0x7ULL << IRDMAQPC_RNRNAK_THRESH_S)
+
+#define IRDMAQPC_TXCQNUM_S 0
+#define IRDMAQPC_TXCQNUM_M (0x7ffffULL << IRDMAQPC_TXCQNUM_S)
+
+#define IRDMAQPC_RXCQNUM_S 32
+#define IRDMAQPC_RXCQNUM_M (0x7ffffULL << IRDMAQPC_RXCQNUM_S)
+
+#define IRDMAQPC_STAT_INDEX_S 0
+#define IRDMAQPC_STAT_INDEX_M (0x7fULL << IRDMAQPC_STAT_INDEX_S)
+
+#define IRDMAQPC_Q2ADDR_S 8
+#define IRDMAQPC_Q2ADDR_M (0xffffffffffffffULL << IRDMAQPC_Q2ADDR_S)
+
+#define IRDMAQPC_LASTBYTESENT_S 0
+#define IRDMAQPC_LASTBYTESENT_M (0xffULL << IRDMAQPC_LASTBYTESENT_S)
+
+#define IRDMAQPC_MACADDRESS_S 16
+#define IRDMAQPC_MACADDRESS_M (0xffffffffffffULL << IRDMAQPC_MACADDRESS_S)
+
+#define IRDMAQPC_ORDSIZE_S 0
+#define IRDMAQPC_ORDSIZE_M (0xffULL << IRDMAQPC_ORDSIZE_S)
+
+#define IRDMAQPC_IRDSIZE_S 16
+#define IRDMAQPC_IRDSIZE_M (0x7ULL << IRDMAQPC_IRDSIZE_S)
+
+#define IRDMAQPC_UDPRIVCQENABLE_S 19
+#define IRDMAQPC_UDPRIVCQENABLE_M BIT_ULL(IRDMAQPC_UDPRIVCQENABLE_S)
+
+#define IRDMAQPC_WRRDRSPOK_S 20
+#define IRDMAQPC_WRRDRSPOK_M BIT_ULL(IRDMAQPC_WRRDRSPOK_S)
+
+#define IRDMAQPC_RDOK_S 21
+#define IRDMAQPC_RDOK_M BIT_ULL(IRDMAQPC_RDOK_S)
+
+#define IRDMAQPC_SNDMARKERS_S 22
+#define IRDMAQPC_SNDMARKERS_M BIT_ULL(IRDMAQPC_SNDMARKERS_S)
+
+#define IRDMAQPC_DCQCNENABLE_S 22
+#define IRDMAQPC_DCQCNENABLE_M BIT_ULL(IRDMAQPC_DCQCNENABLE_S)
+
+#define IRDMAQPC_FW_CC_ENABLE_S 28
+#define IRDMAQPC_FW_CC_ENABLE_M BIT_ULL(IRDMAQPC_FW_CC_ENABLE_S)
+
+#define IRDMAQPC_RCVNOICRC_S 31
+#define IRDMAQPC_RCVNOICRC_M BIT_ULL(IRDMAQPC_RCVNOICRC_S)
+
+#define IRDMAQPC_BINDEN_S 23
+#define IRDMAQPC_BINDEN_M BIT_ULL(IRDMAQPC_BINDEN_S)
+
+#define IRDMAQPC_FASTREGEN_S 24
+#define IRDMAQPC_FASTREGEN_M BIT_ULL(IRDMAQPC_FASTREGEN_S)
+
+#define IRDMAQPC_PRIVEN_S 25
+#define IRDMAQPC_PRIVEN_M BIT_ULL(IRDMAQPC_PRIVEN_S)
+
+#define IRDMAQPC_TIMELYENABLE_S 27
+#define IRDMAQPC_TIMELYENABLE_M BIT_ULL(IRDMAQPC_TIMELYENABLE_S)
+
+#define IRDMAQPC_THIGH_S 52
+#define IRDMAQPC_THIGH_M ((u64)0xfff << IRDMAQPC_THIGH_S)
+
+#define IRDMAQPC_TLOW_S 32
+#define IRDMAQPC_TLOW_M ((u64)0xFF << IRDMAQPC_TLOW_S)
+
+#define IRDMAQPC_USESTATSINSTANCE_S 26
+#define IRDMAQPC_USESTATSINSTANCE_M BIT_ULL(IRDMAQPC_USESTATSINSTANCE_S)
+
+#define IRDMAQPC_IWARPMODE_S 28
+#define IRDMAQPC_IWARPMODE_M BIT_ULL(IRDMAQPC_IWARPMODE_S)
+
+#define IRDMAQPC_RCVMARKERS_S 29
+#define IRDMAQPC_RCVMARKERS_M BIT_ULL(IRDMAQPC_RCVMARKERS_S)
+
+#define IRDMAQPC_ALIGNHDRS_S 30
+#define IRDMAQPC_ALIGNHDRS_M BIT_ULL(IRDMAQPC_ALIGNHDRS_S)
+
+#define IRDMAQPC_RCVNOMPACRC_S 31
+#define IRDMAQPC_RCVNOMPACRC_M BIT_ULL(IRDMAQPC_RCVNOMPACRC_S)
+
+#define IRDMAQPC_RCVMARKOFFSET_S 32
+#define IRDMAQPC_RCVMARKOFFSET_M (0x1ffULL << IRDMAQPC_RCVMARKOFFSET_S)
+
+#define IRDMAQPC_SNDMARKOFFSET_S 48
+#define IRDMAQPC_SNDMARKOFFSET_M (0x1ffULL << IRDMAQPC_SNDMARKOFFSET_S)
+
+#define IRDMAQPC_QPCOMPCTX_S IRDMA_CQPHC_QPCTX_S
+#define IRDMAQPC_QPCOMPCTX_M IRDMA_CQPHC_QPCTX_M
+
+#define IRDMAQPC_SQTPHVAL_S 0
+#define IRDMAQPC_SQTPHVAL_M (0xffULL << IRDMAQPC_SQTPHVAL_S)
+
+#define IRDMAQPC_RQTPHVAL_S 8
+#define IRDMAQPC_RQTPHVAL_M (0xffULL << IRDMAQPC_RQTPHVAL_S)
+
+#define IRDMAQPC_QSHANDLE_S 16
+#define IRDMAQPC_QSHANDLE_M (0x3ffULL << IRDMAQPC_QSHANDLE_S)
+
+#define IRDMAQPC_EXCEPTION_LAN_QUEUE_S 32
+#define IRDMAQPC_EXCEPTION_LAN_QUEUE_M \
+ (0xfffULL << IRDMAQPC_EXCEPTION_LAN_QUEUE_S)
+
+#define IRDMAQPC_LOCAL_IPADDR3_S 0
+#define IRDMAQPC_LOCAL_IPADDR3_M \
+ (0xffffffffULL << IRDMAQPC_LOCAL_IPADDR3_S)
+
+#define IRDMAQPC_LOCAL_IPADDR2_S 32
+#define IRDMAQPC_LOCAL_IPADDR2_M \
+ (0xffffffffULL << IRDMAQPC_LOCAL_IPADDR2_S)
+
+#define IRDMAQPC_LOCAL_IPADDR1_S 0
+#define IRDMAQPC_LOCAL_IPADDR1_M \
+ (0xffffffffULL << IRDMAQPC_LOCAL_IPADDR1_S)
+
+#define IRDMAQPC_LOCAL_IPADDR0_S 32
+#define IRDMAQPC_LOCAL_IPADDR0_M \
+ (0xffffffffULL << IRDMAQPC_LOCAL_IPADDR0_S)
+
+#define IRDMA_FW_VER_MINOR_S 0
+#define IRDMA_FW_VER_MINOR_M \
+ (0xffffULL << IRDMA_FW_VER_MINOR_S)
+
+#define IRDMA_FW_VER_MAJOR_S 16
+#define IRDMA_FW_VER_MAJOR_M \
+ (0xffffULL << IRDMA_FW_VER_MAJOR_S)
+
+#define IRDMA_FEATURE_INFO_S 0
+#define IRDMA_FEATURE_INFO_M \
+ (0xffffffffffULL << IRDMA_FEATURE_INFO_S)
+
+#define IRDMA_FEATURE_CNT_S 32
+#define IRDMA_FEATURE_CNT_M \
+ (0xffffULL << IRDMA_FEATURE_CNT_S)
+
+#define IRDMA_RSVD_S 41
+#define IRDMA_RSVD_M (0x7fffULL << IRDMA_RSVD_S)
+
+/* iwarp QP SQ WQE common fields */
+#define IRDMAQPSQ_OPCODE_S 32
+#define IRDMAQPSQ_OPCODE_M (0x3fULL << IRDMAQPSQ_OPCODE_S)
+
+#define IRDMAQPSQ_COPY_HOST_PBL_S 43
+#define IRDMAQPSQ_COPY_HOST_PBL_M BIT_ULL(IRDMAQPSQ_COPY_HOST_PBL_S)
+
+#define IRDMAQPSQ_ADDFRAGCNT_S 38
+#define IRDMAQPSQ_ADDFRAGCNT_M (0xfULL << IRDMAQPSQ_ADDFRAGCNT_S)
+
+#define IRDMAQPSQ_PUSHWQE_S 56
+#define IRDMAQPSQ_PUSHWQE_M BIT_ULL(IRDMAQPSQ_PUSHWQE_S)
+
+#define IRDMAQPSQ_STREAMMODE_S 58
+#define IRDMAQPSQ_STREAMMODE_M BIT_ULL(IRDMAQPSQ_STREAMMODE_S)
+
+#define IRDMAQPSQ_WAITFORRCVPDU_S 59
+#define IRDMAQPSQ_WAITFORRCVPDU_M BIT_ULL(IRDMAQPSQ_WAITFORRCVPDU_S)
+
+#define IRDMAQPSQ_READFENCE_S 60
+#define IRDMAQPSQ_READFENCE_M BIT_ULL(IRDMAQPSQ_READFENCE_S)
+
+#define IRDMAQPSQ_LOCALFENCE_S 61
+#define IRDMAQPSQ_LOCALFENCE_M BIT_ULL(IRDMAQPSQ_LOCALFENCE_S)
+
+#define IRDMAQPSQ_UDPHEADER_S 61
+#define IRDMAQPSQ_UDPHEADER_M BIT_ULL(IRDMAQPSQ_UDPHEADER_S)
+
+#define IRDMAQPSQ_L4LEN_S 42
+#define IRDMAQPSQ_L4LEN_M ((u64)0xF << IRDMAQPSQ_L4LEN_S)
+
+#define IRDMAQPSQ_SIGCOMPL_S 62
+#define IRDMAQPSQ_SIGCOMPL_M BIT_ULL(IRDMAQPSQ_SIGCOMPL_S)
+
+#define IRDMAQPSQ_VALID_S 63
+#define IRDMAQPSQ_VALID_M BIT_ULL(IRDMAQPSQ_VALID_S)
+
+#define IRDMAQPSQ_FRAG_TO_S IRDMA_CQPHC_QPCTX_S
+#define IRDMAQPSQ_FRAG_TO_M IRDMA_CQPHC_QPCTX_M
+
+#define IRDMAQPSQ_FRAG_VALID_S 63
+#define IRDMAQPSQ_FRAG_VALID_M BIT_ULL(IRDMAQPSQ_FRAG_VALID_S)
+
+#define IRDMAQPSQ_FRAG_LEN_S 32
+#define IRDMAQPSQ_FRAG_LEN_M (0x7fffffffULL << IRDMAQPSQ_FRAG_LEN_S)
+
+#define IRDMAQPSQ_FRAG_STAG_S 0
+#define IRDMAQPSQ_FRAG_STAG_M (0xffffffffULL << IRDMAQPSQ_FRAG_STAG_S)
+
+#define IRDMAQPSQ_GEN1_FRAG_LEN_S 0
+#define IRDMAQPSQ_GEN1_FRAG_LEN_M (0xffffffffULL << IRDMAQPSQ_GEN1_FRAG_LEN_S)
+
+#define IRDMAQPSQ_GEN1_FRAG_STAG_S 32
+#define IRDMAQPSQ_GEN1_FRAG_STAG_M (0xffffffffULL << IRDMAQPSQ_GEN1_FRAG_STAG_S)
+
+#define IRDMAQPSQ_REMSTAGINV_S 0
+#define IRDMAQPSQ_REMSTAGINV_M (0xffffffffULL << IRDMAQPSQ_REMSTAGINV_S)
+
+#define IRDMAQPSQ_DESTQKEY_S 0
+#define IRDMAQPSQ_DESTQKEY_M (0xffffffffULL << IRDMAQPSQ_DESTQKEY_S)
+
+#define IRDMAQPSQ_DESTQPN_S 32
+#define IRDMAQPSQ_DESTQPN_M (0x00ffffffULL << IRDMAQPSQ_DESTQPN_S)
+
+#define IRDMAQPSQ_AHID_S 0
+#define IRDMAQPSQ_AHID_M (0x0001ffffULL << IRDMAQPSQ_AHID_S)
+
+#define IRDMAQPSQ_INLINEDATAFLAG_S 57
+#define IRDMAQPSQ_INLINEDATAFLAG_M BIT_ULL(IRDMAQPSQ_INLINEDATAFLAG_S)
+
+#define IRDMA_INLINE_VALID_S 7
+
+#define IRDMAQPSQ_INLINEDATALEN_S 48
+#define IRDMAQPSQ_INLINEDATALEN_M \
+ (0xffULL << IRDMAQPSQ_INLINEDATALEN_S)
+#define IRDMAQPSQ_IMMDATAFLAG_S 47
+#define IRDMAQPSQ_IMMDATAFLAG_M \
+ BIT_ULL(IRDMAQPSQ_IMMDATAFLAG_S)
+#define IRDMAQPSQ_REPORTRTT_S 46
+#define IRDMAQPSQ_REPORTRTT_M \
+ BIT_ULL(IRDMAQPSQ_REPORTRTT_S)
+
+#define IRDMAQPSQ_IMMDATA_S 0
+#define IRDMAQPSQ_IMMDATA_M \
+ (0xffffffffffffffffULL << IRDMAQPSQ_IMMDATA_S)
+
+/* rdma write */
+#define IRDMAQPSQ_REMSTAG_S 0
+#define IRDMAQPSQ_REMSTAG_M (0xffffffffULL << IRDMAQPSQ_REMSTAG_S)
+
+#define IRDMAQPSQ_REMTO_S IRDMA_CQPHC_QPCTX_S
+#define IRDMAQPSQ_REMTO_M IRDMA_CQPHC_QPCTX_M
+
+/* memory window */
+#define IRDMAQPSQ_STAGRIGHTS_S 48
+#define IRDMAQPSQ_STAGRIGHTS_M (0x1fULL << IRDMAQPSQ_STAGRIGHTS_S)
+
+#define IRDMAQPSQ_VABASEDTO_S 53
+#define IRDMAQPSQ_VABASEDTO_M BIT_ULL(IRDMAQPSQ_VABASEDTO_S)
+
+#define IRDMAQPSQ_MEMWINDOWTYPE_S 54
+#define IRDMAQPSQ_MEMWINDOWTYPE_M BIT_ULL(IRDMAQPSQ_MEMWINDOWTYPE_S)
+
+#define IRDMAQPSQ_MWLEN_S IRDMA_CQPHC_QPCTX_S
+#define IRDMAQPSQ_MWLEN_M IRDMA_CQPHC_QPCTX_M
+
+#define IRDMAQPSQ_PARENTMRSTAG_S 32
+#define IRDMAQPSQ_PARENTMRSTAG_M \
+ (0xffffffffULL << IRDMAQPSQ_PARENTMRSTAG_S)
+
+#define IRDMAQPSQ_MWSTAG_S 0
+#define IRDMAQPSQ_MWSTAG_M (0xffffffffULL << IRDMAQPSQ_MWSTAG_S)
+
+#define IRDMAQPSQ_BASEVA_TO_FBO_S IRDMA_CQPHC_QPCTX_S
+#define IRDMAQPSQ_BASEVA_TO_FBO_M IRDMA_CQPHC_QPCTX_M
+
+/* Local Invalidate */
+#define IRDMAQPSQ_LOCSTAG_S 0
+#define IRDMAQPSQ_LOCSTAG_M (0xffffffffULL << IRDMAQPSQ_LOCSTAG_S)
+
+/* Fast Register */
+#define IRDMAQPSQ_STAGKEY_S 0
+#define IRDMAQPSQ_STAGKEY_M (0xffULL << IRDMAQPSQ_STAGKEY_S)
+
+#define IRDMAQPSQ_STAGINDEX_S 8
+#define IRDMAQPSQ_STAGINDEX_M (0xffffffULL << IRDMAQPSQ_STAGINDEX_S)
+
+#define IRDMAQPSQ_COPYHOSTPBLS_S 43
+#define IRDMAQPSQ_COPYHOSTPBLS_M BIT_ULL(IRDMAQPSQ_COPYHOSTPBLS_S)
+
+#define IRDMAQPSQ_LPBLSIZE_S 44
+#define IRDMAQPSQ_LPBLSIZE_M (3ULL << IRDMAQPSQ_LPBLSIZE_S)
+
+#define IRDMAQPSQ_HPAGESIZE_S 46
+#define IRDMAQPSQ_HPAGESIZE_M (3ULL << IRDMAQPSQ_HPAGESIZE_S)
+
+#define IRDMAQPSQ_STAGLEN_S 0
+#define IRDMAQPSQ_STAGLEN_M (0x1ffffffffffULL << IRDMAQPSQ_STAGLEN_S)
+
+#define IRDMAQPSQ_FIRSTPMPBLIDXLO_S 48
+#define IRDMAQPSQ_FIRSTPMPBLIDXLO_M \
+ (0xffffULL << IRDMAQPSQ_FIRSTPMPBLIDXLO_S)
+
+#define IRDMAQPSQ_FIRSTPMPBLIDXHI_S 0
+#define IRDMAQPSQ_FIRSTPMPBLIDXHI_M \
+ (0xfffULL << IRDMAQPSQ_FIRSTPMPBLIDXHI_S)
+
+#define IRDMAQPSQ_PBLADDR_S 12
+#define IRDMAQPSQ_PBLADDR_M (0xfffffffffffffULL << IRDMAQPSQ_PBLADDR_S)
+
+/* iwarp QP RQ WQE common fields */
+#define IRDMAQPRQ_ADDFRAGCNT_S IRDMAQPSQ_ADDFRAGCNT_S
+#define IRDMAQPRQ_ADDFRAGCNT_M IRDMAQPSQ_ADDFRAGCNT_M
+
+#define IRDMAQPRQ_VALID_S IRDMAQPSQ_VALID_S
+#define IRDMAQPRQ_VALID_M IRDMAQPSQ_VALID_M
+
+#define IRDMAQPRQ_COMPLCTX_S IRDMA_CQPHC_QPCTX_S
+#define IRDMAQPRQ_COMPLCTX_M IRDMA_CQPHC_QPCTX_M
+
+#define IRDMAQPRQ_FRAG_LEN_S IRDMAQPSQ_FRAG_LEN_S
+#define IRDMAQPRQ_FRAG_LEN_M IRDMAQPSQ_FRAG_LEN_M
+
+#define IRDMAQPRQ_STAG_S IRDMAQPSQ_FRAG_STAG_S
+#define IRDMAQPRQ_STAG_M IRDMAQPSQ_FRAG_STAG_M
+
+#define IRDMAQPRQ_TO_S IRDMAQPSQ_FRAG_TO_S
+#define IRDMAQPRQ_TO_M IRDMAQPSQ_FRAG_TO_M
+
+/* Query FPM CQP buf */
+#define IRDMA_QUERY_FPM_MAX_QPS_S 0
+#define IRDMA_QUERY_FPM_MAX_QPS_M \
+ (0x7ffffULL << IRDMA_QUERY_FPM_MAX_QPS_S)
+
+#define IRDMA_QUERY_FPM_MAX_CQS_S 0
+#define IRDMA_QUERY_FPM_MAX_CQS_M \
+ (0xfffffULL << IRDMA_QUERY_FPM_MAX_CQS_S)
+
+#define IRDMA_QUERY_FPM_FIRST_PE_SD_INDEX_S 0
+#define IRDMA_QUERY_FPM_FIRST_PE_SD_INDEX_M \
+ (0x3fffULL << IRDMA_QUERY_FPM_FIRST_PE_SD_INDEX_S)
+
+#define IRDMA_QUERY_FPM_MAX_PE_SDS_S 32
+#define IRDMA_QUERY_FPM_MAX_PE_SDS_M \
+ (0x3fffULL << IRDMA_QUERY_FPM_MAX_PE_SDS_S)
+
+#define IRDMA_QUERY_FPM_MAX_CEQS_S 0
+#define IRDMA_QUERY_FPM_MAX_CEQS_M \
+ (0x3ffULL << IRDMA_QUERY_FPM_MAX_CEQS_S)
+
+#define IRDMA_QUERY_FPM_XFBLOCKSIZE_S 32
+#define IRDMA_QUERY_FPM_XFBLOCKSIZE_M \
+ (0xffffffffULL << IRDMA_QUERY_FPM_XFBLOCKSIZE_S)
+
+#define IRDMA_QUERY_FPM_Q1BLOCKSIZE_S 32
+#define IRDMA_QUERY_FPM_Q1BLOCKSIZE_M \
+ (0xffffffffULL << IRDMA_QUERY_FPM_Q1BLOCKSIZE_S)
+
+#define IRDMA_QUERY_FPM_HTMULTIPLIER_S 16
+#define IRDMA_QUERY_FPM_HTMULTIPLIER_M \
+ (0xfULL << IRDMA_QUERY_FPM_HTMULTIPLIER_S)
+
+#define IRDMA_QUERY_FPM_TIMERBUCKET_S 32
+#define IRDMA_QUERY_FPM_TIMERBUCKET_M \
+ (0xffFFULL << IRDMA_QUERY_FPM_TIMERBUCKET_S)
+
+#define IRDMA_QUERY_FPM_RRFBLOCKSIZE_S 32
+#define IRDMA_QUERY_FPM_RRFBLOCKSIZE_M \
+ (0xffffffffULL << IRDMA_QUERY_FPM_RRFBLOCKSIZE_S)
+
+#define IRDMA_QUERY_FPM_RRFFLBLOCKSIZE_S 32
+#define IRDMA_QUERY_FPM_RRFFLBLOCKSIZE_M \
+ (0xffffffffULL << IRDMA_QUERY_FPM_RRFFLBLOCKSIZE_S)
+
+#define IRDMA_QUERY_FPM_OOISCFBLOCKSIZE_S 32
+#define IRDMA_QUERY_FPM_OOISCFBLOCKSIZE_M \
+ (0xffffffffULL << IRDMA_QUERY_FPM_OOISCFBLOCKSIZE_S)
+
+/* Static HMC pages allocated buf */
+#define IRDMA_SHMC_PAGE_ALLOCATED_HMC_FN_ID_S 0
+#define IRDMA_SHMC_PAGE_ALLOCATED_HMC_FN_ID_M \
+ (0x3fULL << IRDMA_SHMC_PAGE_ALLOCATED_HMC_FN_ID_S)
+
+#define IRDMA_CQP_INIT_WQE(wqe) memset(wqe, 0, 64)
+
+#define IRDMA_GET_CURRENT_CQ_ELEM(_cq) \
+ ( \
+ (_cq)->cq_base[IRDMA_RING_CURRENT_HEAD((_cq)->cq_ring)].buf \
+ )
+#define IRDMA_GET_CURRENT_EXTENDED_CQ_ELEM(_cq) \
+ ( \
+ ((struct irdma_extended_cqe *) \
+ ((_cq)->cq_base))[IRDMA_RING_CURRENT_HEAD((_cq)->cq_ring)].buf \
+ )
+
+#define IRDMA_GET_CURRENT_AEQ_ELEM(_aeq) \
+ ( \
+ (_aeq)->aeqe_base[IRDMA_RING_CURRENT_TAIL((_aeq)->aeq_ring)].buf \
+ )
+
+#define IRDMA_GET_CURRENT_CEQ_ELEM(_ceq) \
+ ( \
+ (_ceq)->ceqe_base[IRDMA_RING_CURRENT_TAIL((_ceq)->ceq_ring)].buf \
+ )
+
+#define IRDMA_RING_INIT(_ring, _size) \
+ { \
+ (_ring).head = 0; \
+ (_ring).tail = 0; \
+ (_ring).size = (_size); \
+ }
+#define IRDMA_RING_SIZE(_ring) ((_ring).size)
+#define IRDMA_RING_CURRENT_HEAD(_ring) ((_ring).head)
+#define IRDMA_RING_CURRENT_TAIL(_ring) ((_ring).tail)
+
+#define IRDMA_RING_MOVE_HEAD(_ring, _retcode) \
+ { \
+ register u32 size; \
+ size = (_ring).size; \
+ if (!IRDMA_RING_FULL_ERR(_ring)) { \
+ (_ring).head = ((_ring).head + 1) % size; \
+ (_retcode) = 0; \
+ } else { \
+ (_retcode) = IRDMA_ERR_RING_FULL; \
+ } \
+ }
+#define IRDMA_RING_MOVE_HEAD_BY_COUNT(_ring, _count, _retcode) \
+ { \
+ register u32 size; \
+ size = (_ring).size; \
+ if ((IRDMA_RING_USED_QUANTA(_ring) + (_count)) < size) { \
+ (_ring).head = ((_ring).head + (_count)) % size; \
+ (_retcode) = 0; \
+ } else { \
+ (_retcode) = IRDMA_ERR_RING_FULL; \
+ } \
+ }
+
+#define IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(_ring, _count) \
+ (_ring).head = ((_ring).head + (_count)) % (_ring).size
+
+#define IRDMA_RING_MOVE_TAIL(_ring) \
+ (_ring).tail = ((_ring).tail + 1) % (_ring).size
+
+#define IRDMA_RING_MOVE_HEAD_NOCHECK(_ring) \
+ (_ring).head = ((_ring).head + 1) % (_ring).size
+
+#define IRDMA_RING_MOVE_TAIL_BY_COUNT(_ring, _count) \
+ (_ring).tail = ((_ring).tail + (_count)) % (_ring).size
+
+#define IRDMA_RING_SET_TAIL(_ring, _pos) \
+ (_ring).tail = (_pos) % (_ring).size
+
+#define IRDMA_RING_FULL_ERR(_ring) \
+ ( \
+ (IRDMA_RING_USED_QUANTA(_ring) == ((_ring).size - 1)) \
+ )
+
+#define IRDMA_ERR_RING_FULL2(_ring) \
+ ( \
+ (IRDMA_RING_USED_QUANTA(_ring) == ((_ring).size - 2)) \
+ )
+
+#define IRDMA_ERR_RING_FULL3(_ring) \
+ ( \
+ (IRDMA_RING_USED_QUANTA(_ring) == ((_ring).size - 3)) \
+ )
+
+#define IRDMA_RING_MORE_WORK(_ring) \
+ ( \
+ (IRDMA_RING_USED_QUANTA(_ring) != 0) \
+ )
+
+#define IRDMA_RING_USED_QUANTA(_ring) \
+ ( \
+ (((_ring).head + (_ring).size - (_ring).tail) % (_ring).size) \
+ )
+
+#define IRDMA_RING_FREE_QUANTA(_ring) \
+ ( \
+ ((_ring).size - IRDMA_RING_USED_QUANTA(_ring) - 1) \
+ )
+
+#define IRDMA_ATOMIC_RING_MOVE_HEAD(_ring, index, _retcode) \
+ { \
+ index = IRDMA_RING_CURRENT_HEAD(_ring); \
+ IRDMA_RING_MOVE_HEAD(_ring, _retcode); \
+ }
+
+#define IRDMA_WQE_SIZE_64 64
+#define IRDMA_WQE_SIZE_128 128
+
+enum irdma_ws_node_op {
+ IRDMA_ADD_NODE = 0,
+ IRDMA_MODIFY_NODE,
+ IRDMA_DEL_NODE,
+};
+
+enum {
+ IRDMA_Q_ALIGNMENT_M = (128 - 1),
+ IRDMA_AEQ_ALIGNMENT_M = (256 - 1),
+ IRDMA_Q2_ALIGNMENT_M = (256 - 1),
+ IRDMA_CEQ_ALIGNMENT_M = (256 - 1),
+ IRDMA_CQ0_ALIGNMENT_M = (256 - 1),
+ IRDMA_HOST_CTX_ALIGNMENT_M = (4 - 1),
+ IRDMA_SHADOWAREA_M = (128 - 1),
+ IRDMA_FPM_QUERY_BUF_ALIGNMENT_M = (4 - 1),
+ IRDMA_FPM_COMMIT_BUF_ALIGNMENT_M = (4 - 1),
+};
+
+enum irdma_alignment {
+ IRDMA_CQP_ALIGNMENT = 0x200,
+ IRDMA_AEQ_ALIGNMENT = 0x100,
+ IRDMA_CEQ_ALIGNMENT = 0x100,
+ IRDMA_CQ0_ALIGNMENT = 0x100,
+ IRDMA_SD_BUF_ALIGNMENT = 0x80,
+ IRDMA_FEATURE_BUF_ALIGNMENT = 0x8,
+};
+
+enum icrdma_protocol_used {
+ ICRDMA_ANY_PROTOCOL = 0,
+ ICRDMA_IWARP_PROTOCOL_ONLY = 1,
+ ICRDMA_ROCE_PROTOCOL_ONLY = 2,
+};
+
+/**
+ * set_64bit_val - set 64 bit value to hw wqe
+ * @wqe_words: wqe addr to write
+ * @byte_index: index in wqe
+ * @value: value to write
+ **/
+static inline void set_64bit_val(__le64 *wqe_words, u32 byte_index, u64 val)
+{
+ wqe_words[byte_index >> 3] = cpu_to_le64(val);
+}
+
+/**
+ * set_32bit_val - set 32 bit value to hw wqe
+ * @wqe_words: wqe addr to write
+ * @byte_index: index in wqe
+ * @value: value to write
+ **/
+static inline void set_32bit_val(u32 *wqe_words, u32 byte_index, u32 val)
+{
+ wqe_words[byte_index >> 2] = val;
+}
+
+/**
+ * get_64bit_val - read 64 bit value from wqe
+ * @wqe_words: wqe addr
+ * @byte_index: index to read from
+ * @value: read value
+ **/
+static inline void get_64bit_val(__le64 *wqe_words, u32 byte_index, u64 *val)
+{
+ *val = le64_to_cpu(wqe_words[byte_index >> 3]);
+}
+
+/**
+ * get_32bit_val - read 32 bit value from wqe
+ * @wqe_words: wqe addr
+ * @byte_index: index to reaad from
+ * @value: return 32 bit value
+ **/
+static inline void get_32bit_val(u32 *wqe_words, u32 byte_index, u32 *val)
+{
+ *val = wqe_words[byte_index >> 2];
+}
+#endif /* IRDMA_DEFS_H */
diff --git a/drivers/infiniband/hw/irdma/irdma.h b/drivers/infiniband/hw/irdma/irdma.h
new file mode 100644
index 0000000..cc1b122
--- /dev/null
+++ b/drivers/infiniband/hw/irdma/irdma.h
@@ -0,0 +1,187 @@
+/* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB */
+/* Copyright (c) 2019, Intel Corporation. */
+
+#ifndef IRDMA_H
+#define IRDMA_H
+
+#define IRDMA_CQPTAIL_WQTAIL_S 0
+#define IRDMA_CQPTAIL_WQTAIL_M MAKEMASK(0x7ff, IRDMA_CQPTAIL_WQTAIL_S)
+
+#define IRDMA_CQPTAIL_CQP_OP_ERR_S 31
+#define IRDMA_CQPTAIL_CQP_OP_ERR_M MAKEMASK(0x1, IRDMA_CQPTAIL_CQP_OP_ERR_S)
+
+#define IRDMA_CQPERRCODES_CQP_MINOR_CODE_S 0
+#define IRDMA_CQPERRCODES_CQP_MINOR_CODE_M MAKEMASK(0xffff, IRDMA_CQPERRCODES_CQP_MINOR_CODE_S)
+#define IRDMA_CQPERRCODES_CQP_MAJOR_CODE_S 16
+#define IRDMA_CQPERRCODES_CQP_MAJOR_CODE_M MAKEMASK(0xffff, IRDMA_CQPERRCODES_CQP_MAJOR_CODE_S)
+
+#define IRDMA_GLPCI_LBARCTRL_PE_DB_SIZE_S 4
+#define IRDMA_GLPCI_LBARCTRL_PE_DB_SIZE_M MAKEMASK(0x3, IRDMA_GLPCI_LBARCTRL_PE_DB_SIZE_S)
+
+#define IRDMA_GLINT_DYN_CTL_INTENA_S 0
+#define IRDMA_GLINT_DYN_CTL_INTENA_M MAKEMASK(0x1, IRDMA_GLINT_DYN_CTL_INTENA_S)
+
+#define IRDMA_GLINT_DYN_CTL_CLEARPBA_S 1
+#define IRDMA_GLINT_DYN_CTL_CLEARPBA_M MAKEMASK(0x1, IRDMA_GLINT_DYN_CTL_CLEARPBA_S)
+
+#define IRDMA_GLINT_DYN_CTL_ITR_INDX_S 3
+#define IRDMA_GLINT_DYN_CTL_ITR_INDX_M MAKEMASK(0x3, IRDMA_GLINT_DYN_CTL_ITR_INDX_S)
+
+#define IRDMA_GLINT_CEQCTL_ITR_INDX_S 11
+#define IRDMA_GLINT_CEQCTL_ITR_INDX_M MAKEMASK(0x3, IRDMA_GLINT_CEQCTL_ITR_INDX_S)
+
+#define IRDMA_GLINT_CEQCTL_CAUSE_ENA_S 30
+#define IRDMA_GLINT_CEQCTL_CAUSE_ENA_M MAKEMASK(0x1, IRDMA_GLINT_CEQCTL_CAUSE_ENA_S)
+
+#define IRDMA_GLINT_CEQCTL_MSIX_INDX_S 0
+#define IRDMA_GLINT_CEQCTL_MSIX_INDX_M MAKEMASK(0x7ff, IRDMA_GLINT_CEQCTL_MSIX_INDX_S)
+
+#define IRDMA_PFINT_AEQCTL_MSIX_INDX_S 0
+#define IRDMA_PFINT_AEQCTL_MSIX_INDX_M MAKEMASK(0x7ff, IRDMA_PFINT_AEQCTL_MSIX_INDX_S)
+
+#define IRDMA_PFINT_AEQCTL_ITR_INDX_S 11
+#define IRDMA_PFINT_AEQCTL_ITR_INDX_M MAKEMASK(0x3, IRDMA_PFINT_AEQCTL_ITR_INDX_S)
+
+#define IRDMA_PFINT_AEQCTL_CAUSE_ENA_S 30
+#define IRDMA_PFINT_AEQCTL_CAUSE_ENA_M MAKEMASK(0x1, IRDMA_PFINT_AEQCTL_CAUSE_ENA_S)
+
+#define IRDMA_WQEALLOC_WQE_DESC_INDEX_S 20
+#define IRDMA_WQEALLOC_WQE_DESC_INDEX_M MAKEMASK(0xfff, IRDMA_WQEALLOC_WQE_DESC_INDEX_S)
+
+#define IRDMA_PFHMC_PDINV_PMSDIDX_S 0
+#define IRDMA_PFHMC_PDINV_PMSDIDX_M MAKEMASK(0xfff, IRDMA_PFHMC_PDINV_PMSDIDX_S)
+
+#define IRDMA_PFHMC_PDINV_PMSDPARTSEL_S 15
+#define IRDMA_PFHMC_PDINV_PMSDPARTSEL_M MAKEMASK(0x1, IRDMA_PFHMC_PDINV_PMSDPARTSEL_S)
+
+#define IRDMA_PFHMC_PDINV_PMPDIDX_S 16
+#define IRDMA_PFHMC_PDINV_PMPDIDX_M MAKEMASK(0x1ff, IRDMA_PFHMC_PDINV_PMPDIDX_S)
+
+#define IRDMA_PFHMC_SDDATALOW_PMSDVALID_S 0
+#define IRDMA_PFHMC_SDDATALOW_PMSDVALID_M MAKEMASK(0x1, IRDMA_PFHMC_SDDATALOW_PMSDVALID_S)
+#define IRDMA_PFHMC_SDDATALOW_PMSDTYPE_S 1
+#define IRDMA_PFHMC_SDDATALOW_PMSDTYPE_M MAKEMASK(0x1, IRDMA_PFHMC_SDDATALOW_PMSDTYPE_S)
+#define IRDMA_PFHMC_SDDATALOW_PMSDBPCOUNT_S 2
+#define IRDMA_PFHMC_SDDATALOW_PMSDBPCOUNT_M MAKEMASK(0x3ff, IRDMA_PFHMC_SDDATALOW_PMSDBPCOUNT_S)
+#define IRDMA_PFHMC_SDDATALOW_PMSDDATALOW_S 12
+#define IRDMA_PFHMC_SDDATALOW_PMSDDATALOW_M MAKEMASK(0xfffff, IRDMA_PFHMC_SDDATALOW_PMSDDATALOW_S)
+
+#define IRDMA_PFHMC_SDCMD_PMSDWR_S 31
+#define IRDMA_PFHMC_SDCMD_PMSDWR_M MAKEMASK(0x1, IRDMA_PFHMC_SDCMD_PMSDWR_S)
+
+#define IRDMA_INVALID_CQ_IDX 0xffffffff
+
+#define IRDMA_I40IW_RTS_AE_FEATURE_FW_VER 0x30010
+#define IRDMA_RTS_AE_FEATURE_FW_VER 0x1000D
+enum irdma_registers {
+ IRDMA_CQPTAIL,
+ IRDMA_CQPDB,
+ IRDMA_CCQPSTATUS,
+ IRDMA_CCQPHIGH,
+ IRDMA_CCQPLOW,
+ IRDMA_CQARM,
+ IRDMA_CQACK,
+ IRDMA_AEQALLOC,
+ IRDMA_CQPERRCODES,
+ IRDMA_WQEALLOC,
+ IRDMA_GLINT_DYN_CTL,
+ IRDMA_DB_ADDR_OFFSET,
+ IRDMA_GLPCI_LBARCTRL,
+ IRDMA_GLPE_CPUSTATUS0,
+ IRDMA_GLPE_CPUSTATUS1,
+ IRDMA_GLPE_CPUSTATUS2,
+ IRDMA_PFINT_AEQCTL,
+ IRDMA_GLINT_CEQCTL,
+ IRDMA_VSIQF_PE_CTL1,
+ IRDMA_PFHMC_PDINV,
+ IRDMA_GLHMC_VFPDINV,
+ IRDMA_MAX_REGS, /* Must be last entry */
+};
+
+enum irdma_vers {
+ IRDMA_GEN_RSVD,
+ IRDMA_GEN_1,
+ IRDMA_GEN_2,
+ IRDMA_GEN_3,
+};
+
+/**
+ * has_feature_rts_ae - determine if FW version supports RTS AE
+ * @fw_ver: version of FW
+ */
+static inline bool has_feature_rts_ae(u32 fw_ver, u8 hw_rev)
+{
+ if (hw_rev == IRDMA_GEN_1 &&
+ fw_ver >= IRDMA_I40IW_RTS_AE_FEATURE_FW_VER)
+ return true;
+ else if (hw_rev != IRDMA_GEN_1 &&
+ fw_ver >= IRDMA_RTS_AE_FEATURE_FW_VER)
+ return true;
+ else
+ return false;
+}
+
+enum irdma_shifts {
+ IRDMA_CCQPSTATUS_CCQP_DONE_S,
+ IRDMA_CCQPSTATUS_CCQP_ERR_S,
+ IRDMA_CQPSQ_STAG_PDID_S,
+ IRDMA_CQPSQ_CQ_CEQID_S,
+ IRDMA_MAX_SHIFTS,
+};
+
+enum irdma_masks {
+ IRDMA_CCQPSTATUS_CCQP_DONE_M,
+ IRDMA_CCQPSTATUS_CCQP_ERR_M,
+ IRDMA_CQPSQ_STAG_PDID_M,
+ IRDMA_CQPSQ_CQ_CEQID_M,
+ IRDMA_MAX_MASKS, /* Must be last entry */
+};
+
+#pragma pack(push, 1)
+
+struct irdma_hw_attrs {
+ u64 max_hw_outbound_msg_size;
+ u64 max_hw_inbound_msg_size;
+ u64 feature_flags;
+ u64 reserved64[15];
+ u32 min_hw_qp_id;
+ u32 min_hw_aeq_size;
+ u32 max_hw_aeq_size;
+ u32 min_hw_ceq_size;
+ u32 max_hw_ceq_size;
+ u32 min_hw_cq_size;
+ u32 max_hw_cq_size;
+ u32 max_hw_wq_frags;
+ u32 max_hw_read_sges;
+ u32 max_hw_device_pages;
+ u32 max_hw_vf_fpm_id;
+ u32 first_hw_vf_fpm_id;
+ u32 max_hw_inline;
+ u32 max_hw_push_inline;
+ u32 max_hw_ird;
+ u32 max_hw_ord;
+ u32 max_hw_wqes;
+ u32 max_hw_sq_quanta;
+ u32 max_hw_rq_quanta;
+ u32 max_hw_wq_quanta;
+ u32 max_hw_pds;
+ u32 max_hw_ena_vf_count;
+ u32 max_qp_wr;
+ u32 max_pe_ready_count;
+ u32 max_done_count;
+ u32 max_sleep_count;
+ u32 max_cqp_compl_wait_time_ms;
+ u32 reserved32[11];
+ u16 max_hw_wq_size;
+ u16 min_sw_wq_size;
+ u16 max_stat_inst;
+ u16 max_hw_sq_chunk;
+ u16 reserved16[12];
+ u8 hw_rev;
+ u8 reserved8[15];
+};
+
+#pragma pack(pop)
+
+void icrdma_init_hw(struct irdma_sc_dev *dev);
+#endif /* IRDMA_H*/
diff --git a/drivers/infiniband/hw/irdma/type.h b/drivers/infiniband/hw/irdma/type.h
new file mode 100644
index 0000000..d280b2d
--- /dev/null
+++ b/drivers/infiniband/hw/irdma/type.h
@@ -0,0 +1,1705 @@
+/* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB */
+/* Copyright (c) 2019, Intel Corporation. */
+
+#ifndef IRDMA_TYPE_H
+#define IRDMA_TYPE_H
+#include "osdep.h"
+#include "irdma.h"
+#include "user.h"
+#include "hmc.h"
+#include "uda.h"
+
+#ifndef CONFIG_DYNAMIC_DEBUG
+enum irdma_debug_flag {
+ IRDMA_DEBUG_NONE = 0x00000000,
+ IRDMA_DEBUG_ERR = 0x00000001,
+ IRDMA_DEBUG_INIT = 0x00000002,
+ IRDMA_DEBUG_DEV = 0x00000004,
+ IRDMA_DEBUG_CM = 0x00000008,
+ IRDMA_DEBUG_VERBS = 0x00000010,
+ IRDMA_DEBUG_PUDA = 0x00000020,
+ IRDMA_DEBUG_ILQ = 0x00000040,
+ IRDMA_DEBUG_IEQ = 0x00000080,
+ IRDMA_DEBUG_QP = 0x00000100,
+ IRDMA_DEBUG_CQ = 0x00000200,
+ IRDMA_DEBUG_MR = 0x00000400,
+ IRDMA_DEBUG_PBLE = 0x00000800,
+ IRDMA_DEBUG_WQE = 0x00001000,
+ IRDMA_DEBUG_AEQ = 0x00002000,
+ IRDMA_DEBUG_CQP = 0x00004000,
+ IRDMA_DEBUG_HMC = 0x00008000,
+ IRDMA_DEBUG_USER = 0x00010000,
+ IRDMA_DEBUG_VIRT = 0x00020000,
+ IRDMA_DEBUG_DCB = 0x00040000,
+ IRDMA_DEBUG_CQE = 0x00800000,
+ IRDMA_DEBUG_CLNT = 0x01000000,
+ IRDMA_DEBUG_WS = 0x02000000,
+ IRDMA_DEBUG_STATS = 0x04000000,
+ IRDMA_DEBUG_ALL = 0xFFFFFFFF,
+};
+#else
+#define IRDMA_DEBUG_ERR "IRDMA_ERR"
+#define IRDMA_DEBUG_INIT "IRDMA_INIT"
+#define IRDMA_DEBUG_DEV "IRDMA_DEV"
+#define IRDMA_DEBUG_CM "IRDMA_CM"
+#define IRDMA_DEBUG_VERBS "IRDMA_VERBS"
+#define IRDMA_DEBUG_PUDA "IRDMA_PUDA"
+#define IRDMA_DEBUG_ILQ "IRDMA_ILQ"
+#define IRDMA_DEBUG_IEQ "IRDMA_IEQ"
+#define IRDMA_DEBUG_QP "IRDMA_QP"
+#define IRDMA_DEBUG_CQ "IRDMA_CQ"
+#define IRDMA_DEBUG_MR "IRDMA_MR"
+#define IRDMA_DEBUG_PBLE "IRDMA_PBLE"
+#define IRDMA_DEBUG_WQE "IRDMA_WQE"
+#define IRDMA_DEBUG_AEQ "IRDMA_AEQ"
+#define IRDMA_DEBUG_CQP "IRDMA_CQP"
+#define IRDMA_DEBUG_HMC "IRDMA_HMC"
+#define IRDMA_DEBUG_USER "IRDMA_USER"
+#define IRDMA_DEBUG_VIRT "IRDMA_VIRT"
+#define IRDMA_DEBUG_DCB "IRDMA_DCB"
+#define IRDMA_DEBUG_CQE "IRDMA_CQE"
+#define IRDMA_DEBUG_CLNT "IRDMA_CLNT"
+#define IRDMA_DEBUG_WS "IRDMA_WS"
+#define IRDMA_DEBUG_STATS "IRDMA_DEBUG_STATS"
+#endif
+
+enum irdma_page_size {
+ IRDMA_PAGE_SIZE_4K = 0,
+ IRDMA_PAGE_SIZE_2M,
+ IRDMA_PAGE_SIZE_1G,
+};
+
+enum irdma_hdrct_flags {
+ DDP_LEN_FLAG = 0x80,
+ DDP_HDR_FLAG = 0x40,
+ RDMA_HDR_FLAG = 0x20,
+};
+
+enum irdma_term_layers {
+ LAYER_RDMA = 0,
+ LAYER_DDP = 1,
+ LAYER_MPA = 2,
+};
+
+enum irdma_term_error_types {
+ RDMAP_REMOTE_PROT = 1,
+ RDMAP_REMOTE_OP = 2,
+ DDP_CATASTROPHIC = 0,
+ DDP_TAGGED_BUF = 1,
+ DDP_UNTAGGED_BUF = 2,
+ DDP_LLP = 3,
+};
+
+enum irdma_term_rdma_errors {
+ RDMAP_INV_STAG = 0x00,
+ RDMAP_INV_BOUNDS = 0x01,
+ RDMAP_ACCESS = 0x02,
+ RDMAP_UNASSOC_STAG = 0x03,
+ RDMAP_TO_WRAP = 0x04,
+ RDMAP_INV_RDMAP_VER = 0x05,
+ RDMAP_UNEXPECTED_OP = 0x06,
+ RDMAP_CATASTROPHIC_LOCAL = 0x07,
+ RDMAP_CATASTROPHIC_GLOBAL = 0x08,
+ RDMAP_CANT_INV_STAG = 0x09,
+ RDMAP_UNSPECIFIED = 0xff,
+};
+
+enum irdma_term_ddp_errors {
+ DDP_CATASTROPHIC_LOCAL = 0x00,
+ DDP_TAGGED_INV_STAG = 0x00,
+ DDP_TAGGED_BOUNDS = 0x01,
+ DDP_TAGGED_UNASSOC_STAG = 0x02,
+ DDP_TAGGED_TO_WRAP = 0x03,
+ DDP_TAGGED_INV_DDP_VER = 0x04,
+ DDP_UNTAGGED_INV_QN = 0x01,
+ DDP_UNTAGGED_INV_MSN_NO_BUF = 0x02,
+ DDP_UNTAGGED_INV_MSN_RANGE = 0x03,
+ DDP_UNTAGGED_INV_MO = 0x04,
+ DDP_UNTAGGED_INV_TOO_LONG = 0x05,
+ DDP_UNTAGGED_INV_DDP_VER = 0x06,
+};
+
+enum irdma_term_mpa_errors {
+ MPA_CLOSED = 0x01,
+ MPA_CRC = 0x02,
+ MPA_MARKER = 0x03,
+ MPA_REQ_RSP = 0x04,
+};
+
+enum irdma_flush_opcode {
+ FLUSH_INVALID = 0,
+ FLUSH_PROT_ERR,
+ FLUSH_REM_ACCESS_ERR,
+ FLUSH_LOC_QP_OP_ERR,
+ FLUSH_REM_OP_ERR,
+ FLUSH_LOC_LEN_ERR,
+ FLUSH_GENERAL_ERR,
+ FLUSH_FATAL_ERR,
+};
+
+enum irdma_term_eventtypes {
+ TERM_EVENT_QP_FATAL,
+ TERM_EVENT_QP_ACCESS_ERR,
+};
+
+enum irdma_hw_stats_index_32b {
+ IRDMA_HW_STAT_INDEX_IP4RXDISCARD = 0,
+ IRDMA_HW_STAT_INDEX_IP4RXTRUNC = 1,
+ IRDMA_HW_STAT_INDEX_IP4TXNOROUTE = 2,
+ IRDMA_HW_STAT_INDEX_IP6RXDISCARD = 3,
+ IRDMA_HW_STAT_INDEX_IP6RXTRUNC = 4,
+ IRDMA_HW_STAT_INDEX_IP6TXNOROUTE = 5,
+ IRDMA_HW_STAT_INDEX_TCPRTXSEG = 6,
+ IRDMA_HW_STAT_INDEX_TCPRXOPTERR = 7,
+ IRDMA_HW_STAT_INDEX_TCPRXPROTOERR = 8,
+ IRDMA_HW_STAT_INDEX_MAX_32_GEN_1 = 9, /* Must be same value as next entry */
+ IRDMA_HW_STAT_INDEX_RXVLANERR = 9,
+ IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED = 10,
+ IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED = 11,
+ IRDMA_HW_STAT_INDEX_TXNPCNPSENT = 12,
+ IRDMA_HW_STAT_INDEX_MAX_32, /* Must be last entry */
+};
+
+enum irdma_hw_stats_index_64b {
+ IRDMA_HW_STAT_INDEX_IP4RXOCTS = 0,
+ IRDMA_HW_STAT_INDEX_IP4RXPKTS = 1,
+ IRDMA_HW_STAT_INDEX_IP4RXFRAGS = 2,
+ IRDMA_HW_STAT_INDEX_IP4RXMCPKTS = 3,
+ IRDMA_HW_STAT_INDEX_IP4TXOCTS = 4,
+ IRDMA_HW_STAT_INDEX_IP4TXPKTS = 5,
+ IRDMA_HW_STAT_INDEX_IP4TXFRAGS = 6,
+ IRDMA_HW_STAT_INDEX_IP4TXMCPKTS = 7,
+ IRDMA_HW_STAT_INDEX_IP6RXOCTS = 8,
+ IRDMA_HW_STAT_INDEX_IP6RXPKTS = 9,
+ IRDMA_HW_STAT_INDEX_IP6RXFRAGS = 10,
+ IRDMA_HW_STAT_INDEX_IP6RXMCPKTS = 11,
+ IRDMA_HW_STAT_INDEX_IP6TXOCTS = 12,
+ IRDMA_HW_STAT_INDEX_IP6TXPKTS = 13,
+ IRDMA_HW_STAT_INDEX_IP6TXFRAGS = 14,
+ IRDMA_HW_STAT_INDEX_IP6TXMCPKTS = 15,
+ IRDMA_HW_STAT_INDEX_TCPRXSEGS = 16,
+ IRDMA_HW_STAT_INDEX_TCPTXSEG = 17,
+ IRDMA_HW_STAT_INDEX_RDMARXRDS = 18,
+ IRDMA_HW_STAT_INDEX_RDMARXSNDS = 19,
+ IRDMA_HW_STAT_INDEX_RDMARXWRS = 20,
+ IRDMA_HW_STAT_INDEX_RDMATXRDS = 21,
+ IRDMA_HW_STAT_INDEX_RDMATXSNDS = 22,
+ IRDMA_HW_STAT_INDEX_RDMATXWRS = 23,
+ IRDMA_HW_STAT_INDEX_RDMAVBND = 24,
+ IRDMA_HW_STAT_INDEX_RDMAVINV = 25,
+ IRDMA_HW_STAT_INDEX_MAX_64_GEN_1 = 26, /* Must be same value as next entry */
+ IRDMA_HW_STAT_INDEX_IP4RXMCOCTS = 26,
+ IRDMA_HW_STAT_INDEX_IP4TXMCOCTS = 27,
+ IRDMA_HW_STAT_INDEX_IP6RXMCOCTS = 28,
+ IRDMA_HW_STAT_INDEX_IP6TXMCOCTS = 29,
+ IRDMA_HW_STAT_INDEX_UDPRXPKTS = 30,
+ IRDMA_HW_STAT_INDEX_UDPTXPKTS = 31,
+ IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS = 32,
+ IRDMA_HW_STAT_INDEX_MAX_64, /* Must be last entry */
+};
+
+enum irdma_feature_type {
+ IRDMA_FEATURE_FW_INFO = 0,
+ IRDMA_MAX_FEATURES, /* Must be last entry */
+};
+
+enum irdma_sched_prio_type {
+ IRDMA_PRIO_WEIGHTED_RR = 1,
+ IRDMA_PRIO_STRICT = 2,
+ IRDMA_PRIO_WEIGHTED_STRICT = 3,
+};
+
+enum irdma_vm_vf_type {
+ IRDMA_VF_TYPE = 0,
+ IRDMA_VM_TYPE,
+ IRDMA_PF_TYPE,
+};
+
+enum irdma_cqp_hmc_profile {
+ IRDMA_HMC_PROFILE_DEFAULT = 1,
+ IRDMA_HMC_PROFILE_FAVOR_VF = 2,
+ IRDMA_HMC_PROFILE_EQUAL = 3,
+};
+
+enum irdma_quad_entry_type {
+ IRDMA_QHASH_TYPE_TCP_ESTABLISHED = 1,
+ IRDMA_QHASH_TYPE_TCP_SYN,
+ IRDMA_QHASH_TYPE_UDP_UNICAST,
+ IRDMA_QHASH_TYPE_UDP_MCAST,
+ IRDMA_QHASH_TYPE_ROCE_MCAST,
+ IRDMA_QHASH_TYPE_ROCEV2_HW,
+};
+
+enum irdma_quad_hash_manage_type {
+ IRDMA_QHASH_MANAGE_TYPE_DELETE = 0,
+ IRDMA_QHASH_MANAGE_TYPE_ADD,
+ IRDMA_QHASH_MANAGE_TYPE_MODIFY,
+};
+
+struct irdma_sc_dev;
+struct irdma_vsi_pestat;
+struct irdma_irq_ops;
+struct irdma_cqp_ops;
+struct irdma_ccq_ops;
+struct irdma_ceq_ops;
+struct irdma_aeq_ops;
+struct irdma_mr_ops;
+struct irdma_cqp_misc_ops;
+struct irdma_pd_ops;
+struct irdma_ah_ops;
+struct irdma_priv_qp_ops;
+struct irdma_priv_cq_ops;
+struct irdma_hmc_ops;
+
+struct irdma_cqp_init_info {
+ u64 cqp_compl_ctx;
+ u64 host_ctx_pa;
+ u64 sq_pa;
+ struct irdma_sc_dev *dev;
+ struct irdma_cqp_quanta *sq;
+ __le64 *host_ctx;
+ u64 *scratch_array;
+ u32 sq_size;
+ u16 hw_maj_ver;
+ u16 hw_min_ver;
+ u8 struct_ver;
+ bool en_datacenter_tcp;
+ u8 hmc_profile;
+ u8 ena_vf_count;
+ u8 ceqs_per_vf;
+ bool disable_packed;
+ bool rocev2_rto_policy;
+ enum irdma_protocol_used protocol_used;
+};
+
+struct irdma_terminate_hdr {
+ u8 layer_etype;
+ u8 error_code;
+ u8 hdrct;
+ u8 rsvd;
+};
+
+struct irdma_cqp_sq_wqe {
+ __le64 buf[IRDMA_CQP_WQE_SIZE];
+};
+
+struct irdma_sc_aeqe {
+ __le64 buf[IRDMA_AEQE_SIZE];
+};
+
+struct irdma_ceqe {
+ __le64 buf[IRDMA_CEQE_SIZE];
+};
+
+struct irdma_cqp_ctx {
+ __le64 buf[IRDMA_CQP_CTX_SIZE];
+};
+
+struct irdma_cq_shadow_area {
+ __le64 buf[IRDMA_SHADOW_AREA_SIZE];
+};
+
+struct irdma_dev_hw_stats_offsets {
+ u32 stats_offset_32[IRDMA_HW_STAT_INDEX_MAX_32];
+ u32 stats_offset_64[IRDMA_HW_STAT_INDEX_MAX_64];
+};
+
+struct irdma_dev_hw_stats {
+ u64 stats_val_32[IRDMA_HW_STAT_INDEX_MAX_32];
+ u64 stats_val_64[IRDMA_HW_STAT_INDEX_MAX_64];
+};
+
+struct irdma_gather_stats {
+ u32 rsvd1;
+ u32 rxvlanerr;
+ u64 ip4rxocts;
+ u64 ip4rxpkts;
+ u32 ip4rxtrunc;
+ u32 ip4rxdiscard;
+ u64 ip4rxfrags;
+ u64 ip4rxmcocts;
+ u64 ip4rxmcpkts;
+ u64 ip6rxocts;
+ u64 ip6rxpkts;
+ u32 ip6rxtrunc;
+ u32 ip6rxdiscard;
+ u64 ip6rxfrags;
+ u64 ip6rxmcocts;
+ u64 ip6rxmcpkts;
+ u64 ip4txocts;
+ u64 ip4txpkts;
+ u64 ip4txfrag;
+ u64 ip4txmcocts;
+ u64 ip4txmcpkts;
+ u64 ip6txocts;
+ u64 ip6txpkts;
+ u64 ip6txfrags;
+ u64 ip6txmcocts;
+ u64 ip6txmcpkts;
+ u32 ip6txnoroute;
+ u32 ip4txnoroute;
+ u64 tcprxsegs;
+ u32 tcprxprotoerr;
+ u32 tcprxopterr;
+ u64 tcptxsegs;
+ u32 rsvd2;
+ u32 tcprtxseg;
+ u64 udprxpkts;
+ u64 udptxpkts;
+ u64 rdmarxwrs;
+ u64 rdmarxrds;
+ u64 rdmarxsnds;
+ u64 rdmatxwrs;
+ u64 rdmatxrds;
+ u64 rdmatxsnds;
+ u64 rdmavbn;
+ u64 rdmavinv;
+ u64 rxnpecnmrkpkts;
+ u32 rxrpcnphandled;
+ u32 rxrpcnpignored;
+ u32 txnpcnpsent;
+ u32 rsvd3[88];
+};
+
+struct irdma_stats_gather_info {
+ bool use_hmc_fcn_index;
+ bool use_stats_inst;
+ u8 hmc_fcn_index;
+ u8 stats_inst_index;
+ struct irdma_dma_mem stats_buff_mem;
+ struct irdma_gather_stats *gather_stats;
+ struct irdma_gather_stats *last_gather_stats;
+};
+
+struct irdma_vsi_pestat {
+ struct irdma_hw *hw;
+ struct irdma_dev_hw_stats hw_stats;
+ struct irdma_stats_gather_info gather_info;
+ struct timer_list stats_timer;
+ struct irdma_sc_vsi *vsi;
+ struct irdma_dev_hw_stats last_hw_stats;
+ spinlock_t lock; /* rdma stats lock */
+};
+
+struct irdma_hw {
+ u8 __iomem *hw_addr;
+ void *dev_context;
+ struct irdma_hmc_info hmc;
+};
+
+struct irdma_pfpdu {
+ struct list_head rxlist;
+ u32 rcv_nxt;
+ u32 fps;
+ u32 max_fpdu_data;
+ u32 nextseqnum;
+ bool mode;
+ bool mpa_crc_err;
+ u64 total_ieq_bufs;
+ u64 fpdu_processed;
+ u64 bad_seq_num;
+ u64 crc_err;
+ u64 no_tx_bufs;
+ u64 tx_err;
+ u64 out_of_order;
+ u64 pmode_count;
+ struct irdma_sc_ah *ah;
+ struct irdma_puda_buf *ah_buf;
+ spinlock_t lock; /* fpdu processing lock */
+ struct irdma_puda_buf *lastrcv_buf;
+};
+
+struct irdma_sc_pd {
+ struct irdma_sc_dev *dev;
+ u32 pd_id;
+ int abi_ver;
+};
+
+struct irdma_cqp_quanta {
+ __le64 elem[IRDMA_CQP_WQE_SIZE];
+};
+
+struct irdma_sc_cqp {
+ u32 size;
+ u64 sq_pa;
+ u64 host_ctx_pa;
+ void *back_cqp;
+ struct irdma_sc_dev *dev;
+ enum irdma_status_code (*process_cqp_sds)(struct irdma_sc_dev *dev,
+ struct irdma_update_sds_info *info);
+ struct irdma_dma_mem sdbuf;
+ struct irdma_ring sq_ring;
+ struct irdma_cqp_quanta *sq_base;
+ __le64 *host_ctx;
+ u64 *scratch_array;
+ u32 cqp_id;
+ u32 sq_size;
+ u32 hw_sq_size;
+ u16 hw_maj_ver;
+ u16 hw_min_ver;
+ u8 struct_ver;
+ u8 polarity;
+ u8 hmc_profile;
+ u8 ena_vf_count;
+ u8 timeout_count;
+ u8 ceqs_per_vf;
+ bool en_datacenter_tcp;
+ bool disable_packed;
+ bool rocev2_rto_policy;
+ enum irdma_protocol_used protocol_used;
+};
+
+struct irdma_sc_aeq {
+ u32 size;
+ u64 aeq_elem_pa;
+ struct irdma_sc_dev *dev;
+ struct irdma_sc_aeqe *aeqe_base;
+ void *pbl_list;
+ u32 elem_cnt;
+ struct irdma_ring aeq_ring;
+ bool virtual_map;
+ u8 pbl_chunk_size;
+ u32 first_pm_pbl_idx;
+ u8 polarity;
+};
+
+struct irdma_sc_ceq {
+ u32 size;
+ u64 ceq_elem_pa;
+ struct irdma_sc_dev *dev;
+ struct irdma_ceqe *ceqe_base;
+ void *pbl_list;
+ u32 ceq_id;
+ u32 elem_cnt;
+ struct irdma_ring ceq_ring;
+ bool virtual_map;
+ u8 pbl_chunk_size;
+ bool tph_en;
+ u8 tph_val;
+ u32 first_pm_pbl_idx;
+ u8 polarity;
+ bool itr_no_expire;
+ struct irdma_sc_vsi *vsi;
+ struct irdma_sc_cq **reg_cq;
+ u32 reg_cq_size;
+ spinlock_t req_cq_lock; /* protect access to reg_cq array */
+};
+
+struct irdma_sc_cq {
+ struct irdma_cq_uk cq_uk;
+ u64 cq_pa;
+ u64 shadow_area_pa;
+ struct irdma_sc_dev *dev;
+ struct irdma_sc_vsi *vsi;
+ void *pbl_list;
+ void *back_cq;
+ u32 ceq_id;
+ u32 shadow_read_threshold;
+ bool ceqe_mask;
+ bool virtual_map;
+ u8 pbl_chunk_size;
+ u8 cq_type;
+ bool ceq_id_valid;
+ bool tph_en;
+ u8 tph_val;
+ u32 first_pm_pbl_idx;
+ bool check_overflow;
+};
+
+struct irdma_sc_qp {
+ struct irdma_qp_uk qp_uk;
+ u64 sq_pa;
+ u64 rq_pa;
+ u64 hw_host_ctx_pa;
+ u64 shadow_area_pa;
+ u64 q2_pa;
+ struct irdma_sc_dev *dev;
+ struct irdma_sc_vsi *vsi;
+ struct irdma_sc_pd *pd;
+ __le64 *hw_host_ctx;
+ void *llp_stream_handle;
+ void *back_qp;
+ struct irdma_pfpdu pfpdu;
+ u32 ieq_qp;
+ u8 *q2_buf;
+ u64 qp_compl_ctx;
+ u16 qs_handle;
+ u16 push_idx;
+ u8 sq_tph_val;
+ u8 rq_tph_val;
+ u8 qp_state;
+ u8 qp_type;
+ u8 hw_sq_size;
+ u8 hw_rq_size;
+ u8 src_mac_addr_idx;
+ bool sq_tph_en;
+ bool rq_tph_en;
+ bool rcv_tph_en;
+ bool xmit_tph_en;
+ bool virtual_map;
+ bool flush_sq;
+ bool flush_rq;
+ u8 user_pri;
+ struct list_head list;
+ bool on_qoslist;
+ bool sq_flush;
+ enum irdma_flush_opcode flush_code;
+ enum irdma_term_eventtypes eventtype;
+ u8 term_flags;
+};
+
+struct irdma_stats_inst_info {
+ bool use_hmc_fcn_index;
+ u8 hmc_fn_id;
+ u8 stats_idx;
+};
+
+struct irdma_up_info {
+ u8 map[8];
+ u8 cnp_up_override;
+ u8 hmc_fcn_idx;
+ bool use_vlan;
+ bool use_cnp_up_override;
+};
+
+#define IRDMA_MAX_WS_NODES 0x3FF
+#define IRDMA_WS_NODE_INVALID 0xFFFF
+
+struct irdma_ws_node_info {
+ u16 id;
+ u16 vsi;
+ u16 parent_id;
+ u16 qs_handle;
+ bool type_leaf;
+ bool enable;
+ u8 prio_type;
+ u8 tc;
+ u8 weight;
+};
+
+struct irdma_hmc_fpm_misc {
+ u32 max_ceqs;
+ u32 max_sds;
+ u32 xf_block_size;
+ u32 q1_block_size;
+ u32 ht_multiplier;
+ u32 timer_bucket;
+ u32 rrf_block_size;
+ u32 ooiscf_block_size;
+};
+
+#define IRDMA_LEAF_DEFAULT_REL_BW 64
+#define IRDMA_PARENT_DEFAULT_REL_BW 1
+
+struct irdma_qos {
+ struct list_head qplist;
+ spinlock_t lock; /* protect qos list */
+ u64 lan_qos_handle;
+ u32 l2_sched_node_id;
+ u16 qs_handle;
+ u8 traffic_class;
+ u8 rel_bw;
+ u8 prio_type;
+};
+
+#define IRDMA_INVALID_FCN_ID 0xff
+struct irdma_sc_vsi {
+ u16 vsi_idx;
+ struct irdma_sc_dev *dev;
+ void *back_vsi;
+ u32 ilq_count;
+ struct irdma_virt_mem ilq_mem;
+ struct irdma_puda_rsrc *ilq;
+ u32 ieq_count;
+ struct irdma_virt_mem ieq_mem;
+ struct irdma_puda_rsrc *ieq;
+ u32 exception_lan_q;
+ u16 mtu;
+ u16 vm_id;
+ u8 fcn_id;
+ enum irdma_vm_vf_type vm_vf_type;
+ bool stats_fcn_id_alloc;
+ struct irdma_qos qos[IRDMA_MAX_USER_PRIORITY];
+ struct irdma_vsi_pestat *pestat;
+ atomic_t qp_suspend_reqs;
+ bool tc_change_pending;
+ u8 qos_rel_bw;
+ u8 qos_prio_type;
+};
+
+struct irdma_sc_dev {
+ struct list_head cqp_cmd_head; /* head of the CQP command list */
+ spinlock_t cqp_lock; /* protect CQP list access */
+ struct irdma_dev_uk dev_uk;
+ bool fcn_id_array[IRDMA_MAX_STATS_COUNT];
+ struct irdma_dma_mem vf_fpm_query_buf[IRDMA_MAX_PE_ENA_VF_COUNT];
+ u64 fpm_query_buf_pa;
+ u64 fpm_commit_buf_pa;
+ __le64 *fpm_query_buf;
+ __le64 *fpm_commit_buf;
+ void *back_dev;
+ struct irdma_hw *hw;
+ u8 __iomem *db_addr;
+ u32 __iomem *wqe_alloc_db;
+ u32 __iomem *cq_arm_db;
+ u32 __iomem *aeq_alloc_db;
+ u32 __iomem *cqp_db;
+ u32 __iomem *cq_ack_db;
+ u32 __iomem *ceq_itr_mask_db;
+ u32 __iomem *aeq_itr_mask_db;
+ u32 hw_regs[IRDMA_MAX_REGS];
+ u64 hw_masks[IRDMA_MAX_MASKS];
+ u64 hw_shifts[IRDMA_MAX_SHIFTS];
+ u64 hw_stats_regs_32[IRDMA_HW_STAT_INDEX_MAX_32];
+ u64 hw_stats_regs_64[IRDMA_HW_STAT_INDEX_MAX_64];
+ u64 feature_info[IRDMA_MAX_FEATURES];
+ u64 cqp_cmd_stats[IRDMA_OP_SIZE_CQP_STAT_ARRAY];
+ struct irdma_hw_attrs hw_attrs;
+ struct irdma_hmc_info *hmc_info;
+ struct irdma_vfdev *vf_dev[IRDMA_MAX_PE_ENA_VF_COUNT];
+ struct irdma_sc_cqp *cqp;
+ struct irdma_sc_aeq *aeq;
+ struct irdma_sc_ceq *ceq[IRDMA_CEQ_MAX_COUNT];
+ struct irdma_sc_cq *ccq;
+ struct irdma_irq_ops *irq_ops;
+ struct irdma_cqp_ops *cqp_ops;
+ struct irdma_ccq_ops *ccq_ops;
+ struct irdma_ceq_ops *ceq_ops;
+ struct irdma_aeq_ops *aeq_ops;
+ struct irdma_pd_ops *iw_pd_ops;
+ struct irdma_ah_ops *iw_ah_ops;
+ struct irdma_priv_qp_ops *iw_priv_qp_ops;
+ struct irdma_priv_cq_ops *iw_priv_cq_ops;
+ struct irdma_mr_ops *mr_ops;
+ struct irdma_cqp_misc_ops *cqp_misc_ops;
+ struct irdma_hmc_ops *hmc_ops;
+ struct irdma_uda_ops *iw_uda_ops;
+ struct irdma_hmc_fpm_misc hmc_fpm_misc;
+ struct irdma_ws_node *ws_tree_root;
+ struct mutex ws_mutex; /* ws tree mutex */
+#ifndef CONFIG_DYNAMIC_DEBUG
+ u32 debug_mask;
+#endif
+ u16 num_vfs;
+ u8 hmc_fn_id;
+ u8 vf_id;
+ bool is_pf;
+ bool vchnl_up;
+ bool ceq_valid;
+ u8 pci_rev;
+ enum irdma_status_code (*ws_add)(struct irdma_sc_vsi *vsi, u8 user_pri);
+ void (*ws_remove)(struct irdma_sc_vsi *vsi, u8 user_pri);
+};
+
+struct irdma_modify_cq_info {
+ u64 cq_pa;
+ struct irdma_cqe *cq_base;
+ void *pbl_list;
+ u32 ceq_id;
+ u32 cq_size;
+ u32 shadow_read_threshold;
+ bool virtual_map;
+ u8 pbl_chunk_size;
+ bool check_overflow;
+ bool cq_resize;
+ bool ceq_change;
+ bool check_overflow_change;
+ u32 first_pm_pbl_idx;
+ bool ceq_valid;
+};
+
+struct irdma_create_qp_info {
+ bool ord_valid;
+ bool tcp_ctx_valid;
+ bool cq_num_valid;
+ bool arp_cache_idx_valid;
+ bool mac_valid;
+ bool force_lpb;
+ u8 next_iwarp_state;
+};
+
+struct irdma_modify_qp_info {
+ u64 rx_win0;
+ u64 rx_win1;
+ u16 new_mss;
+ u8 next_iwarp_state;
+ u8 curr_iwarp_state;
+ u8 termlen;
+ bool ord_valid;
+ bool tcp_ctx_valid;
+ bool udp_ctx_valid;
+ bool cq_num_valid;
+ bool arp_cache_idx_valid;
+ bool reset_tcp_conn;
+ bool remove_hash_idx;
+ bool dont_send_term;
+ bool dont_send_fin;
+ bool cached_var_valid;
+ bool mss_change;
+ bool force_lpb;
+ bool mac_valid;
+};
+
+struct irdma_ccq_cqe_info {
+ struct irdma_sc_cqp *cqp;
+ u64 scratch;
+ u32 op_ret_val;
+ u16 maj_err_code;
+ u16 min_err_code;
+ u8 op_code;
+ bool error;
+};
+
+struct irdma_dcb_app_info {
+ u8 priority;
+ u8 selector;
+ u16 prot_id;
+};
+
+struct irdma_qos_tc_info {
+ u64 tc_ctx;
+ u8 rel_bw;
+ u8 prio_type;
+ u8 egress_virt_up;
+ u8 ingress_virt_up;
+};
+
+struct irdma_l2params {
+ struct irdma_qos_tc_info tc_info[IRDMA_MAX_USER_PRIORITY];
+ struct irdma_dcb_app_info apps[IRDMA_MAX_APPS];
+ u32 num_apps;
+ u16 qs_handle_list[IRDMA_MAX_USER_PRIORITY];
+ u16 mtu;
+ u8 up2tc[IRDMA_MAX_USER_PRIORITY];
+ u8 num_tc;
+ u8 vsi_rel_bw;
+ u8 vsi_prio_type;
+ bool mtu_changed;
+ bool tc_changed;
+};
+
+struct irdma_vsi_init_info {
+ struct irdma_sc_dev *dev;
+ void *back_vsi;
+ struct irdma_l2params *params;
+ u16 exception_lan_q;
+ u16 pf_data_vsi_num;
+ enum irdma_vm_vf_type vm_vf_type;
+ u16 vm_id;
+};
+
+struct irdma_vsi_stats_info {
+ struct irdma_vsi_pestat *pestat;
+ u8 fcn_id;
+ bool alloc_fcn_id;
+};
+
+struct irdma_device_init_info {
+ u64 fpm_query_buf_pa;
+ u64 fpm_commit_buf_pa;
+ __le64 *fpm_query_buf;
+ __le64 *fpm_commit_buf;
+ struct irdma_hw *hw;
+ void __iomem *bar0;
+ enum irdma_status_code (*vchnl_send)(struct irdma_sc_dev *dev,
+ u32 vf_id, u8 *msg, u16 len);
+ void (*init_hw)(struct irdma_sc_dev *dev);
+ u8 hmc_fn_id;
+ bool is_pf;
+#ifndef CONFIG_DYNAMIC_DEBUG
+ u32 debug_mask;
+#endif
+};
+
+struct irdma_ceq_init_info {
+ u64 ceqe_pa;
+ struct irdma_sc_dev *dev;
+ u64 *ceqe_base;
+ void *pbl_list;
+ u32 elem_cnt;
+ u32 ceq_id;
+ bool virtual_map;
+ u8 pbl_chunk_size;
+ bool tph_en;
+ u8 tph_val;
+ u32 first_pm_pbl_idx;
+ bool itr_no_expire;
+ struct irdma_sc_vsi *vsi;
+ struct irdma_sc_cq **reg_cq;
+ u32 reg_cq_idx;
+};
+
+struct irdma_aeq_init_info {
+ u64 aeq_elem_pa;
+ struct irdma_sc_dev *dev;
+ u32 *aeqe_base;
+ void *pbl_list;
+ u32 elem_cnt;
+ bool virtual_map;
+ u8 pbl_chunk_size;
+ u32 first_pm_pbl_idx;
+};
+
+struct irdma_ccq_init_info {
+ u64 cq_pa;
+ u64 shadow_area_pa;
+ struct irdma_sc_dev *dev;
+ struct irdma_cqe *cq_base;
+ __le64 *shadow_area;
+ void *pbl_list;
+ u32 num_elem;
+ u32 ceq_id;
+ u32 shadow_read_threshold;
+ bool ceqe_mask;
+ bool ceq_id_valid;
+ bool tph_en;
+ u8 tph_val;
+ bool avoid_mem_cflct;
+ bool virtual_map;
+ u8 pbl_chunk_size;
+ u32 first_pm_pbl_idx;
+ struct irdma_sc_vsi *vsi;
+};
+
+struct irdma_udp_offload_info {
+ bool ipv4;
+ bool insert_vlan_tag;
+ u8 ttl;
+ u8 tos;
+ u16 src_port;
+ u16 dst_port;
+ u32 dest_ip_addr0;
+ u32 dest_ip_addr1;
+ u32 dest_ip_addr2;
+ u32 dest_ip_addr3;
+ u32 snd_mss;
+ u16 vlan_tag;
+ u16 arp_idx;
+ u32 flow_label;
+ u8 udp_state;
+ u32 psn_nxt;
+ u32 lsn;
+ u32 epsn;
+ u32 psn_max;
+ u32 psn_una;
+ u32 local_ipaddr0;
+ u32 local_ipaddr1;
+ u32 local_ipaddr2;
+ u32 local_ipaddr3;
+ u32 cwnd;
+ u8 rexmit_thresh;
+ u8 rnr_nak_thresh;
+};
+
+struct irdma_roce_offload_info {
+ u16 p_key;
+ u16 err_rq_idx;
+ u32 qkey;
+ u32 dest_qp;
+ u32 local_qp;
+ bool is_qp1;
+ bool udprivcq_en;
+ u8 roce_tver;
+ u8 ack_credits;
+ u8 err_rq_idx_valid;
+ u32 pd_id;
+ u8 ord_size;
+ u8 ird_size;
+ bool dcqcn_en;
+ bool rcv_no_icrc;
+ bool wr_rdresp_en;
+ bool bind_en;
+ bool fast_reg_en;
+ bool priv_mode_en;
+ bool rd_en;
+ bool timely_en;
+ u16 t_high;
+ u16 t_low;
+ bool use_stats_inst;
+ u8 last_byte_sent;
+ u8 mac_addr[ETH_ALEN];
+ bool ecn_en;
+ bool dctcp_en;
+ bool fw_cc_enable;
+};
+
+struct irdma_iwarp_offload_info {
+ u16 rcv_mark_offset;
+ u16 snd_mark_offset;
+ u8 ddp_ver;
+ u8 rdmap_ver;
+ bool snd_mark_en;
+ bool rcv_mark_en;
+ bool ib_rd_en;
+ u8 iwarp_mode;
+ bool align_hdrs;
+ bool rcv_no_mpa_crc;
+
+ bool err_rq_idx_valid;
+ u16 err_rq_idx;
+ u32 pd_id;
+ u8 ord_size;
+ u8 ird_size;
+ bool wr_rdresp_en;
+ bool bind_en;
+ bool fast_reg_en;
+ bool priv_mode_en;
+ bool rd_en;
+ bool timely_en;
+ u16 t_high;
+ u16 t_low;
+ bool use_stats_inst;
+ u8 last_byte_sent;
+ u8 mac_addr[ETH_ALEN];
+ bool ecn_en;
+ bool dctcp_en;
+};
+
+struct irdma_tcp_offload_info {
+ bool ipv4;
+ bool no_nagle;
+ bool insert_vlan_tag;
+ bool time_stamp;
+ u8 cwnd_inc_limit;
+ bool drop_ooo_seg;
+ u8 dup_ack_thresh;
+ u8 ttl;
+ u8 src_mac_addr_idx;
+ bool avoid_stretch_ack;
+ u8 tos;
+ u16 src_port;
+ u16 dst_port;
+ u32 dest_ip_addr0;
+ u32 dest_ip_addr1;
+ u32 dest_ip_addr2;
+ u32 dest_ip_addr3;
+ u32 snd_mss;
+ u16 vlan_tag;
+ u16 arp_idx;
+ u32 flow_label;
+ bool wscale;
+ u8 tcp_state;
+ u8 snd_wscale;
+ u8 rcv_wscale;
+ u32 time_stamp_recent;
+ u32 time_stamp_age;
+ u32 snd_nxt;
+ u32 snd_wnd;
+ u32 rcv_nxt;
+ u32 rcv_wnd;
+ u32 snd_max;
+ u32 snd_una;
+ u32 srtt;
+ u32 rtt_var;
+ u32 ss_thresh;
+ u32 cwnd;
+ u32 snd_wl1;
+ u32 snd_wl2;
+ u32 max_snd_window;
+ u8 rexmit_thresh;
+ u32 local_ipaddr0;
+ u32 local_ipaddr1;
+ u32 local_ipaddr2;
+ u32 local_ipaddr3;
+ bool ignore_tcp_opt;
+ bool ignore_tcp_uns_opt;
+};
+
+struct irdma_qp_host_ctx_info {
+ u64 qp_compl_ctx;
+ union {
+ struct irdma_tcp_offload_info *tcp_info;
+ struct irdma_udp_offload_info *udp_info;
+ };
+ union {
+ struct irdma_iwarp_offload_info *iwarp_info;
+ struct irdma_roce_offload_info *roce_info;
+ };
+ u32 send_cq_num;
+ u32 rcv_cq_num;
+ u16 push_idx;
+ u8 stats_idx;
+ bool push_mode_en;
+ bool tcp_info_valid;
+ bool iwarp_info_valid;
+ bool stats_idx_valid;
+ bool add_to_qoslist;
+ u8 user_pri;
+};
+
+struct irdma_aeqe_info {
+ u64 compl_ctx;
+ u32 qp_cq_id;
+ u16 ae_id;
+ u16 wqe_idx;
+ u8 tcp_state;
+ u8 iwarp_state;
+ bool qp;
+ bool cq;
+ bool sq;
+ bool in_rdrsp_wr;
+ bool out_rdrsp;
+ u8 q2_data_written;
+ bool aeqe_overflow;
+};
+
+struct irdma_allocate_stag_info {
+ u64 total_len;
+ u64 first_pm_pbl_idx;
+ u32 chunk_size;
+ u32 stag_idx;
+ u32 page_size;
+ u32 pd_id;
+ u16 access_rights;
+ bool remote_access;
+ bool use_hmc_fcn_index;
+ u8 hmc_fcn_index;
+ bool use_pf_rid;
+};
+
+struct irdma_mw_alloc_info {
+ u32 mw_stag_index;
+ u32 page_size;
+ u32 pd_id;
+ bool remote_access;
+ bool mw_wide;
+ bool mw1_bind_dont_vldt_key;
+};
+
+struct irdma_reg_ns_stag_info {
+ u64 reg_addr_pa;
+ u64 fbo;
+ void *va;
+ u64 total_len;
+ u32 page_size;
+ u32 chunk_size;
+ u32 first_pm_pbl_index;
+ enum irdma_addressing_type addr_type;
+ irdma_stag_index stag_idx;
+ u16 access_rights;
+ u32 pd_id;
+ irdma_stag_key stag_key;
+ bool use_hmc_fcn_index;
+ u8 hmc_fcn_index;
+ bool use_pf_rid;
+};
+
+struct irdma_fast_reg_stag_info {
+ u64 wr_id;
+ u64 reg_addr_pa;
+ u64 fbo;
+ void *va;
+ u64 total_len;
+ u32 page_size;
+ u32 chunk_size;
+ u32 first_pm_pbl_index;
+ enum irdma_addressing_type addr_type;
+ irdma_stag_index stag_idx;
+ u16 access_rights;
+ u32 pd_id;
+ irdma_stag_key stag_key;
+ bool local_fence;
+ bool read_fence;
+ bool signaled;
+ bool push_wqe;
+ bool use_hmc_fcn_index;
+ u8 hmc_fcn_index;
+ bool use_pf_rid;
+ bool defer_flag;
+};
+
+struct irdma_dealloc_stag_info {
+ u32 stag_idx;
+ u32 pd_id;
+ bool mr;
+ bool dealloc_pbl;
+};
+
+struct irdma_register_shared_stag {
+ void *va;
+ enum irdma_addressing_type addr_type;
+ irdma_stag_index new_stag_idx;
+ irdma_stag_index parent_stag_idx;
+ u32 access_rights;
+ u32 pd_id;
+ irdma_stag_key new_stag_key;
+};
+
+struct irdma_qp_init_info {
+ struct irdma_qp_uk_init_info qp_uk_init_info;
+ struct irdma_sc_pd *pd;
+ struct irdma_sc_vsi *vsi;
+ __le64 *host_ctx;
+ u8 *q2;
+ u64 sq_pa;
+ u64 rq_pa;
+ u64 host_ctx_pa;
+ u64 q2_pa;
+ u64 shadow_area_pa;
+ u8 sq_tph_val;
+ u8 rq_tph_val;
+ u8 type;
+ bool sq_tph_en;
+ bool rq_tph_en;
+ bool rcv_tph_en;
+ bool xmit_tph_en;
+ bool virtual_map;
+};
+
+struct irdma_cq_init_info {
+ struct irdma_sc_dev *dev;
+ u64 cq_base_pa;
+ u64 shadow_area_pa;
+ u32 ceq_id;
+ u32 shadow_read_threshold;
+ bool virtual_map;
+ bool ceqe_mask;
+ u8 pbl_chunk_size;
+ u32 first_pm_pbl_idx;
+ bool ceq_id_valid;
+ bool tph_en;
+ u8 tph_val;
+ u8 type;
+ struct irdma_cq_uk_init_info cq_uk_init_info;
+ struct irdma_sc_vsi *vsi;
+};
+
+struct irdma_upload_context_info {
+ u64 buf_pa;
+ bool freeze_qp;
+ bool raw_format;
+ u32 qp_id;
+ u8 qp_type;
+};
+
+struct irdma_local_mac_entry_info {
+ u8 mac_addr[6];
+ u16 entry_idx;
+};
+
+struct irdma_add_arp_cache_entry_info {
+ u8 mac_addr[ETH_ALEN];
+ u32 reach_max;
+ u16 arp_index;
+ bool permanent;
+};
+
+struct irdma_apbvt_info {
+ u16 port;
+ bool add;
+};
+
+struct irdma_qhash_table_info {
+ struct irdma_sc_vsi *vsi;
+ enum irdma_quad_hash_manage_type manage;
+ enum irdma_quad_entry_type entry_type;
+ bool vlan_valid;
+ bool ipv4_valid;
+ u8 mac_addr[ETH_ALEN];
+ u16 vlan_id;
+ u8 user_pri;
+ u32 qp_num;
+ u32 dest_ip[4];
+ u32 src_ip[4];
+ u16 dest_port;
+ u16 src_port;
+};
+
+struct irdma_cqp_manage_push_page_info {
+ u32 push_idx;
+ u16 qs_handle;
+ u8 free_page;
+ u8 push_page_type;
+};
+
+struct irdma_qp_flush_info {
+ u16 sq_minor_code;
+ u16 sq_major_code;
+ u16 rq_minor_code;
+ u16 rq_major_code;
+ u16 ae_code;
+ u8 ae_src;
+ bool sq;
+ bool rq;
+ bool userflushcode;
+ bool generate_ae;
+};
+
+struct irdma_gen_ae_info {
+ u16 ae_code;
+ u8 ae_src;
+};
+
+struct irdma_cqp_timeout {
+ u64 compl_cqp_cmds;
+ u32 count;
+};
+
+struct irdma_irq_ops {
+ void (*irdma_en_irq)(struct irdma_sc_dev *dev, u32 idx);
+ void (*irdma_dis_irq)(struct irdma_sc_dev *dev, u32 idx);
+ void (*irdma_cfg_aeq)(struct irdma_sc_dev *dev, u32 idx);
+ void (*irdma_cfg_ceq)(struct irdma_sc_dev *dev, u32 ceq_id, u32 idx);
+};
+
+struct irdma_cqp_ops {
+ enum irdma_status_code (*cqp_init)(struct irdma_sc_cqp *cqp,
+ struct irdma_cqp_init_info *info);
+ enum irdma_status_code (*cqp_create)(struct irdma_sc_cqp *cqp,
+ u16 *maj_err, u16 *min_err);
+ void (*cqp_post_sq)(struct irdma_sc_cqp *cqp);
+ __le64 *(*cqp_get_next_send_wqe)(struct irdma_sc_cqp *cqp, u64 scratch);
+ enum irdma_status_code (*cqp_destroy)(struct irdma_sc_cqp *cqp);
+ enum irdma_status_code (*poll_for_cqp_op_done)(struct irdma_sc_cqp *cqp,
+ u8 opcode,
+ struct irdma_ccq_cqe_info *cmpl_info);
+ void (*check_cqp_progress)(struct irdma_cqp_timeout *cqp_timeout,
+ struct irdma_sc_dev *dev);
+};
+
+struct irdma_ccq_ops {
+ enum irdma_status_code (*ccq_init)(struct irdma_sc_cq *ccq,
+ struct irdma_ccq_init_info *info);
+ enum irdma_status_code (*ccq_create)(struct irdma_sc_cq *ccq,
+ u64 scratch,
+ bool check_overflow, bool post_sq);
+ enum irdma_status_code (*ccq_destroy)(struct irdma_sc_cq *ccq, u64 scratch, bool post_sq);
+ enum irdma_status_code (*ccq_create_done)(struct irdma_sc_cq *ccq);
+ enum irdma_status_code (*ccq_get_cqe_info)(struct irdma_sc_cq *ccq,
+ struct irdma_ccq_cqe_info *info);
+ void (*ccq_arm)(struct irdma_sc_cq *ccq);
+};
+
+struct irdma_ceq_ops {
+ enum irdma_status_code (*ceq_init)(struct irdma_sc_ceq *ceq,
+ struct irdma_ceq_init_info *info);
+ enum irdma_status_code (*ceq_create)(struct irdma_sc_ceq *ceq, u64 scratch, bool post_sq);
+ enum irdma_status_code (*cceq_create_done)(struct irdma_sc_ceq *ceq);
+ enum irdma_status_code (*cceq_destroy_done)(struct irdma_sc_ceq *ceq);
+ enum irdma_status_code (*cceq_create)(struct irdma_sc_ceq *ceq, u64 scratch);
+ enum irdma_status_code (*ceq_destroy)(struct irdma_sc_ceq *ceq, u64 scratch, bool post_sq);
+ void *(*process_ceq)(struct irdma_sc_dev *dev, struct irdma_sc_ceq *ceq);
+};
+
+struct irdma_aeq_ops {
+ enum irdma_status_code (*aeq_init)(struct irdma_sc_aeq *aeq,
+ struct irdma_aeq_init_info *info);
+ enum irdma_status_code (*aeq_create)(struct irdma_sc_aeq *aeq, u64 scratch, bool post_sq);
+ enum irdma_status_code (*aeq_destroy)(struct irdma_sc_aeq *aeq, u64 scratch, bool post_sq);
+ enum irdma_status_code (*get_next_aeqe)(struct irdma_sc_aeq *aeq,
+ struct irdma_aeqe_info *info);
+ enum irdma_status_code (*repost_aeq_entries)(struct irdma_sc_dev *dev, u32 count);
+ enum irdma_status_code (*aeq_create_done)(struct irdma_sc_aeq *aeq);
+ enum irdma_status_code (*aeq_destroy_done)(struct irdma_sc_aeq *aeq);
+};
+
+struct irdma_pd_ops {
+ void (*pd_init)(struct irdma_sc_dev *dev, struct irdma_sc_pd *pd, u32 pd_id, int abi_ver);
+};
+
+struct irdma_priv_qp_ops {
+ enum irdma_status_code (*qp_init)(struct irdma_sc_qp *qp,
+ struct irdma_qp_init_info *info);
+ enum irdma_status_code (*qp_create)(struct irdma_sc_qp *qp,
+ struct irdma_create_qp_info *info,
+ u64 scratch, bool post_sq);
+ enum irdma_status_code (*qp_modify)(struct irdma_sc_qp *qp,
+ struct irdma_modify_qp_info *info,
+ u64 scratch, bool post_sq);
+ enum irdma_status_code (*qp_destroy)(struct irdma_sc_qp *qp, u64 scratch,
+ bool remove_hash_idx,
+ bool ignore_mw_bnd, bool post_sq);
+ enum irdma_status_code (*qp_flush_wqes)(struct irdma_sc_qp *qp,
+ struct irdma_qp_flush_info *info,
+ u64 scratch, bool post_sq);
+ enum irdma_status_code (*qp_upload_context)(struct irdma_sc_dev *dev,
+ struct irdma_upload_context_info *info,
+ u64 scratch, bool post_sq);
+ enum irdma_status_code (*qp_setctx)(struct irdma_sc_qp *qp, __le64 *qp_ctx,
+ struct irdma_qp_host_ctx_info *info);
+ enum irdma_status_code (*update_suspend_qp)(struct irdma_sc_cqp *cqp,
+ struct irdma_sc_qp *qp, u64 scratch);
+ enum irdma_status_code (*update_resume_qp)(struct irdma_sc_cqp *cqp,
+ struct irdma_sc_qp *qp, u64 scratch);
+ enum irdma_status_code (*qp_setctx_roce)(struct irdma_sc_qp *qp, __le64 *qp_ctx,
+ struct irdma_qp_host_ctx_info *info);
+ void (*qp_send_lsmm)(struct irdma_sc_qp *qp, void *lsmm_buf, u32 size, irdma_stag stag);
+ void (*qp_send_lsmm_nostag)(struct irdma_sc_qp *qp, void *lsmm_buf, u32 size);
+ void (*qp_send_rtt)(struct irdma_sc_qp *qp, bool read);
+ enum irdma_status_code (*iw_mr_fast_register)(struct irdma_sc_qp *qp,
+ struct irdma_fast_reg_stag_info *info,
+ bool post_sq);
+};
+
+struct irdma_priv_cq_ops {
+ enum irdma_status_code (*cq_init)(struct irdma_sc_cq *cq,
+ struct irdma_cq_init_info *info);
+ enum irdma_status_code (*cq_create)(struct irdma_sc_cq *cq, u64 scratch,
+ bool check_overflow, bool post_sq);
+ enum irdma_status_code (*cq_destroy)(struct irdma_sc_cq *cq, u64 scratch, bool post_sq);
+ enum irdma_status_code (*cq_modify)(struct irdma_sc_cq *cq,
+ struct irdma_modify_cq_info *info,
+ u64 scratch, bool post_sq);
+ void (*cq_ack)(struct irdma_sc_cq *cq);
+};
+
+struct irdma_mr_ops {
+ enum irdma_status_code (*alloc_stag)(struct irdma_sc_dev *dev,
+ struct irdma_allocate_stag_info *info,
+ u64 scratch, bool post_sq);
+ enum irdma_status_code (*mr_reg_non_shared)(struct irdma_sc_dev *dev,
+ struct irdma_reg_ns_stag_info *info,
+ u64 scratch, bool post_sq);
+ enum irdma_status_code (*mr_reg_shared)(struct irdma_sc_dev *dev,
+ struct irdma_register_shared_stag *stag,
+ u64 scratch, bool post_sq);
+ enum irdma_status_code (*dealloc_stag)(struct irdma_sc_dev *dev,
+ struct irdma_dealloc_stag_info *info,
+ u64 scratch, bool post_sq);
+ enum irdma_status_code (*query_stag)(struct irdma_sc_dev *dev, u64 scratch,
+ u32 stag_index, bool post_sq);
+ enum irdma_status_code (*mw_alloc)(struct irdma_sc_dev *dev,
+ struct irdma_mw_alloc_info *info,
+ u64 scratch, bool post_sq);
+};
+
+struct irdma_cqp_misc_ops {
+ enum irdma_status_code (*manage_push_page)(struct irdma_sc_cqp *cqp,
+ struct irdma_cqp_manage_push_page_info *info,
+ u64 scratch, bool post_sq);
+ enum irdma_status_code (*alloc_local_mac_entry)(struct irdma_sc_cqp *cqp,
+ u64 scratch,
+ bool post_sq);
+ enum irdma_status_code (*add_local_mac_entry)(struct irdma_sc_cqp *cqp,
+ struct irdma_local_mac_entry_info *info,
+ u64 scratch, bool post_sq);
+ enum irdma_status_code (*del_local_mac_entry)(struct irdma_sc_cqp *cqp, u64 scratch,
+ u16 entry_idx,
+ u8 ignore_ref_count,
+ bool post_sq);
+ enum irdma_status_code (*add_arp_cache_entry)(struct irdma_sc_cqp *cqp,
+ struct irdma_add_arp_cache_entry_info *info,
+ u64 scratch, bool post_sq);
+ enum irdma_status_code (*del_arp_cache_entry)(struct irdma_sc_cqp *cqp,
+ u64 scratch, u16 arp_index, bool post_sq);
+ enum irdma_status_code (*query_arp_cache_entry)(struct irdma_sc_cqp *cqp,
+ u64 scratch, u16 arp_index, bool post_sq);
+ enum irdma_status_code (*manage_apbvt_entry)(struct irdma_sc_cqp *cqp,
+ struct irdma_apbvt_info *info,
+ u64 scratch, bool post_sq);
+ enum irdma_status_code (*manage_qhash_table_entry)(struct irdma_sc_cqp *cqp,
+ struct irdma_qhash_table_info *info,
+ u64 scratch, bool post_sq);
+ enum irdma_status_code (*cqp_nop)(struct irdma_sc_cqp *cqp, u64 scratch, bool post_sq);
+ enum irdma_status_code (*manage_stats_instance)(struct irdma_sc_cqp *cqp,
+ struct irdma_stats_inst_info *info,
+ bool alloc, u64 scratch);
+ enum irdma_status_code (*gather_stats)(struct irdma_sc_cqp *cqp,
+ struct irdma_stats_gather_info *info,
+ u64 scratch);
+ enum irdma_status_code (*manage_ws_node)(struct irdma_sc_cqp *cqp,
+ struct irdma_ws_node_info *info,
+ enum irdma_ws_node_op node_op,
+ u64 scratch);
+ enum irdma_status_code (*set_up_map)(struct irdma_sc_cqp *cqp,
+ struct irdma_up_info *info,
+ u64 scratch);
+ enum irdma_status_code (*query_rdma_features)(struct irdma_sc_cqp *cqp,
+ struct irdma_dma_mem *buf,
+ u64 scratch);
+};
+
+struct irdma_hmc_ops {
+ enum irdma_status_code (*init_iw_hmc)(struct irdma_sc_dev *dev, u8 hmc_fn_id);
+ enum irdma_status_code (*parse_fpm_query_buf)(struct irdma_sc_dev *dev,
+ __le64 *buf,
+ struct irdma_hmc_info *hmc_info,
+ struct irdma_hmc_fpm_misc *hmc_fpm_misc);
+ enum irdma_status_code (*cfg_iw_fpm)(struct irdma_sc_dev *dev, u8 hmc_fn_id);
+ enum irdma_status_code (*parse_fpm_commit_buf)(struct irdma_sc_dev *dev,
+ __le64 *buf,
+ struct irdma_hmc_obj_info *info,
+ u32 *sd);
+ enum irdma_status_code (*manage_hmc_pm_func_table)(struct irdma_sc_cqp *cqp,
+ u64 scratch,
+ u8 vf_index,
+ bool free_pm_fcn,
+ bool post_sq);
+ enum irdma_status_code (*commit_fpm_val)(struct irdma_sc_cqp *cqp,
+ u64 scratch, u8 hmc_fn_id,
+ struct irdma_dma_mem *commit_fpm_mem,
+ bool post_sq, u8 wait_type);
+ enum irdma_status_code (*query_fpm_val)(struct irdma_sc_cqp *cqp,
+ u64 scratch,
+ u8 hmc_fn_id,
+ struct irdma_dma_mem *query_fpm_mem,
+ bool post_sq, u8 wait_type);
+ enum irdma_status_code (*static_hmc_pages_allocated)(struct irdma_sc_cqp *cqp,
+ u64 scratch,
+ u8 hmc_fn_id,
+ bool post_sq,
+ bool poll_registers);
+ enum irdma_status_code (*commit_fpm_val_done)(struct irdma_sc_cqp *cqp);
+ enum irdma_status_code (*query_fpm_val_done)(struct irdma_sc_cqp *cqp);
+ enum irdma_status_code (*manage_hmc_pm_func_table_done)(struct irdma_sc_cqp *cqp);
+ enum irdma_status_code (*create_hmc_object)(struct irdma_sc_dev *dev,
+ struct irdma_hmc_create_obj_info *info);
+ enum irdma_status_code (*del_hmc_object)(struct irdma_sc_dev *dev,
+ struct irdma_hmc_del_obj_info *info,
+ bool reset);
+ enum irdma_status_code (*pf_init_vfhmc)(struct irdma_sc_dev *dev,
+ u8 vf_hmc_fn_id,
+ u32 *vf_cnt_array);
+ enum irdma_status_code (*vf_cfg_vffpm)(struct irdma_sc_dev *dev, u32 *vf_cnt_array);
+};
+
+struct cqp_info {
+ union {
+ struct {
+ struct irdma_sc_qp *qp;
+ struct irdma_create_qp_info info;
+ u64 scratch;
+ } qp_create;
+
+ struct {
+ struct irdma_sc_qp *qp;
+ struct irdma_modify_qp_info info;
+ u64 scratch;
+ } qp_modify;
+
+ struct {
+ struct irdma_sc_qp *qp;
+ u64 scratch;
+ bool remove_hash_idx;
+ bool ignore_mw_bnd;
+ } qp_destroy;
+
+ struct {
+ struct irdma_sc_cq *cq;
+ u64 scratch;
+ bool check_overflow;
+ } cq_create;
+
+ struct {
+ struct irdma_sc_cq *cq;
+ u64 scratch;
+ } cq_destroy;
+
+ struct {
+ struct irdma_sc_dev *dev;
+ struct irdma_allocate_stag_info info;
+ u64 scratch;
+ } alloc_stag;
+
+ struct {
+ struct irdma_sc_dev *dev;
+ struct irdma_mw_alloc_info info;
+ u64 scratch;
+ } mw_alloc;
+
+ struct {
+ struct irdma_sc_dev *dev;
+ struct irdma_reg_ns_stag_info info;
+ u64 scratch;
+ } mr_reg_non_shared;
+
+ struct {
+ struct irdma_sc_dev *dev;
+ struct irdma_dealloc_stag_info info;
+ u64 scratch;
+ } dealloc_stag;
+
+ struct {
+ struct irdma_sc_cqp *cqp;
+ struct irdma_add_arp_cache_entry_info info;
+ u64 scratch;
+ } add_arp_cache_entry;
+
+ struct {
+ struct irdma_sc_cqp *cqp;
+ u64 scratch;
+ u16 arp_index;
+ } del_arp_cache_entry;
+
+ struct {
+ struct irdma_sc_cqp *cqp;
+ struct irdma_local_mac_entry_info info;
+ u64 scratch;
+ } add_local_mac_entry;
+
+ struct {
+ struct irdma_sc_cqp *cqp;
+ u64 scratch;
+ u8 entry_idx;
+ u8 ignore_ref_count;
+ } del_local_mac_entry;
+
+ struct {
+ struct irdma_sc_cqp *cqp;
+ u64 scratch;
+ } alloc_local_mac_entry;
+
+ struct {
+ struct irdma_sc_cqp *cqp;
+ struct irdma_cqp_manage_push_page_info info;
+ u64 scratch;
+ } manage_push_page;
+
+ struct {
+ struct irdma_sc_dev *dev;
+ struct irdma_upload_context_info info;
+ u64 scratch;
+ } qp_upload_context;
+
+ struct {
+ struct irdma_sc_dev *dev;
+ struct irdma_hmc_fcn_info info;
+ u64 scratch;
+ } manage_hmc_pm;
+
+ struct {
+ struct irdma_sc_ceq *ceq;
+ u64 scratch;
+ } ceq_create;
+
+ struct {
+ struct irdma_sc_ceq *ceq;
+ u64 scratch;
+ } ceq_destroy;
+
+ struct {
+ struct irdma_sc_aeq *aeq;
+ u64 scratch;
+ } aeq_create;
+
+ struct {
+ struct irdma_sc_aeq *aeq;
+ u64 scratch;
+ } aeq_destroy;
+
+ struct {
+ struct irdma_sc_qp *qp;
+ struct irdma_qp_flush_info info;
+ u64 scratch;
+ } qp_flush_wqes;
+
+ struct {
+ struct irdma_sc_qp *qp;
+ struct irdma_gen_ae_info info;
+ u64 scratch;
+ } gen_ae;
+
+ struct {
+ struct irdma_sc_cqp *cqp;
+ void *fpm_val_va;
+ u64 fpm_val_pa;
+ u8 hmc_fn_id;
+ u64 scratch;
+ } query_fpm_val;
+
+ struct {
+ struct irdma_sc_cqp *cqp;
+ void *fpm_val_va;
+ u64 fpm_val_pa;
+ u8 hmc_fn_id;
+ u64 scratch;
+ } commit_fpm_val;
+
+ struct {
+ struct irdma_sc_cqp *cqp;
+ struct irdma_apbvt_info info;
+ u64 scratch;
+ } manage_apbvt_entry;
+
+ struct {
+ struct irdma_sc_cqp *cqp;
+ struct irdma_qhash_table_info info;
+ u64 scratch;
+ } manage_qhash_table_entry;
+
+ struct {
+ struct irdma_sc_dev *dev;
+ struct irdma_update_sds_info info;
+ u64 scratch;
+ } update_pe_sds;
+
+ struct {
+ struct irdma_sc_cqp *cqp;
+ struct irdma_sc_qp *qp;
+ u64 scratch;
+ } suspend_resume;
+
+ struct {
+ struct irdma_sc_cqp *cqp;
+ struct irdma_ah_info info;
+ u64 scratch;
+ } ah_create;
+
+ struct {
+ struct irdma_sc_cqp *cqp;
+ struct irdma_ah_info info;
+ u64 scratch;
+ } ah_destroy;
+
+ struct {
+ struct irdma_sc_cqp *cqp;
+ struct irdma_mcast_grp_info info;
+ u64 scratch;
+ } mc_create;
+
+ struct {
+ struct irdma_sc_cqp *cqp;
+ struct irdma_mcast_grp_info info;
+ u64 scratch;
+ } mc_destroy;
+
+ struct {
+ struct irdma_sc_cqp *cqp;
+ struct irdma_mcast_grp_info info;
+ u64 scratch;
+ } mc_modify;
+
+ struct {
+ struct irdma_sc_cqp *cqp;
+ struct irdma_stats_inst_info info;
+ u64 scratch;
+ } stats_manage;
+
+ struct {
+ struct irdma_sc_cqp *cqp;
+ struct irdma_stats_gather_info info;
+ u64 scratch;
+ } stats_gather;
+
+ struct {
+ struct irdma_sc_cqp *cqp;
+ struct irdma_ws_node_info info;
+ u64 scratch;
+ } ws_node;
+
+ struct {
+ struct irdma_sc_cqp *cqp;
+ struct irdma_up_info info;
+ u64 scratch;
+ } up_map;
+
+ struct {
+ struct irdma_sc_cqp *cqp;
+ struct irdma_dma_mem query_buff_mem;
+ u64 scratch;
+ } query_rdma;
+ } u;
+};
+
+struct cqp_cmds_info {
+ struct list_head cqp_cmd_entry;
+ u8 cqp_cmd;
+ u8 post_sq;
+ struct cqp_info in;
+};
+
+struct irdma_virtchnl_work_info {
+ void (*callback_fcn)(void *vf_dev);
+ void *worker_vf_dev;
+};
+#endif /* IRDMA_TYPE_H */
--
1.8.3.1
^ permalink raw reply related
* [RFC 10/19] RDMA/irdma: Add connection manager
From: Shiraz Saleem @ 2019-02-12 21:43 UTC (permalink / raw)
To: dledford, jgg, davem
Cc: linux-rdma, netdev, mustafa.ismail, jeffrey.t.kirsher,
Shiraz Saleem
In-Reply-To: <20190212214402.23284-1-shiraz.saleem@intel.com>
From: Mustafa Ismail <mustafa.ismail@intel.com>
Add connection management (CM) implementation for
iWARP including accept, reject, connect, create_listen,
destroy_listen and CM utility functions
Signed-off-by: Mustafa Ismail <mustafa.ismail@intel.com>
Signed-off-by: Shiraz Saleem <shiraz.saleem@intel.com>
---
drivers/infiniband/hw/irdma/cm.c | 4621 ++++++++++++++++++++++++++++++++++++++
drivers/infiniband/hw/irdma/cm.h | 425 ++++
2 files changed, 5046 insertions(+)
create mode 100644 drivers/infiniband/hw/irdma/cm.c
create mode 100644 drivers/infiniband/hw/irdma/cm.h
diff --git a/drivers/infiniband/hw/irdma/cm.c b/drivers/infiniband/hw/irdma/cm.c
new file mode 100644
index 0000000..db76da7
--- /dev/null
+++ b/drivers/infiniband/hw/irdma/cm.c
@@ -0,0 +1,4621 @@
+// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
+/* Copyright (c) 2019, Intel Corporation. */
+
+#include <linux/highmem.h>
+#include <net/addrconf.h>
+#include <net/ip6_route.h>
+#include <net/flow.h>
+#include <net/secure_seq.h>
+
+#include "main.h"
+#include "trace.h"
+
+static void irdma_rem_ref_cm_node(struct irdma_cm_node *);
+static void irdma_cm_post_event(struct irdma_cm_event *event);
+static void irdma_disconnect_worker(struct work_struct *work);
+
+/**
+ * irdma_free_sqbuf - put back puda buffer if refcount is 0
+ * @dev: HW device
+ * @buf: puda buffer to free
+ */
+void irdma_free_sqbuf(struct irdma_sc_vsi *vsi, void *bufp)
+{
+ struct irdma_puda_buf *buf = (struct irdma_puda_buf *)bufp;
+ struct irdma_puda_rsrc *ilq = vsi->ilq;
+
+ if (!atomic_dec_return(&buf->refcount))
+ irdma_puda_ret_bufpool(ilq, buf);
+}
+
+/**
+ * irdma_derive_hw_ird_setting - Calculate IRD
+ * @cm_ird: IRD of connection's node
+ *
+ * The ird from the connection is rounded to a supported HW
+ * setting (2,8,32,64,128) and then encoded for ird_size field
+ * of qp_ctx
+ */
+u8 irdma_derive_hw_ird_setting(u16 cm_ird)
+{
+ /* ird_size field is encoded in qp_ctx */
+ switch (cm_ird ? roundup_pow_of_two(cm_ird) : 0) {
+ case IRDMA_HW_IRD_SETTING_128:
+ return 4;
+ case IRDMA_HW_IRD_SETTING_64:
+ return 3;
+ case IRDMA_HW_IRD_SETTING_32:
+ case IRDMA_HW_IRD_SETTING_16:
+ return 2;
+ case IRDMA_HW_IRD_SETTING_8:
+ case IRDMA_HW_IRD_SETTING_4:
+ return 1;
+ case IRDMA_HW_IRD_SETTING_2:
+ default:
+ break;
+ }
+
+ return 0;
+}
+
+/**
+ * irdma_record_ird_ord - Record IRD/ORD passed in
+ * @cm_node: connection's node
+ * @conn_ird: connection IRD
+ * @conn_ord: connection ORD
+ */
+static void irdma_record_ird_ord(struct irdma_cm_node *cm_node,
+ u32 conn_ird,
+ u32 conn_ord)
+{
+ if (conn_ird > cm_node->dev->hw_attrs.max_hw_ird)
+ cm_node->ird_size = cm_node->dev->hw_attrs.max_hw_ird;
+
+ if (conn_ord > cm_node->dev->hw_attrs.max_hw_ord)
+ cm_node->ord_size = cm_node->dev->hw_attrs.max_hw_ord;
+ else if (!conn_ord && cm_node->send_rdma0_op == SEND_RDMA_READ_ZERO)
+ cm_node->ord_size = 1;
+}
+
+/**
+ * irdma_copy_ip_ntohl - copy IP address from network to host
+ * @dst: IP address in host order
+ * @src: IP address in network order (big endian)
+ */
+void irdma_copy_ip_ntohl(u32 *dst, __be32 *src)
+{
+ *dst++ = ntohl(*src++);
+ *dst++ = ntohl(*src++);
+ *dst++ = ntohl(*src++);
+ *dst = ntohl(*src);
+}
+
+/**
+ * irdma_copy_ip_htonl - copy IP address from host to network order
+ * @dst: IP address in network order (big endian)
+ * @src: IP address in host order
+ */
+void irdma_copy_ip_htonl(__be32 *dst, u32 *src)
+{
+ *dst++ = htonl(*src++);
+ *dst++ = htonl(*src++);
+ *dst++ = htonl(*src++);
+ *dst = htonl(*src);
+}
+
+/**
+ * irdma_get_addr_info
+ * @cm_node: contains ip/tcp info
+ * @cm_info: to get a copy of the cm_node ip/tcp info
+ */
+static void irdma_get_addr_info(struct irdma_cm_node *cm_node,
+ struct irdma_cm_info *cm_info)
+{
+ cm_info->ipv4 = cm_node->ipv4;
+ cm_info->vlan_id = cm_node->vlan_id;
+ memcpy(cm_info->loc_addr, cm_node->loc_addr,
+ sizeof(cm_info->loc_addr));
+ memcpy(cm_info->rem_addr, cm_node->rem_addr,
+ sizeof(cm_info->rem_addr));
+ cm_info->loc_port = cm_node->loc_port;
+ cm_info->rem_port = cm_node->rem_port;
+ cm_info->user_pri = cm_node->user_pri;
+}
+
+/**
+ * irdma_fill_sockaddr4 - fill in addr info for IPv4 connection
+ * @cm_node: connection's node
+ * @event: upper layer's cm event
+ */
+static inline void irdma_fill_sockaddr4(struct irdma_cm_node *cm_node,
+ struct iw_cm_event *event)
+{
+ struct sockaddr_in *laddr = (struct sockaddr_in *)
+ &event->local_addr;
+ struct sockaddr_in *raddr = (struct sockaddr_in *)
+ &event->remote_addr;
+
+ laddr->sin_family = AF_INET;
+ raddr->sin_family = AF_INET;
+
+ laddr->sin_port = htons(cm_node->loc_port);
+ raddr->sin_port = htons(cm_node->rem_port);
+
+ laddr->sin_addr.s_addr = htonl(cm_node->loc_addr[0]);
+ raddr->sin_addr.s_addr = htonl(cm_node->rem_addr[0]);
+}
+
+/**
+ * irdma_fill_sockaddr6 - fill in addr info for IPv6 connection
+ * @cm_node: connection's node
+ * @event: upper layer's cm event
+ */
+static inline void irdma_fill_sockaddr6(struct irdma_cm_node *cm_node,
+ struct iw_cm_event *event)
+{
+ struct sockaddr_in6 *laddr6 = (struct sockaddr_in6 *)
+ &event->local_addr;
+ struct sockaddr_in6 *raddr6 = (struct sockaddr_in6 *)
+ &event->remote_addr;
+
+ laddr6->sin6_family = AF_INET6;
+ raddr6->sin6_family = AF_INET6;
+
+ laddr6->sin6_port = htons(cm_node->loc_port);
+ raddr6->sin6_port = htons(cm_node->rem_port);
+
+ irdma_copy_ip_htonl(laddr6->sin6_addr.in6_u.u6_addr32,
+ cm_node->loc_addr);
+ irdma_copy_ip_htonl(raddr6->sin6_addr.in6_u.u6_addr32,
+ cm_node->rem_addr);
+}
+
+/**
+ * irdma_get_cmevent_info - for cm event upcall
+ * @cm_node: connection's node
+ * @cm_id: upper layers cm struct for the event
+ * @event: upper layer's cm event
+ */
+static inline void irdma_get_cmevent_info(struct irdma_cm_node *cm_node,
+ struct iw_cm_id *cm_id,
+ struct iw_cm_event *event)
+{
+ memcpy(&event->local_addr, &cm_id->m_local_addr,
+ sizeof(event->local_addr));
+ memcpy(&event->remote_addr, &cm_id->m_remote_addr,
+ sizeof(event->remote_addr));
+ if (cm_node) {
+ event->private_data = (void *)cm_node->pdata_buf;
+ event->private_data_len = (u8)cm_node->pdata.size;
+ event->ird = cm_node->ird_size;
+ event->ord = cm_node->ord_size;
+ }
+}
+
+/**
+ * irdma_send_cm_event - upcall cm's event handler
+ * @cm_node: connection's node
+ * @cm_id: upper layer's cm info struct
+ * @type: Event type to indicate
+ * @status: status for the event type
+ */
+static int irdma_send_cm_event(struct irdma_cm_node *cm_node,
+ struct iw_cm_id *cm_id,
+ enum iw_cm_event_type type,
+ int status)
+{
+ struct iw_cm_event event = {};
+
+ event.event = type;
+ event.status = status;
+ switch (type) {
+ case IW_CM_EVENT_CONNECT_REQUEST:
+ trace_irdma_send_cm_event(cm_node, cm_id, type, status,
+ __builtin_return_address(0));
+ if (cm_node->ipv4)
+ irdma_fill_sockaddr4(cm_node, &event);
+ else
+ irdma_fill_sockaddr6(cm_node, &event);
+ event.provider_data = (void *)cm_node;
+ event.private_data = (void *)cm_node->pdata_buf;
+ event.private_data_len = (u8)cm_node->pdata.size;
+ event.ird = cm_node->ird_size;
+ break;
+ case IW_CM_EVENT_CONNECT_REPLY:
+ trace_irdma_send_cm_event(cm_node, cm_id, type, status,
+ __builtin_return_address(0));
+ irdma_get_cmevent_info(cm_node, cm_id, &event);
+ break;
+ case IW_CM_EVENT_ESTABLISHED:
+ trace_irdma_send_cm_event(cm_node, cm_id, type, status,
+ __builtin_return_address(0));
+ event.ird = cm_node->ird_size;
+ event.ord = cm_node->ord_size;
+ break;
+ case IW_CM_EVENT_DISCONNECT:
+ trace_irdma_send_cm_event_no_node(cm_id, type, status,
+ __builtin_return_address(0));
+ break;
+ case IW_CM_EVENT_CLOSE:
+ trace_irdma_send_cm_event_no_node(cm_id, type, status,
+ __builtin_return_address(0));
+ break;
+ default:
+ irdma_debug(cm_node->dev, IRDMA_DEBUG_ERR,
+ "Unsupported event type received type = %d\n",
+ type);
+ return -1;
+ }
+
+ return cm_id->event_handler(cm_id, &event);
+}
+
+/**
+ * irdma_create_event - create cm event
+ * @cm_node: connection's node
+ * @type: Event type to generate
+ */
+static struct irdma_cm_event *irdma_create_event(struct irdma_cm_node *cm_node,
+ enum irdma_cm_event_type type)
+{
+ struct irdma_cm_event *event;
+
+ if (!cm_node->cm_id)
+ return NULL;
+
+ event = kzalloc(sizeof(*event), GFP_ATOMIC);
+
+ if (!event)
+ return NULL;
+
+ event->type = type;
+ event->cm_node = cm_node;
+ memcpy(event->cm_info.rem_addr, cm_node->rem_addr,
+ sizeof(event->cm_info.rem_addr));
+ memcpy(event->cm_info.loc_addr, cm_node->loc_addr,
+ sizeof(event->cm_info.loc_addr));
+ event->cm_info.rem_port = cm_node->rem_port;
+ event->cm_info.loc_port = cm_node->loc_port;
+ event->cm_info.cm_id = cm_node->cm_id;
+ irdma_debug(cm_node->dev,
+ IRDMA_DEBUG_CM,
+ "node=%p event=%p type=%u dst=%pI4 src=%pI4\n",
+ cm_node,
+ event,
+ type,
+ event->cm_info.loc_addr,
+ event->cm_info.rem_addr);
+ trace_irdma_create_event(cm_node, type, __builtin_return_address(0));
+ irdma_cm_post_event(event);
+
+ return event;
+}
+
+/**
+ * irdma_free_retrans_entry - free send entry
+ * @cm_node: connection's node
+ */
+static void irdma_free_retrans_entry(struct irdma_cm_node *cm_node)
+{
+ struct irdma_device *iwdev = cm_node->iwdev;
+ struct irdma_timer_entry *send_entry;
+
+ send_entry = cm_node->send_entry;
+ if (!send_entry)
+ return;
+
+ cm_node->send_entry = NULL;
+ irdma_free_sqbuf(&iwdev->vsi, (void *)send_entry->sqbuf);
+ kfree(send_entry);
+ atomic_dec(&cm_node->ref_count);
+}
+
+/**
+ * irdma_cleanup_retrans_entry - free send entry with lock
+ * @cm_node: connection's node
+ */
+static void irdma_cleanup_retrans_entry(struct irdma_cm_node *cm_node)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&cm_node->retrans_list_lock, flags);
+ irdma_free_retrans_entry(cm_node);
+ spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags);
+}
+
+/**
+ * irdma_form_ah_cm_frame - get a free packet and build frame with address handle
+ * @cm_node: connection's node ionfo to use in frame
+ * @options: pointer to options info
+ * @hdr: pointer mpa header
+ * @pdata: pointer to private data
+ * @flags: indicates FIN or ACK
+ */
+static struct irdma_puda_buf *irdma_form_ah_cm_frame(struct irdma_cm_node *cm_node,
+ struct irdma_kmem_info *options,
+ struct irdma_kmem_info *hdr,
+ struct irdma_kmem_info *pdata,
+ u8 flags)
+{
+ struct irdma_puda_buf *sqbuf;
+ struct irdma_sc_vsi *vsi = &cm_node->iwdev->vsi;
+ u8 *buf;
+ struct tcphdr *tcph;
+ u16 pktsize;
+ u32 opts_len = 0;
+ u32 pd_len = 0;
+ u32 hdr_len = 0;
+
+ if (!cm_node->ah || !cm_node->ah->ah_info.ah_valid) {
+ irdma_debug(cm_node->dev, IRDMA_DEBUG_ERR, "AH invalid\n");
+ return NULL;
+ }
+
+ sqbuf = irdma_puda_get_bufpool(vsi->ilq);
+ if (!sqbuf) {
+ irdma_debug(cm_node->dev, IRDMA_DEBUG_ERR, "SQ buf NULL\n");
+ return NULL;
+ }
+
+ sqbuf->ah_id = cm_node->ah->ah_info.ah_idx;
+ buf = sqbuf->mem.va;
+ if (options)
+ opts_len = (u32)options->size;
+
+ if (hdr)
+ hdr_len = hdr->size;
+
+ if (pdata)
+ pd_len = pdata->size;
+
+ pktsize = sizeof(*tcph) + opts_len + hdr_len + pd_len;
+
+ memset(buf, 0, pktsize);
+
+ sqbuf->totallen = pktsize;
+ sqbuf->tcphlen = sizeof(*tcph) + opts_len;
+ sqbuf->scratch = (void *)cm_node;
+
+ tcph = (struct tcphdr *)buf;
+ buf += sizeof(*tcph);
+
+ tcph->source = htons(cm_node->loc_port);
+ tcph->dest = htons(cm_node->rem_port);
+ tcph->seq = htonl(cm_node->tcp_cntxt.loc_seq_num);
+
+ if (flags & SET_ACK) {
+ cm_node->tcp_cntxt.loc_ack_num = cm_node->tcp_cntxt.rcv_nxt;
+ tcph->ack_seq = htonl(cm_node->tcp_cntxt.loc_ack_num);
+ tcph->ack = 1;
+ } else {
+ tcph->ack_seq = 0;
+ }
+
+ if (flags & SET_SYN) {
+ cm_node->tcp_cntxt.loc_seq_num++;
+ tcph->syn = 1;
+ } else {
+ cm_node->tcp_cntxt.loc_seq_num += hdr_len + pd_len;
+ }
+
+ if (flags & SET_FIN) {
+ cm_node->tcp_cntxt.loc_seq_num++;
+ tcph->fin = 1;
+ }
+
+ if (flags & SET_RST)
+ tcph->rst = 1;
+
+ tcph->doff = (u16)((sizeof(*tcph) + opts_len + 3) >> 2);
+ sqbuf->tcphlen = tcph->doff << 2;
+ tcph->window = htons(cm_node->tcp_cntxt.rcv_wnd);
+ tcph->urg_ptr = 0;
+
+ if (opts_len) {
+ memcpy(buf, options->addr, opts_len);
+ buf += opts_len;
+ }
+
+ if (hdr_len) {
+ memcpy(buf, hdr->addr, hdr_len);
+ buf += hdr_len;
+ }
+
+ if (pdata && pdata->addr)
+ memcpy(buf, pdata->addr, pdata->size);
+
+ atomic_set(&sqbuf->refcount, 1);
+
+ irdma_debug_buf(vsi->dev,
+ IRDMA_DEBUG_ILQ,
+ "TRANSMIT ILQ BUFFER",
+ sqbuf->mem.va,
+ sqbuf->totallen);
+
+ return sqbuf;
+}
+
+/**
+ * irdma_form_uda_cm_frame - get a free packet and build frame full tcpip packet
+ * @cm_node: connection's node ionfo to use in frame
+ * @options: pointer to options info
+ * @hdr: pointer mpa header
+ * @pdata: pointer to private data
+ * @flags: indicates FIN or ACK
+ */
+static struct irdma_puda_buf *irdma_form_uda_cm_frame(struct irdma_cm_node *cm_node,
+ struct irdma_kmem_info *options,
+ struct irdma_kmem_info *hdr,
+ struct irdma_kmem_info *pdata,
+ u8 flags)
+{
+ struct irdma_puda_buf *sqbuf;
+ struct irdma_sc_vsi *vsi = &cm_node->iwdev->vsi;
+ u8 *buf;
+
+ struct tcphdr *tcph;
+ struct iphdr *iph;
+ struct ipv6hdr *ip6h;
+ struct ethhdr *ethh;
+ u16 pktsize;
+ u16 eth_hlen = ETH_HLEN;
+ u32 opts_len = 0;
+ u32 pd_len = 0;
+ u32 hdr_len = 0;
+
+ u16 vtag;
+
+ sqbuf = irdma_puda_get_bufpool(vsi->ilq);
+ if (!sqbuf)
+ return NULL;
+
+ buf = sqbuf->mem.va;
+
+ if (options)
+ opts_len = (u32)options->size;
+
+ if (hdr)
+ hdr_len = hdr->size;
+
+ if (pdata)
+ pd_len = pdata->size;
+
+ if (cm_node->vlan_id <= VLAN_VID_MASK)
+ eth_hlen += 4;
+
+ if (cm_node->ipv4)
+ pktsize = sizeof(*iph) + sizeof(*tcph);
+ else
+ pktsize = sizeof(*ip6h) + sizeof(*tcph);
+ pktsize += opts_len + hdr_len + pd_len;
+
+ memset(buf, 0, eth_hlen + pktsize);
+
+ sqbuf->totallen = pktsize + eth_hlen;
+ sqbuf->maclen = eth_hlen;
+ sqbuf->tcphlen = sizeof(*tcph) + opts_len;
+ sqbuf->scratch = (void *)cm_node;
+
+ ethh = (struct ethhdr *)buf;
+ buf += eth_hlen;
+
+ if (cm_node->ipv4) {
+ sqbuf->ipv4 = true;
+
+ iph = (struct iphdr *)buf;
+ buf += sizeof(*iph);
+ tcph = (struct tcphdr *)buf;
+ buf += sizeof(*tcph);
+
+ ether_addr_copy(ethh->h_dest, cm_node->rem_mac);
+ ether_addr_copy(ethh->h_source, cm_node->loc_mac);
+ if (cm_node->vlan_id <= VLAN_VID_MASK) {
+ ((struct vlan_ethhdr *)ethh)->h_vlan_proto =
+ htons(ETH_P_8021Q);
+ vtag = (cm_node->user_pri << VLAN_PRIO_SHIFT)
+ | cm_node->vlan_id;
+ ((struct vlan_ethhdr *)ethh)->h_vlan_TCI = htons(vtag);
+
+ ((struct vlan_ethhdr *)ethh)->h_vlan_encapsulated_proto =
+ htons(ETH_P_IP);
+ } else {
+ ethh->h_proto = htons(ETH_P_IP);
+ }
+
+ iph->version = IPVERSION;
+ iph->ihl = 5; /* 5 * 4Byte words, IP headr len */
+ iph->tos = cm_node->tos;
+ iph->tot_len = htons(pktsize);
+ iph->id = htons(++cm_node->tcp_cntxt.loc_id);
+
+ iph->frag_off = htons(0x4000);
+ iph->ttl = 0x40;
+ iph->protocol = IPPROTO_TCP;
+ iph->saddr = htonl(cm_node->loc_addr[0]);
+ iph->daddr = htonl(cm_node->rem_addr[0]);
+ } else {
+ sqbuf->ipv4 = false;
+ ip6h = (struct ipv6hdr *)buf;
+ buf += sizeof(*ip6h);
+ tcph = (struct tcphdr *)buf;
+ buf += sizeof(*tcph);
+
+ ether_addr_copy(ethh->h_dest, cm_node->rem_mac);
+ ether_addr_copy(ethh->h_source, cm_node->loc_mac);
+ if (cm_node->vlan_id <= VLAN_VID_MASK) {
+ ((struct vlan_ethhdr *)ethh)->h_vlan_proto =
+ htons(ETH_P_8021Q);
+ vtag = (cm_node->user_pri << VLAN_PRIO_SHIFT)
+ | cm_node->vlan_id;
+ ((struct vlan_ethhdr *)ethh)->h_vlan_TCI = htons(vtag);
+ ((struct vlan_ethhdr *)ethh)->h_vlan_encapsulated_proto =
+ htons(ETH_P_IPV6);
+ } else {
+ ethh->h_proto = htons(ETH_P_IPV6);
+ }
+ ip6h->version = 6;
+ ip6h->priority = cm_node->tos >> 4;
+ ip6h->flow_lbl[0] = cm_node->tos << 4;
+ ip6h->flow_lbl[1] = 0;
+ ip6h->flow_lbl[2] = 0;
+ ip6h->payload_len = htons(pktsize - sizeof(*ip6h));
+ ip6h->nexthdr = 6;
+ ip6h->hop_limit = 128;
+ irdma_copy_ip_htonl(ip6h->saddr.in6_u.u6_addr32,
+ cm_node->loc_addr);
+ irdma_copy_ip_htonl(ip6h->daddr.in6_u.u6_addr32,
+ cm_node->rem_addr);
+ }
+
+ tcph->source = htons(cm_node->loc_port);
+ tcph->dest = htons(cm_node->rem_port);
+ tcph->seq = htonl(cm_node->tcp_cntxt.loc_seq_num);
+
+ if (flags & SET_ACK) {
+ cm_node->tcp_cntxt.loc_ack_num = cm_node->tcp_cntxt.rcv_nxt;
+ tcph->ack_seq = htonl(cm_node->tcp_cntxt.loc_ack_num);
+ tcph->ack = 1;
+ } else {
+ tcph->ack_seq = 0;
+ }
+
+ if (flags & SET_SYN) {
+ cm_node->tcp_cntxt.loc_seq_num++;
+ tcph->syn = 1;
+ } else {
+ cm_node->tcp_cntxt.loc_seq_num += hdr_len + pd_len;
+ }
+
+ if (flags & SET_FIN) {
+ cm_node->tcp_cntxt.loc_seq_num++;
+ tcph->fin = 1;
+ }
+
+ if (flags & SET_RST)
+ tcph->rst = 1;
+
+ tcph->doff = (u16)((sizeof(*tcph) + opts_len + 3) >> 2);
+ sqbuf->tcphlen = tcph->doff << 2;
+ tcph->window = htons(cm_node->tcp_cntxt.rcv_wnd);
+ tcph->urg_ptr = 0;
+
+ if (opts_len) {
+ memcpy(buf, options->addr, opts_len);
+ buf += opts_len;
+ }
+
+ if (hdr_len) {
+ memcpy(buf, hdr->addr, hdr_len);
+ buf += hdr_len;
+ }
+
+ if (pdata && pdata->addr)
+ memcpy(buf, pdata->addr, pdata->size);
+
+ atomic_set(&sqbuf->refcount, 1);
+
+ irdma_debug_buf(vsi->dev,
+ IRDMA_DEBUG_ILQ,
+ "TRANSMIT ILQ BUFFER",
+ sqbuf->mem.va,
+ sqbuf->totallen);
+
+ return sqbuf;
+}
+
+/**
+ * irdma_send_reset - Send RST packet
+ * @cm_node: connection's node
+ */
+int irdma_send_reset(struct irdma_cm_node *cm_node)
+{
+ struct irdma_puda_buf *sqbuf;
+ int flags = SET_RST | SET_ACK;
+
+ trace_irdma_send_reset(cm_node, 0, __builtin_return_address(0));
+ sqbuf = cm_node->cm_core->form_cm_frame(cm_node, NULL, NULL, NULL, flags);
+ if (!sqbuf)
+ return -1;
+
+ return irdma_schedule_cm_timer(cm_node, sqbuf,
+ IRDMA_TIMER_TYPE_SEND, 0, 1);
+}
+
+/**
+ * irdma_active_open_err - send event for active side cm error
+ * @cm_node: connection's node
+ * @reset: Flag to send reset or not
+ */
+static void irdma_active_open_err(struct irdma_cm_node *cm_node, bool reset)
+{
+ trace_irdma_active_open_err(cm_node, reset,
+ __builtin_return_address(0));
+ irdma_cleanup_retrans_entry(cm_node);
+ cm_node->cm_core->stats_connect_errs++;
+ if (reset) {
+ irdma_debug(cm_node->dev,
+ IRDMA_DEBUG_CM,
+ "cm_node=%p state=%d\n",
+ cm_node,
+ cm_node->state);
+ atomic_inc(&cm_node->ref_count);
+ irdma_send_reset(cm_node);
+ }
+
+ cm_node->state = IRDMA_CM_STATE_CLOSED;
+ irdma_create_event(cm_node, IRDMA_CM_EVENT_ABORTED);
+}
+
+/**
+ * irdma_passive_open_err - handle passive side cm error
+ * @cm_node: connection's node
+ * @reset: send reset or just free cm_node
+ */
+static void irdma_passive_open_err(struct irdma_cm_node *cm_node, bool reset)
+{
+ irdma_cleanup_retrans_entry(cm_node);
+ cm_node->cm_core->stats_passive_errs++;
+ cm_node->state = IRDMA_CM_STATE_CLOSED;
+ irdma_debug(cm_node->dev,
+ IRDMA_DEBUG_CM,
+ "cm_node=%p state =%d\n",
+ cm_node,
+ cm_node->state);
+ trace_irdma_passive_open_err(cm_node, reset,
+ __builtin_return_address(0));
+ if (reset)
+ irdma_send_reset(cm_node);
+ else
+ irdma_rem_ref_cm_node(cm_node);
+}
+
+/**
+ * irdma_event_connect_error - to create connect error event
+ * @event: cm information for connect event
+ */
+static void irdma_event_connect_error(struct irdma_cm_event *event)
+{
+ struct irdma_qp *iwqp;
+ struct iw_cm_id *cm_id;
+
+ cm_id = event->cm_node->cm_id;
+ if (!cm_id)
+ return;
+
+ iwqp = cm_id->provider_data;
+
+ if (!iwqp || !iwqp->iwdev)
+ return;
+
+ iwqp->cm_id = NULL;
+ cm_id->provider_data = NULL;
+ irdma_send_cm_event(event->cm_node, cm_id,
+ IW_CM_EVENT_CONNECT_REPLY,
+ -ECONNRESET);
+ cm_id->rem_ref(cm_id);
+ irdma_rem_ref_cm_node(event->cm_node);
+}
+
+/**
+ * irdma_process_options - process options from TCP header
+ * @cm_node: connection's node
+ * @optionsloc: point to start of options
+ * @optionsize: size of all options
+ * @syn_packet: flag if syn packet
+ */
+static int irdma_process_options(struct irdma_cm_node *cm_node,
+ u8 *optionsloc,
+ u32 optionsize,
+ u32 syn_pkt)
+{
+ u32 tmp;
+ u32 offset = 0;
+ union all_known_options *all_options;
+ char got_mss_option = 0;
+
+ while (offset < optionsize) {
+ all_options = (union all_known_options *)(optionsloc + offset);
+ switch (all_options->base.optionnum) {
+ case OPTION_NUM_EOL:
+ offset = optionsize;
+ break;
+ case OPTION_NUM_NONE:
+ offset += 1;
+ continue;
+ case OPTION_NUM_MSS:
+ irdma_debug(cm_node->dev,
+ IRDMA_DEBUG_CM,
+ "MSS Length: %d Offset: %d Size: %d\n",
+ all_options->mss.len,
+ offset,
+ optionsize);
+ got_mss_option = 1;
+ if (all_options->mss.len != 4)
+ return -EINVAL;
+ tmp = ntohs(all_options->mss.mss);
+ if ((cm_node->ipv4 &&
+ (tmp + IRDMA_MTU_TO_MSS_IPV4) < IRDMA_MIN_MTU_IPV4) ||
+ (!cm_node->ipv4 &&
+ (tmp + IRDMA_MTU_TO_MSS_IPV6) < IRDMA_MIN_MTU_IPV6))
+ return -EINVAL;
+ if (tmp < cm_node->tcp_cntxt.mss)
+ cm_node->tcp_cntxt.mss = tmp;
+ break;
+ case OPTION_NUM_WINDOW_SCALE:
+ cm_node->tcp_cntxt.snd_wscale = all_options->windowscale.shiftcount;
+ break;
+ default:
+ irdma_debug(cm_node->dev,
+ IRDMA_DEBUG_CM,
+ "Unsupported TCP Option: %x\n",
+ all_options->base.optionnum);
+ break;
+ }
+ offset += all_options->base.len;
+ }
+ if (!got_mss_option && syn_pkt)
+ cm_node->tcp_cntxt.mss = IRDMA_CM_DEFAULT_MSS;
+
+ return 0;
+}
+
+/**
+ * irdma_handle_tcp_options - setup TCP context info after parsing TCP options
+ * @cm_node: connection's node
+ * @tcph: pointer tcp header
+ * @optionsize: size of options rcvd
+ * @passive: active or passive flag
+ */
+static int irdma_handle_tcp_options(struct irdma_cm_node *cm_node,
+ struct tcphdr *tcph,
+ int optionsize,
+ int passive)
+{
+ u8 *optionsloc = (u8 *)&tcph[1];
+
+ if (optionsize) {
+ if (irdma_process_options(cm_node,
+ optionsloc,
+ optionsize,
+ (u32)tcph->syn)) {
+ irdma_debug(cm_node->dev,
+ IRDMA_DEBUG_CM,
+ "Node %p, Sending Reset\n",
+ cm_node);
+ if (passive)
+ irdma_passive_open_err(cm_node, true);
+ else
+ irdma_active_open_err(cm_node, true);
+ return -1;
+ }
+ }
+
+ cm_node->tcp_cntxt.snd_wnd = ntohs(tcph->window) <<
+ cm_node->tcp_cntxt.snd_wscale;
+
+ if (cm_node->tcp_cntxt.snd_wnd > cm_node->tcp_cntxt.max_snd_wnd)
+ cm_node->tcp_cntxt.max_snd_wnd = cm_node->tcp_cntxt.snd_wnd;
+
+ return 0;
+}
+
+/**
+ * irdma_build_mpa_v1 - build a MPA V1 frame
+ * @cm_node: connection's node
+ * @mpa_key: to do read0 or write0
+ */
+static void irdma_build_mpa_v1(struct irdma_cm_node *cm_node,
+ void *start_addr,
+ u8 mpa_key)
+{
+ struct ietf_mpa_v1 *mpa_frame = (struct ietf_mpa_v1 *)start_addr;
+
+ switch (mpa_key) {
+ case MPA_KEY_REQUEST:
+ memcpy(mpa_frame->key, IEFT_MPA_KEY_REQ, IETF_MPA_KEY_SIZE);
+ break;
+ case MPA_KEY_REPLY:
+ memcpy(mpa_frame->key, IEFT_MPA_KEY_REP, IETF_MPA_KEY_SIZE);
+ break;
+ default:
+ break;
+ }
+ mpa_frame->flags = IETF_MPA_FLAGS_CRC;
+ mpa_frame->rev = cm_node->mpa_frame_rev;
+ mpa_frame->priv_data_len = htons(cm_node->pdata.size);
+}
+
+/**
+ * irdma_build_mpa_v2 - build a MPA V2 frame
+ * @cm_node: connection's node
+ * @start_addr: buffer start address
+ * @mpa_key: to do read0 or write0
+ */
+static void irdma_build_mpa_v2(struct irdma_cm_node *cm_node,
+ void *start_addr,
+ u8 mpa_key)
+{
+ struct ietf_mpa_v2 *mpa_frame = (struct ietf_mpa_v2 *)start_addr;
+ struct ietf_rtr_msg *rtr_msg = &mpa_frame->rtr_msg;
+ u16 ctrl_ird, ctrl_ord;
+
+ /* initialize the upper 5 bytes of the frame */
+ irdma_build_mpa_v1(cm_node, start_addr, mpa_key);
+ mpa_frame->flags |= IETF_MPA_V2_FLAG;
+ mpa_frame->priv_data_len = cpu_to_be16(be16_to_cpu(mpa_frame->priv_data_len) +
+ IETF_RTR_MSG_SIZE);
+
+ /* initialize RTR msg */
+ if (cm_node->mpav2_ird_ord == IETF_NO_IRD_ORD) {
+ ctrl_ird = IETF_NO_IRD_ORD;
+ ctrl_ord = IETF_NO_IRD_ORD;
+ } else {
+ ctrl_ird = (cm_node->ird_size > IETF_NO_IRD_ORD) ?
+ IETF_NO_IRD_ORD : cm_node->ird_size;
+ ctrl_ord = (cm_node->ord_size > IETF_NO_IRD_ORD) ?
+ IETF_NO_IRD_ORD : cm_node->ord_size;
+ }
+ ctrl_ird |= IETF_PEER_TO_PEER;
+
+ switch (mpa_key) {
+ case MPA_KEY_REQUEST:
+ ctrl_ord |= IETF_RDMA0_WRITE;
+ ctrl_ord |= IETF_RDMA0_READ;
+ break;
+ case MPA_KEY_REPLY:
+ switch (cm_node->send_rdma0_op) {
+ case SEND_RDMA_WRITE_ZERO:
+ ctrl_ord |= IETF_RDMA0_WRITE;
+ break;
+ case SEND_RDMA_READ_ZERO:
+ ctrl_ord |= IETF_RDMA0_READ;
+ break;
+ }
+ break;
+ default:
+ break;
+ }
+ rtr_msg->ctrl_ird = htons(ctrl_ird);
+ rtr_msg->ctrl_ord = htons(ctrl_ord);
+}
+
+/**
+ * irdma_cm_build_mpa_frame - build mpa frame for mpa version 1 or version 2
+ * @cm_node: connection's node
+ * @mpa: mpa: data buffer
+ * @mpa_key: to do read0 or write0
+ */
+static int irdma_cm_build_mpa_frame(struct irdma_cm_node *cm_node,
+ struct irdma_kmem_info *mpa,
+ u8 mpa_key)
+{
+ int hdr_len = 0;
+
+ switch (cm_node->mpa_frame_rev) {
+ case IETF_MPA_V1:
+ hdr_len = sizeof(struct ietf_mpa_v1);
+ irdma_build_mpa_v1(cm_node, mpa->addr, mpa_key);
+ break;
+ case IETF_MPA_V2:
+ hdr_len = sizeof(struct ietf_mpa_v2);
+ irdma_build_mpa_v2(cm_node, mpa->addr, mpa_key);
+ break;
+ default:
+ break;
+ }
+
+ return hdr_len;
+}
+
+/**
+ * irdma_send_mpa_request - active node send mpa request to passive node
+ * @cm_node: connection's node
+ */
+static int irdma_send_mpa_request(struct irdma_cm_node *cm_node)
+{
+ struct irdma_puda_buf *sqbuf;
+
+ if (!cm_node) {
+ irdma_pr_err("cm_node == NULL\n");
+ return -1;
+ }
+
+ cm_node->mpa_hdr.addr = &cm_node->mpa_frame;
+ cm_node->mpa_hdr.size = irdma_cm_build_mpa_frame(cm_node,
+ &cm_node->mpa_hdr,
+ MPA_KEY_REQUEST);
+ if (!cm_node->mpa_hdr.size) {
+ irdma_debug(cm_node->dev, IRDMA_DEBUG_ERR, "mpa size = %d\n",
+ cm_node->mpa_hdr.size);
+ return -1;
+ }
+
+ sqbuf = cm_node->cm_core->form_cm_frame(cm_node,
+ NULL,
+ &cm_node->mpa_hdr,
+ &cm_node->pdata,
+ SET_ACK);
+ if (!sqbuf)
+ return -1;
+
+ return irdma_schedule_cm_timer(cm_node, sqbuf,
+ IRDMA_TIMER_TYPE_SEND, 1, 0);
+}
+
+/**
+ * irdma_send_mpa_reject -
+ * @cm_node: connection's node
+ * @pdata: reject data for connection
+ * @plen: length of reject data
+ */
+static int irdma_send_mpa_reject(struct irdma_cm_node *cm_node,
+ const void *pdata,
+ u8 plen)
+{
+ struct irdma_puda_buf *sqbuf;
+ struct irdma_kmem_info priv_info;
+
+ cm_node->mpa_hdr.addr = &cm_node->mpa_frame;
+ cm_node->mpa_hdr.size = irdma_cm_build_mpa_frame(cm_node,
+ &cm_node->mpa_hdr,
+ MPA_KEY_REPLY);
+
+ cm_node->mpa_frame.flags |= IETF_MPA_FLAGS_REJECT;
+ priv_info.addr = (void *)pdata;
+ priv_info.size = plen;
+
+ sqbuf = cm_node->cm_core->form_cm_frame(cm_node,
+ NULL,
+ &cm_node->mpa_hdr,
+ &priv_info,
+ SET_ACK | SET_FIN);
+ if (!sqbuf)
+ return -ENOMEM;
+
+ cm_node->state = IRDMA_CM_STATE_FIN_WAIT1;
+
+ return irdma_schedule_cm_timer(cm_node, sqbuf,
+ IRDMA_TIMER_TYPE_SEND, 1, 0);
+}
+
+/**
+ * irdma_mpav2_negotiate_ird_ord - negotiate MPAv2 IRD/ORD
+ * @cm_node: connection's node
+ * @buffer: Data pointer
+ */
+static int irdma_negotiate_mpa_v2_ird_ord(struct irdma_cm_node *cm_node, u8 *buf)
+{
+ struct ietf_mpa_v2 *mpa_v2_frame;
+ struct ietf_rtr_msg *rtr_msg;
+ u16 ird_size;
+ u16 ord_size;
+ u16 ctrl_ord;
+ u16 ctrl_ird;
+
+ mpa_v2_frame = (struct ietf_mpa_v2 *)buf;
+ rtr_msg = &mpa_v2_frame->rtr_msg;
+
+ /* parse rtr message */
+ ctrl_ord = ntohs(rtr_msg->ctrl_ord);
+ ctrl_ird = ntohs(rtr_msg->ctrl_ird);
+ ird_size = ctrl_ird & IETF_NO_IRD_ORD;
+ ord_size = ctrl_ord & IETF_NO_IRD_ORD;
+
+ if (!(ctrl_ird & IETF_PEER_TO_PEER))
+ return -1;
+
+ if (ird_size == IETF_NO_IRD_ORD || ord_size == IETF_NO_IRD_ORD) {
+ cm_node->mpav2_ird_ord = IETF_NO_IRD_ORD;
+ goto negotiate_done;
+ }
+
+ if (cm_node->state != IRDMA_CM_STATE_MPAREQ_SENT) {
+ /* responder */
+ if (!ord_size && (ctrl_ord & IETF_RDMA0_READ))
+ cm_node->ird_size = 1;
+ if (cm_node->ord_size > ird_size)
+ cm_node->ord_size = ird_size;
+ } else {
+ /* initiator */
+ if (!ird_size && (ctrl_ord & IETF_RDMA0_READ))
+ /* Remote peer doesn't support RDMA0_READ */
+ return -1;
+
+ if (cm_node->ord_size > ird_size)
+ cm_node->ord_size = ird_size;
+
+ if (cm_node->ird_size < ord_size)
+ /* no resources available */
+ return -1;
+ }
+
+negotiate_done:
+ if (ctrl_ord & IETF_RDMA0_READ)
+ cm_node->send_rdma0_op = SEND_RDMA_READ_ZERO;
+ else if (ctrl_ord & IETF_RDMA0_WRITE)
+ cm_node->send_rdma0_op = SEND_RDMA_WRITE_ZERO;
+ else
+ /* Not supported RDMA0 operation */
+ return -1;
+
+ irdma_debug(cm_node->dev, IRDMA_DEBUG_CM,
+ "MPAV2 Negotiated ORD: %d, IRD: %d\n",
+ cm_node->ord_size, cm_node->ird_size);
+ trace_irdma_negotiate_mpa_v2(cm_node);
+ return 0;
+}
+
+/**
+ * recv_mpa - process an IETF MPA frame
+ * @cm_node: connection's node
+ * @buffer: Data pointer
+ * @type: to return accept or reject
+ * @len: Len of mpa buffer
+ */
+static int irdma_parse_mpa(struct irdma_cm_node *cm_node,
+ u8 *buf,
+ u32 *type,
+ u32 len)
+{
+ struct ietf_mpa_v1 *mpa_frame;
+ int mpa_hdr_len;
+ int priv_data_len;
+
+ *type = IRDMA_MPA_REQUEST_ACCEPT;
+
+ if (len < sizeof(struct ietf_mpa_v1)) {
+ irdma_debug(cm_node->dev, IRDMA_DEBUG_ERR,
+ "ietf buffer small (%x)\n", len);
+ return -1;
+ }
+
+ mpa_frame = (struct ietf_mpa_v1 *)buf;
+ mpa_hdr_len = sizeof(struct ietf_mpa_v1);
+ priv_data_len = ntohs(mpa_frame->priv_data_len);
+
+ if (priv_data_len > IETF_MAX_PRIV_DATA_LEN) {
+ irdma_debug(cm_node->dev, IRDMA_DEBUG_ERR,
+ "large pri_data %d\n", priv_data_len);
+ return -1;
+ }
+
+ if (mpa_frame->rev != IETF_MPA_V1 && mpa_frame->rev != IETF_MPA_V2) {
+ irdma_debug(cm_node->dev, IRDMA_DEBUG_ERR,
+ "unsupported mpa rev = %d\n", mpa_frame->rev);
+ return -1;
+ }
+
+ if (mpa_frame->rev > cm_node->mpa_frame_rev) {
+ irdma_debug(cm_node->dev, IRDMA_DEBUG_ERR,
+ "rev %d\n", mpa_frame->rev);
+ return -1;
+ }
+
+ cm_node->mpa_frame_rev = mpa_frame->rev;
+ if (cm_node->state != IRDMA_CM_STATE_MPAREQ_SENT) {
+ if (memcmp(mpa_frame->key, IEFT_MPA_KEY_REQ,
+ IETF_MPA_KEY_SIZE)) {
+ irdma_debug(cm_node->dev, IRDMA_DEBUG_ERR,
+ "Unexpected MPA Key received\n");
+ return -1;
+ }
+ } else {
+ if (memcmp(mpa_frame->key, IEFT_MPA_KEY_REP,
+ IETF_MPA_KEY_SIZE)) {
+ irdma_debug(cm_node->dev, IRDMA_DEBUG_ERR,
+ "Unexpected MPA Key received\n");
+ return -1;
+ }
+ }
+
+ if (priv_data_len + mpa_hdr_len > len) {
+ irdma_debug(cm_node->dev, IRDMA_DEBUG_ERR,
+ "ietf buffer len(%x + %x != %x)\n",
+ priv_data_len, mpa_hdr_len, len);
+ return -1;
+ }
+
+ if (len > IRDMA_MAX_CM_BUF) {
+ irdma_debug(cm_node->dev, IRDMA_DEBUG_ERR,
+ "ietf buffer large len = %d\n", len);
+ return -1;
+ }
+
+ switch (mpa_frame->rev) {
+ case IETF_MPA_V2:
+ mpa_hdr_len += IETF_RTR_MSG_SIZE;
+ if (irdma_negotiate_mpa_v2_ird_ord(cm_node, buf))
+ return -1;
+ break;
+ case IETF_MPA_V1:
+ default:
+ break;
+ }
+
+ memcpy(cm_node->pdata_buf, buf + mpa_hdr_len, priv_data_len);
+ cm_node->pdata.size = priv_data_len;
+
+ if (mpa_frame->flags & IETF_MPA_FLAGS_REJECT)
+ *type = IRDMA_MPA_REQUEST_REJECT;
+
+ if (mpa_frame->flags & IETF_MPA_FLAGS_MARKERS)
+ cm_node->snd_mark_en = true;
+
+ return 0;
+}
+
+/**
+ * irdma_schedule_cm_timer
+ * @cm_node: connection's node
+ * @sqbuf: buffer to send
+ * @type: if it is send or close
+ * @send_retrans: if rexmits to be done
+ * @close_when_complete: is cm_node to be removed
+ *
+ * note - cm_node needs to be protected before calling this. Encase in:
+ * irdma_rem_ref_cm_node(cm_core, cm_node);
+ * irdma_schedule_cm_timer(...)
+ * atomic_inc(&cm_node->ref_count);
+ */
+int irdma_schedule_cm_timer(struct irdma_cm_node *cm_node,
+ struct irdma_puda_buf *sqbuf,
+ enum irdma_timer_type type,
+ int send_retrans,
+ int close_when_complete)
+{
+ struct irdma_sc_vsi *vsi = &cm_node->iwdev->vsi;
+ struct irdma_cm_core *cm_core = cm_node->cm_core;
+ struct irdma_timer_entry *new_send;
+ u32 was_timer_set;
+ unsigned long flags;
+
+ new_send = kzalloc(sizeof(*new_send), GFP_ATOMIC);
+ if (!new_send) {
+ if (type != IRDMA_TIMER_TYPE_CLOSE)
+ irdma_free_sqbuf(vsi, (void *)sqbuf);
+ return -ENOMEM;
+ }
+
+ new_send->retrycount = IRDMA_DEFAULT_RETRYS;
+ new_send->retranscount = IRDMA_DEFAULT_RETRANS;
+ new_send->sqbuf = sqbuf;
+ new_send->timetosend = jiffies;
+ new_send->type = type;
+ new_send->send_retrans = send_retrans;
+ new_send->close_when_complete = close_when_complete;
+
+ if (type == IRDMA_TIMER_TYPE_CLOSE) {
+ new_send->timetosend += (HZ / 10);
+ if (cm_node->close_entry) {
+ kfree(new_send);
+ irdma_debug(cm_node->dev, IRDMA_DEBUG_ERR,
+ "already close entry\n");
+ return -EINVAL;
+ }
+
+ cm_node->close_entry = new_send;
+ }
+
+ if (type == IRDMA_TIMER_TYPE_SEND) {
+ spin_lock_irqsave(&cm_node->retrans_list_lock, flags);
+ cm_node->send_entry = new_send;
+ atomic_inc(&cm_node->ref_count);
+ spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags);
+ new_send->timetosend = jiffies + IRDMA_RETRY_TIMEOUT;
+
+ atomic_inc(&sqbuf->refcount);
+ irdma_puda_send_buf(vsi->ilq, sqbuf);
+ if (!send_retrans) {
+ irdma_cleanup_retrans_entry(cm_node);
+ if (close_when_complete)
+ irdma_rem_ref_cm_node(cm_node);
+ return 0;
+ }
+ }
+
+ spin_lock_irqsave(&cm_core->ht_lock, flags);
+ was_timer_set = timer_pending(&cm_core->tcp_timer);
+
+ if (!was_timer_set) {
+ cm_core->tcp_timer.expires = new_send->timetosend;
+ add_timer(&cm_core->tcp_timer);
+ }
+ spin_unlock_irqrestore(&cm_core->ht_lock, flags);
+
+ return 0;
+}
+
+/**
+ * irdma_retrans_expired - Could not rexmit the packet
+ * @cm_node: connection's node
+ */
+static void irdma_retrans_expired(struct irdma_cm_node *cm_node)
+{
+ struct iw_cm_id *cm_id = cm_node->cm_id;
+ enum irdma_cm_node_state state = cm_node->state;
+
+ cm_node->state = IRDMA_CM_STATE_CLOSED;
+ switch (state) {
+ case IRDMA_CM_STATE_SYN_RCVD:
+ case IRDMA_CM_STATE_CLOSING:
+ irdma_rem_ref_cm_node(cm_node);
+ break;
+ case IRDMA_CM_STATE_FIN_WAIT1:
+ case IRDMA_CM_STATE_LAST_ACK:
+ if (cm_node->cm_id)
+ cm_id->rem_ref(cm_id);
+ irdma_send_reset(cm_node);
+ break;
+ default:
+ atomic_inc(&cm_node->ref_count);
+ irdma_send_reset(cm_node);
+ irdma_create_event(cm_node, IRDMA_CM_EVENT_ABORTED);
+ break;
+ }
+}
+
+/**
+ * irdma_handle_close_entry - for handling retry/timeouts
+ * @cm_node: connection's node
+ * @rem_node: flag for remove cm_node
+ */
+static void irdma_handle_close_entry(struct irdma_cm_node *cm_node,
+ u32 rem_node)
+{
+ struct irdma_timer_entry *close_entry = cm_node->close_entry;
+ struct iw_cm_id *cm_id = cm_node->cm_id;
+ struct irdma_qp *iwqp;
+ unsigned long flags;
+
+ if (!close_entry)
+ return;
+ iwqp = (struct irdma_qp *)close_entry->sqbuf;
+ if (iwqp) {
+ spin_lock_irqsave(&iwqp->lock, flags);
+ if (iwqp->cm_id) {
+ iwqp->hw_tcp_state = IRDMA_TCP_STATE_CLOSED;
+ iwqp->hw_iwarp_state = IRDMA_QP_STATE_ERROR;
+ iwqp->last_aeq = IRDMA_AE_RESET_SENT;
+ iwqp->ibqp_state = IB_QPS_ERR;
+ spin_unlock_irqrestore(&iwqp->lock, flags);
+ irdma_cm_disconn(iwqp);
+ } else {
+ spin_unlock_irqrestore(&iwqp->lock, flags);
+ }
+ } else if (rem_node) {
+ /* TIME_WAIT state */
+ irdma_rem_ref_cm_node(cm_node);
+ }
+ if (cm_id)
+ cm_id->rem_ref(cm_id);
+ kfree(close_entry);
+ cm_node->close_entry = NULL;
+}
+
+/**
+ * irdma_build_timer_list - Add cm_nodes to timer list
+ * @timer_list: ptr to timer list
+ * @hte: ptr to accelerated or non-accelerated list
+ */
+static void irdma_build_timer_list(struct list_head *timer_list,
+ struct list_head *hte)
+{
+ struct irdma_cm_node *cm_node;
+ struct list_head *list_core_temp, *list_node;
+
+ list_for_each_safe(list_node, list_core_temp, hte) {
+ cm_node = container_of(list_node, struct irdma_cm_node, list);
+ if (cm_node->close_entry || cm_node->send_entry) {
+ atomic_inc(&cm_node->ref_count);
+ list_add(&cm_node->timer_entry, timer_list);
+ }
+ }
+}
+
+/**
+ * irdma_cm_timer_tick - system's timer expired callback
+ * @t: Pointer to timer_list
+ */
+static void irdma_cm_timer_tick(struct timer_list *t)
+{
+ unsigned long nexttimeout = jiffies + IRDMA_LONG_TIME;
+ struct irdma_cm_node *cm_node;
+ struct irdma_timer_entry *send_entry, *close_entry;
+ struct list_head *list_core_temp;
+ struct list_head *list_node;
+ struct irdma_cm_core *cm_core = from_timer(cm_core, t, tcp_timer);
+ struct irdma_sc_vsi *vsi;
+ u32 settimer = 0;
+ unsigned long timetosend;
+ struct irdma_sc_dev *dev;
+ unsigned long flags;
+ struct list_head timer_list;
+
+ INIT_LIST_HEAD(&timer_list);
+
+ spin_lock_irqsave(&cm_core->ht_lock, flags);
+ irdma_build_timer_list(&timer_list, &cm_core->non_accelerated_list);
+ irdma_build_timer_list(&timer_list, &cm_core->accelerated_list);
+ spin_unlock_irqrestore(&cm_core->ht_lock, flags);
+
+ list_for_each_safe(list_node, list_core_temp, &timer_list) {
+ cm_node = container_of(list_node,
+ struct irdma_cm_node,
+ timer_entry);
+ close_entry = cm_node->close_entry;
+
+ if (close_entry) {
+ if (time_after(close_entry->timetosend, jiffies)) {
+ if (nexttimeout > close_entry->timetosend ||
+ !settimer) {
+ nexttimeout = close_entry->timetosend;
+ settimer = 1;
+ }
+ } else {
+ irdma_handle_close_entry(cm_node, 1);
+ }
+ }
+
+ spin_lock_irqsave(&cm_node->retrans_list_lock, flags);
+
+ send_entry = cm_node->send_entry;
+ if (!send_entry)
+ goto done;
+ if (time_after(send_entry->timetosend, jiffies)) {
+ if (cm_node->state != IRDMA_CM_STATE_OFFLOADED) {
+ if (nexttimeout > send_entry->timetosend ||
+ !settimer) {
+ nexttimeout = send_entry->timetosend;
+ settimer = 1;
+ }
+ } else {
+ irdma_free_retrans_entry(cm_node);
+ }
+ goto done;
+ }
+
+ if (cm_node->state == IRDMA_CM_STATE_OFFLOADED ||
+ cm_node->state == IRDMA_CM_STATE_CLOSED) {
+ irdma_free_retrans_entry(cm_node);
+ goto done;
+ }
+
+ if (!send_entry->retranscount || !send_entry->retrycount) {
+ irdma_free_retrans_entry(cm_node);
+
+ spin_unlock_irqrestore(&cm_node->retrans_list_lock,
+ flags);
+ irdma_retrans_expired(cm_node);
+ cm_node->state = IRDMA_CM_STATE_CLOSED;
+ spin_lock_irqsave(&cm_node->retrans_list_lock, flags);
+ goto done;
+ }
+ spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags);
+
+ vsi = &cm_node->iwdev->vsi;
+ dev = cm_node->dev;
+ if (!cm_node->ack_rcvd) {
+ atomic_inc(&send_entry->sqbuf->refcount);
+ irdma_puda_send_buf(vsi->ilq, send_entry->sqbuf);
+ cm_node->cm_core->stats_pkt_retrans++;
+ }
+
+ spin_lock_irqsave(&cm_node->retrans_list_lock, flags);
+ if (send_entry->send_retrans) {
+ send_entry->retranscount--;
+ timetosend = (IRDMA_RETRY_TIMEOUT <<
+ (IRDMA_DEFAULT_RETRANS -
+ send_entry->retranscount));
+
+ send_entry->timetosend = jiffies +
+ min(timetosend, IRDMA_MAX_TIMEOUT);
+ if (nexttimeout > send_entry->timetosend || !settimer) {
+ nexttimeout = send_entry->timetosend;
+ settimer = 1;
+ }
+ } else {
+ int close_when_complete;
+
+ close_when_complete = send_entry->close_when_complete;
+ irdma_free_retrans_entry(cm_node);
+ if (close_when_complete)
+ irdma_rem_ref_cm_node(cm_node);
+ }
+done:
+ spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags);
+ irdma_rem_ref_cm_node(cm_node);
+ }
+
+ if (settimer) {
+ spin_lock_irqsave(&cm_core->ht_lock, flags);
+ if (!timer_pending(&cm_core->tcp_timer)) {
+ cm_core->tcp_timer.expires = nexttimeout;
+ add_timer(&cm_core->tcp_timer);
+ }
+ spin_unlock_irqrestore(&cm_core->ht_lock, flags);
+ }
+}
+
+/**
+ * irdma_send_syn - send SYN packet
+ * @cm_node: connection's node
+ * @sendack: flag to set ACK bit or not
+ */
+int irdma_send_syn(struct irdma_cm_node *cm_node, u32 sendack)
+{
+ struct irdma_puda_buf *sqbuf;
+ int flags = SET_SYN;
+ char optionsbuf[sizeof(struct option_mss) +
+ sizeof(struct option_windowscale) +
+ sizeof(struct option_base) + TCP_OPTIONS_PADDING];
+ struct irdma_kmem_info opts;
+ int optionssize = 0;
+ /* Sending MSS option */
+ union all_known_options *options;
+
+ opts.addr = optionsbuf;
+ if (!cm_node)
+ return -EINVAL;
+
+ options = (union all_known_options *)&optionsbuf[optionssize];
+ options->mss.optionnum = OPTION_NUM_MSS;
+ options->mss.len = sizeof(struct option_mss);
+ options->mss.mss = htons(cm_node->tcp_cntxt.mss);
+ optionssize += sizeof(struct option_mss);
+
+ options = (union all_known_options *)&optionsbuf[optionssize];
+ options->windowscale.optionnum = OPTION_NUM_WINDOW_SCALE;
+ options->windowscale.len = sizeof(struct option_windowscale);
+ options->windowscale.shiftcount = cm_node->tcp_cntxt.rcv_wscale;
+ optionssize += sizeof(struct option_windowscale);
+ options = (union all_known_options *)&optionsbuf[optionssize];
+ options->eol = OPTION_NUM_EOL;
+ optionssize += 1;
+
+ if (sendack)
+ flags |= SET_ACK;
+
+ opts.size = optionssize;
+
+ sqbuf = cm_node->cm_core->form_cm_frame(cm_node, &opts, NULL, NULL, flags);
+ if (!sqbuf)
+ return -1;
+
+ return irdma_schedule_cm_timer(cm_node, sqbuf,
+ IRDMA_TIMER_TYPE_SEND, 1, 0);
+}
+
+/**
+ * irdma_send_ack - Send ACK packet
+ * @cm_node: connection's node
+ */
+void irdma_send_ack(struct irdma_cm_node *cm_node)
+{
+ struct irdma_puda_buf *sqbuf;
+ struct irdma_sc_vsi *vsi = &cm_node->iwdev->vsi;
+
+ sqbuf = cm_node->cm_core->form_cm_frame(cm_node, NULL, NULL, NULL, SET_ACK);
+ if (sqbuf)
+ irdma_puda_send_buf(vsi->ilq, sqbuf);
+}
+
+/**
+ * irdma_send_fin - Send FIN pkt
+ * @cm_node: connection's node
+ */
+static int irdma_send_fin(struct irdma_cm_node *cm_node)
+{
+ struct irdma_puda_buf *sqbuf;
+
+ sqbuf = cm_node->cm_core->form_cm_frame(cm_node, NULL, NULL, NULL, SET_ACK | SET_FIN);
+ if (!sqbuf)
+ return -1;
+
+ return irdma_schedule_cm_timer(cm_node, sqbuf,
+ IRDMA_TIMER_TYPE_SEND, 1, 0);
+}
+
+/**
+ * irdma_find_node - find a cm node that matches the reference
+ * cm node
+ * @cm_core: cm's core
+ * @rem_port: remote tcp port num
+ * @rem_addr: remote ip addr
+ * @loc_port: local tcp port num
+ * @loc_addr: loc ip addr
+ * @add_refcnt: flag to increment refcount of cm_node
+ * @accelerated_list: flag for accelerated vs non-accelerated list to search
+ */
+struct irdma_cm_node *irdma_find_node(struct irdma_cm_core *cm_core,
+ u16 rem_port,
+ u32 *rem_addr,
+ u16 loc_port,
+ u32 *loc_addr,
+ bool add_refcnt,
+ bool accelerated_list)
+{
+ struct list_head *hte;
+ struct irdma_cm_node *cm_node;
+ unsigned long flags;
+
+ hte = accelerated_list ? &cm_core->accelerated_list : &cm_core->non_accelerated_list;
+
+ spin_lock_irqsave(&cm_core->ht_lock, flags);
+ list_for_each_entry(cm_node, hte, list) {
+ if (!memcmp(cm_node->loc_addr, loc_addr, sizeof(cm_node->loc_addr)) &&
+ cm_node->loc_port == loc_port &&
+ !memcmp(cm_node->rem_addr, rem_addr, sizeof(cm_node->rem_addr)) &&
+ cm_node->rem_port == rem_port) {
+ if (add_refcnt)
+ atomic_inc(&cm_node->ref_count);
+ spin_unlock_irqrestore(&cm_core->ht_lock, flags);
+ trace_irdma_find_node(cm_node, 0, NULL);
+ return cm_node;
+ }
+ }
+ spin_unlock_irqrestore(&cm_core->ht_lock, flags);
+
+ /* no owner node */
+ return NULL;
+}
+
+/**
+ * irdma_find_listener - find a cm node listening on this addr-port pair
+ * @cm_core: cm's core
+ * @dst_port: listener tcp port num
+ * @dst_addr: listener ip addr
+ * @listener_state: state to match with listen node's
+ */
+static struct irdma_cm_listener *irdma_find_listener(struct irdma_cm_core *cm_core,
+ u32 *dst_addr,
+ u16 dst_port,
+ u16 vlan_id,
+ enum irdma_cm_listener_state listener_state)
+{
+ struct irdma_cm_listener *listen_node;
+ static const u32 ip_zero[4] = { 0, 0, 0, 0 };
+ u32 listen_addr[4];
+ u16 listen_port;
+ unsigned long flags;
+
+ /* walk list and find cm_node associated with this session ID */
+ spin_lock_irqsave(&cm_core->listen_list_lock, flags);
+ list_for_each_entry(listen_node, &cm_core->listen_nodes, list) {
+ memcpy(listen_addr, listen_node->loc_addr, sizeof(listen_addr));
+ listen_port = listen_node->loc_port;
+ /* compare node pair, return node handle if a match */
+ if ((!memcmp(listen_addr, dst_addr, sizeof(listen_addr)) ||
+ !memcmp(listen_addr, ip_zero, sizeof(listen_addr))) &&
+ listen_port == dst_port && vlan_id == listen_node->vlan_id &&
+ (listener_state & listen_node->listener_state)) {
+ atomic_inc(&listen_node->ref_count);
+ spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
+ trace_irdma_find_listener(listen_node);
+ return listen_node;
+ }
+ }
+ spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
+
+ return NULL;
+}
+
+/**
+ * irdma_add_hte_node - add a cm node to the hash table
+ * @cm_core: cm's core
+ * @cm_node: connection's node
+ */
+static void irdma_add_hte_node(struct irdma_cm_core *cm_core,
+ struct irdma_cm_node *cm_node)
+{
+ unsigned long flags;
+
+ if (!cm_node || !cm_core)
+ return;
+
+ spin_lock_irqsave(&cm_core->ht_lock, flags);
+ list_add_tail(&cm_node->list, &cm_core->non_accelerated_list);
+ spin_unlock_irqrestore(&cm_core->ht_lock, flags);
+}
+
+/**
+ * irdma_find_port - find port that matches reference port
+ * @hte: ptr to accelerated or non-accelerated list
+ * @port: port number
+ */
+static bool irdma_find_port(struct list_head *hte, u16 port)
+{
+ struct irdma_cm_node *cm_node;
+
+ list_for_each_entry(cm_node, hte, list) {
+ if (cm_node->loc_port == port)
+ return true;
+ }
+
+ return false;
+}
+
+/**
+ * irdma_port_in_use - determine if port is in use
+ * @cm_core: cm's core
+ * @port: port number
+ */
+bool irdma_port_in_use(struct irdma_cm_core *cm_core, u16 port)
+{
+ struct irdma_cm_listener *listen_node;
+ unsigned long flags;
+
+ spin_lock_irqsave(&cm_core->ht_lock, flags);
+ if (irdma_find_port(&cm_core->accelerated_list, port) ||
+ irdma_find_port(&cm_core->non_accelerated_list, port)) {
+ spin_unlock_irqrestore(&cm_core->ht_lock, flags);
+ return true;
+ }
+ spin_unlock_irqrestore(&cm_core->ht_lock, flags);
+
+ spin_lock_irqsave(&cm_core->listen_list_lock, flags);
+ list_for_each_entry(listen_node, &cm_core->listen_nodes, list) {
+ if (listen_node->loc_port == port) {
+ spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
+ return true;
+ }
+ }
+ spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
+
+ return false;
+}
+
+/**
+ * irdma_del_multiple_qhash - Remove qhash and child listens
+ * @iwdev: iWarp device
+ * @cm_info: CM info for parent listen node
+ * @cm_parent_listen_node: The parent listen node
+ */
+static enum irdma_status_code
+irdma_del_multiple_qhash(struct irdma_device *iwdev,
+ struct irdma_cm_info *cm_info,
+ struct irdma_cm_listener *cm_parent_listen_node)
+{
+ struct irdma_cm_listener *child_listen_node;
+ enum irdma_status_code ret = IRDMA_ERR_CFG;
+ struct list_head *pos, *tpos;
+ unsigned long flags;
+
+ spin_lock_irqsave(&iwdev->cm_core.listen_list_lock, flags);
+ list_for_each_safe(pos, tpos,
+ &cm_parent_listen_node->child_listen_list) {
+ child_listen_node = list_entry(pos, struct irdma_cm_listener,
+ child_listen_list);
+ if (child_listen_node->ipv4)
+ irdma_debug(&iwdev->rf->sc_dev,
+ IRDMA_DEBUG_CM,
+ "removing child listen for IP=%pI4, port=%d, vlan=%d\n",
+ child_listen_node->loc_addr,
+ child_listen_node->loc_port,
+ child_listen_node->vlan_id);
+ else
+ irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
+ "removing child listen for IP=%pI6, port=%d, vlan=%d\n",
+ child_listen_node->loc_addr,
+ child_listen_node->loc_port,
+ child_listen_node->vlan_id);
+ trace_irdma_del_multiple_qhash(child_listen_node);
+ list_del(pos);
+ memcpy(cm_info->loc_addr, child_listen_node->loc_addr,
+ sizeof(cm_info->loc_addr));
+ cm_info->vlan_id = child_listen_node->vlan_id;
+ if (child_listen_node->qhash_set) {
+ ret = irdma_manage_qhash(iwdev, cm_info,
+ IRDMA_QHASH_TYPE_TCP_SYN,
+ IRDMA_QHASH_MANAGE_TYPE_DELETE,
+ NULL, false);
+ child_listen_node->qhash_set = false;
+ } else {
+ ret = 0;
+ }
+ irdma_debug(&iwdev->rf->sc_dev,
+ IRDMA_DEBUG_CM,
+ "Child listen node freed = %p\n",
+ child_listen_node);
+ kfree(child_listen_node);
+ cm_parent_listen_node->cm_core->stats_listen_nodes_destroyed++;
+ }
+ spin_unlock_irqrestore(&iwdev->cm_core.listen_list_lock, flags);
+
+ return ret;
+}
+
+/**
+ * irdma_netdev_vlan_ipv6 - Gets the netdev and mac
+ * @addr: local IPv6 address
+ * @vlan_id: vlan id for the given IPv6 address
+ * @mac: mac address for the given IPv6 address
+ *
+ * Returns the net_device of the IPv6 address and also sets the
+ * vlan id and mac for that address.
+ */
+struct net_device *irdma_netdev_vlan_ipv6(u32 *addr, u16 *vlan_id, u8 *mac)
+{
+ struct net_device *ip_dev = NULL;
+ struct in6_addr laddr6;
+
+ if (!IS_ENABLED(CONFIG_IPV6))
+ return NULL;
+
+ irdma_copy_ip_htonl(laddr6.in6_u.u6_addr32, addr);
+ if (vlan_id)
+ *vlan_id = IRDMA_NO_VLAN;
+ if (mac)
+ eth_zero_addr(mac);
+
+ rcu_read_lock();
+ for_each_netdev_rcu(&init_net, ip_dev) {
+ if (ipv6_chk_addr(&init_net, &laddr6, ip_dev, 1)) {
+ if (vlan_id)
+ *vlan_id = rdma_vlan_dev_vlan_id(ip_dev);
+ if (ip_dev->dev_addr && mac)
+ ether_addr_copy(mac, ip_dev->dev_addr);
+ break;
+ }
+ }
+ rcu_read_unlock();
+
+ return ip_dev;
+}
+
+/**
+ * irdma_get_vlan_ipv4 - Returns the vlan_id for IPv4 address
+ * @addr: local IPv4 address
+ */
+u16 irdma_get_vlan_ipv4(u32 *addr)
+{
+ struct net_device *netdev;
+ u16 vlan_id = IRDMA_NO_VLAN;
+
+ netdev = ip_dev_find(&init_net, htonl(addr[0]));
+ if (netdev) {
+ vlan_id = rdma_vlan_dev_vlan_id(netdev);
+ dev_put(netdev);
+ }
+
+ return vlan_id;
+}
+
+/**
+ * irdma_add_mqh_6 - Adds multiple qhashes for IPv6
+ * @iwdev: iWarp device
+ * @cm_info: CM info for parent listen node
+ * @cm_parent_listen_node: The parent listen node
+ *
+ * Adds a qhash and a child listen node for every IPv6 address
+ * on the adapter and adds the associated qhash filter
+ */
+static enum irdma_status_code
+irdma_add_mqh_6(struct irdma_device *iwdev,
+ struct irdma_cm_info *cm_info,
+ struct irdma_cm_listener *cm_parent_listen_node)
+{
+ struct net_device *ip_dev;
+ struct inet6_dev *idev;
+ struct inet6_ifaddr *ifp, *tmp;
+ enum irdma_status_code ret = 0;
+ struct irdma_cm_listener *child_listen_node;
+ unsigned long flags;
+
+ rtnl_lock();
+ for_each_netdev(&init_net, ip_dev) {
+ if (!(ip_dev->flags & IFF_UP))
+ continue;
+
+ if (((rdma_vlan_dev_vlan_id(ip_dev) >= IRDMA_NO_VLAN) ||
+ (rdma_vlan_dev_real_dev(ip_dev) != iwdev->netdev)) &&
+ ip_dev != iwdev->netdev)
+ continue;
+
+ idev = __in6_dev_get(ip_dev);
+ if (!idev) {
+ irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
+ "idev == NULL\n");
+ break;
+ }
+ list_for_each_entry_safe(ifp, tmp, &idev->addr_list, if_list) {
+ irdma_debug(&iwdev->rf->sc_dev,
+ IRDMA_DEBUG_CM,
+ "IP=%pI6, vlan_id=%d, MAC=%pM\n",
+ &ifp->addr,
+ rdma_vlan_dev_vlan_id(ip_dev),
+ ip_dev->dev_addr);
+ child_listen_node = kzalloc(sizeof(*child_listen_node), GFP_KERNEL);
+ irdma_debug(&iwdev->rf->sc_dev,
+ IRDMA_DEBUG_CM,
+ "Allocating child listener %p\n",
+ child_listen_node);
+ if (!child_listen_node) {
+ irdma_debug(&iwdev->rf->sc_dev,
+ IRDMA_DEBUG_ERR,
+ "listener memory allocation\n");
+ ret = IRDMA_ERR_NO_MEMORY;
+ goto exit;
+ }
+
+ cm_info->vlan_id = rdma_vlan_dev_vlan_id(ip_dev);
+ cm_parent_listen_node->vlan_id = cm_info->vlan_id;
+ memcpy(child_listen_node, cm_parent_listen_node,
+ sizeof(*child_listen_node));
+ irdma_copy_ip_ntohl(child_listen_node->loc_addr,
+ ifp->addr.in6_u.u6_addr32);
+ memcpy(cm_info->loc_addr, child_listen_node->loc_addr,
+ sizeof(cm_info->loc_addr));
+ ret = irdma_manage_qhash(iwdev, cm_info,
+ IRDMA_QHASH_TYPE_TCP_SYN,
+ IRDMA_QHASH_MANAGE_TYPE_ADD,
+ NULL, true);
+ if (ret) {
+ kfree(child_listen_node);
+ continue;
+ }
+
+ trace_irdma_add_mqh_6(iwdev, child_listen_node,
+ ip_dev->dev_addr);
+
+ child_listen_node->qhash_set = true;
+ spin_lock_irqsave(&iwdev->cm_core.listen_list_lock, flags);
+ list_add(&child_listen_node->child_listen_list,
+ &cm_parent_listen_node->child_listen_list);
+ spin_unlock_irqrestore(&iwdev->cm_core.listen_list_lock, flags);
+ cm_parent_listen_node->cm_core->stats_listen_nodes_created++;
+ }
+ }
+exit:
+ rtnl_unlock();
+
+ return ret;
+}
+
+/**
+ * irdma_add_mqh_4 - Adds multiple qhashes for IPv4
+ * @iwdev: iWarp device
+ * @cm_info: CM info for parent listen node
+ * @cm_parent_listen_node: The parent listen node
+ *
+ * Adds a qhash and a child listen node for every IPv4 address
+ * on the adapter and adds the associated qhash filter
+ */
+static enum irdma_status_code
+irdma_add_mqh_4(struct irdma_device *iwdev,
+ struct irdma_cm_info *cm_info,
+ struct irdma_cm_listener *cm_parent_listen_node)
+{
+ struct net_device *dev;
+ struct in_device *idev;
+ struct irdma_cm_listener *child_listen_node;
+ enum irdma_status_code ret = 0;
+ unsigned long flags;
+
+ rtnl_lock();
+ for_each_netdev(&init_net, dev) {
+ if (!(dev->flags & IFF_UP))
+ continue;
+
+ if (((rdma_vlan_dev_vlan_id(dev) >= IRDMA_NO_VLAN) ||
+ (rdma_vlan_dev_real_dev(dev) != iwdev->netdev)) &&
+ dev != iwdev->netdev)
+ continue;
+
+ idev = in_dev_get(dev);
+ for_ifa(idev) {
+ irdma_debug(&iwdev->rf->sc_dev,
+ IRDMA_DEBUG_CM,
+ "Allocating child CM Listener forIP=%pI4, vlan_id=%d, MAC=%pM\n",
+ &ifa->ifa_address,
+ rdma_vlan_dev_vlan_id(dev),
+ dev->dev_addr);
+ child_listen_node = kzalloc(sizeof(*child_listen_node), GFP_KERNEL);
+ cm_parent_listen_node->cm_core->stats_listen_nodes_created++;
+ irdma_debug(&iwdev->rf->sc_dev,
+ IRDMA_DEBUG_CM,
+ "Allocating child listener %p\n",
+ child_listen_node);
+ if (!child_listen_node) {
+ irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_ERR,
+ "listener memory allocation\n");
+ in_dev_put(idev);
+ ret = IRDMA_ERR_NO_MEMORY;
+ goto exit;
+ }
+
+ cm_info->vlan_id = rdma_vlan_dev_vlan_id(dev);
+ cm_parent_listen_node->vlan_id = cm_info->vlan_id;
+ memcpy(child_listen_node, cm_parent_listen_node,
+ sizeof(*child_listen_node));
+ child_listen_node->loc_addr[0] = ntohl(ifa->ifa_address);
+ memcpy(cm_info->loc_addr, child_listen_node->loc_addr,
+ sizeof(cm_info->loc_addr));
+ ret = irdma_manage_qhash(iwdev,
+ cm_info,
+ IRDMA_QHASH_TYPE_TCP_SYN,
+ IRDMA_QHASH_MANAGE_TYPE_ADD,
+ NULL,
+ true);
+ if (ret) {
+ kfree(child_listen_node);
+ cm_parent_listen_node->cm_core->stats_listen_nodes_created--;
+ continue;
+ }
+
+ trace_irdma_add_mqh_4(iwdev, child_listen_node,
+ dev->dev_addr);
+
+ child_listen_node->qhash_set = true;
+ spin_lock_irqsave(&iwdev->cm_core.listen_list_lock, flags);
+ list_add(&child_listen_node->child_listen_list,
+ &cm_parent_listen_node->child_listen_list);
+ spin_unlock_irqrestore(&iwdev->cm_core.listen_list_lock, flags);
+ }
+ endfor_ifa(idev);
+ in_dev_put(idev);
+ }
+exit:
+ rtnl_unlock();
+
+ return ret;
+}
+
+/**
+ * irdma_dec_refcnt_listen - delete listener and associated cm nodes
+ * @cm_core: cm's core
+ * @free_hanging_nodes: to free associated cm_nodes
+ * @apbvt_del: flag to delete the apbvt
+ */
+static int irdma_dec_refcnt_listen(struct irdma_cm_core *cm_core,
+ struct irdma_cm_listener *listener,
+ int free_hanging_nodes, bool apbvt_del)
+{
+ int err;
+ struct list_head *list_pos;
+ struct list_head *list_temp;
+ struct irdma_cm_node *cm_node;
+ struct list_head reset_list;
+ struct irdma_cm_info nfo;
+ enum irdma_cm_node_state old_state;
+ unsigned long flags;
+
+ trace_irdma_dec_refcnt_listen(listener, __builtin_return_address(0));
+ /* free non-accelerated child nodes for this listener */
+ INIT_LIST_HEAD(&reset_list);
+ if (free_hanging_nodes) {
+ spin_lock_irqsave(&cm_core->ht_lock, flags);
+ list_for_each_safe(list_pos, list_temp,
+ &cm_core->non_accelerated_list) {
+ cm_node = container_of(list_pos, struct irdma_cm_node, list);
+ if (cm_node->listener == listener && !cm_node->accelerated) {
+ atomic_inc(&cm_node->ref_count);
+ list_add(&cm_node->reset_entry, &reset_list);
+ }
+ }
+ spin_unlock_irqrestore(&cm_core->ht_lock, flags);
+ }
+
+ list_for_each_safe(list_pos, list_temp, &reset_list) {
+ cm_node = container_of(list_pos, struct irdma_cm_node, reset_entry);
+ if (cm_node->state >= IRDMA_CM_STATE_FIN_WAIT1) {
+ irdma_rem_ref_cm_node(cm_node);
+ continue;
+ }
+
+ irdma_cleanup_retrans_entry(cm_node);
+ err = irdma_send_reset(cm_node);
+ if (err) {
+ cm_node->state = IRDMA_CM_STATE_CLOSED;
+ irdma_debug(cm_node->dev,
+ IRDMA_DEBUG_ERR,
+ "send reset failed\n");
+ } else {
+ old_state = cm_node->state;
+ cm_node->state = IRDMA_CM_STATE_LISTENER_DESTROYED;
+ if (old_state != IRDMA_CM_STATE_MPAREQ_RCVD)
+ irdma_rem_ref_cm_node(cm_node);
+ }
+ }
+
+ if (!atomic_dec_return(&listener->ref_count)) {
+ spin_lock_irqsave(&cm_core->listen_list_lock, flags);
+ list_del(&listener->list);
+ spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
+
+ if (listener->iwdev) {
+ if (apbvt_del)
+ irdma_manage_apbvt(listener->iwdev,
+ listener->loc_port,
+ IRDMA_MANAGE_APBVT_DEL);
+ memcpy(nfo.loc_addr, listener->loc_addr,
+ sizeof(nfo.loc_addr));
+ nfo.loc_port = listener->loc_port;
+ nfo.ipv4 = listener->ipv4;
+ nfo.vlan_id = listener->vlan_id;
+ nfo.user_pri = listener->user_pri;
+ nfo.qh_qpid = listener->iwdev->vsi.ilq->qp_id;
+
+ if (!list_empty(&listener->child_listen_list)) {
+ irdma_del_multiple_qhash(listener->iwdev, &nfo,
+ listener);
+ } else {
+ if (listener->qhash_set)
+ irdma_manage_qhash(listener->iwdev,
+ &nfo,
+ IRDMA_QHASH_TYPE_TCP_SYN,
+ IRDMA_QHASH_MANAGE_TYPE_DELETE,
+ NULL,
+ false);
+ }
+ }
+
+ cm_core->stats_listen_destroyed++;
+ kfree(listener);
+ cm_core->stats_listen_nodes_destroyed++;
+ listener = NULL;
+ return 0;
+ }
+
+ return -EINVAL;
+}
+
+/**
+ * irdma_cm_del_listen - delete a listener
+ * @cm_core: cm's core
+ * @listener: passive connection's listener
+ * @apbvt_del: flag to delete apbvt
+ */
+static int irdma_cm_del_listen(struct irdma_cm_core *cm_core,
+ struct irdma_cm_listener *listener,
+ bool apbvt_del)
+{
+ listener->listener_state = IRDMA_CM_LISTENER_PASSIVE_STATE;
+ listener->cm_id = NULL;
+
+ return irdma_dec_refcnt_listen(cm_core, listener, 1, apbvt_del);
+}
+
+/**
+ * irdma_addr_resolve_neigh - resolve neighbor address
+ * @iwdev: iwarp device structure
+ * @src_ip: local ip address
+ * @dst_ip: remote ip address
+ * @arpindex: if there is an arp entry
+ */
+static int irdma_addr_resolve_neigh(struct irdma_device *iwdev,
+ u32 src_ip,
+ u32 dst_ip,
+ int arpindex)
+{
+ struct rtable *rt;
+ struct neighbour *neigh;
+ int rc = arpindex;
+ __be32 dst_ipaddr = htonl(dst_ip);
+ __be32 src_ipaddr = htonl(src_ip);
+
+ rt = ip_route_output(&init_net, dst_ipaddr, src_ipaddr, 0, 0);
+ if (IS_ERR(rt)) {
+ irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_ERR,
+ "ip_route_output fail\n");
+ return -EINVAL;
+ }
+
+ neigh = dst_neigh_lookup(&rt->dst, &dst_ipaddr);
+
+ rcu_read_lock();
+ if (!neigh)
+ goto exit;
+
+ if (neigh->nud_state & NUD_VALID)
+ rc = irdma_add_arp(iwdev->rf, &dst_ip, true, neigh->ha);
+ else
+ neigh_event_send(neigh, NULL);
+exit:
+ rcu_read_unlock();
+
+ if (neigh)
+ neigh_release(neigh);
+
+ ip_rt_put(rt);
+
+ return rc;
+}
+
+/**
+ * irdma_get_dst_ipv6 - get destination cache entry via ipv6 lookup
+ * @src_addr: local ipv6 sock address
+ * @dst_addr: destination ipv6 sock address
+ */
+static struct dst_entry *irdma_get_dst_ipv6(struct sockaddr_in6 *src_addr,
+ struct sockaddr_in6 *dst_addr)
+{
+ struct dst_entry *dst = NULL;
+
+ if ((IS_ENABLED(CONFIG_IPV6))) {
+ struct flowi6 fl6 = {};
+
+ fl6.daddr = dst_addr->sin6_addr;
+ fl6.saddr = src_addr->sin6_addr;
+ if (ipv6_addr_type(&fl6.daddr) & IPV6_ADDR_LINKLOCAL)
+ fl6.flowi6_oif = dst_addr->sin6_scope_id;
+
+ dst = ip6_route_output(&init_net, NULL, &fl6);
+ }
+
+ return dst;
+}
+
+/**
+ * irdma_addr_resolve_neigh_ipv6 - resolve neighbor ipv6 address
+ * @iwdev: iwarp device structure
+ * @src: local ip address
+ * @dst: remote ip address
+ * @arpindex: if there is an arp entry
+ */
+static int irdma_addr_resolve_neigh_ipv6(struct irdma_device *iwdev,
+ u32 *src,
+ u32 *dest,
+ int arpindex)
+{
+ struct neighbour *neigh;
+ int rc = arpindex;
+ struct dst_entry *dst;
+ struct sockaddr_in6 dst_addr = {};
+ struct sockaddr_in6 src_addr = {};
+
+ dst_addr.sin6_family = AF_INET6;
+ irdma_copy_ip_htonl(dst_addr.sin6_addr.in6_u.u6_addr32, dest);
+ src_addr.sin6_family = AF_INET6;
+ irdma_copy_ip_htonl(src_addr.sin6_addr.in6_u.u6_addr32, src);
+ dst = irdma_get_dst_ipv6(&src_addr, &dst_addr);
+ if (!dst || dst->error) {
+ if (dst) {
+ dst_release(dst);
+ irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_ERR,
+ "ip6_route_output returned dst->error = %d\n",
+ dst->error);
+ }
+ return -EINVAL;
+ }
+
+ neigh = dst_neigh_lookup(dst, dst_addr.sin6_addr.in6_u.u6_addr32);
+
+ rcu_read_lock();
+ if (!neigh)
+ goto exit;
+
+ irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
+ "dst_neigh_lookup MAC=%pM\n", neigh->ha);
+
+ trace_irdma_addr_resolve(iwdev, neigh->ha);
+
+ if (neigh->nud_state & NUD_VALID)
+ rc = irdma_add_arp(iwdev->rf, dest, false, neigh->ha);
+ else
+ neigh_event_send(neigh, NULL);
+exit:
+
+ rcu_read_unlock();
+ if (neigh)
+ neigh_release(neigh);
+ dst_release(dst);
+
+ return rc;
+}
+
+/**
+ * irdma_ipv4_is_lpb - check if loopback
+ * @loc_addr: local addr to compare
+ * @rem_addr: remote address
+ */
+bool irdma_ipv4_is_lpb(u32 loc_addr, u32 rem_addr)
+{
+ return ipv4_is_loopback(htonl(rem_addr)) || (loc_addr == rem_addr);
+}
+
+/**
+ * irdma_ipv6_is_loopback - check if loopback
+ * @loc_addr: local addr to compare
+ * @rem_addr: remote address
+ */
+bool irdma_ipv6_is_lpb(u32 *loc_addr, u32 *rem_addr)
+{
+ struct in6_addr raddr6;
+
+ irdma_copy_ip_htonl(raddr6.in6_u.u6_addr32, rem_addr);
+
+ return !memcmp(loc_addr, rem_addr, 16) || ipv6_addr_loopback(&raddr6);
+}
+
+/**
+ * irdma_cm_create_ah - create a cm address handle
+ * @cm_node: The connection manager node to create AH for
+ * @wait: Provides option to wait for ah creation or not
+ */
+static int irdma_cm_create_ah(struct irdma_cm_node *cm_node, bool wait)
+{
+ struct irdma_ah_info ah_info = {};
+ struct irdma_device *iwdev = cm_node->iwdev;
+
+ ether_addr_copy(ah_info.mac_addr, iwdev->netdev->dev_addr);
+
+ ah_info.hop_ttl = 0x40;
+ ah_info.tc_tos = cm_node->tos;
+ ah_info.vsi = &iwdev->vsi;
+
+ if (cm_node->ipv4) {
+ ah_info.ipv4_valid = true;
+ ah_info.dest_ip_addr[0] = cm_node->rem_addr[0];
+ ah_info.src_ip_addr[0] = cm_node->loc_addr[0];
+ ah_info.do_lpbk = irdma_ipv4_is_lpb(ah_info.src_ip_addr[0],
+ ah_info.dest_ip_addr[0]);
+ } else {
+ memcpy(ah_info.dest_ip_addr, cm_node->rem_addr,
+ sizeof(ah_info.dest_ip_addr));
+ memcpy(ah_info.src_ip_addr, cm_node->loc_addr,
+ sizeof(ah_info.src_ip_addr));
+ ah_info.do_lpbk = irdma_ipv6_is_lpb(ah_info.src_ip_addr,
+ ah_info.dest_ip_addr);
+ }
+
+ ah_info.vlan_tag = cm_node->vlan_id;
+ if (cm_node->vlan_id <= VLAN_VID_MASK)
+ ah_info.insert_vlan_tag = 1;
+
+ ah_info.dst_arpindex = irdma_arp_table(iwdev->rf,
+ ah_info.dest_ip_addr,
+ ah_info.ipv4_valid,
+ NULL,
+ IRDMA_ARP_RESOLVE);
+
+ if (irdma_puda_create_ah(&iwdev->rf->sc_dev,
+ &ah_info,
+ wait,
+ IRDMA_PUDA_RSRC_TYPE_ILQ,
+ cm_node,
+ &cm_node->ah))
+ return -ENOMEM;
+
+ trace_irdma_create_ah(cm_node);
+ return 0;
+}
+
+/**
+ * irdma_cm_free_ah - free a cm address handle
+ * @cm_node: The connection manager node to create AH for
+ */
+static void irdma_cm_free_ah(struct irdma_cm_node *cm_node)
+{
+ struct irdma_device *iwdev = cm_node->iwdev;
+
+ trace_irdma_cm_free_ah(cm_node);
+ irdma_puda_free_ah(&iwdev->rf->sc_dev, cm_node->ah);
+ cm_node->ah = NULL;
+}
+
+/**
+ * irdma_make_cm_node - create a new instance of a cm node
+ * @cm_core: cm's core
+ * @iwdev: iwarp device structure
+ * @cm_info: quad info for connection
+ * @listener: passive connection's listener
+ */
+static struct irdma_cm_node *irdma_make_cm_node(struct irdma_cm_core *cm_core,
+ struct irdma_device *iwdev,
+ struct irdma_cm_info *cm_info,
+ struct irdma_cm_listener *listener)
+{
+ struct irdma_cm_node *cm_node;
+ int oldarpindex;
+ int arpindex;
+ struct net_device *netdev = iwdev->netdev;
+
+ /* create an hte and cm_node for this instance */
+ cm_node = kzalloc(sizeof(*cm_node), GFP_ATOMIC);
+ if (!cm_node)
+ return NULL;
+
+ /* set our node specific transport info */
+ cm_node->ipv4 = cm_info->ipv4;
+ cm_node->vlan_id = cm_info->vlan_id;
+ if (cm_node->vlan_id == IRDMA_NO_VLAN && iwdev->dcb)
+ cm_node->vlan_id = 0;
+ cm_node->tos = cm_info->tos;
+ cm_node->user_pri = cm_info->user_pri;
+ if (listener) {
+ if (listener->tos != cm_info->tos)
+ irdma_dev_warn(&iwdev->rf->sc_dev,
+ "application TOS[%d] and remote client TOS[%d] mismatch\n",
+ listener->tos, cm_info->tos);
+ cm_node->tos = max(listener->tos, cm_info->tos);
+ cm_node->user_pri = rt_tos2priority(cm_node->tos);
+ irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_DCB,
+ "listener: TOS:[%d] UP:[%d]\n",
+ cm_node->tos, cm_node->user_pri);
+ trace_irdma_listener_tos(iwdev, cm_node->tos,
+ cm_node->user_pri);
+ }
+ memcpy(cm_node->loc_addr, cm_info->loc_addr, sizeof(cm_node->loc_addr));
+ memcpy(cm_node->rem_addr, cm_info->rem_addr, sizeof(cm_node->rem_addr));
+ cm_node->loc_port = cm_info->loc_port;
+ cm_node->rem_port = cm_info->rem_port;
+
+ cm_node->mpa_frame_rev = IRDMA_CM_DEFAULT_MPA_VER;
+ cm_node->send_rdma0_op = SEND_RDMA_READ_ZERO;
+ cm_node->iwdev = iwdev;
+ cm_node->dev = &iwdev->rf->sc_dev;
+
+ cm_node->ird_size = cm_node->dev->hw_attrs.max_hw_ird;
+ cm_node->ord_size = cm_node->dev->hw_attrs.max_hw_ord;
+
+ cm_node->listener = listener;
+ cm_node->cm_id = cm_info->cm_id;
+ ether_addr_copy(cm_node->loc_mac, netdev->dev_addr);
+ spin_lock_init(&cm_node->retrans_list_lock);
+ cm_node->ack_rcvd = false;
+
+ atomic_set(&cm_node->ref_count, 1);
+ /* associate our parent CM core */
+ cm_node->cm_core = cm_core;
+ cm_node->tcp_cntxt.loc_id = IRDMA_CM_DEFAULT_LOCAL_ID;
+ cm_node->tcp_cntxt.rcv_wscale = iwdev->rcv_wscale;
+ cm_node->tcp_cntxt.rcv_wnd = iwdev->rcv_wnd >> cm_node->tcp_cntxt.rcv_wscale;
+ if (cm_node->ipv4) {
+ cm_node->tcp_cntxt.loc_seq_num = secure_tcp_seq(htonl(cm_node->loc_addr[0]),
+ htonl(cm_node->rem_addr[0]),
+ htons(cm_node->loc_port),
+ htons(cm_node->rem_port));
+ cm_node->tcp_cntxt.mss = iwdev->vsi.mtu - IRDMA_MTU_TO_MSS_IPV4;
+ } else if (IS_ENABLED(CONFIG_IPV6)) {
+ __be32 loc[4] = {
+ htonl(cm_node->loc_addr[0]), htonl(cm_node->loc_addr[1]),
+ htonl(cm_node->loc_addr[2]), htonl(cm_node->loc_addr[3])
+ };
+ __be32 rem[4] = {
+ htonl(cm_node->rem_addr[0]), htonl(cm_node->rem_addr[1]),
+ htonl(cm_node->rem_addr[2]), htonl(cm_node->rem_addr[3])
+ };
+ cm_node->tcp_cntxt.loc_seq_num = secure_tcpv6_seq(loc, rem,
+ htons(cm_node->loc_port),
+ htons(cm_node->rem_port));
+ cm_node->tcp_cntxt.mss = iwdev->vsi.mtu - IRDMA_MTU_TO_MSS_IPV6;
+ }
+
+ if ((cm_node->ipv4 &&
+ irdma_ipv4_is_lpb(cm_node->loc_addr[0], cm_node->rem_addr[0])) ||
+ (!cm_node->ipv4 && irdma_ipv6_is_lpb(cm_node->loc_addr, cm_node->rem_addr))) {
+ arpindex = irdma_arp_table(iwdev->rf,
+ cm_node->rem_addr,
+ cm_node->ipv4,
+ NULL,
+ IRDMA_ARP_RESOLVE);
+ } else {
+ oldarpindex = irdma_arp_table(iwdev->rf,
+ cm_node->rem_addr,
+ cm_node->ipv4,
+ NULL,
+ IRDMA_ARP_RESOLVE);
+ if (cm_node->ipv4)
+ arpindex = irdma_addr_resolve_neigh(iwdev,
+ cm_info->loc_addr[0],
+ cm_info->rem_addr[0],
+ oldarpindex);
+ else if (IS_ENABLED(CONFIG_IPV6))
+ arpindex = irdma_addr_resolve_neigh_ipv6(iwdev,
+ cm_info->loc_addr,
+ cm_info->rem_addr,
+ oldarpindex);
+ else
+ arpindex = -EINVAL;
+ }
+
+ if (arpindex < 0)
+ goto err;
+
+ ether_addr_copy(cm_node->rem_mac,
+ iwdev->rf->arp_table[arpindex].mac_addr);
+ irdma_add_hte_node(cm_core, cm_node);
+ cm_core->stats_nodes_created++;
+ return cm_node;
+
+err:
+ kfree(cm_node);
+
+ return NULL;
+}
+
+/**
+ * irdma_rem_ref_cm_node - destroy an instance of a cm node
+ * @cm_node: connection's node
+ */
+static void irdma_rem_ref_cm_node(struct irdma_cm_node *cm_node)
+{
+ struct irdma_cm_core *cm_core = cm_node->cm_core;
+ struct irdma_qp *iwqp;
+ struct irdma_cm_info nfo;
+ unsigned long flags;
+
+ trace_irdma_rem_ref_cm_node(cm_node, 0, __builtin_return_address(0));
+ spin_lock_irqsave(&cm_node->cm_core->ht_lock, flags);
+ if (atomic_dec_return(&cm_node->ref_count)) {
+ spin_unlock_irqrestore(&cm_node->cm_core->ht_lock, flags);
+ return;
+ }
+
+ list_del(&cm_node->list);
+ spin_unlock_irqrestore(&cm_node->cm_core->ht_lock, flags);
+
+ /* if the node is destroyed before connection was accelerated */
+ if (!cm_node->accelerated && cm_node->accept_pend) {
+ irdma_debug(cm_node->dev, IRDMA_DEBUG_CM,
+ "node destroyed before established\n");
+ atomic_dec(&cm_node->listener->pend_accepts_cnt);
+ }
+ if (cm_node->close_entry)
+ irdma_handle_close_entry(cm_node, 0);
+ if (cm_node->listener) {
+ irdma_dec_refcnt_listen(cm_core, cm_node->listener, 0, true);
+ } else {
+ if (cm_node->apbvt_set) {
+ irdma_manage_apbvt(cm_node->iwdev,
+ cm_node->loc_port,
+ IRDMA_MANAGE_APBVT_DEL);
+ cm_node->apbvt_set = 0;
+ }
+ irdma_get_addr_info(cm_node, &nfo);
+ if (cm_node->qhash_set) {
+ nfo.qh_qpid = cm_node->iwdev->vsi.ilq->qp_id;
+ irdma_manage_qhash(cm_node->iwdev,
+ &nfo,
+ IRDMA_QHASH_TYPE_TCP_ESTABLISHED,
+ IRDMA_QHASH_MANAGE_TYPE_DELETE,
+ NULL,
+ false);
+ cm_node->qhash_set = 0;
+ }
+ }
+
+ iwqp = cm_node->iwqp;
+ if (iwqp) {
+ iwqp->cm_node = NULL;
+ irdma_rem_ref(&iwqp->ibqp);
+ cm_node->iwqp = NULL;
+ } else if (cm_node->qhash_set) {
+ irdma_get_addr_info(cm_node, &nfo);
+ nfo.qh_qpid = cm_node->iwdev->vsi.ilq->qp_id;
+ irdma_manage_qhash(cm_node->iwdev,
+ &nfo,
+ IRDMA_QHASH_TYPE_TCP_ESTABLISHED,
+ IRDMA_QHASH_MANAGE_TYPE_DELETE,
+ NULL,
+ false);
+ cm_node->qhash_set = 0;
+ }
+ cm_core->cm_free_ah(cm_node);
+ cm_node->cm_core->stats_nodes_destroyed++;
+ kfree(cm_node);
+}
+
+/**
+ * irdma_handle_fin_pkt - FIN packet received
+ * @cm_node: connection's node
+ */
+static void irdma_handle_fin_pkt(struct irdma_cm_node *cm_node)
+{
+ switch (cm_node->state) {
+ case IRDMA_CM_STATE_SYN_RCVD:
+ case IRDMA_CM_STATE_SYN_SENT:
+ case IRDMA_CM_STATE_ESTABLISHED:
+ case IRDMA_CM_STATE_MPAREJ_RCVD:
+ cm_node->tcp_cntxt.rcv_nxt++;
+ irdma_cleanup_retrans_entry(cm_node);
+ cm_node->state = IRDMA_CM_STATE_LAST_ACK;
+ irdma_send_fin(cm_node);
+ break;
+ case IRDMA_CM_STATE_MPAREQ_SENT:
+ irdma_create_event(cm_node, IRDMA_CM_EVENT_ABORTED);
+ cm_node->tcp_cntxt.rcv_nxt++;
+ irdma_cleanup_retrans_entry(cm_node);
+ cm_node->state = IRDMA_CM_STATE_CLOSED;
+ atomic_inc(&cm_node->ref_count);
+ irdma_send_reset(cm_node);
+ break;
+ case IRDMA_CM_STATE_FIN_WAIT1:
+ cm_node->tcp_cntxt.rcv_nxt++;
+ irdma_cleanup_retrans_entry(cm_node);
+ cm_node->state = IRDMA_CM_STATE_CLOSING;
+ irdma_send_ack(cm_node);
+ /*
+ * Wait for ACK as this is simultaneous close.
+ * After we receive ACK, do not send anything.
+ * Just rm the node.
+ */
+ break;
+ case IRDMA_CM_STATE_FIN_WAIT2:
+ cm_node->tcp_cntxt.rcv_nxt++;
+ irdma_cleanup_retrans_entry(cm_node);
+ cm_node->state = IRDMA_CM_STATE_TIME_WAIT;
+ irdma_send_ack(cm_node);
+ irdma_schedule_cm_timer(cm_node, NULL, IRDMA_TIMER_TYPE_CLOSE, 1, 0);
+ break;
+ case IRDMA_CM_STATE_TIME_WAIT:
+ cm_node->tcp_cntxt.rcv_nxt++;
+ irdma_cleanup_retrans_entry(cm_node);
+ cm_node->state = IRDMA_CM_STATE_CLOSED;
+ irdma_rem_ref_cm_node(cm_node);
+ break;
+ case IRDMA_CM_STATE_OFFLOADED:
+ default:
+ irdma_debug(cm_node->dev, IRDMA_DEBUG_ERR,
+ "bad state node state = %d\n", cm_node->state);
+ break;
+ }
+}
+
+/**
+ * irdma_handle_rst_pkt - process received RST packet
+ * @cm_node: connection's node
+ * @rbuf: receive buffer
+ */
+static void irdma_handle_rst_pkt(struct irdma_cm_node *cm_node,
+ struct irdma_puda_buf *rbuf)
+{
+ irdma_cleanup_retrans_entry(cm_node);
+ switch (cm_node->state) {
+ case IRDMA_CM_STATE_SYN_SENT:
+ case IRDMA_CM_STATE_MPAREQ_SENT:
+ switch (cm_node->mpa_frame_rev) {
+ case IETF_MPA_V2:
+ /* Drop down to MPA_V1*/
+ cm_node->mpa_frame_rev = IETF_MPA_V1;
+ /* send a syn and goto syn sent state */
+ cm_node->state = IRDMA_CM_STATE_SYN_SENT;
+ if (irdma_send_syn(cm_node, 0))
+ irdma_active_open_err(cm_node, false);
+ break;
+ case IETF_MPA_V1:
+ default:
+ irdma_active_open_err(cm_node, false);
+ break;
+ }
+ break;
+ case IRDMA_CM_STATE_MPAREQ_RCVD:
+ atomic_add_return(1, &cm_node->passive_state);
+ break;
+ case IRDMA_CM_STATE_ESTABLISHED:
+ case IRDMA_CM_STATE_SYN_RCVD:
+ case IRDMA_CM_STATE_LISTENING:
+ irdma_passive_open_err(cm_node, false);
+ break;
+ case IRDMA_CM_STATE_OFFLOADED:
+ irdma_active_open_err(cm_node, false);
+ break;
+ case IRDMA_CM_STATE_CLOSED:
+ break;
+ case IRDMA_CM_STATE_FIN_WAIT2:
+ case IRDMA_CM_STATE_FIN_WAIT1:
+ case IRDMA_CM_STATE_LAST_ACK:
+ cm_node->cm_id->rem_ref(cm_node->cm_id);
+ case IRDMA_CM_STATE_TIME_WAIT:
+ cm_node->state = IRDMA_CM_STATE_CLOSED;
+ irdma_rem_ref_cm_node(cm_node);
+ break;
+ default:
+ break;
+ }
+}
+
+/**
+ * irdma_handle_rcv_mpa - Process a recv'd mpa buffer
+ * @cm_node: connection's node
+ * @rbuf: receive buffer
+ */
+static void irdma_handle_rcv_mpa(struct irdma_cm_node *cm_node,
+ struct irdma_puda_buf *rbuf)
+{
+ int err;
+ int datasize = rbuf->datalen;
+ u8 *dataloc = rbuf->data;
+
+ enum irdma_cm_event_type type = IRDMA_CM_EVENT_UNKNOWN;
+ u32 res_type;
+
+ err = irdma_parse_mpa(cm_node, dataloc, &res_type, datasize);
+ if (err) {
+ if (cm_node->state == IRDMA_CM_STATE_MPAREQ_SENT)
+ irdma_active_open_err(cm_node, true);
+ else
+ irdma_passive_open_err(cm_node, true);
+ return;
+ }
+
+ switch (cm_node->state) {
+ case IRDMA_CM_STATE_ESTABLISHED:
+ if (res_type == IRDMA_MPA_REQUEST_REJECT)
+ irdma_debug(cm_node->dev, IRDMA_DEBUG_CM,
+ "state for reject\n");
+ cm_node->state = IRDMA_CM_STATE_MPAREQ_RCVD;
+ type = IRDMA_CM_EVENT_MPA_REQ;
+ irdma_send_ack(cm_node); /* ACK received MPA request */
+ atomic_set(&cm_node->passive_state,
+ IRDMA_PASSIVE_STATE_INDICATED);
+ break;
+ case IRDMA_CM_STATE_MPAREQ_SENT:
+ irdma_cleanup_retrans_entry(cm_node);
+ if (res_type == IRDMA_MPA_REQUEST_REJECT) {
+ type = IRDMA_CM_EVENT_MPA_REJECT;
+ cm_node->state = IRDMA_CM_STATE_MPAREJ_RCVD;
+ } else {
+ type = IRDMA_CM_EVENT_CONNECTED;
+ cm_node->state = IRDMA_CM_STATE_OFFLOADED;
+ }
+ irdma_send_ack(cm_node);
+ break;
+ default:
+ irdma_debug(cm_node->dev, IRDMA_DEBUG_ERR,
+ "wrong cm_node state =%d\n", cm_node->state);
+ break;
+ }
+ irdma_create_event(cm_node, type);
+}
+
+/**
+ * irdma_check_syn - Check for error on received syn ack
+ * @cm_node: connection's node
+ * @tcph: pointer tcp header
+ */
+static int irdma_check_syn(struct irdma_cm_node *cm_node, struct tcphdr *tcph)
+{
+ if (ntohl(tcph->ack_seq) != cm_node->tcp_cntxt.loc_seq_num) {
+ irdma_active_open_err(cm_node, true);
+ return 1;
+ }
+
+ return 0;
+}
+
+/**
+ * irdma_check_seq - check seq numbers if OK
+ * @cm_node: connection's node
+ * @tcph: pointer tcp header
+ */
+static int irdma_check_seq(struct irdma_cm_node *cm_node, struct tcphdr *tcph)
+{
+ u32 seq;
+ u32 ack_seq;
+ u32 loc_seq_num = cm_node->tcp_cntxt.loc_seq_num;
+ u32 rcv_nxt = cm_node->tcp_cntxt.rcv_nxt;
+ u32 rcv_wnd;
+ int err = 0;
+
+ seq = ntohl(tcph->seq);
+ ack_seq = ntohl(tcph->ack_seq);
+ rcv_wnd = cm_node->tcp_cntxt.rcv_wnd;
+ if (ack_seq != loc_seq_num ||
+ !between(seq, rcv_nxt, (rcv_nxt + rcv_wnd)))
+ err = -1;
+ if (err)
+ irdma_debug(cm_node->dev, IRDMA_DEBUG_ERR, "seq number err\n");
+
+ return err;
+}
+
+/**
+ * irdma_handle_syn_pkt - is for Passive node
+ * @cm_node: connection's node
+ * @rbuf: receive buffer
+ */
+static void irdma_handle_syn_pkt(struct irdma_cm_node *cm_node,
+ struct irdma_puda_buf *rbuf)
+{
+ struct tcphdr *tcph = (struct tcphdr *)rbuf->tcph;
+ int err;
+ u32 inc_sequence;
+ int optionsize;
+ struct irdma_cm_info nfo;
+
+ optionsize = (tcph->doff << 2) - sizeof(struct tcphdr);
+ inc_sequence = ntohl(tcph->seq);
+
+ switch (cm_node->state) {
+ case IRDMA_CM_STATE_SYN_SENT:
+ case IRDMA_CM_STATE_MPAREQ_SENT:
+ /* Rcvd syn on active open connection */
+ irdma_active_open_err(cm_node, 1);
+ break;
+ case IRDMA_CM_STATE_LISTENING:
+ /* Passive OPEN */
+ if (atomic_read(&cm_node->listener->pend_accepts_cnt) >
+ cm_node->listener->backlog) {
+ cm_node->cm_core->stats_backlog_drops++;
+ irdma_passive_open_err(cm_node, false);
+ break;
+ }
+ err = irdma_handle_tcp_options(cm_node, tcph, optionsize, 1);
+ if (err) {
+ irdma_passive_open_err(cm_node, false);
+ /* drop pkt */
+ break;
+ }
+ err = cm_node->cm_core->cm_create_ah(cm_node, false);
+ if (err) {
+ irdma_passive_open_err(cm_node, false);
+ /* drop pkt */
+ break;
+ }
+ cm_node->tcp_cntxt.rcv_nxt = inc_sequence + 1;
+ cm_node->accept_pend = 1;
+ atomic_inc(&cm_node->listener->pend_accepts_cnt);
+
+ cm_node->state = IRDMA_CM_STATE_SYN_RCVD;
+ irdma_get_addr_info(cm_node, &nfo);
+ nfo.qh_qpid = cm_node->iwdev->vsi.ilq->qp_id;
+ err = irdma_manage_qhash(cm_node->iwdev,
+ &nfo,
+ IRDMA_QHASH_TYPE_TCP_ESTABLISHED,
+ IRDMA_QHASH_MANAGE_TYPE_ADD,
+ (void *)cm_node,
+ false);
+ cm_node->qhash_set = true;
+ break;
+ case IRDMA_CM_STATE_CLOSED:
+ irdma_cleanup_retrans_entry(cm_node);
+ atomic_inc(&cm_node->ref_count);
+ irdma_send_reset(cm_node);
+ break;
+ case IRDMA_CM_STATE_OFFLOADED:
+ case IRDMA_CM_STATE_ESTABLISHED:
+ case IRDMA_CM_STATE_FIN_WAIT1:
+ case IRDMA_CM_STATE_FIN_WAIT2:
+ case IRDMA_CM_STATE_MPAREQ_RCVD:
+ case IRDMA_CM_STATE_LAST_ACK:
+ case IRDMA_CM_STATE_CLOSING:
+ case IRDMA_CM_STATE_UNKNOWN:
+ default:
+ break;
+ }
+}
+
+/**
+ * irdma_handle_synack_pkt - Process SYN+ACK packet (active side)
+ * @cm_node: connection's node
+ * @rbuf: receive buffer
+ */
+static void irdma_handle_synack_pkt(struct irdma_cm_node *cm_node,
+ struct irdma_puda_buf *rbuf)
+{
+ struct tcphdr *tcph = (struct tcphdr *)rbuf->tcph;
+ int err;
+ u32 inc_sequence;
+ int optionsize;
+
+ optionsize = (tcph->doff << 2) - sizeof(struct tcphdr);
+ inc_sequence = ntohl(tcph->seq);
+ switch (cm_node->state) {
+ case IRDMA_CM_STATE_SYN_SENT:
+ irdma_cleanup_retrans_entry(cm_node);
+ /* active open */
+ if (irdma_check_syn(cm_node, tcph)) {
+ irdma_debug(cm_node->dev, IRDMA_DEBUG_ERR,
+ "check syn fail\n");
+ return;
+ }
+ cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->ack_seq);
+ /* setup options */
+ err = irdma_handle_tcp_options(cm_node, tcph, optionsize, 0);
+ if (err) {
+ irdma_debug(cm_node->dev,
+ IRDMA_DEBUG_CM,
+ "cm_node=%p tcp_options failed\n",
+ cm_node);
+ break;
+ }
+ irdma_cleanup_retrans_entry(cm_node);
+ cm_node->tcp_cntxt.rcv_nxt = inc_sequence + 1;
+ irdma_send_ack(cm_node); /* ACK for the syn_ack */
+ err = irdma_send_mpa_request(cm_node);
+ if (err) {
+ irdma_debug(cm_node->dev,
+ IRDMA_DEBUG_CM,
+ "cm_node=%p irdma_send_mpa_request failed\n",
+ cm_node);
+ break;
+ }
+ cm_node->state = IRDMA_CM_STATE_MPAREQ_SENT;
+ break;
+ case IRDMA_CM_STATE_MPAREQ_RCVD:
+ irdma_passive_open_err(cm_node, true);
+ break;
+ case IRDMA_CM_STATE_LISTENING:
+ cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->ack_seq);
+ irdma_cleanup_retrans_entry(cm_node);
+ cm_node->state = IRDMA_CM_STATE_CLOSED;
+ irdma_send_reset(cm_node);
+ break;
+ case IRDMA_CM_STATE_CLOSED:
+ cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->ack_seq);
+ irdma_cleanup_retrans_entry(cm_node);
+ atomic_inc(&cm_node->ref_count);
+ irdma_send_reset(cm_node);
+ break;
+ case IRDMA_CM_STATE_ESTABLISHED:
+ case IRDMA_CM_STATE_FIN_WAIT1:
+ case IRDMA_CM_STATE_FIN_WAIT2:
+ case IRDMA_CM_STATE_LAST_ACK:
+ case IRDMA_CM_STATE_OFFLOADED:
+ case IRDMA_CM_STATE_CLOSING:
+ case IRDMA_CM_STATE_UNKNOWN:
+ case IRDMA_CM_STATE_MPAREQ_SENT:
+ default:
+ break;
+ }
+}
+
+/**
+ * irdma_handle_ack_pkt - process packet with ACK
+ * @cm_node: connection's node
+ * @rbuf: receive buffer
+ */
+static int irdma_handle_ack_pkt(struct irdma_cm_node *cm_node,
+ struct irdma_puda_buf *rbuf)
+{
+ struct tcphdr *tcph = (struct tcphdr *)rbuf->tcph;
+ u32 inc_sequence;
+ int ret;
+ int optionsize;
+ u32 datasize = rbuf->datalen;
+
+ optionsize = (tcph->doff << 2) - sizeof(struct tcphdr);
+
+ if (irdma_check_seq(cm_node, tcph))
+ return -EINVAL;
+
+ inc_sequence = ntohl(tcph->seq);
+ switch (cm_node->state) {
+ case IRDMA_CM_STATE_SYN_RCVD:
+ irdma_cleanup_retrans_entry(cm_node);
+ ret = irdma_handle_tcp_options(cm_node, tcph, optionsize, 1);
+ if (ret)
+ return ret;
+ cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->ack_seq);
+ cm_node->state = IRDMA_CM_STATE_ESTABLISHED;
+ if (datasize) {
+ cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize;
+ irdma_handle_rcv_mpa(cm_node, rbuf);
+ }
+ break;
+ case IRDMA_CM_STATE_ESTABLISHED:
+ irdma_cleanup_retrans_entry(cm_node);
+ if (datasize) {
+ cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize;
+ irdma_handle_rcv_mpa(cm_node, rbuf);
+ }
+ break;
+ case IRDMA_CM_STATE_MPAREQ_SENT:
+ cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->ack_seq);
+ if (datasize) {
+ cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize;
+ cm_node->ack_rcvd = false;
+ irdma_handle_rcv_mpa(cm_node, rbuf);
+ } else {
+ cm_node->ack_rcvd = true;
+ }
+ break;
+ case IRDMA_CM_STATE_LISTENING:
+ irdma_cleanup_retrans_entry(cm_node);
+ cm_node->state = IRDMA_CM_STATE_CLOSED;
+ irdma_send_reset(cm_node);
+ break;
+ case IRDMA_CM_STATE_CLOSED:
+ irdma_cleanup_retrans_entry(cm_node);
+ atomic_inc(&cm_node->ref_count);
+ irdma_send_reset(cm_node);
+ break;
+ case IRDMA_CM_STATE_LAST_ACK:
+ case IRDMA_CM_STATE_CLOSING:
+ irdma_cleanup_retrans_entry(cm_node);
+ cm_node->state = IRDMA_CM_STATE_CLOSED;
+ if (!cm_node->accept_pend)
+ cm_node->cm_id->rem_ref(cm_node->cm_id);
+ irdma_rem_ref_cm_node(cm_node);
+ break;
+ case IRDMA_CM_STATE_FIN_WAIT1:
+ irdma_cleanup_retrans_entry(cm_node);
+ cm_node->state = IRDMA_CM_STATE_FIN_WAIT2;
+ break;
+ case IRDMA_CM_STATE_SYN_SENT:
+ case IRDMA_CM_STATE_FIN_WAIT2:
+ case IRDMA_CM_STATE_OFFLOADED:
+ case IRDMA_CM_STATE_MPAREQ_RCVD:
+ case IRDMA_CM_STATE_UNKNOWN:
+ default:
+ irdma_cleanup_retrans_entry(cm_node);
+ break;
+ }
+
+ return 0;
+}
+
+/**
+ * irdma_process_packet - process cm packet
+ * @cm_node: connection's node
+ * @rbuf: receive buffer
+ */
+static void irdma_process_pkt(struct irdma_cm_node *cm_node,
+ struct irdma_puda_buf *rbuf)
+{
+ enum irdma_tcpip_pkt_type pkt_type = IRDMA_PKT_TYPE_UNKNOWN;
+ struct tcphdr *tcph = (struct tcphdr *)rbuf->tcph;
+ u32 fin_set = 0;
+ int err;
+
+ if (tcph->rst) {
+ pkt_type = IRDMA_PKT_TYPE_RST;
+ } else if (tcph->syn) {
+ pkt_type = IRDMA_PKT_TYPE_SYN;
+ if (tcph->ack)
+ pkt_type = IRDMA_PKT_TYPE_SYNACK;
+ } else if (tcph->ack) {
+ pkt_type = IRDMA_PKT_TYPE_ACK;
+ }
+ if (tcph->fin)
+ fin_set = 1;
+
+ switch (pkt_type) {
+ case IRDMA_PKT_TYPE_SYN:
+ irdma_handle_syn_pkt(cm_node, rbuf);
+ break;
+ case IRDMA_PKT_TYPE_SYNACK:
+ irdma_handle_synack_pkt(cm_node, rbuf);
+ break;
+ case IRDMA_PKT_TYPE_ACK:
+ err = irdma_handle_ack_pkt(cm_node, rbuf);
+ if (fin_set && !err)
+ irdma_handle_fin_pkt(cm_node);
+ break;
+ case IRDMA_PKT_TYPE_RST:
+ irdma_handle_rst_pkt(cm_node, rbuf);
+ break;
+ default:
+ if (fin_set &&
+ (!irdma_check_seq(cm_node, (struct tcphdr *)rbuf->tcph)))
+ irdma_handle_fin_pkt(cm_node);
+ break;
+ }
+}
+
+/**
+ * irdma_make_listen_node - create a listen node with params
+ * @cm_core: cm's core
+ * @iwdev: iwarp device structure
+ * @cm_info: quad info for connection
+ */
+static struct irdma_cm_listener *irdma_make_listen_node(struct irdma_cm_core *cm_core,
+ struct irdma_device *iwdev,
+ struct irdma_cm_info *cm_info)
+{
+ struct irdma_cm_listener *listener;
+ unsigned long flags;
+
+ /* cannot have multiple matching listeners */
+ listener = irdma_find_listener(cm_core, cm_info->loc_addr,
+ cm_info->loc_port,
+ cm_info->vlan_id,
+ IRDMA_CM_LISTENER_EITHER_STATE);
+ if (listener &&
+ listener->listener_state == IRDMA_CM_LISTENER_ACTIVE_STATE) {
+ atomic_dec(&listener->ref_count);
+ return NULL;
+ }
+
+ if (!listener) {
+ /* create a CM listen node
+ * 1/2 node to compare incoming traffic to
+ */
+ listener = kzalloc(sizeof(*listener), GFP_KERNEL);
+ if (!listener)
+ return NULL;
+ cm_core->stats_listen_nodes_created++;
+ memcpy(listener->loc_addr, cm_info->loc_addr,
+ sizeof(listener->loc_addr));
+ listener->loc_port = cm_info->loc_port;
+
+ INIT_LIST_HEAD(&listener->child_listen_list);
+
+ atomic_set(&listener->ref_count, 1);
+ } else {
+ listener->reused_node = 1;
+ }
+
+ listener->cm_id = cm_info->cm_id;
+ listener->ipv4 = cm_info->ipv4;
+ listener->vlan_id = cm_info->vlan_id;
+ atomic_set(&listener->pend_accepts_cnt, 0);
+ listener->cm_core = cm_core;
+ listener->iwdev = iwdev;
+
+ listener->backlog = cm_info->backlog;
+ listener->listener_state = IRDMA_CM_LISTENER_ACTIVE_STATE;
+
+ if (!listener->reused_node) {
+ spin_lock_irqsave(&cm_core->listen_list_lock, flags);
+ list_add(&listener->list, &cm_core->listen_nodes);
+ spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
+ }
+
+ return listener;
+}
+
+/**
+ * irdma_create_cm_node - make a connection node with params
+ * @cm_core: cm's core
+ * @iwdev: iwarp device structure
+ * @private_data_len: len to private data for mpa request
+ * @private_data: pointer to private data for connection
+ * @cm_info: quad info for connection
+ * @caller_cm_node: pointer to cm_node structure to return
+ */
+static int irdma_create_cm_node(struct irdma_cm_core *cm_core,
+ struct irdma_device *iwdev,
+ struct iw_cm_conn_param *conn_param,
+ struct irdma_cm_info *cm_info,
+ struct irdma_cm_node **caller_cm_node)
+{
+ struct irdma_cm_node *cm_node;
+ u16 private_data_len = conn_param->private_data_len;
+ void *private_data = (void *)conn_param->private_data;
+
+ /* create a CM connection node */
+ cm_node = irdma_make_cm_node(cm_core, iwdev, cm_info, NULL);
+ if (!cm_node)
+ return -ENOMEM;
+
+ /* set our node side to client (active) side */
+ cm_node->tcp_cntxt.client = 1;
+ cm_node->tcp_cntxt.rcv_wscale = IRDMA_CM_DEFAULT_RCV_WND_SCALE;
+
+ irdma_record_ird_ord(cm_node, conn_param->ird, conn_param->ord);
+
+ cm_node->pdata.size = private_data_len;
+ cm_node->pdata.addr = cm_node->pdata_buf;
+
+ memcpy(cm_node->pdata_buf, private_data, private_data_len);
+ *caller_cm_node = cm_node;
+
+ return 0;
+}
+
+/**
+ * irdma_cm_reject - reject and teardown a connection
+ * @cm_node: connection's node
+ * @pdate: ptr to private data for reject
+ * @plen: size of private data
+ */
+static int irdma_cm_reject(struct irdma_cm_node *cm_node,
+ const void *pdata,
+ u8 plen)
+{
+ int ret;
+ int passive_state;
+ struct iw_cm_id *cm_id = cm_node->cm_id;
+
+ if (cm_node->tcp_cntxt.client)
+ return 0;
+
+ irdma_cleanup_retrans_entry(cm_node);
+
+ passive_state = atomic_add_return(1, &cm_node->passive_state);
+ if (passive_state == IRDMA_SEND_RESET_EVENT) {
+ cm_node->state = IRDMA_CM_STATE_CLOSED;
+ irdma_rem_ref_cm_node(cm_node);
+ return 0;
+ }
+
+ if (cm_node->state == IRDMA_CM_STATE_LISTENER_DESTROYED) {
+ irdma_rem_ref_cm_node(cm_node);
+ return 0;
+ }
+
+ ret = irdma_send_mpa_reject(cm_node, pdata, plen);
+ if (!ret) {
+ cm_id->add_ref(cm_id);
+ return 0;
+ }
+
+ cm_node->state = IRDMA_CM_STATE_CLOSED;
+ if (irdma_send_reset(cm_node))
+ irdma_debug(cm_node->dev, IRDMA_DEBUG_ERR,
+ "send reset failed\n");
+ return ret;
+}
+
+/**
+ * irdma_cm_close - close of cm connection
+ * @cm_node: connection's node
+ */
+static int irdma_cm_close(struct irdma_cm_node *cm_node)
+{
+ if (!cm_node)
+ return -EINVAL;
+
+ switch (cm_node->state) {
+ case IRDMA_CM_STATE_SYN_RCVD:
+ case IRDMA_CM_STATE_SYN_SENT:
+ case IRDMA_CM_STATE_ONE_SIDE_ESTABLISHED:
+ case IRDMA_CM_STATE_ESTABLISHED:
+ case IRDMA_CM_STATE_ACCEPTING:
+ case IRDMA_CM_STATE_MPAREQ_SENT:
+ case IRDMA_CM_STATE_MPAREQ_RCVD:
+ irdma_cleanup_retrans_entry(cm_node);
+ irdma_send_reset(cm_node);
+ break;
+ case IRDMA_CM_STATE_CLOSE_WAIT:
+ cm_node->state = IRDMA_CM_STATE_LAST_ACK;
+ irdma_send_fin(cm_node);
+ break;
+ case IRDMA_CM_STATE_FIN_WAIT1:
+ case IRDMA_CM_STATE_FIN_WAIT2:
+ case IRDMA_CM_STATE_LAST_ACK:
+ case IRDMA_CM_STATE_TIME_WAIT:
+ case IRDMA_CM_STATE_CLOSING:
+ return -1;
+ case IRDMA_CM_STATE_LISTENING:
+ irdma_cleanup_retrans_entry(cm_node);
+ irdma_send_reset(cm_node);
+ break;
+ case IRDMA_CM_STATE_MPAREJ_RCVD:
+ case IRDMA_CM_STATE_UNKNOWN:
+ case IRDMA_CM_STATE_INITED:
+ case IRDMA_CM_STATE_CLOSED:
+ case IRDMA_CM_STATE_LISTENER_DESTROYED:
+ irdma_rem_ref_cm_node(cm_node);
+ break;
+ case IRDMA_CM_STATE_OFFLOADED:
+ if (cm_node->send_entry)
+ irdma_debug(cm_node->dev, IRDMA_DEBUG_ERR,
+ "CM send_entry in OFFLOADED state\n");
+ irdma_rem_ref_cm_node(cm_node);
+ break;
+ }
+
+ return 0;
+}
+
+/**
+ * irdma_receive_ilq - recv an ETHERNET packet, and process it
+ * through CM
+ * @dev: HW dev struct
+ * @rbuf: receive buffer
+ */
+void irdma_receive_ilq(struct irdma_sc_vsi *vsi, struct irdma_puda_buf *rbuf)
+{
+ struct irdma_cm_node *cm_node;
+ struct irdma_cm_listener *listener;
+ struct iphdr *iph;
+ struct ipv6hdr *ip6h;
+ struct tcphdr *tcph;
+ struct irdma_cm_info cm_info = {};
+ struct irdma_device *iwdev = (struct irdma_device *)vsi->back_vsi;
+ struct irdma_cm_core *cm_core = &iwdev->cm_core;
+ struct vlan_ethhdr *ethh;
+ u16 vtag;
+
+ /* if vlan, then maclen = 18 else 14 */
+ iph = (struct iphdr *)rbuf->iph;
+ irdma_debug_buf(vsi->dev,
+ IRDMA_DEBUG_ILQ,
+ "RECEIVE ILQ BUFFER",
+ rbuf->mem.va,
+ rbuf->totallen);
+ if (iwdev->rf->sc_dev.hw_attrs.hw_rev > IRDMA_GEN_1) {
+ if (rbuf->vlan_valid) {
+ vtag = rbuf->vlan_id;
+ cm_info.user_pri = (vtag & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
+ cm_info.vlan_id = vtag & VLAN_VID_MASK;
+ } else {
+ cm_info.vlan_id = IRDMA_NO_VLAN;
+ }
+ } else {
+ ethh = (struct vlan_ethhdr *)rbuf->mem.va;
+
+ if (ethh->h_vlan_proto == htons(ETH_P_8021Q)) {
+ vtag = ntohs(ethh->h_vlan_TCI);
+ cm_info.user_pri = (vtag & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
+ cm_info.vlan_id = vtag & VLAN_VID_MASK;
+ irdma_debug(cm_core->dev,
+ IRDMA_DEBUG_CM,
+ "vlan_id=%d\n",
+ cm_info.vlan_id);
+ } else {
+ cm_info.vlan_id = IRDMA_NO_VLAN;
+ }
+ }
+ tcph = (struct tcphdr *)rbuf->tcph;
+
+ if (rbuf->ipv4) {
+ cm_info.loc_addr[0] = ntohl(iph->daddr);
+ cm_info.rem_addr[0] = ntohl(iph->saddr);
+ cm_info.ipv4 = true;
+ cm_info.tos = iph->tos;
+ } else {
+ ip6h = (struct ipv6hdr *)rbuf->iph;
+ irdma_copy_ip_ntohl(cm_info.loc_addr,
+ ip6h->daddr.in6_u.u6_addr32);
+ irdma_copy_ip_ntohl(cm_info.rem_addr,
+ ip6h->saddr.in6_u.u6_addr32);
+ cm_info.ipv4 = false;
+ cm_info.tos = (ip6h->priority << 4) | (ip6h->flow_lbl[0] >> 4);
+ }
+ cm_info.loc_port = ntohs(tcph->dest);
+ cm_info.rem_port = ntohs(tcph->source);
+ cm_node = irdma_find_node(cm_core,
+ cm_info.rem_port,
+ cm_info.rem_addr,
+ cm_info.loc_port,
+ cm_info.loc_addr,
+ true, false);
+
+ if (!cm_node) {
+ /* Only type of packet accepted are for the
+ * PASSIVE open (syn only)
+ */
+ if (!tcph->syn || tcph->ack)
+ return;
+
+ listener = irdma_find_listener(cm_core,
+ cm_info.loc_addr,
+ cm_info.loc_port,
+ cm_info.vlan_id,
+ IRDMA_CM_LISTENER_ACTIVE_STATE);
+ if (!listener) {
+ cm_info.cm_id = NULL;
+ irdma_debug(cm_core->dev, IRDMA_DEBUG_CM,
+ "no listener found\n");
+ return;
+ }
+
+ cm_info.cm_id = listener->cm_id;
+ cm_node = irdma_make_cm_node(cm_core, iwdev, &cm_info,
+ listener);
+ if (!cm_node) {
+ irdma_debug(cm_core->dev,
+ IRDMA_DEBUG_CM,
+ "allocate node failed\n");
+ atomic_dec(&listener->ref_count);
+ return;
+ }
+
+ if (!tcph->rst && !tcph->fin) {
+ cm_node->state = IRDMA_CM_STATE_LISTENING;
+ } else {
+ irdma_rem_ref_cm_node(cm_node);
+ return;
+ }
+
+ atomic_inc(&cm_node->ref_count);
+ } else if (cm_node->state == IRDMA_CM_STATE_OFFLOADED) {
+ irdma_rem_ref_cm_node(cm_node);
+ return;
+ }
+
+ irdma_process_pkt(cm_node, rbuf);
+ irdma_rem_ref_cm_node(cm_node);
+}
+
+static int irdma_cm_create_ah_nop(struct irdma_cm_node *cm_node, bool wait)
+{
+ return 0;
+}
+
+static void irdma_cm_free_ah_nop(struct irdma_cm_node *cm_node)
+{
+}
+
+/**
+ * irdma_setup_cm_core - allocate a top level instance of a cm
+ * core
+ * @iwdev: iwarp device structure
+ */
+void irdma_setup_cm_core(struct irdma_device *iwdev, u8 rdma_ver)
+{
+ struct irdma_cm_core *cm_core = &iwdev->cm_core;
+
+ cm_core->iwdev = iwdev;
+ cm_core->dev = &iwdev->rf->sc_dev;
+
+ INIT_LIST_HEAD(&cm_core->accelerated_list);
+ INIT_LIST_HEAD(&cm_core->non_accelerated_list);
+ INIT_LIST_HEAD(&cm_core->listen_nodes);
+
+ timer_setup(&cm_core->tcp_timer, irdma_cm_timer_tick, 0);
+
+ spin_lock_init(&cm_core->ht_lock);
+ spin_lock_init(&cm_core->listen_list_lock);
+ spin_lock_init(&cm_core->apbvt_lock);
+
+ cm_core->event_wq = create_singlethread_workqueue("iwewq");
+ cm_core->disconn_wq = create_singlethread_workqueue("iwdwq");
+ switch (rdma_ver) {
+ case IRDMA_GEN_1:
+ cm_core->form_cm_frame = irdma_form_uda_cm_frame;
+ cm_core->cm_create_ah = irdma_cm_create_ah_nop;
+ cm_core->cm_free_ah = irdma_cm_free_ah_nop;
+ break;
+ case IRDMA_GEN_2:
+ default:
+ cm_core->form_cm_frame = irdma_form_ah_cm_frame;
+ cm_core->cm_create_ah = irdma_cm_create_ah;
+ cm_core->cm_free_ah = irdma_cm_free_ah;
+ }
+}
+
+/**
+ * irdma_cleanup_cm_core - deallocate a top level instance of a
+ * cm core
+ * @cm_core: cm's core
+ */
+void irdma_cleanup_cm_core(struct irdma_cm_core *cm_core)
+{
+ unsigned long flags;
+
+ if (!cm_core)
+ return;
+
+ spin_lock_irqsave(&cm_core->ht_lock, flags);
+ if (timer_pending(&cm_core->tcp_timer))
+ del_timer_sync(&cm_core->tcp_timer);
+ spin_unlock_irqrestore(&cm_core->ht_lock, flags);
+
+ destroy_workqueue(cm_core->event_wq);
+ destroy_workqueue(cm_core->disconn_wq);
+}
+
+/**
+ * irdma_init_tcp_ctx - setup qp context
+ * @cm_node: connection's node
+ * @tcp_info: offload info for tcp
+ * @iwqp: associate qp for the connection
+ */
+static void irdma_init_tcp_ctx(struct irdma_cm_node *cm_node,
+ struct irdma_tcp_offload_info *tcp_info,
+ struct irdma_qp *iwqp)
+{
+ tcp_info->ipv4 = cm_node->ipv4;
+ tcp_info->drop_ooo_seg = !iwqp->iwdev->rf->ooo;
+ tcp_info->wscale = true;
+ tcp_info->ignore_tcp_opt = true;
+ tcp_info->ignore_tcp_uns_opt = true;
+ tcp_info->no_nagle = false;
+
+ tcp_info->ttl = IRDMA_DEFAULT_TTL;
+ tcp_info->rtt_var = IRDMA_DEFAULT_RTT_VAR;
+ tcp_info->ss_thresh = IRDMA_DEFAULT_SS_THRESH;
+ tcp_info->rexmit_thresh = IRDMA_DEFAULT_REXMIT_THRESH;
+
+ tcp_info->tcp_state = IRDMA_TCP_STATE_ESTABLISHED;
+ tcp_info->snd_wscale = cm_node->tcp_cntxt.snd_wscale;
+ tcp_info->rcv_wscale = cm_node->tcp_cntxt.rcv_wscale;
+
+ tcp_info->snd_nxt = cm_node->tcp_cntxt.loc_seq_num;
+ tcp_info->snd_wnd = cm_node->tcp_cntxt.snd_wnd;
+ tcp_info->rcv_nxt = cm_node->tcp_cntxt.rcv_nxt;
+ tcp_info->snd_max = cm_node->tcp_cntxt.loc_seq_num;
+
+ tcp_info->snd_una = cm_node->tcp_cntxt.loc_seq_num;
+ tcp_info->cwnd = 2 * cm_node->tcp_cntxt.mss;
+ tcp_info->snd_wl1 = cm_node->tcp_cntxt.rcv_nxt;
+ tcp_info->snd_wl2 = cm_node->tcp_cntxt.loc_seq_num;
+ tcp_info->max_snd_window = cm_node->tcp_cntxt.max_snd_wnd;
+ tcp_info->rcv_wnd = cm_node->tcp_cntxt.rcv_wnd <<
+ cm_node->tcp_cntxt.rcv_wscale;
+
+ tcp_info->flow_label = 0;
+ tcp_info->snd_mss = (u32)cm_node->tcp_cntxt.mss;
+ if (cm_node->vlan_id <= VLAN_VID_MASK) {
+ tcp_info->insert_vlan_tag = true;
+ tcp_info->vlan_tag = cm_node->vlan_id;
+ }
+ if (cm_node->ipv4) {
+ tcp_info->src_port = cm_node->loc_port;
+ tcp_info->dst_port = cm_node->rem_port;
+
+ tcp_info->dest_ip_addr3 = cm_node->rem_addr[0];
+ tcp_info->local_ipaddr3 = cm_node->loc_addr[0];
+ tcp_info->arp_idx = (u16)irdma_arp_table(iwqp->iwdev->rf,
+ &tcp_info->dest_ip_addr3,
+ true,
+ NULL,
+ IRDMA_ARP_RESOLVE);
+ } else {
+ tcp_info->src_port = cm_node->loc_port;
+ tcp_info->dst_port = cm_node->rem_port;
+ tcp_info->dest_ip_addr0 = cm_node->rem_addr[0];
+ tcp_info->dest_ip_addr1 = cm_node->rem_addr[1];
+ tcp_info->dest_ip_addr2 = cm_node->rem_addr[2];
+ tcp_info->dest_ip_addr3 = cm_node->rem_addr[3];
+ tcp_info->local_ipaddr0 = cm_node->loc_addr[0];
+ tcp_info->local_ipaddr1 = cm_node->loc_addr[1];
+ tcp_info->local_ipaddr2 = cm_node->loc_addr[2];
+ tcp_info->local_ipaddr3 = cm_node->loc_addr[3];
+ tcp_info->arp_idx = (u16)irdma_arp_table(iwqp->iwdev->rf,
+ &tcp_info->dest_ip_addr0,
+ false,
+ NULL,
+ IRDMA_ARP_RESOLVE);
+ }
+}
+
+/**
+ * irdma_cm_init_tsa_conn - setup qp for RTS
+ * @iwqp: associate qp for the connection
+ * @cm_node: connection's node
+ */
+static void irdma_cm_init_tsa_conn(struct irdma_qp *iwqp,
+ struct irdma_cm_node *cm_node)
+{
+ struct irdma_tcp_offload_info tcp_info = {};
+ struct irdma_iwarp_offload_info *iwarp_info;
+ struct irdma_qp_host_ctx_info *ctx_info;
+ struct irdma_sc_dev *dev = &iwqp->iwdev->rf->sc_dev;
+
+ iwarp_info = &iwqp->iwarp_info;
+ ctx_info = &iwqp->ctx_info;
+
+ ctx_info->tcp_info = &tcp_info;
+ ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id;
+ ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id;
+
+ iwarp_info->ord_size = cm_node->ord_size;
+ iwarp_info->ird_size = irdma_derive_hw_ird_setting(cm_node->ird_size);
+ iwarp_info->rd_en = true;
+ iwarp_info->rdmap_ver = 1;
+ iwarp_info->ddp_ver = 1;
+ iwarp_info->pd_id = iwqp->iwpd->sc_pd.pd_id;
+
+ ctx_info->tcp_info_valid = true;
+ ctx_info->iwarp_info_valid = true;
+ ctx_info->add_to_qoslist = true;
+ ctx_info->user_pri = cm_node->user_pri;
+
+ irdma_init_tcp_ctx(cm_node, &tcp_info, iwqp);
+ if (cm_node->snd_mark_en) {
+ iwarp_info->snd_mark_en = true;
+ iwarp_info->snd_mark_offset =
+ (tcp_info.snd_nxt & SNDMARKER_SEQNMASK) + cm_node->lsmm_size;
+ }
+
+ cm_node->state = IRDMA_CM_STATE_OFFLOADED;
+ tcp_info.tcp_state = IRDMA_TCP_STATE_ESTABLISHED;
+ tcp_info.src_mac_addr_idx = iwqp->iwdev->mac_ip_table_idx;
+
+ dev->iw_priv_qp_ops->qp_setctx(&iwqp->sc_qp,
+ iwqp->host_ctx.va, ctx_info);
+
+ /* once tcp_info is set, no need to do it again */
+ ctx_info->tcp_info_valid = false;
+ ctx_info->iwarp_info_valid = false;
+ ctx_info->add_to_qoslist = false;
+}
+
+/**
+ * irdma_cm_disconn - when a connection is being closed
+ * @iwqp: associated qp for the connection
+ */
+void irdma_cm_disconn(struct irdma_qp *iwqp)
+{
+ struct disconn_work *work;
+ struct irdma_device *iwdev = iwqp->iwdev;
+ struct irdma_cm_core *cm_core = &iwdev->cm_core;
+ unsigned long flags;
+
+ work = kzalloc(sizeof(*work), GFP_ATOMIC);
+ if (!work)
+ return;
+
+ spin_lock_irqsave(&iwdev->rf->qptable_lock, flags);
+ if (!iwdev->rf->qp_table[iwqp->ibqp.qp_num]) {
+ spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags);
+ irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
+ "qp_id %d is already freed\n",
+ iwqp->ibqp.qp_num);
+ kfree(work);
+ return;
+ }
+ irdma_add_ref(&iwqp->ibqp);
+ spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags);
+
+ work->iwqp = iwqp;
+ INIT_WORK(&work->work, irdma_disconnect_worker);
+ queue_work(cm_core->disconn_wq, &work->work);
+}
+
+/**
+ * irdma_qp_disconnect - free qp and close cm
+ * @iwqp: associate qp for the connection
+ */
+static void irdma_qp_disconnect(struct irdma_qp *iwqp)
+{
+ struct irdma_device *iwdev;
+ struct irdma_ib_device *iwibdev;
+
+ iwdev = iwqp->iwdev;
+ if (!iwdev) {
+ irdma_pr_err("iwdev == NULL\n");
+ return;
+ }
+
+ iwibdev = iwdev->iwibdev;
+
+ if (iwqp->active_conn) {
+ /* indicate this connection is NOT active */
+ iwqp->active_conn = 0;
+ } else {
+ /* Need to free the Last Streaming Mode Message */
+ if (iwqp->ietf_mem.va) {
+ if (iwqp->lsmm_mr)
+ iwibdev->ibdev.ops.dereg_mr(iwqp->lsmm_mr);
+ irdma_free_dma_mem(iwdev->rf->sc_dev.hw,
+ &iwqp->ietf_mem);
+ }
+ }
+
+ /* close the CM node down if it is still active */
+ if (iwqp->cm_node) {
+ irdma_debug(&iwdev->rf->sc_dev,
+ IRDMA_DEBUG_CM,
+ "Call close API\n");
+ irdma_cm_close(iwqp->cm_node);
+ }
+}
+
+/**
+ * irdma_cm_disconn_true - called by worker thread to disconnect qp
+ * @iwqp: associate qp for the connection
+ */
+static void irdma_cm_disconn_true(struct irdma_qp *iwqp)
+{
+ struct iw_cm_id *cm_id;
+ struct irdma_device *iwdev;
+ struct irdma_sc_qp *qp = &iwqp->sc_qp;
+ u16 last_ae;
+ u8 original_hw_tcp_state;
+ u8 original_ibqp_state;
+ int disconn_status = 0;
+ int issue_disconn = 0;
+ int issue_close = 0;
+ int issue_flush = 0;
+ struct ib_event ibevent;
+ unsigned long flags;
+ int err;
+
+ if (!iwqp) {
+ irdma_pr_err("iwqp == NULL\n");
+ return;
+ }
+
+ iwdev = iwqp->iwdev;
+ spin_lock_irqsave(&iwqp->lock, flags);
+ if (rdma_protocol_roce(&iwdev->iwibdev->ibdev, 1)) {
+ struct ib_qp_attr attr;
+
+ if (iwqp->flush_issued || iwqp->destroyed) {
+ spin_unlock_irqrestore(&iwqp->lock, flags);
+ return;
+ }
+
+ spin_unlock_irqrestore(&iwqp->lock, flags);
+
+ attr.qp_state = IB_QPS_ERR;
+ irdma_modify_qp_roce(&iwqp->ibqp, &attr, IB_QP_STATE, NULL);
+ if (iwqp->ibqp.event_handler) {
+ ibevent.device = iwqp->ibqp.device;
+ ibevent.event = IB_EVENT_QP_FATAL;
+ ibevent.element.qp = &iwqp->ibqp;
+ iwqp->ibqp.event_handler(&ibevent,
+ iwqp->ibqp.qp_context);
+ }
+ return;
+ }
+
+ cm_id = iwqp->cm_id;
+ /* make sure we havent already closed this connection */
+ if (!cm_id) {
+ spin_unlock_irqrestore(&iwqp->lock, flags);
+ return;
+ }
+
+ original_hw_tcp_state = iwqp->hw_tcp_state;
+ original_ibqp_state = iwqp->ibqp_state;
+ last_ae = iwqp->last_aeq;
+
+ if (qp->term_flags) {
+ issue_disconn = 1;
+ issue_close = 1;
+ iwqp->cm_id = NULL;
+ irdma_terminate_del_timer(qp);
+ if (!iwqp->flush_issued) {
+ iwqp->flush_issued = 1;
+ issue_flush = 1;
+ }
+ } else if ((original_hw_tcp_state == IRDMA_TCP_STATE_CLOSE_WAIT) ||
+ ((original_ibqp_state == IB_QPS_RTS) &&
+ (last_ae == IRDMA_AE_LLP_CONNECTION_RESET))) {
+ issue_disconn = 1;
+ if (last_ae == IRDMA_AE_LLP_CONNECTION_RESET)
+ disconn_status = -ECONNRESET;
+ }
+
+ if ((original_hw_tcp_state == IRDMA_TCP_STATE_CLOSED ||
+ original_hw_tcp_state == IRDMA_TCP_STATE_TIME_WAIT ||
+ last_ae == IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE ||
+ last_ae == IRDMA_AE_LLP_CONNECTION_RESET ||
+ iwdev->reset)) {
+ issue_close = 1;
+ iwqp->cm_id = NULL;
+ qp->term_flags = 0;
+ if (!iwqp->flush_issued) {
+ iwqp->flush_issued = 1;
+ issue_flush = 1;
+ }
+ }
+
+ spin_unlock_irqrestore(&iwqp->lock, flags);
+ if (issue_flush && !iwqp->destroyed) {
+ /* Flush the queues */
+ irdma_flush_wqes(iwdev->rf, iwqp);
+
+ if (qp->term_flags && iwqp->ibqp.event_handler) {
+ ibevent.device = iwqp->ibqp.device;
+ ibevent.event =
+ (qp->eventtype == TERM_EVENT_QP_FATAL) ?
+ IB_EVENT_QP_FATAL : IB_EVENT_QP_ACCESS_ERR;
+ ibevent.element.qp = &iwqp->ibqp;
+ iwqp->ibqp.event_handler(&ibevent,
+ iwqp->ibqp.qp_context);
+ }
+ }
+
+ if (!cm_id || !cm_id->event_handler)
+ return;
+ if (issue_disconn) {
+ err = irdma_send_cm_event(NULL,
+ cm_id,
+ IW_CM_EVENT_DISCONNECT,
+ disconn_status);
+ if (err)
+ irdma_debug(&iwdev->rf->sc_dev,
+ IRDMA_DEBUG_CM,
+ "disconnect event failed: - cm_id = %p\n",
+ cm_id);
+ }
+ if (issue_close) {
+ irdma_qp_disconnect(iwqp);
+ cm_id->provider_data = iwqp;
+ err = irdma_send_cm_event(NULL,
+ cm_id,
+ IW_CM_EVENT_CLOSE,
+ 0);
+ if (err)
+ irdma_debug(&iwdev->rf->sc_dev,
+ IRDMA_DEBUG_CM,
+ "close event failed: - cm_id = %p\n",
+ cm_id);
+ cm_id->rem_ref(cm_id);
+ }
+}
+
+/**
+ * irdma_disconnect_worker - worker for connection close
+ * @work: points or disconn structure
+ */
+static void irdma_disconnect_worker(struct work_struct *work)
+{
+ struct disconn_work *dwork =
+ container_of(work, struct disconn_work, work);
+ struct irdma_qp *iwqp = dwork->iwqp;
+
+ kfree(dwork);
+ irdma_cm_disconn_true(iwqp);
+ irdma_rem_ref(&iwqp->ibqp);
+}
+
+/**
+ * irdma_accept - registered call for connection to be accepted
+ * @cm_id: cm information for passive connection
+ * @conn_param: accpet parameters
+ */
+int irdma_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
+{
+ struct ib_qp *ibqp;
+ struct irdma_qp *iwqp;
+ struct irdma_device *iwdev;
+ struct irdma_sc_dev *dev;
+ struct irdma_cm_core *cm_core;
+ struct irdma_cm_node *cm_node;
+ struct ib_qp_attr attr = {};
+ int passive_state;
+ struct ib_mr *ibmr;
+ struct irdma_pd *iwpd;
+ u16 buf_len = 0;
+ struct irdma_kmem_info accept;
+ enum irdma_status_code status;
+ unsigned long flags;
+ u64 tagged_offset;
+ int wait_ret = 0;
+
+ ibqp = irdma_get_qp(cm_id->device, conn_param->qpn);
+ if (!ibqp)
+ return -EINVAL;
+
+ iwqp = to_iwqp(ibqp);
+ iwdev = iwqp->iwdev;
+ dev = &iwdev->rf->sc_dev;
+ cm_core = &iwdev->cm_core;
+ cm_node = (struct irdma_cm_node *)cm_id->provider_data;
+
+ if (((struct sockaddr_in *)&cm_id->local_addr)->sin_family == AF_INET) {
+ cm_node->ipv4 = true;
+ cm_node->vlan_id = irdma_get_vlan_ipv4(cm_node->loc_addr);
+ } else {
+ cm_node->ipv4 = false;
+ irdma_netdev_vlan_ipv6(cm_node->loc_addr, &cm_node->vlan_id,
+ NULL);
+ }
+ irdma_debug(cm_node->dev,
+ IRDMA_DEBUG_CM,
+ "Accept vlan_id=%d\n",
+ cm_node->vlan_id);
+
+ trace_irdma_accept(cm_node, 0, NULL);
+
+ if (cm_node->state == IRDMA_CM_STATE_LISTENER_DESTROYED) {
+ irdma_rem_ref_cm_node(cm_node);
+ return -EINVAL;
+ }
+
+ passive_state = atomic_add_return(1, &cm_node->passive_state);
+ if (passive_state == IRDMA_SEND_RESET_EVENT) {
+ irdma_rem_ref_cm_node(cm_node);
+ return -ECONNRESET;
+ }
+
+ cm_node->cm_core->stats_accepts++;
+ iwqp->cm_node = (void *)cm_node;
+ cm_node->iwqp = iwqp;
+
+ if (cm_node->dev->ws_add(iwqp->sc_qp.vsi, cm_node->user_pri))
+ return -ENOMEM;
+
+ buf_len = conn_param->private_data_len + IRDMA_MAX_IETF_SIZE;
+
+ status = irdma_allocate_dma_mem(dev->hw, &iwqp->ietf_mem, buf_len, 1);
+ if (status)
+ return -ENOMEM;
+
+ cm_node->pdata.size = conn_param->private_data_len;
+ accept.addr = iwqp->ietf_mem.va;
+ accept.size = irdma_cm_build_mpa_frame(cm_node, &accept, MPA_KEY_REPLY);
+ memcpy(accept.addr + accept.size, conn_param->private_data,
+ conn_param->private_data_len);
+
+ /* setup our first outgoing iWarp send WQE (the IETF frame response) */
+ iwpd = iwqp->iwpd;
+ tagged_offset = (uintptr_t)iwqp->ietf_mem.va;
+ ibmr = irdma_reg_phys_mr(&iwpd->ibpd,
+ iwqp->ietf_mem.pa,
+ buf_len,
+ IB_ACCESS_LOCAL_WRITE,
+ &tagged_offset);
+ if (IS_ERR(ibmr)) {
+ irdma_free_dma_mem(dev->hw, &iwqp->ietf_mem);
+ return -ENOMEM;
+ }
+
+ ibmr->pd = &iwpd->ibpd;
+ ibmr->device = iwpd->ibpd.device;
+ iwqp->lsmm_mr = ibmr;
+ if (iwqp->page)
+ iwqp->sc_qp.qp_uk.sq_base = kmap(iwqp->page);
+ dev->iw_priv_qp_ops->qp_send_lsmm(&iwqp->sc_qp,
+ iwqp->ietf_mem.va,
+ (accept.size + conn_param->private_data_len),
+ ibmr->lkey);
+
+ if (iwqp->page)
+ kunmap(iwqp->page);
+
+ iwqp->cm_id = cm_id;
+ cm_node->cm_id = cm_id;
+
+ cm_id->provider_data = (void *)iwqp;
+ iwqp->active_conn = 0;
+
+ cm_node->lsmm_size = accept.size + conn_param->private_data_len;
+
+ irdma_cm_init_tsa_conn(iwqp, cm_node);
+ cm_id->add_ref(cm_id);
+ irdma_add_ref(&iwqp->ibqp);
+
+ attr.qp_state = IB_QPS_RTS;
+ cm_node->qhash_set = false;
+ cm_node->cm_core->cm_free_ah(cm_node);
+
+ irdma_modify_qp(&iwqp->ibqp, &attr, IB_QP_STATE, NULL);
+ spin_lock_irqsave(&cm_core->ht_lock, flags);
+ list_move_tail(&cm_node->list, &cm_core->accelerated_list);
+ spin_unlock_irqrestore(&cm_core->ht_lock, flags);
+
+ if (dev->hw_attrs.feature_flags & IRDMA_FEATURE_RTS_AE) {
+ wait_ret = wait_event_interruptible_timeout(iwqp->waitq,
+ iwqp->rts_ae_rcvd,
+ IRDMA_MAX_TIMEOUT);
+ if (!wait_ret)
+ irdma_debug(cm_node->dev, IRDMA_DEBUG_CM,
+ "Slow Connection: cm_node=%p, loc_port=%d, rem_port=%d, cm_id=%p\n",
+ cm_node,
+ cm_node->loc_port,
+ cm_node->rem_port,
+ cm_node->cm_id);
+ }
+
+ cm_node->accelerated = 1;
+ irdma_send_cm_event(cm_node, cm_id, IW_CM_EVENT_ESTABLISHED, 0);
+
+ if (cm_node->accept_pend) {
+ atomic_dec(&cm_node->listener->pend_accepts_cnt);
+ cm_node->accept_pend = 0;
+ }
+
+ irdma_debug(cm_node->dev,
+ IRDMA_DEBUG_CM,
+ "rem_port=0x%04x, loc_port=0x%04x\n",
+ cm_node->rem_port,
+ cm_node->loc_port);
+
+ return 0;
+}
+
+/**
+ * irdma_reject - registered call for connection to be rejected
+ * @cm_id: cm information for passive connection
+ * @pdata: private data to be sent
+ * @pdata_len: private data length
+ */
+int irdma_reject(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len)
+{
+ struct irdma_device *iwdev;
+ struct irdma_cm_node *cm_node;
+
+ cm_node = (struct irdma_cm_node *)cm_id->provider_data;
+ cm_node->cm_id = cm_id;
+ cm_node->pdata.size = pdata_len;
+
+ trace_irdma_reject(cm_node, 0, NULL);
+
+ iwdev = to_iwdev(cm_id->device);
+ if (!iwdev)
+ return -EINVAL;
+
+ cm_node->cm_core->stats_rejects++;
+
+ if (pdata_len + sizeof(struct ietf_mpa_v2) > IRDMA_MAX_CM_BUF)
+ return -EINVAL;
+
+ return irdma_cm_reject(cm_node, pdata, pdata_len);
+}
+
+/**
+ * irdma_connect - registered call for connection to be established
+ * @cm_id: cm information for passive connection
+ * @conn_param: Information about the connection
+ */
+int irdma_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
+{
+ struct ib_qp *ibqp;
+ struct irdma_qp *iwqp;
+ struct irdma_device *iwdev;
+ struct irdma_cm_node *cm_node;
+ struct irdma_cm_info cm_info;
+ struct sockaddr_in *laddr;
+ struct sockaddr_in *raddr;
+ struct sockaddr_in6 *laddr6;
+ struct sockaddr_in6 *raddr6;
+ int ret = 0;
+
+ ibqp = irdma_get_qp(cm_id->device, conn_param->qpn);
+ if (!ibqp)
+ return -EINVAL;
+ iwqp = to_iwqp(ibqp);
+ if (!iwqp)
+ return -EINVAL;
+ iwdev = iwqp->iwdev;
+ if (!iwdev)
+ return -EINVAL;
+
+ laddr = (struct sockaddr_in *)&cm_id->m_local_addr;
+ raddr = (struct sockaddr_in *)&cm_id->m_remote_addr;
+ laddr6 = (struct sockaddr_in6 *)&cm_id->m_local_addr;
+ raddr6 = (struct sockaddr_in6 *)&cm_id->m_remote_addr;
+
+ if (!(laddr->sin_port) || !(raddr->sin_port))
+ return -EINVAL;
+
+ iwqp->active_conn = 1;
+ iwqp->cm_id = NULL;
+ cm_id->provider_data = iwqp;
+
+ /* set up the connection params for the node */
+ if (cm_id->remote_addr.ss_family == AF_INET) {
+ if (iwdev->vsi.mtu < IRDMA_MIN_MTU_IPV4)
+ return -EINVAL;
+
+ cm_info.ipv4 = true;
+ memset(cm_info.loc_addr, 0, sizeof(cm_info.loc_addr));
+ memset(cm_info.rem_addr, 0, sizeof(cm_info.rem_addr));
+ cm_info.loc_addr[0] = ntohl(laddr->sin_addr.s_addr);
+ cm_info.rem_addr[0] = ntohl(raddr->sin_addr.s_addr);
+ cm_info.loc_port = ntohs(laddr->sin_port);
+ cm_info.rem_port = ntohs(raddr->sin_port);
+ cm_info.vlan_id = irdma_get_vlan_ipv4(cm_info.loc_addr);
+ } else {
+ if (iwdev->vsi.mtu < IRDMA_MIN_MTU_IPV6)
+ return -EINVAL;
+
+ cm_info.ipv4 = false;
+ irdma_copy_ip_ntohl(cm_info.loc_addr,
+ laddr6->sin6_addr.in6_u.u6_addr32);
+ irdma_copy_ip_ntohl(cm_info.rem_addr,
+ raddr6->sin6_addr.in6_u.u6_addr32);
+ cm_info.loc_port = ntohs(laddr6->sin6_port);
+ cm_info.rem_port = ntohs(raddr6->sin6_port);
+ irdma_netdev_vlan_ipv6(cm_info.loc_addr, &cm_info.vlan_id, NULL);
+ }
+ cm_info.cm_id = cm_id;
+ cm_info.qh_qpid = iwdev->vsi.ilq->qp_id;
+ cm_info.tos = cm_id->tos;
+ cm_info.user_pri = rt_tos2priority(cm_id->tos);
+
+ if (iwqp->sc_qp.dev->ws_add(iwqp->sc_qp.vsi, cm_info.user_pri))
+ return -ENOMEM;
+
+ irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_DCB, "DCB: TOS:[%d] UP:[%d]\n",
+ cm_id->tos, cm_info.user_pri);
+
+ trace_irdma_dcb_tos(iwdev, cm_id->tos, cm_info.user_pri);
+
+ cm_id->add_ref(cm_id);
+ ret = irdma_create_cm_node(&iwdev->cm_core, iwdev,
+ conn_param, &cm_info, &cm_node);
+ if (ret) {
+ cm_id->rem_ref(cm_id);
+ return ret;
+ }
+ ret = cm_node->cm_core->cm_create_ah(cm_node, true);
+ if (ret)
+ goto err;
+ if (irdma_manage_qhash(iwdev,
+ &cm_info,
+ IRDMA_QHASH_TYPE_TCP_ESTABLISHED,
+ IRDMA_QHASH_MANAGE_TYPE_ADD,
+ NULL,
+ true)) {
+ ret = -EINVAL;
+ goto err;
+ }
+ cm_node->qhash_set = true;
+
+ if (irdma_manage_apbvt(iwdev, cm_info.loc_port,
+ IRDMA_MANAGE_APBVT_ADD)) {
+ ret = -EINVAL;
+ goto err;
+ }
+
+ cm_node->apbvt_set = true;
+ iwqp->cm_node = cm_node;
+ cm_node->iwqp = iwqp;
+ iwqp->cm_id = cm_id;
+ irdma_add_ref(&iwqp->ibqp);
+
+ if (cm_node->state != IRDMA_CM_STATE_OFFLOADED) {
+ cm_node->state = IRDMA_CM_STATE_SYN_SENT;
+ ret = irdma_send_syn(cm_node, 0);
+ if (ret)
+ goto err;
+ }
+
+ irdma_debug(cm_node->dev,
+ IRDMA_DEBUG_CM,
+ "rem_port=0x%04x, loc_port=0x%04x, cm_node=%p, cm_id = %p.\n",
+ cm_node->rem_port,
+ cm_node->loc_port,
+ cm_node,
+ cm_node->cm_id);
+
+ trace_irdma_connect(cm_node, 0, NULL);
+
+ return 0;
+
+err:
+ if (cm_info.ipv4)
+ irdma_debug(&iwdev->rf->sc_dev,
+ IRDMA_DEBUG_CM,
+ "connect() FAILED: dest addr=%pI4",
+ cm_info.rem_addr);
+ else
+ irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
+ "connect() FAILED: dest addr=%pI6",
+ cm_info.rem_addr);
+ irdma_rem_ref_cm_node(cm_node);
+ cm_id->rem_ref(cm_id);
+ iwdev->cm_core.stats_connect_errs++;
+
+ return ret;
+}
+
+/**
+ * irdma_create_listen - registered call creating listener
+ * @cm_id: cm information for passive connection
+ * @backlog: to max accept pending count
+ */
+int irdma_create_listen(struct iw_cm_id *cm_id, int backlog)
+{
+ struct irdma_device *iwdev;
+ struct irdma_cm_listener *cm_listen_node;
+ struct irdma_cm_info cm_info = {};
+ enum irdma_status_code err;
+ struct sockaddr_in *laddr;
+ struct sockaddr_in6 *laddr6;
+ bool wildcard = false;
+
+ iwdev = to_iwdev(cm_id->device);
+ if (!iwdev)
+ return -EINVAL;
+
+ laddr = (struct sockaddr_in *)&cm_id->m_local_addr;
+ laddr6 = (struct sockaddr_in6 *)&cm_id->m_local_addr;
+ cm_info.qh_qpid = iwdev->vsi.ilq->qp_id;
+
+ if (laddr->sin_family == AF_INET) {
+ if (iwdev->vsi.mtu < IRDMA_MIN_MTU_IPV4)
+ return -EINVAL;
+
+ cm_info.ipv4 = true;
+ cm_info.loc_addr[0] = ntohl(laddr->sin_addr.s_addr);
+ cm_info.loc_port = ntohs(laddr->sin_port);
+
+ if (laddr->sin_addr.s_addr != htonl(INADDR_ANY))
+ cm_info.vlan_id = irdma_get_vlan_ipv4(cm_info.loc_addr);
+ else
+ wildcard = true;
+
+ } else {
+ if (iwdev->vsi.mtu < IRDMA_MIN_MTU_IPV6)
+ return -EINVAL;
+
+ cm_info.ipv4 = false;
+ irdma_copy_ip_ntohl(cm_info.loc_addr,
+ laddr6->sin6_addr.in6_u.u6_addr32);
+ cm_info.loc_port = ntohs(laddr6->sin6_port);
+ if (ipv6_addr_type(&laddr6->sin6_addr) != IPV6_ADDR_ANY)
+ irdma_netdev_vlan_ipv6(cm_info.loc_addr,
+ &cm_info.vlan_id,
+ NULL);
+ else
+ wildcard = true;
+ }
+ cm_info.backlog = backlog;
+ cm_info.cm_id = cm_id;
+
+ trace_irdma_create_listen(iwdev, &cm_info);
+
+ cm_listen_node = irdma_make_listen_node(&iwdev->cm_core, iwdev,
+ &cm_info);
+ if (!cm_listen_node) {
+ irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_ERR,
+ "cm_listen_node == NULL\n");
+ return -ENOMEM;
+ }
+
+ cm_id->provider_data = cm_listen_node;
+
+ cm_listen_node->tos = cm_id->tos;
+ cm_listen_node->user_pri = rt_tos2priority(cm_id->tos);
+ cm_info.user_pri = cm_listen_node->user_pri;
+ if (!cm_listen_node->reused_node) {
+ if (wildcard) {
+ if (cm_info.ipv4)
+ err = irdma_add_mqh_4(iwdev,
+ &cm_info,
+ cm_listen_node);
+ else
+ err = irdma_add_mqh_6(iwdev,
+ &cm_info,
+ cm_listen_node);
+ if (err)
+ goto error;
+
+ err = irdma_manage_apbvt(iwdev,
+ cm_info.loc_port,
+ IRDMA_MANAGE_APBVT_ADD);
+
+ if (err)
+ goto error;
+ } else {
+ err = irdma_manage_qhash(iwdev,
+ &cm_info,
+ IRDMA_QHASH_TYPE_TCP_SYN,
+ IRDMA_QHASH_MANAGE_TYPE_ADD,
+ NULL,
+ true);
+ if (err)
+ goto error;
+
+ cm_listen_node->qhash_set = true;
+ err = irdma_manage_apbvt(iwdev,
+ cm_info.loc_port,
+ IRDMA_MANAGE_APBVT_ADD);
+ if (err)
+ goto error;
+ }
+ }
+ cm_id->add_ref(cm_id);
+ cm_listen_node->cm_core->stats_listen_created++;
+ return 0;
+
+error:
+
+ irdma_cm_del_listen(&iwdev->cm_core, (void *)cm_listen_node, false);
+
+ return -EINVAL;
+}
+
+/**
+ * irdma_destroy_listen - registered call to destroy listener
+ * @cm_id: cm information for passive connection
+ */
+int irdma_destroy_listen(struct iw_cm_id *cm_id)
+{
+ struct irdma_device *iwdev;
+
+ iwdev = to_iwdev(cm_id->device);
+ if (cm_id->provider_data)
+ irdma_cm_del_listen(&iwdev->cm_core, cm_id->provider_data,
+ true);
+ else
+ irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_ERR,
+ "cm_id->provider_data was NULL\n");
+
+ cm_id->rem_ref(cm_id);
+
+ return 0;
+}
+
+/**
+ * irdma_cm_event_connected - handle connected active node
+ * @event: the info for cm_node of connection
+ */
+static void irdma_cm_event_connected(struct irdma_cm_event *event)
+{
+ struct irdma_qp *iwqp;
+ struct irdma_device *iwdev;
+ struct irdma_cm_core *cm_core;
+ struct irdma_cm_node *cm_node;
+ struct irdma_sc_dev *dev;
+ struct ib_qp_attr attr = {};
+ struct iw_cm_id *cm_id;
+ unsigned long flags;
+ int status;
+ bool read0;
+ int wait_ret = 0;
+
+ cm_node = event->cm_node;
+ cm_id = cm_node->cm_id;
+ iwqp = (struct irdma_qp *)cm_id->provider_data;
+ iwdev = iwqp->iwdev;
+ dev = &iwdev->rf->sc_dev;
+ cm_core = &iwdev->cm_core;
+
+ if (iwqp->destroyed) {
+ status = -ETIMEDOUT;
+ goto error;
+ }
+
+ irdma_cm_init_tsa_conn(iwqp, cm_node);
+ read0 = (cm_node->send_rdma0_op == SEND_RDMA_READ_ZERO);
+ if (iwqp->page)
+ iwqp->sc_qp.qp_uk.sq_base = kmap(iwqp->page);
+ dev->iw_priv_qp_ops->qp_send_rtt(&iwqp->sc_qp, read0);
+ if (iwqp->page)
+ kunmap(iwqp->page);
+
+ attr.qp_state = IB_QPS_RTS;
+ cm_node->qhash_set = false;
+ irdma_modify_qp(&iwqp->ibqp, &attr, IB_QP_STATE, NULL);
+ spin_lock_irqsave(&cm_core->ht_lock, flags);
+ list_move_tail(&cm_node->list, &cm_core->accelerated_list);
+ spin_unlock_irqrestore(&cm_core->ht_lock, flags);
+
+ if (dev->hw_attrs.feature_flags & IRDMA_FEATURE_RTS_AE) {
+ wait_ret = wait_event_interruptible_timeout(iwqp->waitq,
+ iwqp->rts_ae_rcvd,
+ IRDMA_MAX_TIMEOUT);
+ if (!wait_ret)
+ irdma_debug(cm_node->dev, IRDMA_DEBUG_CM,
+ "Slow Connection: cm_node=%p, loc_port=%d, rem_port=%d, cm_id=%p\n",
+ cm_node,
+ cm_node->loc_port,
+ cm_node->rem_port,
+ cm_node->cm_id);
+ }
+
+ cm_node->accelerated = 1;
+ irdma_send_cm_event(cm_node, cm_id, IW_CM_EVENT_CONNECT_REPLY, 0);
+ cm_node->cm_core->cm_free_ah(cm_node);
+ return;
+
+error:
+ iwqp->cm_id = NULL;
+ cm_id->provider_data = NULL;
+ irdma_send_cm_event(event->cm_node,
+ cm_id,
+ IW_CM_EVENT_CONNECT_REPLY,
+ status);
+ cm_id->rem_ref(cm_id);
+ irdma_rem_ref_cm_node(event->cm_node);
+}
+
+/**
+ * irdma_cm_event_reset - handle reset
+ * @event: the info for cm_node of connection
+ */
+static void irdma_cm_event_reset(struct irdma_cm_event *event)
+{
+ struct irdma_cm_node *cm_node = event->cm_node;
+ struct iw_cm_id *cm_id = cm_node->cm_id;
+ struct irdma_qp *iwqp;
+
+ if (!cm_id)
+ return;
+
+ iwqp = cm_id->provider_data;
+ if (!iwqp)
+ return;
+
+ irdma_debug(cm_node->dev,
+ IRDMA_DEBUG_CM,
+ "reset event %p - cm_id = %p\n",
+ event->cm_node, cm_id);
+ iwqp->cm_id = NULL;
+
+ irdma_send_cm_event(cm_node, cm_node->cm_id,
+ IW_CM_EVENT_DISCONNECT, -ECONNRESET);
+ irdma_send_cm_event(cm_node, cm_node->cm_id,
+ IW_CM_EVENT_CLOSE, 0);
+}
+
+/**
+ * irdma_cm_event_handler - send event to cm upper layer
+ * @work: pointer of cm event info.
+ */
+static void irdma_cm_event_handler(struct work_struct *work)
+{
+ struct irdma_cm_event *event = container_of(work,
+ struct irdma_cm_event,
+ event_work);
+ struct irdma_cm_node *cm_node;
+
+ if (!event || !event->cm_node || !event->cm_node->cm_core)
+ return;
+
+ cm_node = event->cm_node;
+ trace_irdma_cm_event_handler(cm_node, event->type, NULL);
+
+ switch (event->type) {
+ case IRDMA_CM_EVENT_MPA_REQ:
+ irdma_send_cm_event(cm_node,
+ cm_node->cm_id,
+ IW_CM_EVENT_CONNECT_REQUEST,
+ 0);
+ break;
+ case IRDMA_CM_EVENT_RESET:
+ irdma_cm_event_reset(event);
+ break;
+ case IRDMA_CM_EVENT_CONNECTED:
+ if (!event->cm_node->cm_id ||
+ event->cm_node->state != IRDMA_CM_STATE_OFFLOADED)
+ break;
+ irdma_cm_event_connected(event);
+ break;
+ case IRDMA_CM_EVENT_MPA_REJECT:
+ if (!event->cm_node->cm_id ||
+ cm_node->state == IRDMA_CM_STATE_OFFLOADED)
+ break;
+ irdma_send_cm_event(cm_node,
+ cm_node->cm_id,
+ IW_CM_EVENT_CONNECT_REPLY,
+ -ECONNREFUSED);
+ break;
+ case IRDMA_CM_EVENT_ABORTED:
+ if (!event->cm_node->cm_id ||
+ event->cm_node->state == IRDMA_CM_STATE_OFFLOADED)
+ break;
+ irdma_event_connect_error(event);
+ break;
+ default:
+ irdma_debug(cm_node->dev, IRDMA_DEBUG_ERR,
+ "bad event type = %d\n", event->type);
+ break;
+ }
+
+ event->cm_info.cm_id->rem_ref(event->cm_info.cm_id);
+ irdma_rem_ref_cm_node(event->cm_node);
+ kfree(event);
+}
+
+/**
+ * irdma_cm_post_event - queue event request for worker thread
+ * @event: cm node's info for up event call
+ */
+static void irdma_cm_post_event(struct irdma_cm_event *event)
+{
+ atomic_inc(&event->cm_node->ref_count);
+ event->cm_info.cm_id->add_ref(event->cm_info.cm_id);
+ INIT_WORK(&event->event_work, irdma_cm_event_handler);
+ queue_work(event->cm_node->cm_core->event_wq, &event->event_work);
+}
+
+/**
+ * irdma_qhash_ctrl - enable/disable qhash for list
+ * @iwdev: device pointer
+ * @parent_listen_node: parent listen node
+ * @nfo: cm info node
+ * @ipaddr: Pointer to IPv4 or IPv6 address
+ * @ipv4: flag indicating IPv4 when true
+ * @ifup: flag indicating interface up when true
+ *
+ * Enables or disables the qhash for the node in the child
+ * listen list that matches ipaddr. If no matching IP was found
+ * it will allocate and add a new child listen node to the
+ * parent listen node. The listen_list_lock is assumed to be
+ * held when called.
+ */
+static void irdma_qhash_ctrl(struct irdma_device *iwdev,
+ struct irdma_cm_listener *parent_listen_node,
+ struct irdma_cm_info *nfo,
+ u32 *ipaddr, bool ipv4, bool ifup)
+{
+ struct list_head *child_listen_list = &parent_listen_node->child_listen_list;
+ struct irdma_cm_listener *child_listen_node;
+ struct list_head *pos, *tpos;
+ enum irdma_status_code err;
+ bool node_allocated = false;
+ enum irdma_quad_hash_manage_type op = ifup ?
+ IRDMA_QHASH_MANAGE_TYPE_ADD :
+ IRDMA_QHASH_MANAGE_TYPE_DELETE;
+
+ list_for_each_safe(pos, tpos, child_listen_list) {
+ child_listen_node = list_entry(pos,
+ struct irdma_cm_listener,
+ child_listen_list);
+ if (!memcmp(child_listen_node->loc_addr, ipaddr, ipv4 ? 4 : 16))
+ goto set_qhash;
+ }
+
+ /* if not found then add a child listener if interface is going up */
+ if (!ifup)
+ return;
+ child_listen_node = kzalloc(sizeof(*child_listen_node), GFP_ATOMIC);
+ if (!child_listen_node)
+ return;
+
+ node_allocated = true;
+ memcpy(child_listen_node, parent_listen_node,
+ sizeof(*child_listen_node));
+
+ memcpy(child_listen_node->loc_addr, ipaddr, ipv4 ? 4 : 16);
+
+set_qhash:
+ memcpy(nfo->loc_addr,
+ child_listen_node->loc_addr,
+ sizeof(nfo->loc_addr));
+ nfo->vlan_id = child_listen_node->vlan_id;
+ err = irdma_manage_qhash(iwdev, nfo,
+ IRDMA_QHASH_TYPE_TCP_SYN,
+ op,
+ NULL,
+ false);
+ if (!err) {
+ child_listen_node->qhash_set = ifup;
+ if (node_allocated)
+ list_add(&child_listen_node->child_listen_list,
+ &parent_listen_node->child_listen_list);
+ } else if (node_allocated) {
+ kfree(child_listen_node);
+ }
+}
+
+/**
+ * irdma_cm_teardown_connections - teardown QPs
+ * @iwdev: device pointer
+ * @ipaddr: Pointer to IPv4 or IPv6 address
+ * @ipv4: flag indicating IPv4 when true
+ * @disconnect_all: flag indicating disconnect all QPs
+ * teardown QPs where source or destination addr matches ip addr
+ */
+void irdma_cm_teardown_connections(struct irdma_device *iwdev, u32 *ipaddr,
+ struct irdma_cm_info *nfo,
+ bool disconnect_all)
+{
+ struct irdma_cm_core *cm_core = &iwdev->cm_core;
+ struct list_head *list_core_temp;
+ struct list_head *list_node;
+ struct irdma_cm_node *cm_node;
+ struct list_head teardown_list;
+ struct ib_qp_attr attr;
+ struct irdma_sc_vsi *vsi = &iwdev->vsi;
+ struct irdma_sc_qp *sc_qp;
+ struct irdma_qp *qp;
+ int i;
+ unsigned long flags;
+
+ INIT_LIST_HEAD(&teardown_list);
+
+ spin_lock_irqsave(&cm_core->ht_lock, flags);
+ list_for_each_safe(list_node, list_core_temp,
+ &cm_core->accelerated_list) {
+ cm_node = container_of(list_node, struct irdma_cm_node, list);
+ if (disconnect_all ||
+ (nfo->vlan_id == cm_node->vlan_id &&
+ !memcmp(cm_node->loc_addr, ipaddr, nfo->ipv4 ? 4 : 16))) {
+ atomic_inc(&cm_node->ref_count);
+ list_add(&cm_node->teardown_entry, &teardown_list);
+ }
+ }
+ list_for_each_safe(list_node, list_core_temp,
+ &cm_core->non_accelerated_list) {
+ cm_node = container_of(list_node, struct irdma_cm_node, list);
+ if (disconnect_all ||
+ (nfo->vlan_id == cm_node->vlan_id &&
+ !memcmp(cm_node->loc_addr, ipaddr, nfo->ipv4 ? 4 : 16))) {
+ atomic_inc(&cm_node->ref_count);
+ list_add(&cm_node->teardown_entry, &teardown_list);
+ }
+ }
+ spin_unlock_irqrestore(&cm_core->ht_lock, flags);
+
+ list_for_each_safe(list_node, list_core_temp, &teardown_list) {
+ cm_node = container_of(list_node, struct irdma_cm_node,
+ teardown_entry);
+ attr.qp_state = IB_QPS_ERR;
+ irdma_modify_qp(&cm_node->iwqp->ibqp, &attr, IB_QP_STATE, NULL);
+ if (iwdev->reset)
+ irdma_cm_disconn(cm_node->iwqp);
+ irdma_rem_ref_cm_node(cm_node);
+ }
+ if (!iwdev->roce_mode)
+ return;
+
+ INIT_LIST_HEAD(&teardown_list);
+ for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) {
+ spin_lock_irqsave(&vsi->qos[i].lock, flags);
+ list_for_each_safe(list_node, list_core_temp, &vsi->qos[i].qplist) {
+ u32 qp_ip[4];
+
+ sc_qp = container_of(list_node, struct irdma_sc_qp, list);
+ if (sc_qp->qp_type != IRDMA_QP_TYPE_ROCE_RC)
+ continue;
+
+ qp = sc_qp->back_qp;
+ if (!disconnect_all) {
+ if (nfo->ipv4)
+ qp_ip[0] = qp->udp_info.local_ipaddr3;
+ else
+ memcpy(qp_ip,
+ &qp->udp_info.local_ipaddr0,
+ sizeof(qp_ip));
+ }
+
+ if (disconnect_all ||
+ (nfo->vlan_id == qp->udp_info.vlan_tag &&
+ !memcmp(qp_ip, ipaddr, nfo->ipv4 ? 4 : 16))) {
+ spin_lock_irqsave(&iwdev->rf->qptable_lock, flags);
+ if (iwdev->rf->qp_table[sc_qp->qp_uk.qp_id]) {
+ irdma_add_ref(&qp->ibqp);
+ list_add(&qp->teardown_entry, &teardown_list);
+ }
+ spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags);
+ }
+ }
+ spin_unlock_irqrestore(&vsi->qos[i].lock, flags);
+ }
+
+ list_for_each_safe(list_node, list_core_temp, &teardown_list) {
+ qp = container_of(list_node, struct irdma_qp, teardown_entry);
+ attr.qp_state = IB_QPS_ERR;
+ irdma_modify_qp_roce(&qp->ibqp, &attr, IB_QP_STATE, NULL);
+ irdma_rem_ref(&qp->ibqp);
+ }
+}
+
+/**
+ * irdma_ifdown_notify - process an ifdown on an interface
+ * @iwdev: device pointer
+ * @ipaddr: Pointer to IPv4 or IPv6 address
+ * @ipv4: flag indicating IPv4 when true
+ * @ifup: flag indicating interface up when true
+ */
+void irdma_if_notify(struct irdma_device *iwdev,
+ struct net_device *netdev,
+ u32 *ipaddr,
+ bool ipv4,
+ bool ifup)
+{
+ struct irdma_cm_core *cm_core = &iwdev->cm_core;
+ unsigned long flags;
+ struct irdma_cm_listener *listen_node;
+ static const u32 ip_zero[4] = { 0, 0, 0, 0 };
+ struct irdma_cm_info nfo;
+ u16 vlan_id = rdma_vlan_dev_vlan_id(netdev);
+ enum irdma_status_code ret;
+ enum irdma_quad_hash_manage_type op = ifup ?
+ IRDMA_QHASH_MANAGE_TYPE_ADD :
+ IRDMA_QHASH_MANAGE_TYPE_DELETE;
+
+ nfo.vlan_id = vlan_id;
+ nfo.ipv4 = ipv4;
+
+ /* Disable or enable qhash for listeners */
+ spin_lock_irqsave(&cm_core->listen_list_lock, flags);
+ list_for_each_entry(listen_node, &cm_core->listen_nodes, list) {
+ if (vlan_id != listen_node->vlan_id)
+ continue;
+
+ if (memcmp(listen_node->loc_addr, ipaddr, ipv4 ? 4 : 16) &&
+ memcmp(listen_node->loc_addr, ip_zero, ipv4 ? 4 : 16))
+ continue;
+
+ memcpy(nfo.loc_addr, listen_node->loc_addr, sizeof(nfo.loc_addr));
+ nfo.loc_port = listen_node->loc_port;
+ nfo.user_pri = listen_node->user_pri;
+ if (!list_empty(&listen_node->child_listen_list)) {
+ irdma_qhash_ctrl(iwdev,
+ listen_node,
+ &nfo,
+ ipaddr, ipv4, ifup);
+ } else if (memcmp(listen_node->loc_addr, ip_zero, ipv4 ? 4 : 16)) {
+ ret = irdma_manage_qhash(iwdev,
+ &nfo,
+ IRDMA_QHASH_TYPE_TCP_SYN,
+ op,
+ NULL,
+ false);
+ if (!ret)
+ listen_node->qhash_set = ifup;
+ }
+ }
+ spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
+
+ /* disconnect any connected qp's on ifdown */
+ if (!ifup)
+ irdma_cm_teardown_connections(iwdev, ipaddr, &nfo, false);
+}
diff --git a/drivers/infiniband/hw/irdma/cm.h b/drivers/infiniband/hw/irdma/cm.h
new file mode 100644
index 0000000..407338d
--- /dev/null
+++ b/drivers/infiniband/hw/irdma/cm.h
@@ -0,0 +1,425 @@
+/* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB */
+/* Copyright (c) 2019, Intel Corporation. */
+
+#ifndef IRDMA_CM_H
+#define IRDMA_CM_H
+
+#define IRDMA_MANAGE_APBVT_DEL 0
+#define IRDMA_MANAGE_APBVT_ADD 1
+
+#define IRDMA_MPA_REQUEST_ACCEPT 1
+#define IRDMA_MPA_REQUEST_REJECT 2
+
+/* IETF MPA -- defines */
+#define IEFT_MPA_KEY_REQ "MPA ID Req Frame"
+#define IEFT_MPA_KEY_REP "MPA ID Rep Frame"
+#define IETF_MPA_KEY_SIZE 16
+#define IETF_MPA_VER 1
+#define IETF_MAX_PRIV_DATA_LEN 512
+#define IETF_MPA_FRAME_SIZE 20
+#define IETF_RTR_MSG_SIZE 4
+#define IETF_MPA_V2_FLAG 0x10
+#define SNDMARKER_SEQNMASK 0x000001ff
+#define IRDMA_MAX_IETF_SIZE 32
+
+/* IETF RTR MSG Fields */
+#define IETF_PEER_TO_PEER 0x8000
+#define IETF_FLPDU_ZERO_LEN 0x4000
+#define IETF_RDMA0_WRITE 0x8000
+#define IETF_RDMA0_READ 0x4000
+#define IETF_NO_IRD_ORD 0x3fff
+
+/* HW-supported IRD sizes*/
+#define IRDMA_HW_IRD_SETTING_2 2
+#define IRDMA_HW_IRD_SETTING_4 4
+#define IRDMA_HW_IRD_SETTING_8 8
+#define IRDMA_HW_IRD_SETTING_16 16
+#define IRDMA_HW_IRD_SETTING_32 32
+#define IRDMA_HW_IRD_SETTING_64 64
+#define IRDMA_HW_IRD_SETTING_128 128
+
+#define MAX_PORTS 65536
+
+#define IRDMA_PASSIVE_STATE_INDICATED 0
+#define IRDMA_DO_NOT_SEND_RESET_EVENT 1
+#define IRDMA_SEND_RESET_EVENT 2
+
+#define MAX_IRDMA_IFS 4
+
+#define SET_ACK 1
+#define SET_SYN 2
+#define SET_FIN 4
+#define SET_RST 8
+
+#define TCP_OPTIONS_PADDING 3
+
+#define IRDMA_DEFAULT_RETRYS 64
+#define IRDMA_DEFAULT_RETRANS 8
+#define IRDMA_DEFAULT_TTL 0x40
+#define IRDMA_DEFAULT_RTT_VAR 6
+#define IRDMA_DEFAULT_SS_THRESH 0x3fffffff
+#define IRDMA_DEFAULT_REXMIT_THRESH 8
+
+#define IRDMA_RETRY_TIMEOUT HZ
+#define IRDMA_SHORT_TIME 10
+#define IRDMA_LONG_TIME (2 * HZ)
+#define IRDMA_MAX_TIMEOUT ((unsigned long)(12 * HZ))
+
+#define IRDMA_CM_HASHTABLE_SIZE 1024
+#define IRDMA_CM_TCP_TIMER_INTERVAL 3000
+#define IRDMA_CM_DEFAULT_MTU 1540
+#define IRDMA_CM_DEFAULT_FRAME_CNT 10
+#define IRDMA_CM_THREAD_STACK_SIZE 256
+#define IRDMA_CM_DEFAULT_RCV_WND 64240
+#define IRDMA_CM_DEFAULT_RCV_WND_SCALED 0x3fffc
+#define IRDMA_CM_DEFAULT_RCV_WND_SCALE 2
+#define IRDMA_CM_DEFAULT_FREE_PKTS 10
+#define IRDMA_CM_FREE_PKT_LO_WATERMARK 2
+#define IRDMA_CM_DEFAULT_MSS 536
+#define IRDMA_CM_DEFAULT_MPA_VER 2
+#define IRDMA_CM_DEFAULT_SEQ 0x159bf75f
+#define IRDMA_CM_DEFAULT_LOCAL_ID 0x3b47
+#define IRDMA_CM_DEFAULT_SEQ2 0x18ed5740
+#define IRDMA_CM_DEFAULT_LOCAL_ID2 0xb807
+#define IRDMA_MAX_CM_BUF (IRDMA_MAX_IETF_SIZE + IETF_MAX_PRIV_DATA_LEN)
+
+enum ietf_mpa_flags {
+ IETF_MPA_FLAGS_REJECT = 0x20,
+ IETF_MPA_FLAGS_CRC = 0x40,
+ IETF_MPA_FLAGS_MARKERS = 0x80,
+};
+
+enum irdma_timer_type {
+ IRDMA_TIMER_TYPE_SEND,
+ IRDMA_TIMER_TYPE_RECV,
+ IRDMA_TIMER_NODE_CLEANUP,
+ IRDMA_TIMER_TYPE_CLOSE,
+};
+
+enum option_nums {
+ OPTION_NUM_EOL,
+ OPTION_NUM_NONE,
+ OPTION_NUM_MSS,
+ OPTION_NUM_WINDOW_SCALE,
+ OPTION_NUM_SACK_PERM,
+ OPTION_NUM_SACK,
+ OPTION_NUM_WRITE0 = 0xbc,
+};
+
+/* cm node transition states */
+enum irdma_cm_node_state {
+ IRDMA_CM_STATE_UNKNOWN,
+ IRDMA_CM_STATE_INITED,
+ IRDMA_CM_STATE_LISTENING,
+ IRDMA_CM_STATE_SYN_RCVD,
+ IRDMA_CM_STATE_SYN_SENT,
+ IRDMA_CM_STATE_ONE_SIDE_ESTABLISHED,
+ IRDMA_CM_STATE_ESTABLISHED,
+ IRDMA_CM_STATE_ACCEPTING,
+ IRDMA_CM_STATE_MPAREQ_SENT,
+ IRDMA_CM_STATE_MPAREQ_RCVD,
+ IRDMA_CM_STATE_MPAREJ_RCVD,
+ IRDMA_CM_STATE_OFFLOADED,
+ IRDMA_CM_STATE_FIN_WAIT1,
+ IRDMA_CM_STATE_FIN_WAIT2,
+ IRDMA_CM_STATE_CLOSE_WAIT,
+ IRDMA_CM_STATE_TIME_WAIT,
+ IRDMA_CM_STATE_LAST_ACK,
+ IRDMA_CM_STATE_CLOSING,
+ IRDMA_CM_STATE_LISTENER_DESTROYED,
+ IRDMA_CM_STATE_CLOSED,
+};
+
+enum mpa_frame_ver {
+ IETF_MPA_V1 = 1,
+ IETF_MPA_V2 = 2,
+};
+
+enum mpa_frame_key {
+ MPA_KEY_REQUEST,
+ MPA_KEY_REPLY,
+};
+
+enum send_rdma0 {
+ SEND_RDMA_READ_ZERO = 1,
+ SEND_RDMA_WRITE_ZERO = 2,
+};
+
+enum irdma_tcpip_pkt_type {
+ IRDMA_PKT_TYPE_UNKNOWN,
+ IRDMA_PKT_TYPE_SYN,
+ IRDMA_PKT_TYPE_SYNACK,
+ IRDMA_PKT_TYPE_ACK,
+ IRDMA_PKT_TYPE_FIN,
+ IRDMA_PKT_TYPE_RST,
+};
+
+enum irdma_cm_listener_state {
+ IRDMA_CM_LISTENER_PASSIVE_STATE = 1,
+ IRDMA_CM_LISTENER_ACTIVE_STATE = 2,
+ IRDMA_CM_LISTENER_EITHER_STATE = 3,
+};
+
+/* CM event codes */
+enum irdma_cm_event_type {
+ IRDMA_CM_EVENT_UNKNOWN,
+ IRDMA_CM_EVENT_ESTABLISHED,
+ IRDMA_CM_EVENT_MPA_REQ,
+ IRDMA_CM_EVENT_MPA_CONNECT,
+ IRDMA_CM_EVENT_MPA_ACCEPT,
+ IRDMA_CM_EVENT_MPA_REJECT,
+ IRDMA_CM_EVENT_MPA_ESTABLISHED,
+ IRDMA_CM_EVENT_CONNECTED,
+ IRDMA_CM_EVENT_RESET,
+ IRDMA_CM_EVENT_ABORTED,
+};
+
+struct irdma_bth { /* Base Trasnport Header */
+ u8 opcode;
+ u8 flags;
+ __be16 pkey;
+ __be32 qpn;
+ __be32 apsn;
+};
+
+struct ietf_mpa_v1 {
+ u8 key[IETF_MPA_KEY_SIZE];
+ u8 flags;
+ u8 rev;
+ __be16 priv_data_len;
+ u8 priv_data[0];
+};
+
+struct ietf_rtr_msg {
+ __be16 ctrl_ird;
+ __be16 ctrl_ord;
+};
+
+struct ietf_mpa_v2 {
+ u8 key[IETF_MPA_KEY_SIZE];
+ u8 flags;
+ u8 rev;
+ __be16 priv_data_len;
+ struct ietf_rtr_msg rtr_msg;
+ u8 priv_data[0];
+};
+
+struct option_base {
+ u8 optionnum;
+ u8 len;
+};
+
+struct option_mss {
+ u8 optionnum;
+ u8 len;
+ __be16 mss;
+};
+
+struct option_windowscale {
+ u8 optionnum;
+ u8 len;
+ u8 shiftcount;
+};
+
+union all_known_options {
+ char eol;
+ struct option_base base;
+ struct option_mss mss;
+ struct option_windowscale windowscale;
+};
+
+struct irdma_timer_entry {
+ struct list_head list;
+ unsigned long timetosend; /* jiffies */
+ struct irdma_puda_buf *sqbuf;
+ u32 type;
+ u32 retrycount;
+ u32 retranscount;
+ u32 context;
+ u32 send_retrans;
+ int close_when_complete;
+};
+
+/* CM context params */
+struct irdma_cm_tcp_context {
+ u8 client;
+ u32 loc_seq_num;
+ u32 loc_ack_num;
+ u32 rem_ack_num;
+ u32 rcv_nxt;
+ u32 loc_id;
+ u32 rem_id;
+ u32 snd_wnd;
+ u32 max_snd_wnd;
+ u32 rcv_wnd;
+ u32 mss;
+ u8 snd_wscale;
+ u8 rcv_wscale;
+ struct timeval sent_ts;
+};
+
+struct irdma_cm_listener {
+ struct list_head list;
+ struct iw_cm_id *cm_id;
+ struct irdma_cm_core *cm_core;
+ struct irdma_device *iwdev;
+ struct list_head child_listen_list;
+ enum irdma_cm_listener_state listener_state;
+ atomic_t ref_count;
+ atomic_t pend_accepts_cnt;
+ u32 loc_addr[4];
+ u32 reused_node;
+ int backlog;
+ u16 loc_port;
+ u16 vlan_id;
+ u8 loc_mac[ETH_ALEN];
+ u8 user_pri;
+ u8 tos;
+ bool qhash_set;
+ bool ipv4;
+};
+
+struct irdma_kmem_info {
+ void *addr;
+ u32 size;
+};
+
+struct irdma_cm_node {
+ struct irdma_qp *iwqp;
+ struct irdma_device *iwdev;
+ struct irdma_sc_dev *dev;
+ struct irdma_cm_tcp_context tcp_cntxt;
+ struct irdma_cm_core *cm_core;
+ struct irdma_timer_entry *send_entry;
+ struct irdma_timer_entry *close_entry;
+ struct irdma_cm_listener *listener;
+ struct list_head timer_entry;
+ struct list_head reset_entry;
+ struct list_head teardown_entry;
+ struct irdma_kmem_info pdata;
+ struct irdma_sc_ah *ah;
+ union {
+ struct ietf_mpa_v1 mpa_frame;
+ struct ietf_mpa_v2 mpa_v2_frame;
+ };
+ struct irdma_kmem_info mpa_hdr;
+ struct iw_cm_id *cm_id;
+ struct list_head list;
+ spinlock_t retrans_list_lock; /* protect CM node rexmit updates*/
+ atomic_t passive_state;
+ atomic_t ref_count;
+ enum irdma_cm_node_state state;
+ enum send_rdma0 send_rdma0_op;
+ enum mpa_frame_ver mpa_frame_rev;
+ u32 loc_addr[4], rem_addr[4];
+ u16 loc_port, rem_port;
+ int accelerated;
+ int apbvt_set;
+ int accept_pend;
+ u16 vlan_id;
+ u16 ird_size;
+ u16 ord_size;
+ u16 mpav2_ird_ord;
+ u16 lsmm_size;
+ u8 pdata_buf[IETF_MAX_PRIV_DATA_LEN];
+ u8 loc_mac[ETH_ALEN];
+ u8 rem_mac[ETH_ALEN];
+ u8 user_pri;
+ u8 tos;
+ bool ack_rcvd;
+ bool qhash_set;
+ bool ipv4;
+ bool snd_mark_en;
+};
+
+/* Used by internal CM APIs to pass CM information*/
+struct irdma_cm_info {
+ struct iw_cm_id *cm_id;
+ u16 loc_port;
+ u16 rem_port;
+ u32 loc_addr[4];
+ u32 rem_addr[4];
+ u32 qh_qpid;
+ u16 vlan_id;
+ int backlog;
+ u8 user_pri;
+ u8 tos;
+ bool ipv4;
+};
+
+struct irdma_cm_event {
+ enum irdma_cm_event_type type;
+ struct irdma_cm_info cm_info;
+ struct work_struct event_work;
+ struct irdma_cm_node *cm_node;
+};
+
+struct irdma_cm_core {
+ struct irdma_device *iwdev;
+ struct irdma_sc_dev *dev;
+ struct list_head listen_nodes;
+ struct list_head accelerated_list;
+ struct list_head non_accelerated_list;
+ struct timer_list tcp_timer;
+ struct workqueue_struct *event_wq;
+ struct workqueue_struct *disconn_wq;
+ spinlock_t ht_lock; /* protect CM node (active side) list */
+ spinlock_t listen_list_lock; /* protect listener list */
+ spinlock_t apbvt_lock; /*serialize apbvt add/del entries*/
+ unsigned long ports_in_use[BITS_TO_LONGS(MAX_PORTS)];
+ u64 stats_nodes_created;
+ u64 stats_nodes_destroyed;
+ u64 stats_listen_created;
+ u64 stats_listen_destroyed;
+ u64 stats_listen_nodes_created;
+ u64 stats_listen_nodes_destroyed;
+ u64 stats_lpbs;
+ u64 stats_accepts;
+ u64 stats_rejects;
+ u64 stats_connect_errs;
+ u64 stats_passive_errs;
+ u64 stats_pkt_retrans;
+ u64 stats_backlog_drops;
+ struct irdma_puda_buf * (*form_cm_frame)(struct irdma_cm_node *cm_node,
+ struct irdma_kmem_info *options,
+ struct irdma_kmem_info *hdr,
+ struct irdma_kmem_info *pdata,
+ u8 flags);
+ int (*cm_create_ah)(struct irdma_cm_node *cm_node, bool wait);
+ void (*cm_free_ah)(struct irdma_cm_node *cm_node);
+};
+
+int irdma_schedule_cm_timer(struct irdma_cm_node *cm_node,
+ struct irdma_puda_buf *sqbuf,
+ enum irdma_timer_type type,
+ int send_retrans,
+ int close_when_complete);
+int irdma_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param);
+int irdma_reject(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len);
+int irdma_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param);
+int irdma_create_listen(struct iw_cm_id *cm_id, int backlog);
+int irdma_destroy_listen(struct iw_cm_id *cm_id);
+void irdma_cm_teardown_connections(struct irdma_device *iwdev,
+ u32 *ipaddr,
+ struct irdma_cm_info *nfo,
+ bool disconnect_all);
+int irdma_cm_start(struct irdma_device *dev);
+int irdma_cm_stop(struct irdma_device *dev);
+bool irdma_ipv4_is_lpb(u32 loc_addr, u32 rem_addr);
+bool irdma_ipv6_is_lpb(u32 *loc_addr, u32 *rem_addr);
+int irdma_arp_table(struct irdma_pci_f *rf,
+ u32 *ip_addr,
+ bool ipv4,
+ u8 *mac_addr,
+ u32 action);
+int irdma_add_arp(struct irdma_pci_f *rf, u32 *ip, bool ipv4, u8 *mac);
+void irdma_if_notify(struct irdma_device *iwdev,
+ struct net_device *netdev,
+ u32 *ipaddr,
+ bool ipv4,
+ bool ifup);
+bool irdma_port_in_use(struct irdma_cm_core *cm_core, u16 port);
+void irdma_send_ack(struct irdma_cm_node *cm_node);
+void irdma_lpb_nop(struct irdma_sc_qp *qp);
+u8 irdma_derive_hw_ird_setting(u16 cm_ird);
+#endif /* IRDMA_CM_H */
--
1.8.3.1
^ permalink raw reply related
* [RFC 12/19] RDMA/irdma: Implement device supported verb APIs
From: Shiraz Saleem @ 2019-02-12 21:43 UTC (permalink / raw)
To: dledford, jgg, davem
Cc: linux-rdma, netdev, mustafa.ismail, jeffrey.t.kirsher,
Shiraz Saleem
In-Reply-To: <20190212214402.23284-1-shiraz.saleem@intel.com>
From: Mustafa Ismail <mustafa.ismail@intel.com>
Implement device supported verb APIs. The supported APIs
vary based on the underlying transport the ibdev is
registered as (i.e. iWARP or RoCEv2).
Signed-off-by: Mustafa Ismail <mustafa.ismail@intel.com>
Signed-off-by: Shiraz Saleem <shiraz.saleem@intel.com>
---
drivers/infiniband/hw/irdma/verbs.c | 4267 ++++++++++++++++++++++++++++++
drivers/infiniband/hw/irdma/verbs.h | 191 ++
include/uapi/rdma/rdma_user_ioctl_cmds.h | 1 +
3 files changed, 4459 insertions(+)
create mode 100644 drivers/infiniband/hw/irdma/verbs.c
create mode 100644 drivers/infiniband/hw/irdma/verbs.h
diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c
new file mode 100644
index 0000000..af843b9
--- /dev/null
+++ b/drivers/infiniband/hw/irdma/verbs.c
@@ -0,0 +1,4267 @@
+// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
+/* Copyright (c) 2019, Intel Corporation. */
+
+#include <linux/random.h>
+#include <linux/highmem.h>
+#include <linux/time.h>
+#include <linux/irq.h>
+#include <asm/byteorder.h>
+#include <net/ip.h>
+#include <rdma/ib_verbs.h>
+#include <rdma/iw_cm.h>
+#include <rdma/ib_user_verbs.h>
+#include <rdma/ib_umem.h>
+#include <rdma/ib_cache.h>
+#include "main.h"
+
+/**
+ * irdma_query_device - get device attributes
+ * @ibdev: device pointer from stack
+ * @props: returning device attributes
+ * @udata: user data
+ */
+static int irdma_query_device(struct ib_device *ibdev,
+ struct ib_device_attr *props,
+ struct ib_udata *udata)
+{
+ struct irdma_device *iwdev = to_iwdev(ibdev);
+ struct irdma_pci_f *rf = iwdev->rf;
+ struct pci_dev *pdev = iwdev->rf->pdev;
+
+ if (udata->inlen || udata->outlen)
+ return -EINVAL;
+
+ memset(props, 0, sizeof(*props));
+ ether_addr_copy((u8 *)&props->sys_image_guid, iwdev->netdev->dev_addr);
+ props->fw_ver = (u64)FW_MAJOR_VER(&rf->sc_dev) << 32 |
+ FW_MINOR_VER(&rf->sc_dev) << 16;
+ props->device_cap_flags = iwdev->device_cap_flags;
+ props->vendor_id = pdev->vendor;
+ props->vendor_part_id = pdev->device;
+ props->hw_ver = (u32)rf->sc_dev.pci_rev;
+ props->max_mr_size = rf->sc_dev.hw_attrs.max_hw_outbound_msg_size;
+ props->max_qp = rf->max_qp - rf->used_qps;
+ props->max_qp_wr = rf->sc_dev.hw_attrs.max_qp_wr;
+ props->max_send_sge = rf->sc_dev.hw_attrs.max_hw_wq_frags;
+ props->max_recv_sge = rf->sc_dev.hw_attrs.max_hw_wq_frags;
+ props->max_cq = rf->max_cq - rf->used_cqs;
+ props->max_cqe = rf->max_cqe;
+ props->max_mr = rf->max_mr - rf->used_mrs;
+ props->max_mw = props->max_mr;
+ props->max_pd = rf->max_pd - rf->used_pds;
+ props->max_sge_rd = rf->sc_dev.hw_attrs.max_hw_read_sges;
+ props->max_qp_rd_atom = rf->sc_dev.hw_attrs.max_hw_ird;
+ props->max_qp_init_rd_atom = props->max_qp_rd_atom;
+ props->atomic_cap = IB_ATOMIC_NONE;
+ props->max_map_per_fmr = 1;
+ props->max_ah = rf->max_ah;
+ props->max_mcast_grp = rf->max_mcg;
+ props->max_mcast_qp_attach = IRDMA_MAX_MGS_PER_CTX;
+ props->max_total_mcast_qp_attach = rf->max_qp * IRDMA_MAX_MGS_PER_CTX;
+ props->max_fast_reg_page_list_len = IRDMA_MAX_PAGES_PER_FMR;
+
+ return 0;
+}
+
+/**
+ * irdma_get_eth_speed_and_width - Get IB port speed and width from netdev speed
+ * @link speed: netdev phy link speed
+ * @active_speed: IB port speed
+ * @active width: IB port width
+ */
+static void irdma_get_eth_speed_and_width(u32 link_speed,
+ u8 *active_speed,
+ u8 *active_width)
+{
+ if (link_speed <= SPEED_1000) {
+ *active_width = IB_WIDTH_1X;
+ *active_speed = IB_SPEED_SDR;
+ } else if (link_speed <= SPEED_10000) {
+ *active_width = IB_WIDTH_1X;
+ *active_speed = IB_SPEED_FDR10;
+ } else if (link_speed <= SPEED_20000) {
+ *active_width = IB_WIDTH_4X;
+ *active_speed = IB_SPEED_DDR;
+ } else if (link_speed <= SPEED_25000) {
+ *active_width = IB_WIDTH_1X;
+ *active_speed = IB_SPEED_EDR;
+ } else if (link_speed <= SPEED_40000) {
+ *active_width = IB_WIDTH_4X;
+ *active_speed = IB_SPEED_FDR10;
+ } else {
+ *active_width = IB_WIDTH_4X;
+ *active_speed = IB_SPEED_EDR;
+ }
+}
+
+/**
+ * irdma_query_port - get port attributes
+ * @ibdev: device pointer from stack
+ * @port: port number for query
+ * @props: returning device attributes
+ */
+static int irdma_query_port(struct ib_device *ibdev,
+ u8 port,
+ struct ib_port_attr *props)
+{
+ struct irdma_device *iwdev = to_iwdev(ibdev);
+ struct net_device *netdev = iwdev->netdev;
+
+ memset(props, 0, sizeof(*props));
+ props->max_mtu = IB_MTU_4096;
+ props->active_mtu = ib_mtu_int_to_enum(netdev->mtu);
+
+ props->lid = 1;
+ props->lmc = 0;
+ props->sm_lid = 0;
+ props->sm_sl = 0;
+ if (netif_carrier_ok(netdev) && netif_running(netdev)) {
+ props->state = IB_PORT_ACTIVE;
+ props->phys_state = 5;
+ } else {
+ props->state = IB_PORT_DOWN;
+ props->phys_state = 3;
+ }
+ irdma_get_eth_speed_and_width(SPEED_100000, &props->active_speed,
+ &props->active_width);
+
+ if (rdma_protocol_roce(ibdev, 1)) {
+ props->gid_tbl_len = 32;
+ props->ip_gids = true;
+ } else {
+ props->gid_tbl_len = 1;
+ }
+ props->pkey_tbl_len = 1;
+ props->qkey_viol_cntr = 0;
+ props->port_cap_flags |= IB_PORT_CM_SUP | IB_PORT_REINIT_SUP;
+ props->max_msg_sz = iwdev->rf->sc_dev.hw_attrs.max_hw_outbound_msg_size;
+
+ return 0;
+}
+
+/**
+ * irdma_alloc_ucontext - Allocate the user context data structure
+ * @ibdev: device pointer from stack
+ * @udata: user data
+ *
+ * This keeps track of all objects associated with a particular
+ * user-mode client.
+ */
+static struct ib_ucontext *irdma_alloc_ucontext(struct ib_device *ibdev,
+ struct ib_udata *udata)
+{
+ struct irdma_device *iwdev = to_iwdev(ibdev);
+ struct irdma_alloc_ucontext_req req;
+ struct irdma_alloc_ucontext_resp uresp = {};
+ struct i40iw_alloc_ucontext_resp uresp_gen1 = {};
+ struct irdma_ucontext *ucontext;
+ int err;
+
+ if (ib_copy_from_udata(&req, udata, sizeof(req)))
+ return ERR_PTR(-EINVAL);
+
+ if (req.userspace_ver > IRDMA_ABI_VER) {
+ err = -EINVAL;
+ goto ver_error;
+ }
+
+ ucontext = kzalloc(sizeof(*ucontext), GFP_KERNEL);
+ if (!ucontext)
+ return ERR_PTR(-ENOMEM);
+
+ ucontext->iwdev = iwdev;
+ ucontext->abi_ver = req.userspace_ver;
+
+ if (iwdev->rf->sc_dev.hw_attrs.hw_rev == IRDMA_GEN_1) {
+ uresp_gen1.max_qps = iwdev->rf->max_qp;
+ uresp_gen1.max_pds = iwdev->rf->sc_dev.hw_attrs.max_hw_pds;
+ uresp_gen1.wq_size = iwdev->rf->sc_dev.hw_attrs.max_qp_wr * 2;
+ uresp_gen1.kernel_ver = req.userspace_ver;
+ if (ib_copy_to_udata(udata, &uresp_gen1, sizeof(uresp_gen1))) {
+ kfree(ucontext);
+ return ERR_PTR(-EFAULT);
+ }
+ } else {
+ if (req.userspace_ver < IRDMA_ABI_VER) {
+ err = -EINVAL;
+ goto ver_error;
+ }
+ uresp.kernel_ver = req.userspace_ver;
+ uresp.hw_attrs = iwdev->rf->sc_dev.hw_attrs;
+ if (ib_copy_to_udata(udata, &uresp, sizeof(uresp))) {
+ kfree(ucontext);
+ return ERR_PTR(-EFAULT);
+ }
+ }
+
+ INIT_LIST_HEAD(&ucontext->cq_reg_mem_list);
+ spin_lock_init(&ucontext->cq_reg_mem_list_lock);
+ INIT_LIST_HEAD(&ucontext->qp_reg_mem_list);
+ spin_lock_init(&ucontext->qp_reg_mem_list_lock);
+ INIT_LIST_HEAD(&ucontext->vma_list);
+ mutex_init(&ucontext->vma_list_mutex);
+
+ return &ucontext->ibucontext;
+
+ver_error:
+ irdma_dev_err(&iwdev->rf->sc_dev,
+ "Invalid userspace driver version detected. Detected version %d, should be %d\n",
+ req.userspace_ver, IRDMA_ABI_VER);
+ uresp.kernel_ver = IRDMA_ABI_VER;
+ return ERR_PTR(-EINVAL);
+}
+
+/**
+ * irdma_dealloc_ucontext - deallocate the user context data structure
+ * @context: user context created during alloc
+ */
+static int irdma_dealloc_ucontext(struct ib_ucontext *context)
+{
+ struct irdma_ucontext *ucontext = to_ucontext(context);
+ unsigned long flags;
+
+ spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
+ if (!list_empty(&ucontext->cq_reg_mem_list)) {
+ spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags);
+ return -EBUSY;
+ }
+ spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags);
+
+ spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags);
+ if (!list_empty(&ucontext->qp_reg_mem_list)) {
+ spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags);
+ return -EBUSY;
+ }
+ spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags);
+ kfree(ucontext);
+
+ return 0;
+}
+
+/**
+ * irdma_vma_open -
+ * @vma: User VMA
+ */
+static void irdma_vma_open(struct vm_area_struct *vma)
+{
+ vma->vm_ops = NULL;
+}
+
+/**
+ * irdma_vma_close - Remove vma data from vma list
+ * @vma: User VMA
+ */
+static void irdma_vma_close(struct vm_area_struct *vma)
+{
+ struct irdma_vma_data *vma_data;
+
+ vma_data = (struct irdma_vma_data *)vma->vm_private_data;
+ vma->vm_private_data = NULL;
+ vma_data->vma = NULL;
+ mutex_lock(vma_data->vma_list_mutex);
+ list_del(&vma_data->list);
+ mutex_unlock(vma_data->vma_list_mutex);
+ kfree(vma_data);
+}
+
+static const struct vm_operations_struct irdma_vm_ops = {
+ .open = irdma_vma_open,
+ .close = irdma_vma_close
+};
+
+/**
+ * irdma_set_vma_data - Save vma data in context list
+ * @vma: User VMA
+ * @context: ib user context
+ */
+static int irdma_set_vma_data(struct vm_area_struct *vma,
+ struct irdma_ucontext *context)
+{
+ struct list_head *vma_head = &context->vma_list;
+ struct irdma_vma_data *vma_entry;
+
+ vma_entry = kzalloc(sizeof(*vma_entry), GFP_KERNEL);
+ if (!vma_entry)
+ return -ENOMEM;
+
+ vma->vm_private_data = vma_entry;
+ vma->vm_ops = &irdma_vm_ops;
+
+ vma_entry->vma = vma;
+ vma_entry->vma_list_mutex = &context->vma_list_mutex;
+
+ mutex_lock(&context->vma_list_mutex);
+ list_add(&vma_entry->list, vma_head);
+ mutex_unlock(&context->vma_list_mutex);
+
+ return 0;
+}
+
+/**
+ * irdma_disassociate_ucontext - Disassociate user context
+ * @context: ib user context
+ */
+static void irdma_disassociate_ucontext(struct ib_ucontext *context)
+{
+ struct irdma_ucontext *ucontext = to_ucontext(context);
+
+ struct irdma_vma_data *vma_data, *n;
+ struct vm_area_struct *vma;
+
+ irdma_dev_info(&ucontext->iwdev->rf->sc_dev, "called\n");
+ mutex_lock(&ucontext->vma_list_mutex);
+ list_for_each_entry_safe(vma_data, n, &ucontext->vma_list, list) {
+ vma = vma_data->vma;
+ zap_vma_ptes(vma, vma->vm_start, PAGE_SIZE);
+
+ vma->vm_flags &= ~(VM_SHARED | VM_MAYSHARE);
+ vma->vm_ops = NULL;
+ list_del(&vma_data->list);
+ kfree(vma_data);
+ }
+ mutex_unlock(&ucontext->vma_list_mutex);
+}
+
+/**
+ * irdma_mmap - user memory map
+ * @context: context created during alloc
+ * @vma: kernel info for user memory map
+ */
+static int irdma_mmap(struct ib_ucontext *context,
+ struct vm_area_struct *vma)
+{
+ struct irdma_ucontext *ucontext;
+ u64 db_addr_offset;
+ u64 push_offset;
+
+ ucontext = to_ucontext(context);
+ db_addr_offset = ucontext->iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET];
+ if (ucontext->iwdev->rf->sc_dev.is_pf) {
+ push_offset = IRDMA_PUSH_OFFSET;
+ if (vma->vm_pgoff)
+ vma->vm_pgoff += IRDMA_PF_FIRST_PUSH_PAGE_INDEX - 1;
+ } else {
+ push_offset = IRDMA_VF_PUSH_OFFSET;
+ if (vma->vm_pgoff)
+ vma->vm_pgoff += IRDMA_VF_FIRST_PUSH_PAGE_INDEX - 1;
+ }
+
+ vma->vm_pgoff += db_addr_offset >> PAGE_SHIFT;
+ if (vma->vm_pgoff == (db_addr_offset >> PAGE_SHIFT)) {
+ vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+ vma->vm_private_data = ucontext;
+ } else {
+ if ((vma->vm_pgoff - (push_offset >> PAGE_SHIFT)) % 2)
+ vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+ else
+ vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
+ }
+
+ if (io_remap_pfn_range(vma,
+ vma->vm_start,
+ vma->vm_pgoff + (pci_resource_start(ucontext->iwdev->rf->pdev, 0)
+ >> PAGE_SHIFT),
+ PAGE_SIZE,
+ vma->vm_page_prot))
+ return -EAGAIN;
+
+ return irdma_set_vma_data(vma, ucontext);
+}
+
+/**
+ * irdma_alloc_push_page - allocate a push page for qp
+ * @rf: RDMA PCI function
+ * @qp: hardware control qp
+ */
+static void irdma_alloc_push_page(struct irdma_pci_f *rf,
+ struct irdma_sc_qp *qp)
+{
+ struct irdma_cqp_request *cqp_request;
+ struct cqp_cmds_info *cqp_info;
+ enum irdma_status_code status;
+
+ if (qp->push_idx != IRDMA_INVALID_PUSH_PAGE_INDEX)
+ return;
+
+ cqp_request = irdma_get_cqp_request(&rf->cqp, true);
+ if (!cqp_request)
+ return;
+
+ atomic_inc(&cqp_request->refcount);
+ cqp_info = &cqp_request->info;
+ cqp_info->cqp_cmd = IRDMA_OP_MANAGE_PUSH_PAGE;
+ cqp_info->post_sq = 1;
+ cqp_info->in.u.manage_push_page.info.qs_handle = qp->qs_handle;
+ cqp_info->in.u.manage_push_page.info.free_page = 0;
+ cqp_info->in.u.manage_push_page.cqp = &rf->cqp.sc_cqp;
+ cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request;
+
+ status = irdma_handle_cqp_op(rf, cqp_request);
+ if (!status)
+ qp->push_idx = cqp_request->compl_info.op_ret_val;
+ else
+ irdma_debug(&rf->sc_dev, IRDMA_DEBUG_ERR, "CQP-OP Push page fail");
+
+ irdma_put_cqp_request(&rf->cqp, cqp_request);
+}
+
+/**
+ * irdma_alloc_pd - allocate protection domain
+ * @ibdev: device pointer from stack
+ * @context: user context created during alloc
+ * @udata: user data
+ */
+static struct ib_pd *irdma_alloc_pd(struct ib_device *ibdev,
+ struct ib_ucontext *context,
+ struct ib_udata *udata)
+{
+ struct irdma_pd *iwpd;
+ struct irdma_device *iwdev = to_iwdev(ibdev);
+ struct irdma_sc_dev *dev = &iwdev->rf->sc_dev;
+ struct irdma_pci_f *rf = iwdev->rf;
+ struct irdma_alloc_pd_resp uresp = {};
+ struct irdma_sc_pd *sc_pd;
+ struct irdma_ucontext *ucontext;
+ u32 pd_id = 0;
+ int err;
+
+ if (iwdev->closing)
+ return ERR_PTR(-ENODEV);
+
+ err = irdma_alloc_rsrc(rf, rf->allocated_pds, rf->max_pd, &pd_id,
+ &rf->next_pd);
+ if (err)
+ return ERR_PTR(err);
+
+ iwpd = kzalloc(sizeof(*iwpd), GFP_KERNEL);
+ if (!iwpd) {
+ err = -ENOMEM;
+ goto free_res;
+ }
+
+ sc_pd = &iwpd->sc_pd;
+ if (context) {
+ ucontext = to_ucontext(context);
+ dev->iw_pd_ops->pd_init(dev, sc_pd, pd_id, ucontext->abi_ver);
+ uresp.pd_id = pd_id;
+ if (ib_copy_to_udata(udata, &uresp, sizeof(uresp))) {
+ err = -EFAULT;
+ goto error;
+ }
+ } else {
+ dev->iw_pd_ops->pd_init(dev, sc_pd, pd_id, -1);
+ }
+
+ irdma_add_pdusecount(iwpd);
+ return &iwpd->ibpd;
+
+error:
+ kfree(iwpd);
+free_res:
+ irdma_free_rsrc(rf, rf->allocated_pds, pd_id);
+
+ return ERR_PTR(err);
+}
+
+/**
+ * irdma_dealloc_pd - deallocate pd
+ * @ibpd: ptr of pd to be deallocated
+ */
+static int irdma_dealloc_pd(struct ib_pd *ibpd)
+{
+ struct irdma_pd *iwpd = to_iwpd(ibpd);
+ struct irdma_device *iwdev = to_iwdev(ibpd->device);
+
+ irdma_rem_pdusecount(iwpd, iwdev);
+
+ return 0;
+}
+
+/**
+ * irdma_get_pbl - Retrieve pbl from a list given a virtual
+ * address
+ * @va: user virtual address
+ * @pbl_list: pbl list to search in (QP's or CQ's)
+ */
+static struct irdma_pbl *irdma_get_pbl(unsigned long va,
+ struct list_head *pbl_list)
+{
+ struct irdma_pbl *iwpbl;
+
+ list_for_each_entry(iwpbl, pbl_list, list) {
+ if (iwpbl->user_base == va) {
+ list_del(&iwpbl->list);
+ iwpbl->on_list = false;
+ return iwpbl;
+ }
+ }
+
+ return NULL;
+}
+
+/**
+ * irdma_clean_cqes - clean cq entries for qp
+ * @iwqp: qp ptr (user or kernel)
+ * @iwcq: cq ptr
+ */
+static void irdma_clean_cqes(struct irdma_qp *iwqp,
+ struct irdma_cq *iwcq)
+{
+ struct irdma_cq_uk *ukcq = &iwcq->sc_cq.cq_uk;
+
+ ukcq->ops.iw_cq_clean(&iwqp->sc_qp.qp_uk, ukcq);
+}
+
+/**
+ * irdma_destroy_qp - destroy qp
+ * @ibqp: qp's ib pointer also to get to device's qp address
+ */
+static int irdma_destroy_qp(struct ib_qp *ibqp)
+{
+ struct irdma_qp *iwqp = to_iwqp(ibqp);
+
+ if (atomic_read(&iwqp->mcast_ref_cnt))
+ return -EBUSY;
+
+ iwqp->destroyed = 1;
+ if (iwqp->ibqp_state >= IB_QPS_INIT && iwqp->ibqp_state < IB_QPS_RTS)
+ irdma_next_iw_state(iwqp, IRDMA_QP_STATE_ERROR, 0, 0, 0);
+
+ if (!iwqp->user_mode) {
+ if (iwqp->iwscq) {
+ irdma_clean_cqes(iwqp, iwqp->iwscq);
+ if (iwqp->iwrcq != iwqp->iwscq)
+ irdma_clean_cqes(iwqp, iwqp->iwrcq);
+ }
+ }
+
+ irdma_rem_ref(&iwqp->ibqp);
+
+ return 0;
+}
+
+/**
+ * irdma_setup_virt_qp - setup for allocation of virtual qp
+ * @dev: iwarp device
+ * @qp: qp ptr
+ * @init_info: initialize info to return
+ */
+static int irdma_setup_virt_qp(struct irdma_device *iwdev,
+ struct irdma_qp *iwqp,
+ struct irdma_qp_init_info *init_info)
+{
+ struct irdma_pbl *iwpbl = iwqp->iwpbl;
+ struct irdma_qp_mr *qpmr = &iwpbl->qp_mr;
+
+ iwqp->page = qpmr->sq_page;
+ init_info->shadow_area_pa = qpmr->shadow;
+ if (iwpbl->pbl_allocated) {
+ init_info->virtual_map = true;
+ init_info->sq_pa = qpmr->sq_pbl.idx;
+ init_info->rq_pa = qpmr->rq_pbl.idx;
+ } else {
+ init_info->sq_pa = qpmr->sq_pbl.addr;
+ init_info->rq_pa = qpmr->rq_pbl.addr;
+ }
+
+ return 0;
+}
+
+/**
+ * irdma_setup_kmode_qp - setup initialization for kernel mode qp
+ * @iwdev: iwarp device
+ * @iwqp: qp ptr (user or kernel)
+ * @info: initialize info to return
+ */
+static int irdma_setup_kmode_qp(struct irdma_device *iwdev,
+ struct irdma_qp *iwqp,
+ struct irdma_qp_init_info *info)
+{
+ struct irdma_dma_mem *mem = &iwqp->kqp.dma_mem;
+ u32 sqdepth, rqdepth;
+ u8 sqshift, rqshift;
+ u32 size;
+ enum irdma_status_code status;
+ struct irdma_qp_uk_init_info *ukinfo = &info->qp_uk_init_info;
+
+ irdma_get_wqe_shift(&iwdev->rf->sc_dev.hw_attrs,
+ ukinfo->max_sq_frag_cnt + 1,
+ ukinfo->max_inline_data,
+ &sqshift);
+ status = irdma_get_sqdepth(&iwdev->rf->sc_dev.hw_attrs,
+ ukinfo->sq_size,
+ sqshift,
+ &sqdepth);
+ if (status)
+ return -ENOMEM;
+
+ irdma_get_wqe_shift(&iwdev->rf->sc_dev.hw_attrs,
+ ukinfo->max_rq_frag_cnt,
+ 0,
+ &rqshift);
+ status = irdma_get_rqdepth(&iwdev->rf->sc_dev.hw_attrs,
+ ukinfo->rq_size,
+ rqshift,
+ &rqdepth);
+ if (status)
+ return -ENOMEM;
+
+ size = sqdepth * sizeof(struct irdma_sq_uk_wr_trk_info) +
+ (rqdepth << 3);
+ iwqp->kqp.wrid_mem = kzalloc(size, GFP_KERNEL);
+ ukinfo->sq_wrtrk_array = (struct irdma_sq_uk_wr_trk_info *)
+ iwqp->kqp.wrid_mem;
+ if (!ukinfo->sq_wrtrk_array)
+ return -ENOMEM;
+
+ ukinfo->rq_wrid_array = (u64 *)&ukinfo->sq_wrtrk_array[sqdepth];
+ size = (sqdepth + rqdepth) * IRDMA_QP_WQE_MIN_SIZE;
+ size += (IRDMA_SHADOW_AREA_SIZE << 3);
+
+ status = irdma_allocate_dma_mem(iwdev->rf->sc_dev.hw, mem, size, 256);
+ if (status) {
+ kfree(ukinfo->sq_wrtrk_array);
+ ukinfo->sq_wrtrk_array = NULL;
+ return -ENOMEM;
+ }
+
+ ukinfo->sq = mem->va;
+ info->sq_pa = mem->pa;
+ ukinfo->rq = &ukinfo->sq[sqdepth];
+ info->rq_pa = info->sq_pa + (sqdepth * IRDMA_QP_WQE_MIN_SIZE);
+ ukinfo->shadow_area = ukinfo->rq[rqdepth].elem;
+ info->shadow_area_pa = info->rq_pa + (rqdepth * IRDMA_QP_WQE_MIN_SIZE);
+ ukinfo->sq_size = sqdepth >> sqshift;
+ ukinfo->rq_size = rqdepth >> rqshift;
+ ukinfo->qp_id = iwqp->ibqp.qp_num;
+
+ return 0;
+}
+
+/**
+ * irdma_roce_mtu - set MTU to supported path MTU values
+ * @mtu: MTU
+ */
+static u32 irdma_roce_mtu(u32 mtu)
+{
+ if (mtu > 4096)
+ return 4096;
+ else if (mtu > 2048)
+ return 2048;
+ else if (mtu > 1024)
+ return 1024;
+ else if (mtu > 512)
+ return 512;
+ else
+ return 256;
+}
+
+/**
+ * irdma_create_qp - create qp
+ * @ibpd: ptr of pd
+ * @init_attr: attributes for qp
+ * @udata: user data for create qp
+ */
+static struct ib_qp *irdma_create_qp(struct ib_pd *ibpd,
+ struct ib_qp_init_attr *init_attr,
+ struct ib_udata *udata)
+{
+ struct irdma_pd *iwpd = to_iwpd(ibpd);
+ struct irdma_device *iwdev = to_iwdev(ibpd->device);
+ struct irdma_pci_f *rf = iwdev->rf;
+ struct irdma_cqp *iwcqp = &rf->cqp;
+ struct irdma_qp *iwqp;
+ struct irdma_ucontext *ucontext;
+ struct irdma_create_qp_req req;
+ struct irdma_create_qp_resp uresp = {};
+ struct i40iw_create_qp_resp uresp_gen1 = {};
+ u32 qp_num = 0;
+ void *mem;
+ enum irdma_status_code ret;
+ int err_code = 0;
+ int sq_size;
+ int rq_size;
+ struct irdma_sc_qp *qp;
+ struct irdma_sc_dev *dev = &rf->sc_dev;
+ struct irdma_qp_init_info init_info = {};
+ struct irdma_create_qp_info *qp_info;
+ struct irdma_cqp_request *cqp_request;
+ struct cqp_cmds_info *cqp_info;
+ struct irdma_qp_host_ctx_info *ctx_info;
+ struct irdma_iwarp_offload_info *iwarp_info;
+ struct irdma_roce_offload_info *roce_info;
+ struct irdma_udp_offload_info *udp_info;
+ unsigned long flags;
+
+ if (iwdev->closing)
+ return ERR_PTR(-ENODEV);
+
+ if (init_attr->create_flags)
+ return ERR_PTR(-EINVAL);
+
+ if (init_attr->cap.max_inline_data > dev->hw_attrs.max_hw_inline)
+ init_attr->cap.max_inline_data = dev->hw_attrs.max_hw_inline;
+
+ if (init_attr->cap.max_send_sge > dev->hw_attrs.max_hw_wq_frags)
+ init_attr->cap.max_send_sge = dev->hw_attrs.max_hw_wq_frags;
+
+ if (init_attr->cap.max_recv_sge > dev->hw_attrs.max_hw_wq_frags)
+ init_attr->cap.max_recv_sge = dev->hw_attrs.max_hw_wq_frags;
+
+ sq_size = init_attr->cap.max_send_wr;
+ rq_size = init_attr->cap.max_recv_wr;
+
+ init_info.vsi = &iwdev->vsi;
+ init_info.qp_uk_init_info.hw_attrs = &dev->hw_attrs;
+ init_info.qp_uk_init_info.sq_size = sq_size;
+ init_info.qp_uk_init_info.rq_size = rq_size;
+ init_info.qp_uk_init_info.max_sq_frag_cnt = init_attr->cap.max_send_sge;
+ init_info.qp_uk_init_info.max_rq_frag_cnt = init_attr->cap.max_recv_sge;
+ init_info.qp_uk_init_info.max_inline_data = init_attr->cap.max_inline_data;
+
+ mem = kzalloc(sizeof(*iwqp), GFP_KERNEL);
+ if (!mem)
+ return ERR_PTR(-ENOMEM);
+
+ iwqp = (struct irdma_qp *)mem;
+ iwqp->allocated_buf = mem;
+ qp = &iwqp->sc_qp;
+ qp->back_qp = (void *)iwqp;
+ qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX;
+
+ if (irdma_allocate_dma_mem(dev->hw,
+ &iwqp->q2_ctx_mem,
+ IRDMA_Q2_BUF_SIZE + IRDMA_QP_CTX_SIZE,
+ 256)) {
+ err_code = -ENOMEM;
+ goto error;
+ }
+
+ init_info.q2 = iwqp->q2_ctx_mem.va;
+ init_info.q2_pa = iwqp->q2_ctx_mem.pa;
+ init_info.host_ctx = (void *)init_info.q2 + IRDMA_Q2_BUF_SIZE;
+ init_info.host_ctx_pa = init_info.q2_pa + IRDMA_Q2_BUF_SIZE;
+
+ if (init_attr->qp_type == IB_QPT_GSI && rf->sc_dev.is_pf)
+ qp_num = 1;
+ else
+ err_code = irdma_alloc_rsrc(rf, rf->allocated_qps, rf->max_qp,
+ &qp_num, &rf->next_qp);
+ if (err_code)
+ goto error;
+
+ iwqp->iwdev = iwdev;
+ iwqp->iwpd = iwpd;
+ if (init_attr->qp_type == IB_QPT_GSI && !rf->sc_dev.is_pf)
+ iwqp->ibqp.qp_num = 1;
+ else
+ iwqp->ibqp.qp_num = qp_num;
+
+ qp = &iwqp->sc_qp;
+ iwqp->iwscq = to_iwcq(init_attr->send_cq);
+ iwqp->iwrcq = to_iwcq(init_attr->recv_cq);
+ iwqp->host_ctx.va = init_info.host_ctx;
+ iwqp->host_ctx.pa = init_info.host_ctx_pa;
+ iwqp->host_ctx.size = IRDMA_QP_CTX_SIZE;
+
+ init_info.pd = &iwpd->sc_pd;
+ init_info.qp_uk_init_info.qp_id = iwqp->ibqp.qp_num;
+ if (!rdma_protocol_roce(&iwdev->iwibdev->ibdev, 1))
+ init_info.qp_uk_init_info.first_sq_wq = 1;
+ iwqp->ctx_info.qp_compl_ctx = (uintptr_t)qp;
+ init_waitqueue_head(&iwqp->waitq);
+ init_waitqueue_head(&iwqp->mod_qp_waitq);
+
+ if (rdma_protocol_roce(&iwdev->iwibdev->ibdev, 1)) {
+ if (init_attr->qp_type != IB_QPT_RC &&
+ init_attr->qp_type != IB_QPT_UD &&
+ init_attr->qp_type != IB_QPT_GSI) {
+ err_code = -EINVAL;
+ goto error;
+ }
+ } else {
+ if (init_attr->qp_type != IB_QPT_RC) {
+ err_code = -EINVAL;
+ goto error;
+ }
+ }
+
+ if (iwdev->push_mode)
+ irdma_alloc_push_page(rf, qp);
+
+ if (udata) {
+ err_code = ib_copy_from_udata(&req, udata, sizeof(req));
+ if (err_code) {
+ irdma_debug(dev, IRDMA_DEBUG_ERR,
+ "ib_copy_from_data fail\n");
+ goto error;
+ }
+
+ iwqp->ctx_info.qp_compl_ctx = req.user_compl_ctx;
+ iwqp->user_mode = 1;
+ ucontext = to_ucontext(ibpd->uobject->context);
+ if (req.user_wqe_bufs) {
+ spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags);
+ iwqp->iwpbl = irdma_get_pbl((unsigned long)req.user_wqe_bufs,
+ &ucontext->qp_reg_mem_list);
+ spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags);
+
+ if (!iwqp->iwpbl) {
+ err_code = -ENODATA;
+ irdma_debug(dev, IRDMA_DEBUG_ERR,
+ "no pbl info\n");
+ goto error;
+ }
+ }
+ err_code = irdma_setup_virt_qp(iwdev, iwqp, &init_info);
+ } else {
+ err_code = irdma_setup_kmode_qp(iwdev, iwqp, &init_info);
+ }
+
+ if (err_code) {
+ irdma_debug(dev, IRDMA_DEBUG_ERR, "setup qp failed\n");
+ goto error;
+ }
+
+ if (rdma_protocol_roce(&iwdev->iwibdev->ibdev, 1)) {
+ if (init_attr->qp_type == IB_QPT_RC) {
+ init_info.type = IRDMA_QP_TYPE_ROCE_RC;
+ init_info.qp_uk_init_info.qp_caps =
+ IRDMA_SEND_WITH_IMM | IRDMA_WRITE_WITH_IMM | IRDMA_ROCE;
+ } else {
+ init_info.type = IRDMA_QP_TYPE_ROCE_UD;
+ init_info.qp_uk_init_info.qp_caps = IRDMA_SEND_WITH_IMM | IRDMA_ROCE;
+ }
+ } else {
+ init_info.type = IRDMA_QP_TYPE_IWARP;
+ init_info.qp_uk_init_info.qp_caps = IRDMA_WRITE_WITH_IMM;
+ }
+
+ ret = dev->iw_priv_qp_ops->qp_init(qp, &init_info);
+ if (ret) {
+ err_code = -EPROTO;
+ irdma_debug(dev, IRDMA_DEBUG_ERR, "qp_init fail\n");
+ goto error;
+ }
+
+ ctx_info = &iwqp->ctx_info;
+ if (rdma_protocol_roce(&iwdev->iwibdev->ibdev, 1)) {
+ iwqp->ctx_info.roce_info = &iwqp->roce_info;
+ iwqp->ctx_info.udp_info = &iwqp->udp_info;
+ udp_info = &iwqp->udp_info;
+ udp_info->snd_mss = irdma_roce_mtu(iwdev->vsi.mtu);
+ udp_info->cwnd = 0x400;
+ udp_info->src_port = 0xc000;
+ udp_info->dst_port = ROCE_V2_UDP_DPORT;
+ roce_info = &iwqp->roce_info;
+ ether_addr_copy(roce_info->mac_addr, iwdev->netdev->dev_addr);
+
+ if (init_attr->qp_type == IB_QPT_GSI && !rf->sc_dev.is_pf)
+ roce_info->is_qp1 = true;
+ roce_info->rd_en = true;
+ roce_info->wr_rdresp_en = true;
+ roce_info->dctcp_en = iwdev->dctcp_en;
+ roce_info->ecn_en = iwdev->ecn_en;
+ roce_info->dcqcn_en = iwdev->roce_dcqcn_en;
+ roce_info->timely_en = iwdev->roce_timely_en;
+
+ roce_info->ack_credits = 0x1E;
+ roce_info->ird_size = IRDMA_MAX_ENCODED_IRD_SIZE;
+ roce_info->ord_size = dev->hw_attrs.max_hw_ord;
+
+ if (!iwqp->user_mode) {
+ roce_info->priv_mode_en = true;
+ roce_info->fast_reg_en = true;
+ roce_info->udprivcq_en = true;
+ }
+ roce_info->roce_tver = 0;
+ } else {
+ iwqp->ctx_info.iwarp_info = &iwqp->iwarp_info;
+ iwarp_info = &iwqp->iwarp_info;
+ ether_addr_copy(iwarp_info->mac_addr, iwdev->netdev->dev_addr);
+ iwarp_info->rd_en = true;
+ iwarp_info->wr_rdresp_en = true;
+ iwarp_info->ib_rd_en = true;
+ if (!iwqp->user_mode) {
+ iwarp_info->priv_mode_en = true;
+ iwarp_info->fast_reg_en = true;
+ }
+ iwarp_info->ddp_ver = 1;
+ iwarp_info->rdmap_ver = 1;
+ ctx_info->iwarp_info_valid = true;
+ }
+
+ ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id;
+ ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id;
+ if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX) {
+ ctx_info->push_mode_en = false;
+ } else {
+ ctx_info->push_mode_en = true;
+ ctx_info->push_idx = qp->push_idx;
+ }
+
+ if (rdma_protocol_roce(&iwdev->iwibdev->ibdev, 1)) {
+ ret =
+ dev->iw_priv_qp_ops->qp_setctx_roce(&iwqp->sc_qp,
+ iwqp->host_ctx.va,
+ ctx_info);
+ } else {
+ ret = dev->iw_priv_qp_ops->qp_setctx(&iwqp->sc_qp,
+ iwqp->host_ctx.va,
+ ctx_info);
+ ctx_info->iwarp_info_valid = false;
+ }
+
+ cqp_request = irdma_get_cqp_request(iwcqp, true);
+ if (!cqp_request) {
+ err_code = -ENOMEM;
+ goto error;
+ }
+
+ cqp_info = &cqp_request->info;
+ qp_info = &cqp_request->info.in.u.qp_create.info;
+ memset(qp_info, 0, sizeof(*qp_info));
+ qp_info->mac_valid = true;
+ qp_info->cq_num_valid = true;
+ qp_info->next_iwarp_state = IRDMA_QP_STATE_IDLE;
+
+ cqp_info->cqp_cmd = IRDMA_OP_QP_CREATE;
+ cqp_info->post_sq = 1;
+ cqp_info->in.u.qp_create.qp = qp;
+ cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request;
+ ret = irdma_handle_cqp_op(rf, cqp_request);
+ if (ret) {
+ irdma_debug(dev, IRDMA_DEBUG_ERR, "CQP-OP QP create fail");
+ err_code = -ENOMEM;
+ goto error;
+ }
+
+ irdma_add_ref(&iwqp->ibqp);
+ spin_lock_init(&iwqp->lock);
+ spin_lock_init(&iwqp->sc_qp.pfpdu.lock);
+ iwqp->sig_all = (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ? 1 : 0;
+ rf->qp_table[qp_num] = iwqp;
+ irdma_add_pdusecount(iwqp->iwpd);
+ irdma_add_devusecount(iwdev);
+ if (udata) {
+ if (iwdev->rf->sc_dev.hw_attrs.hw_rev == IRDMA_GEN_1) {
+ uresp_gen1.lsmm = 1;
+ uresp_gen1.actual_sq_size = sq_size;
+ uresp_gen1.actual_rq_size = rq_size;
+ uresp_gen1.qp_id = qp_num;
+ uresp_gen1.push_idx = qp->push_idx;
+ uresp_gen1.lsmm = 1;
+ err_code = ib_copy_to_udata(udata, &uresp_gen1, sizeof(uresp_gen1));
+ } else {
+ if (rdma_protocol_iwarp(&iwdev->iwibdev->ibdev, 1))
+ uresp.lsmm = 1;
+ uresp.actual_sq_size = sq_size;
+ uresp.actual_rq_size = rq_size;
+ uresp.qp_id = qp_num;
+ uresp.push_idx = qp->push_idx;
+ uresp.qp_caps = qp->qp_uk.qp_caps;
+
+ err_code = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
+ }
+ if (err_code) {
+ irdma_debug(dev, IRDMA_DEBUG_ERR, "copy_to_udata failed\n");
+ irdma_destroy_qp(&iwqp->ibqp);
+ return ERR_PTR(err_code);
+ }
+ }
+ init_completion(&iwqp->sq_drained);
+ init_completion(&iwqp->rq_drained);
+ return &iwqp->ibqp;
+
+error:
+ irdma_free_qp_rsrc(iwdev, iwqp, qp_num);
+
+ return ERR_PTR(err_code);
+}
+
+/**
+ * irdma_query - query qp attributes
+ * @ibqp: qp pointer
+ * @attr: attributes pointer
+ * @attr_mask: Not used
+ * @init_attr: qp attributes to return
+ */
+static int irdma_query_qp(struct ib_qp *ibqp,
+ struct ib_qp_attr *attr,
+ int attr_mask,
+ struct ib_qp_init_attr *init_attr)
+{
+ struct irdma_qp *iwqp = to_iwqp(ibqp);
+ struct irdma_sc_qp *qp = &iwqp->sc_qp;
+
+ attr->qp_state = iwqp->ibqp_state;
+ attr->cur_qp_state = iwqp->ibqp_state;
+ attr->qp_access_flags = 0;
+ attr->cap.max_send_wr = qp->qp_uk.sq_size - 1;
+ attr->cap.max_recv_wr = qp->qp_uk.rq_size - 1;
+ attr->cap.max_inline_data = qp->qp_uk.max_inline_data;
+ attr->cap.max_send_sge = qp->qp_uk.max_sq_frag_cnt;
+ attr->cap.max_recv_sge = qp->qp_uk.max_rq_frag_cnt;
+ attr->qkey = iwqp->roce_info.qkey;
+
+ init_attr->event_handler = iwqp->ibqp.event_handler;
+ init_attr->qp_context = iwqp->ibqp.qp_context;
+ init_attr->send_cq = iwqp->ibqp.send_cq;
+ init_attr->recv_cq = iwqp->ibqp.recv_cq;
+ init_attr->srq = iwqp->ibqp.srq;
+ init_attr->cap = attr->cap;
+
+ return 0;
+}
+
+/**
+ * irdma_query_pkey - Query partition key
+ * @ibdev: device pointer from stack
+ * @port: port number
+ * @index: index of pkey
+ * @pkey: pointer to store the pkey
+ */
+static int irdma_query_pkey(struct ib_device *ibdev,
+ u8 port,
+ u16 index,
+ u16 *pkey)
+{
+ struct irdma_device *iwdev = to_iwdev(ibdev);
+
+ if (rdma_protocol_roce(&iwdev->iwibdev->ibdev, 1))
+ *pkey = 0xFFFF;
+ else
+ *pkey = 0;
+
+ return 0;
+}
+
+/**
+ * irdma_modify_qp_roce - modify qp request
+ * @ibqp: qp's pointer for modify
+ * @attr: access attributes
+ * @attr_mask: state mask
+ * @udata: user data
+ */
+int irdma_modify_qp_roce(struct ib_qp *ibqp,
+ struct ib_qp_attr *attr,
+ int attr_mask,
+ struct ib_udata *udata)
+{
+ struct irdma_pd *iwpd = to_iwpd(ibqp->pd);
+ struct irdma_qp *iwqp = to_iwqp(ibqp);
+ struct irdma_device *iwdev = iwqp->iwdev;
+ struct irdma_sc_dev *dev = &iwdev->rf->sc_dev;
+ struct irdma_qp_host_ctx_info *ctx_info;
+ struct irdma_roce_offload_info *roce_info;
+ struct irdma_udp_offload_info *udp_info;
+ struct irdma_modify_qp_info info = {};
+ unsigned long flags;
+ u8 issue_modify_qp = 0;
+ int ret = 0;
+
+ ctx_info = &iwqp->ctx_info;
+ roce_info = &iwqp->roce_info;
+ udp_info = &iwqp->udp_info;
+
+ if (attr_mask & IB_QP_DEST_QPN)
+ roce_info->dest_qp = attr->dest_qp_num;
+
+ if (attr_mask & IB_QP_PKEY_INDEX) {
+ irdma_query_pkey(ibqp->device, 0, attr->pkey_index,
+ &roce_info->p_key);
+ }
+
+ if (attr_mask & IB_QP_QKEY)
+ roce_info->qkey = attr->qkey;
+
+ if (attr_mask & IB_QP_PORT)
+ iwqp->roce_ah.av.attrs.port_num = attr->ah_attr.port_num;
+
+ if (attr_mask & IB_QP_PATH_MTU) {
+ const u16 path_mtu[] = {-1, 256, 512, 1024, 2048, 4096};
+
+ if (attr->path_mtu < IB_MTU_256 ||
+ attr->path_mtu > IB_MTU_4096 ||
+ iwdev->vsi.mtu <= path_mtu[attr->path_mtu]) {
+ irdma_dev_warn(dev, "Invalid MTU %d\n", attr->path_mtu);
+ return -EINVAL;
+ }
+
+ udp_info->snd_mss = path_mtu[attr->path_mtu];
+ }
+
+ if (attr_mask & IB_QP_SQ_PSN) {
+ udp_info->psn_nxt = attr->sq_psn;
+ udp_info->lsn = 0xffff;
+ udp_info->psn_una = attr->sq_psn;
+ udp_info->psn_max = attr->sq_psn;
+ }
+
+ if (attr_mask & IB_QP_RQ_PSN)
+ udp_info->epsn = attr->rq_psn;
+
+ if (attr_mask & IB_QP_RNR_RETRY)
+ udp_info->rnr_nak_thresh = attr->rnr_retry;
+
+ if (attr_mask & IB_QP_RETRY_CNT)
+ udp_info->rexmit_thresh = attr->retry_cnt;
+
+ ctx_info->roce_info->pd_id = iwpd->sc_pd.pd_id;
+
+ if (attr_mask & IB_QP_AV) {
+ struct irdma_av *av = &iwqp->roce_ah.av;
+ const struct ib_gid_attr *sgid_attr;
+ u16 vlan_id = IRDMA_NO_VLAN;
+ u32 local_ip[4];
+
+ memset(&iwqp->roce_ah, 0, sizeof(iwqp->roce_ah));
+ if (attr->ah_attr.ah_flags & IB_AH_GRH) {
+ udp_info->ttl = attr->ah_attr.grh.hop_limit;
+ udp_info->flow_label = attr->ah_attr.grh.flow_label;
+ dev->ws_remove(iwqp->sc_qp.vsi, ctx_info->user_pri);
+ ctx_info->user_pri = attr->ah_attr.grh.traffic_class;
+ if (dev->ws_add(iwqp->sc_qp.vsi, ctx_info->user_pri))
+ return -ENOMEM;
+ irdma_qp_add_qos(&iwqp->sc_qp);
+ }
+ sgid_attr = attr->ah_attr.grh.sgid_attr;
+ if (sgid_attr->ndev) {
+ vlan_id = rdma_vlan_dev_vlan_id(sgid_attr->ndev);
+ ether_addr_copy(ctx_info->roce_info->mac_addr,
+ sgid_attr->ndev->dev_addr);
+ }
+ udp_info->vlan_tag = vlan_id;
+ if (udp_info->vlan_tag != IRDMA_NO_VLAN)
+ udp_info->insert_vlan_tag = true;
+ else
+ udp_info->insert_vlan_tag = false;
+ av->attrs = attr->ah_attr;
+ av->attrs.port_num = attr->ah_attr.port_num;
+ rdma_gid2ip(&av->sgid_addr.saddr, &sgid_attr->gid);
+ rdma_gid2ip(&av->dgid_addr.saddr, &attr->ah_attr.grh.dgid);
+ roce_info->local_qp = ibqp->qp_num;
+ if (av->sgid_addr.saddr.sa_family == AF_INET6) {
+ __be32 *daddr =
+ av->dgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32;
+ __be32 *saddr =
+ av->sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32;
+
+ irdma_copy_ip_ntohl(&udp_info->dest_ip_addr0, daddr);
+ irdma_copy_ip_ntohl(&udp_info->local_ipaddr0, saddr);
+
+ udp_info->ipv4 = false;
+ irdma_copy_ip_ntohl(local_ip, daddr);
+
+ udp_info->arp_idx = irdma_arp_table(iwdev->rf,
+ &local_ip[0],
+ false, NULL,
+ IRDMA_ARP_RESOLVE);
+ } else {
+ __be32 saddr = av->sgid_addr.saddr_in.sin_addr.s_addr;
+ __be32 daddr = av->dgid_addr.saddr_in.sin_addr.s_addr;
+
+ local_ip[0] = ntohl(daddr);
+
+ udp_info->ipv4 = true;
+ udp_info->dest_ip_addr0 = 0;
+ udp_info->dest_ip_addr1 = 0;
+ udp_info->dest_ip_addr2 = 0;
+ udp_info->dest_ip_addr3 = local_ip[0];
+
+ udp_info->local_ipaddr0 = 0;
+ udp_info->local_ipaddr1 = 0;
+ udp_info->local_ipaddr2 = 0;
+ udp_info->local_ipaddr3 = ntohl(saddr);
+ }
+ udp_info->arp_idx =
+ irdma_add_arp(iwdev->rf, local_ip, udp_info->ipv4,
+ attr->ah_attr.roce.dmac);
+ }
+
+ if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
+ if (attr->max_rd_atomic > dev->hw_attrs.max_hw_ord) {
+ irdma_dev_err(dev,
+ "rd_atomic = %d, above max_hw_ord=%d\n",
+ attr->max_rd_atomic,
+ dev->hw_attrs.max_hw_ord);
+ return -EINVAL;
+ }
+ if (attr->max_rd_atomic)
+ roce_info->ord_size = attr->max_rd_atomic;
+ info.ord_valid = true;
+ }
+
+ if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
+ if (attr->max_dest_rd_atomic > dev->hw_attrs.max_hw_ird) {
+ irdma_dev_err(dev,
+ "rd_atomic = %d, above max_hw_ird=%d\n",
+ attr->max_rd_atomic,
+ dev->hw_attrs.max_hw_ird);
+ return -EINVAL;
+ }
+ if (attr->max_dest_rd_atomic)
+ roce_info->ird_size = irdma_derive_hw_ird_setting(attr->max_dest_rd_atomic);
+ }
+
+ if (attr_mask & IB_QP_ACCESS_FLAGS) {
+ if (attr->qp_access_flags & IB_ACCESS_LOCAL_WRITE)
+ roce_info->wr_rdresp_en = true;
+ if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE)
+ roce_info->wr_rdresp_en = true;
+ if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ)
+ roce_info->rd_en = true;
+ if (attr->qp_access_flags & IB_ACCESS_MW_BIND)
+ roce_info->bind_en = true;
+
+ if (iwqp->user_mode) {
+ roce_info->rd_en = true;
+ roce_info->wr_rdresp_en = true;
+ roce_info->priv_mode_en = false;
+ }
+ }
+
+ spin_lock_irqsave(&iwqp->lock, flags);
+ if (attr_mask & IB_QP_STATE) {
+ if (!ib_modify_qp_is_ok(iwqp->ibqp_state, attr->qp_state,
+ iwqp->ibqp.qp_type, attr_mask)) {
+ irdma_dev_warn(dev, "modify_qp invalid for qp_id=%d, old_state=0x%x, new_state=0x%x\n",
+ iwqp->ibqp.qp_num, iwqp->ibqp_state, attr->qp_state);
+ return -EINVAL;
+ }
+ if (iwdev->closing && attr->qp_state != IB_QPS_ERR) {
+ ret = -EINVAL;
+ goto exit;
+ }
+ info.curr_iwarp_state = iwqp->iwarp_state;
+
+ switch (attr->qp_state) {
+ case IB_QPS_INIT:
+ if (iwqp->iwarp_state > (u32)IRDMA_QP_STATE_IDLE) {
+ ret = -EINVAL;
+ goto exit;
+ }
+
+ if (iwqp->iwarp_state == IRDMA_QP_STATE_INVALID) {
+ info.next_iwarp_state = IRDMA_QP_STATE_IDLE;
+ issue_modify_qp = 1;
+ }
+ break;
+ case IB_QPS_RTR:
+ if (iwqp->iwarp_state > (u32)IRDMA_QP_STATE_IDLE) {
+ ret = -EINVAL;
+ goto exit;
+ }
+ info.arp_cache_idx_valid = true;
+ info.cq_num_valid = true;
+ info.next_iwarp_state = IRDMA_QP_STATE_RTR;
+ issue_modify_qp = 1;
+ break;
+ case IB_QPS_RTS:
+ if (iwqp->ibqp_state < IB_QPS_RTR ||
+ iwqp->ibqp_state == IB_QPS_ERR) {
+ ret = -EINVAL;
+ goto exit;
+ }
+
+ info.arp_cache_idx_valid = true;
+ info.cq_num_valid = true;
+ info.next_iwarp_state = IRDMA_QP_STATE_RTS;
+ issue_modify_qp = 1;
+ break;
+ case IB_QPS_SQD:
+ if (iwqp->hw_iwarp_state > (u32)IRDMA_QP_STATE_RTS)
+ goto exit;
+
+ if (iwqp->iwarp_state == (u32)IRDMA_QP_STATE_CLOSING ||
+ iwqp->iwarp_state < (u32)IRDMA_QP_STATE_RTS)
+ goto exit;
+
+ if (iwqp->iwarp_state > (u32)IRDMA_QP_STATE_CLOSING) {
+ ret = -EINVAL;
+ goto exit;
+ }
+
+ info.next_iwarp_state = IRDMA_QP_STATE_ERROR;
+ issue_modify_qp = 1;
+ break;
+
+ case IB_QPS_SQE:
+ case IB_QPS_ERR:
+ case IB_QPS_RESET:
+ if (iwqp->ibqp_state == IB_QPS_SQD)
+ break;
+
+ if (iwqp->iwarp_state == (u32)IRDMA_QP_STATE_ERROR) {
+ ret = -EINVAL;
+ goto exit;
+ }
+
+ info.next_iwarp_state = IRDMA_QP_STATE_ERROR;
+ issue_modify_qp = 1;
+ break;
+ default:
+ ret = -EINVAL;
+ goto exit;
+ }
+
+ iwqp->ibqp_state = attr->qp_state;
+ }
+
+ ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id;
+ ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id;
+ ret = dev->iw_priv_qp_ops->qp_setctx_roce(&iwqp->sc_qp,
+ iwqp->host_ctx.va,
+ ctx_info);
+ spin_unlock_irqrestore(&iwqp->lock, flags);
+
+ if (ret) {
+ irdma_debug(dev, IRDMA_DEBUG_ERR, "setctx_roce\n");
+ return -EINVAL;
+ }
+
+ if (attr_mask & IB_QP_STATE) {
+ if (issue_modify_qp) {
+ if (irdma_hw_modify_qp(iwdev, iwqp, &info, true))
+ return -EINVAL;
+ spin_lock_irqsave(&iwqp->lock, flags);
+ if (iwqp->iwarp_state == info.curr_iwarp_state) {
+ iwqp->iwarp_state = info.next_iwarp_state;
+ iwqp->ibqp_state = attr->qp_state;
+ }
+ if (iwqp->ibqp_state > IB_QPS_RTS && !iwqp->flush_issued) {
+ iwqp->flush_issued = 1;
+ spin_unlock_irqrestore(&iwqp->lock, flags);
+ irdma_flush_wqes(iwdev->rf, iwqp);
+ } else {
+ spin_unlock_irqrestore(&iwqp->lock, flags);
+ }
+ } else {
+ iwqp->ibqp_state = attr->qp_state;
+ }
+ }
+
+ return 0;
+exit:
+ spin_unlock_irqrestore(&iwqp->lock, flags);
+
+ return ret;
+}
+
+/**
+ * irdma_modify_qp - modify qp request
+ * @ibqp: qp's pointer for modify
+ * @attr: access attributes
+ * @attr_mask: state mask
+ * @udata: user data
+ */
+int irdma_modify_qp(struct ib_qp *ibqp,
+ struct ib_qp_attr *attr,
+ int attr_mask,
+ struct ib_udata *udata)
+{
+ struct irdma_qp *iwqp = to_iwqp(ibqp);
+ struct irdma_device *iwdev = iwqp->iwdev;
+ struct irdma_qp_host_ctx_info *ctx_info;
+ struct irdma_iwarp_offload_info *offload_info;
+ struct irdma_modify_qp_info info = {};
+ u8 issue_modify_qp = 0;
+ u8 dont_wait = 0;
+ u32 err;
+ unsigned long flags;
+
+ ctx_info = &iwqp->ctx_info;
+ offload_info = &iwqp->iwarp_info;
+
+ spin_lock_irqsave(&iwqp->lock, flags);
+ if (attr_mask & IB_QP_STATE) {
+ if (iwdev->closing && attr->qp_state != IB_QPS_ERR) {
+ err = -EINVAL;
+ goto exit;
+ }
+
+ info.curr_iwarp_state = iwqp->iwarp_state;
+ switch (attr->qp_state) {
+ case IB_QPS_INIT:
+ case IB_QPS_RTR:
+ if (iwqp->iwarp_state > (u32)IRDMA_QP_STATE_IDLE) {
+ err = -EINVAL;
+ goto exit;
+ }
+
+ if (iwqp->iwarp_state == IRDMA_QP_STATE_INVALID) {
+ info.next_iwarp_state = IRDMA_QP_STATE_IDLE;
+ issue_modify_qp = 1;
+ }
+ break;
+ case IB_QPS_RTS:
+ if (iwqp->iwarp_state > (u32)IRDMA_QP_STATE_RTS ||
+ !iwqp->cm_id) {
+ err = -EINVAL;
+ goto exit;
+ }
+
+ issue_modify_qp = 1;
+ iwqp->hw_tcp_state = IRDMA_TCP_STATE_ESTABLISHED;
+ iwqp->hte_added = 1;
+ info.next_iwarp_state = IRDMA_QP_STATE_RTS;
+ info.tcp_ctx_valid = true;
+ info.ord_valid = true;
+ info.arp_cache_idx_valid = true;
+ info.cq_num_valid = true;
+ break;
+ case IB_QPS_SQD:
+ if (iwqp->hw_iwarp_state > (u32)IRDMA_QP_STATE_RTS) {
+ err = 0;
+ goto exit;
+ }
+
+ if (iwqp->iwarp_state == (u32)IRDMA_QP_STATE_CLOSING ||
+ iwqp->iwarp_state < (u32)IRDMA_QP_STATE_RTS) {
+ err = 0;
+ goto exit;
+ }
+
+ if (iwqp->iwarp_state > (u32)IRDMA_QP_STATE_CLOSING) {
+ err = -EINVAL;
+ goto exit;
+ }
+
+ info.next_iwarp_state = IRDMA_QP_STATE_CLOSING;
+ issue_modify_qp = 1;
+ break;
+ case IB_QPS_SQE:
+ if (iwqp->iwarp_state >= (u32)IRDMA_QP_STATE_TERMINATE) {
+ err = -EINVAL;
+ goto exit;
+ }
+
+ info.next_iwarp_state = IRDMA_QP_STATE_TERMINATE;
+ issue_modify_qp = 1;
+ break;
+ case IB_QPS_ERR:
+ case IB_QPS_RESET:
+ if (iwqp->iwarp_state == (u32)IRDMA_QP_STATE_ERROR) {
+ err = -EINVAL;
+ goto exit;
+ }
+
+ if (iwqp->sc_qp.term_flags)
+ irdma_terminate_del_timer(&iwqp->sc_qp);
+ info.next_iwarp_state = IRDMA_QP_STATE_ERROR;
+ if (iwqp->hw_tcp_state > IRDMA_TCP_STATE_CLOSED &&
+ iwdev->iw_status &&
+ iwqp->hw_tcp_state != IRDMA_TCP_STATE_TIME_WAIT)
+ info.reset_tcp_conn = true;
+ else
+ dont_wait = 1;
+
+ issue_modify_qp = 1;
+ info.next_iwarp_state = IRDMA_QP_STATE_ERROR;
+ break;
+ default:
+ err = -EINVAL;
+ goto exit;
+ }
+
+ iwqp->ibqp_state = attr->qp_state;
+ }
+ if (attr_mask & IB_QP_ACCESS_FLAGS) {
+ ctx_info->iwarp_info_valid = true;
+ if (attr->qp_access_flags & IB_ACCESS_LOCAL_WRITE)
+ offload_info->wr_rdresp_en = true;
+ if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE)
+ offload_info->wr_rdresp_en = true;
+ if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ)
+ offload_info->rd_en = true;
+ if (attr->qp_access_flags & IB_ACCESS_MW_BIND)
+ offload_info->bind_en = true;
+
+ if (iwqp->user_mode) {
+ offload_info->rd_en = true;
+ offload_info->wr_rdresp_en = true;
+ offload_info->priv_mode_en = false;
+ }
+ }
+
+ if (ctx_info->iwarp_info_valid) {
+ struct irdma_sc_dev *dev = &iwdev->rf->sc_dev;
+ int ret;
+
+ ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id;
+ ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id;
+ ret = dev->iw_priv_qp_ops->qp_setctx(&iwqp->sc_qp,
+ iwqp->host_ctx.va,
+ ctx_info);
+ if (ret) {
+ irdma_debug(dev, IRDMA_DEBUG_ERR, "setting QP context\n");
+ err = -EINVAL;
+ goto exit;
+ }
+ }
+ spin_unlock_irqrestore(&iwqp->lock, flags);
+
+ if (attr_mask & IB_QP_STATE) {
+ if (issue_modify_qp) {
+ if (irdma_hw_modify_qp(iwdev, iwqp, &info, true))
+ return -EINVAL;
+ }
+
+ spin_lock_irqsave(&iwqp->lock, flags);
+ if (iwqp->iwarp_state == info.curr_iwarp_state) {
+ iwqp->iwarp_state = info.next_iwarp_state;
+ iwqp->ibqp_state = attr->qp_state;
+ }
+ spin_unlock_irqrestore(&iwqp->lock, flags);
+ }
+
+ if (issue_modify_qp && iwqp->ibqp_state > IB_QPS_RTS) {
+ if (dont_wait) {
+ if (iwqp->cm_id && iwqp->hw_tcp_state) {
+ spin_lock_irqsave(&iwqp->lock, flags);
+ iwqp->hw_tcp_state = IRDMA_TCP_STATE_CLOSED;
+ iwqp->last_aeq = IRDMA_AE_RESET_SENT;
+ spin_unlock_irqrestore(&iwqp->lock, flags);
+ irdma_cm_disconn(iwqp);
+ }
+ } else {
+ spin_lock_irqsave(&iwqp->lock, flags);
+ if (iwqp->cm_id) {
+ if (atomic_inc_return(&iwqp->close_timer_started) == 1) {
+ iwqp->cm_id->add_ref(iwqp->cm_id);
+ irdma_schedule_cm_timer(iwqp->cm_node,
+ (struct irdma_puda_buf *)iwqp,
+ IRDMA_TIMER_TYPE_CLOSE,
+ 1,
+ 0);
+ }
+ }
+ spin_unlock_irqrestore(&iwqp->lock, flags);
+ }
+ }
+
+ return 0;
+exit:
+ spin_unlock_irqrestore(&iwqp->lock, flags);
+
+ return err;
+}
+
+/**
+ * cq_free_resources - free up recources for cq
+ * @rf: RDMA PCI function
+ * @iwcq: cq ptr
+ */
+static void cq_free_rsrc(struct irdma_pci_f *rf, struct irdma_cq *iwcq)
+{
+ struct irdma_sc_cq *cq = &iwcq->sc_cq;
+
+ if (!iwcq->user_mode)
+ irdma_free_dma_mem(rf->sc_dev.hw, &iwcq->kmem);
+ irdma_free_rsrc(rf, rf->allocated_cqs, cq->cq_uk.cq_id);
+}
+
+/**
+ * irdma_destroy_cq - destroy cq
+ * @ib_cq: cq pointer
+ */
+static int irdma_destroy_cq(struct ib_cq *ib_cq)
+{
+ struct irdma_cq *iwcq;
+ struct irdma_device *iwdev;
+ struct irdma_sc_cq *cq;
+
+ if (!ib_cq) {
+ irdma_pr_err("ib_cq == NULL\n");
+ return 0;
+ }
+
+ iwcq = to_iwcq(ib_cq);
+ iwdev = to_iwdev(ib_cq->device);
+ cq = &iwcq->sc_cq;
+ irdma_cq_wq_destroy(iwdev->rf, cq);
+ cq_free_rsrc(iwdev->rf, iwcq);
+ kfree(iwcq);
+ irdma_rem_devusecount(iwdev);
+
+ return 0;
+}
+
+/**
+ * irdma_create_cq - create cq
+ * @ibdev: device pointer from stack
+ * @attr: attributes for cq
+ * @context: user context created during alloc
+ * @udata: user data
+ */
+static struct ib_cq *irdma_create_cq(struct ib_device *ibdev,
+ const struct ib_cq_init_attr *attr,
+ struct ib_ucontext *context,
+ struct ib_udata *udata)
+{
+ struct irdma_device *iwdev = to_iwdev(ibdev);
+ struct irdma_pci_f *rf = iwdev->rf;
+ struct irdma_cq *iwcq;
+ struct irdma_pbl *iwpbl;
+ u32 cq_num = 0;
+ struct irdma_sc_cq *cq;
+ struct irdma_sc_dev *dev = &rf->sc_dev;
+ struct irdma_cq_init_info info = {};
+ enum irdma_status_code status;
+ struct irdma_cqp_request *cqp_request;
+ struct cqp_cmds_info *cqp_info;
+ struct irdma_cq_uk_init_info *ukinfo = &info.cq_uk_init_info;
+ unsigned long flags;
+ int err_code;
+ int entries = attr->cqe;
+
+ if (iwdev->closing)
+ return ERR_PTR(-ENODEV);
+
+ if (entries > rf->max_cqe)
+ return ERR_PTR(-EINVAL);
+
+ iwcq = kzalloc(sizeof(*iwcq), GFP_KERNEL);
+ if (!iwcq)
+ return ERR_PTR(-ENOMEM);
+
+ err_code = irdma_alloc_rsrc(rf, rf->allocated_cqs,
+ rf->max_cq, &cq_num,
+ &rf->next_cq);
+ if (err_code)
+ goto error;
+
+ cq = &iwcq->sc_cq;
+ cq->back_cq = (void *)iwcq;
+ spin_lock_init(&iwcq->lock);
+ info.dev = dev;
+ ukinfo->cq_size = max(entries, 4);
+ ukinfo->cq_id = cq_num;
+ iwcq->ibcq.cqe = info.cq_uk_init_info.cq_size;
+ if (attr->comp_vector < rf->ceqs_count)
+ info.ceq_id = attr->comp_vector;
+ info.ceq_id_valid = true;
+ info.ceqe_mask = 1;
+ info.type = IRDMA_CQ_TYPE_IWARP;
+ info.vsi = &iwdev->vsi;
+
+ if (context) {
+ struct irdma_ucontext *ucontext;
+ struct irdma_create_cq_req req = {};
+ struct irdma_cq_mr *cqmr;
+
+ iwcq->user_mode = true;
+ ucontext = to_ucontext(context);
+ if (ib_copy_from_udata(&req, udata,
+ sizeof(struct irdma_create_cq_req)))
+ goto cq_free_rsrc;
+
+ spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
+ iwpbl = irdma_get_pbl((unsigned long)req.user_cq_buf,
+ &ucontext->cq_reg_mem_list);
+ spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags);
+ if (!iwpbl) {
+ err_code = -EPROTO;
+ goto cq_free_rsrc;
+ }
+
+ iwcq->iwpbl = iwpbl;
+ iwcq->cq_mem_size = 0;
+ cqmr = &iwpbl->cq_mr;
+ info.shadow_area_pa = cqmr->shadow;
+ if (iwpbl->pbl_allocated) {
+ info.virtual_map = true;
+ info.pbl_chunk_size = 1;
+ info.first_pm_pbl_idx = cqmr->cq_pbl.idx;
+ } else {
+ info.cq_base_pa = cqmr->cq_pbl.addr;
+ }
+ } else {
+ /* Kmode allocations */
+ int rsize;
+ int shadow;
+
+ rsize = info.cq_uk_init_info.cq_size * sizeof(struct irdma_cqe);
+ rsize = round_up(rsize, 256);
+ shadow = IRDMA_SHADOW_AREA_SIZE << 3;
+ status = irdma_allocate_dma_mem(dev->hw, &iwcq->kmem,
+ rsize + shadow, 256);
+ if (status) {
+ err_code = -ENOMEM;
+ goto cq_free_rsrc;
+ }
+
+ ukinfo->cq_base = iwcq->kmem.va;
+ info.cq_base_pa = iwcq->kmem.pa;
+ info.shadow_area_pa = info.cq_base_pa + rsize;
+ ukinfo->shadow_area = iwcq->kmem.va + rsize;
+ }
+
+ if (dev->iw_priv_cq_ops->cq_init(cq, &info)) {
+ irdma_debug(dev, IRDMA_DEBUG_ERR, "init cq fail\n");
+ err_code = -EPROTO;
+ goto cq_free_rsrc;
+ }
+
+ cqp_request = irdma_get_cqp_request(&rf->cqp, true);
+ if (!cqp_request) {
+ err_code = -ENOMEM;
+ goto cq_free_rsrc;
+ }
+
+ cqp_info = &cqp_request->info;
+ cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE;
+ cqp_info->post_sq = 1;
+ cqp_info->in.u.cq_create.cq = cq;
+ cqp_info->in.u.cq_create.check_overflow = true;
+ cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request;
+ cqp_info->in.u.cq_create.check_overflow = true;
+ status = irdma_handle_cqp_op(rf, cqp_request);
+ if (status) {
+ irdma_debug(dev, IRDMA_DEBUG_ERR, "CQP-OP Create CQ fail");
+ err_code = -ENOMEM;
+ goto cq_free_rsrc;
+ }
+
+ if (context) {
+ struct irdma_create_cq_resp resp = {};
+
+ resp.cq_id = info.cq_uk_init_info.cq_id;
+ resp.cq_size = info.cq_uk_init_info.cq_size;
+ if (ib_copy_to_udata(udata, &resp, sizeof(resp))) {
+ irdma_debug(dev, IRDMA_DEBUG_ERR, "copy to user data\n");
+ err_code = -EPROTO;
+ goto cq_destroy;
+ }
+ }
+
+ irdma_add_devusecount(iwdev);
+
+ return (struct ib_cq *)iwcq;
+
+cq_destroy:
+ irdma_cq_wq_destroy(rf, cq);
+cq_free_rsrc:
+ cq_free_rsrc(rf, iwcq);
+error:
+ kfree(iwcq);
+
+ return ERR_PTR(err_code);
+}
+
+/**
+ * irdma_get_user_access - get hw access from IB access
+ * @acc: IB access to return hw access
+ */
+static inline u16 irdma_get_user_access(int acc)
+{
+ u16 access = 0;
+
+ access |= (acc & IB_ACCESS_LOCAL_WRITE) ?
+ IRDMA_ACCESS_FLAGS_LOCALWRITE : 0;
+ access |= (acc & IB_ACCESS_REMOTE_WRITE) ?
+ IRDMA_ACCESS_FLAGS_REMOTEWRITE : 0;
+ access |= (acc & IB_ACCESS_REMOTE_READ) ?
+ IRDMA_ACCESS_FLAGS_REMOTEREAD : 0;
+ access |= (acc & IB_ACCESS_MW_BIND) ?
+ IRDMA_ACCESS_FLAGS_BIND_WINDOW : 0;
+
+ return access;
+}
+
+/**
+ * irdma_free_stag - free stag resource
+ * @iwdev: iwarp device
+ * @stag: stag to free
+ */
+static void irdma_free_stag(struct irdma_device *iwdev, u32 stag)
+{
+ u32 stag_idx;
+
+ stag_idx = (stag & iwdev->rf->mr_stagmask) >> IRDMA_CQPSQ_STAG_IDX_S;
+ irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_mrs, stag_idx);
+ irdma_rem_devusecount(iwdev);
+}
+
+/**
+ * irdma_create_stag - create random stag
+ * @iwdev: iwarp device
+ */
+static u32 irdma_create_stag(struct irdma_device *iwdev)
+{
+ u32 stag = 0;
+ u32 stag_index = 0;
+ u32 next_stag_index;
+ u32 driver_key;
+ u32 random;
+ u8 consumer_key;
+ int ret;
+
+ get_random_bytes(&random, sizeof(random));
+ consumer_key = (u8)random;
+
+ driver_key = random & ~iwdev->rf->mr_stagmask;
+ next_stag_index = (random & iwdev->rf->mr_stagmask) >> 8;
+ next_stag_index %= iwdev->rf->max_mr;
+
+ ret = irdma_alloc_rsrc(iwdev->rf,
+ iwdev->rf->allocated_mrs, iwdev->rf->max_mr,
+ &stag_index, &next_stag_index);
+ if (!ret) {
+ stag = stag_index << IRDMA_CQPSQ_STAG_IDX_S;
+ stag |= driver_key;
+ stag += (u32)consumer_key;
+ irdma_add_devusecount(iwdev);
+ }
+ return stag;
+}
+
+/**
+ * irdma_next_pbl_addr - Get next pbl address
+ * @pbl: pointer to a pble
+ * @pinfo: info pointer
+ * @idx: index
+ */
+static inline u64 *irdma_next_pbl_addr(u64 *pbl,
+ struct irdma_pble_info **pinfo,
+ u32 *idx)
+{
+ *idx += 1;
+ if ((!(*pinfo)) || (*idx != (*pinfo)->cnt))
+ return ++pbl;
+ *idx = 0;
+ (*pinfo)++;
+
+ return (u64 *)(*pinfo)->addr;
+}
+
+/**
+ * irdma_copy_user_pgaddrs - copy user page address to pble's os locally
+ * @iwmr: iwmr for IB's user page addresses
+ * @pbl: ple pointer to save 1 level or 0 level pble
+ * @level: indicated level 0, 1 or 2
+ */
+static void irdma_copy_user_pgaddrs(struct irdma_mr *iwmr,
+ u64 *pbl,
+ enum irdma_pble_level level)
+{
+ struct ib_umem *region = iwmr->region;
+ struct irdma_pbl *iwpbl = &iwmr->iwpbl;
+ int chunk_pages, entry, i;
+ struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc;
+ struct irdma_pble_info *pinfo;
+ struct scatterlist *sg;
+ u64 pg_addr = 0;
+ u32 idx = 0;
+
+ pinfo = (level == PBLE_LEVEL_1) ? NULL : palloc->level2.leaf;
+
+ for_each_sg(region->sg_head.sgl, sg, region->nmap, entry) {
+ chunk_pages = DIV_ROUND_UP(sg_dma_len(sg), iwmr->page_size);
+ if (iwmr->type == IW_MEMREG_TYPE_QP &&
+ !iwpbl->qp_mr.sq_page)
+ iwpbl->qp_mr.sq_page = sg_page(sg);
+ for (i = 0; i < chunk_pages; i++) {
+ pg_addr = sg_dma_address(sg) + (i * iwmr->page_size);
+ if ((entry + i) == 0)
+ *pbl = pg_addr & iwmr->page_msk;
+ else if (!(pg_addr & ~iwmr->page_msk))
+ *pbl = pg_addr;
+ else
+ continue;
+ pbl = irdma_next_pbl_addr(pbl, &pinfo, &idx);
+ }
+ }
+}
+
+/**
+ * irdma_check_mem_contiguous - check if pbls stored in arr are contiguous
+ * @arr: lvl1 pbl array
+ * @npages: page count
+ * pg_size: page size
+ *
+ */
+static bool irdma_check_mem_contiguous(u64 *arr, u32 npages, u32 pg_size)
+{
+ u32 pg_idx;
+
+ for (pg_idx = 0; pg_idx < npages; pg_idx++) {
+ if ((*arr + (pg_size * pg_idx)) != arr[pg_idx])
+ return false;
+ }
+
+ return true;
+}
+
+/**
+ * irdma_check_mr_contiguous - check if MR is physically contiguous
+ * @palloc: pbl allocation struct
+ * pg_size: page size
+ */
+static bool irdma_check_mr_contiguous(struct irdma_pble_alloc *palloc,
+ u32 pg_size)
+{
+ struct irdma_pble_level2 *lvl2 = &palloc->level2;
+ struct irdma_pble_info *leaf = lvl2->leaf;
+ u64 *arr = NULL;
+ u64 *start_addr = NULL;
+ int i;
+ bool ret;
+
+ if (palloc->level == PBLE_LEVEL_1) {
+ arr = (u64 *)palloc->level1.addr;
+ ret = irdma_check_mem_contiguous(arr, palloc->total_cnt,
+ pg_size);
+ return ret;
+ }
+
+ start_addr = (u64 *)leaf->addr;
+
+ for (i = 0; i < lvl2->leaf_cnt; i++, leaf++) {
+ arr = (u64 *)leaf->addr;
+ if ((*start_addr + (i * pg_size * PBLE_PER_PAGE)) != *arr)
+ return false;
+ ret = irdma_check_mem_contiguous(arr, leaf->cnt, pg_size);
+ if (!ret)
+ return false;
+ }
+
+ return true;
+}
+
+/**
+ * irdma_setup_pbles - copy user pg address to pble's
+ * @rf: RDMA PCI function
+ * @iwmr: mr pointer for this memory registration
+ * @use_pbles: flag if to use pble's
+ */
+static int irdma_setup_pbles(struct irdma_pci_f *rf,
+ struct irdma_mr *iwmr,
+ bool use_pbles)
+{
+ struct irdma_pbl *iwpbl = &iwmr->iwpbl;
+ struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc;
+ struct irdma_pble_info *pinfo;
+ u64 *pbl;
+ enum irdma_status_code status;
+ enum irdma_pble_level level = PBLE_LEVEL_1;
+
+ if (use_pbles) {
+ status = irdma_get_pble(rf->pble_rsrc, palloc, iwmr->page_cnt,
+ false);
+ if (status)
+ return -ENOMEM;
+
+ iwpbl->pbl_allocated = true;
+ level = palloc->level;
+ pinfo = (level == PBLE_LEVEL_1) ?
+ &palloc->level1 : palloc->level2.leaf;
+ pbl = (u64 *)pinfo->addr;
+ } else {
+ pbl = iwmr->pgaddrmem;
+ }
+
+ irdma_copy_user_pgaddrs(iwmr, pbl, level);
+
+ if (use_pbles)
+ iwmr->pgaddrmem[0] = *pbl;
+
+ return 0;
+}
+
+/**
+ * irdma_handle_q_mem - handle memory for qp and cq
+ * @iwdev: iwarp device
+ * @req: information for q memory management
+ * @iwpbl: pble struct
+ * @use_pbles: flag to use pble
+ */
+static int irdma_handle_q_mem(struct irdma_device *iwdev,
+ struct irdma_mem_reg_req *req,
+ struct irdma_pbl *iwpbl,
+ bool use_pbles)
+{
+ struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc;
+ struct irdma_mr *iwmr = iwpbl->iwmr;
+ struct irdma_qp_mr *qpmr = &iwpbl->qp_mr;
+ struct irdma_cq_mr *cqmr = &iwpbl->cq_mr;
+ struct irdma_hmc_pble *hmc_p;
+ u64 *arr = iwmr->pgaddrmem;
+ u32 pg_size;
+ int err;
+ int total;
+ bool ret = true;
+
+ total = req->sq_pages + req->rq_pages + req->cq_pages;
+ pg_size = iwmr->page_size;
+ err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles);
+ if (err)
+ return err;
+
+ if (use_pbles && palloc->level != PBLE_LEVEL_1) {
+ irdma_free_pble(iwdev->rf->pble_rsrc, palloc);
+ iwpbl->pbl_allocated = false;
+ return -ENOMEM;
+ }
+
+ if (use_pbles)
+ arr = (u64 *)palloc->level1.addr;
+
+ if (iwmr->type == IW_MEMREG_TYPE_QP) {
+ hmc_p = &qpmr->sq_pbl;
+ qpmr->shadow = (dma_addr_t)arr[total];
+
+ if (use_pbles) {
+ ret = irdma_check_mem_contiguous(arr,
+ req->sq_pages,
+ pg_size);
+ if (ret)
+ ret = irdma_check_mem_contiguous(&arr[req->sq_pages],
+ req->rq_pages,
+ pg_size);
+ }
+
+ if (!ret) {
+ hmc_p->idx = palloc->level1.idx;
+ hmc_p = &qpmr->rq_pbl;
+ hmc_p->idx = palloc->level1.idx + req->sq_pages;
+ } else {
+ hmc_p->addr = arr[0];
+ hmc_p = &qpmr->rq_pbl;
+ hmc_p->addr = arr[req->sq_pages];
+ }
+ } else { /* CQ */
+ hmc_p = &cqmr->cq_pbl;
+ cqmr->shadow = (dma_addr_t)arr[total];
+
+ if (use_pbles)
+ ret = irdma_check_mem_contiguous(arr,
+ req->cq_pages,
+ pg_size);
+
+ if (!ret)
+ hmc_p->idx = palloc->level1.idx;
+ else
+ hmc_p->addr = arr[0];
+ }
+
+ if (use_pbles && ret) {
+ irdma_free_pble(iwdev->rf->pble_rsrc, palloc);
+ iwpbl->pbl_allocated = false;
+ }
+
+ return err;
+}
+
+/**
+ * irdma_hw_alloc_mw - create the hw memory window
+ * @rf: RDMA PCI function
+ * @iwmr: pointer to memory window info
+ */
+static int irdma_hw_alloc_mw(struct irdma_pci_f *rf,
+ struct irdma_mr *iwmr)
+{
+ struct irdma_mw_alloc_info *info;
+ struct irdma_pd *iwpd = to_iwpd(iwmr->ibmr.pd);
+ struct irdma_cqp_request *cqp_request;
+ struct cqp_cmds_info *cqp_info;
+
+ cqp_request = irdma_get_cqp_request(&rf->cqp, true);
+ if (!cqp_request)
+ return -ENOMEM;
+
+ cqp_info = &cqp_request->info;
+ info = &cqp_info->in.u.mw_alloc.info;
+ memset(info, 0, sizeof(*info));
+ if (iwmr->ibmw.type == IB_MW_TYPE_1)
+ info->mw_wide = true;
+
+ info->page_size = PAGE_SIZE;
+ info->mw_stag_index = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S;
+ info->pd_id = iwpd->sc_pd.pd_id;
+ info->remote_access = true;
+ cqp_info->cqp_cmd = IRDMA_OP_MW_ALLOC;
+ cqp_info->post_sq = 1;
+ cqp_info->in.u.mw_alloc.dev = &rf->sc_dev;
+ cqp_info->in.u.mw_alloc.scratch = (uintptr_t)cqp_request;
+ if (irdma_handle_cqp_op(rf, cqp_request)) {
+ irdma_debug(&rf->sc_dev, IRDMA_DEBUG_ERR, "CQP-OP MR Reg fail");
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+/**
+ * irdma_alloc_mw
+ * @pd: Protection domain
+ * @type: Window type
+ * @udata: user data pointer
+ */
+static struct ib_mw *irdma_alloc_mw(struct ib_pd *pd,
+ enum ib_mw_type type,
+ struct ib_udata *udata)
+{
+ struct irdma_pd *iwpd = to_iwpd(pd);
+ struct irdma_device *iwdev = to_iwdev(pd->device);
+ struct irdma_mr *iwmr;
+ int err_code;
+ u32 stag;
+
+ iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);
+ if (!iwmr)
+ return ERR_PTR(-ENOMEM);
+
+ stag = irdma_create_stag(iwdev);
+ if (!stag) {
+ err_code = -ENOMEM;
+ goto err;
+ }
+
+ iwmr->stag = stag;
+ iwmr->ibmw.rkey = stag;
+ iwmr->ibmw.pd = pd;
+ iwmr->ibmw.type = type;
+ iwmr->ibmw.device = pd->device;
+ iwmr->type = IW_MEMREG_TYPE_MW;
+
+ err_code = irdma_hw_alloc_mw(iwdev->rf, iwmr);
+ if (err_code)
+ goto err1;
+
+ irdma_add_pdusecount(iwpd);
+
+ return &iwmr->ibmw;
+
+err1:
+ irdma_free_stag(iwdev, stag);
+err:
+ kfree(iwmr);
+
+ return ERR_PTR(err_code);
+}
+
+/**
+ * irdma_dealloc_mw
+ */
+static int irdma_dealloc_mw(struct ib_mw *ibmw)
+{
+ struct ib_pd *ibpd = ibmw->pd;
+ struct irdma_pd *iwpd = to_iwpd(ibpd);
+ struct irdma_mr *iwmr = to_iwmr((struct ib_mr *)ibmw);
+ struct irdma_device *iwdev = to_iwdev(ibmw->device);
+ struct irdma_cqp_request *cqp_request;
+ struct cqp_cmds_info *cqp_info;
+ struct irdma_dealloc_stag_info *info;
+
+ cqp_request = irdma_get_cqp_request(&iwdev->rf->cqp, true);
+ if (!cqp_request)
+ return -ENOMEM;
+
+ cqp_info = &cqp_request->info;
+ info = &cqp_info->in.u.dealloc_stag.info;
+ memset(info, 0, sizeof(*info));
+ info->pd_id = iwpd->sc_pd.pd_id & 0x00007fff;
+ info->stag_idx = RS_64_1(ibmw->rkey, IRDMA_CQPSQ_STAG_IDX_S);
+ info->mr = false;
+ cqp_info->cqp_cmd = IRDMA_OP_DEALLOC_STAG;
+ cqp_info->post_sq = 1;
+ cqp_info->in.u.dealloc_stag.dev = &iwdev->rf->sc_dev;
+ cqp_info->in.u.dealloc_stag.scratch = (uintptr_t)cqp_request;
+ if (irdma_handle_cqp_op(iwdev->rf, cqp_request))
+ irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_ERR,
+ "CQP-OP dealloc MW failed for stag_idx = 0x%x\n",
+ info->stag_idx);
+ irdma_rem_pdusecount(iwpd, iwdev);
+ irdma_free_stag(iwdev, iwmr->stag);
+ kfree(iwmr);
+
+ return 0;
+}
+
+/**
+ * irdma_hw_alloc_stag - cqp command to allocate stag
+ * @iwdev: iwarp device
+ * @iwmr: iwarp mr pointer
+ */
+static int irdma_hw_alloc_stag(struct irdma_device *iwdev,
+ struct irdma_mr *iwmr)
+{
+ struct irdma_allocate_stag_info *info;
+ struct irdma_pd *iwpd = to_iwpd(iwmr->ibmr.pd);
+ enum irdma_status_code status;
+ int err = 0;
+ struct irdma_cqp_request *cqp_request;
+ struct cqp_cmds_info *cqp_info;
+
+ cqp_request = irdma_get_cqp_request(&iwdev->rf->cqp, true);
+ if (!cqp_request)
+ return -ENOMEM;
+
+ cqp_info = &cqp_request->info;
+ info = &cqp_info->in.u.alloc_stag.info;
+ memset(info, 0, sizeof(*info));
+ info->page_size = PAGE_SIZE;
+ info->stag_idx = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S;
+ info->pd_id = iwpd->sc_pd.pd_id;
+ info->total_len = iwmr->len;
+ info->remote_access = true;
+ cqp_info->cqp_cmd = IRDMA_OP_ALLOC_STAG;
+ cqp_info->post_sq = 1;
+ cqp_info->in.u.alloc_stag.dev = &iwdev->rf->sc_dev;
+ cqp_info->in.u.alloc_stag.scratch = (uintptr_t)cqp_request;
+ status = irdma_handle_cqp_op(iwdev->rf, cqp_request);
+ if (status) {
+ err = -ENOMEM;
+ irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_ERR,
+ "CQP-OP MR Reg fail");
+ }
+
+ return err;
+}
+
+/**
+ * irdma_alloc_mr - register stag for fast memory registration
+ * @pd: ibpd pointer
+ * @mr_type: memory for stag registrion
+ * @max_num_sg: man number of pages
+ */
+static struct ib_mr *irdma_alloc_mr(struct ib_pd *pd,
+ enum ib_mr_type mr_type,
+ u32 max_num_sg)
+{
+ struct irdma_pd *iwpd = to_iwpd(pd);
+ struct irdma_device *iwdev = to_iwdev(pd->device);
+ struct irdma_pble_alloc *palloc;
+ struct irdma_pbl *iwpbl;
+ struct irdma_mr *iwmr;
+ enum irdma_status_code status;
+ u32 stag;
+ int err_code = -ENOMEM;
+
+ iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);
+ if (!iwmr)
+ return ERR_PTR(-ENOMEM);
+
+ stag = irdma_create_stag(iwdev);
+ if (!stag) {
+ err_code = -ENOMEM;
+ goto err;
+ }
+
+ iwmr->stag = stag;
+ iwmr->ibmr.rkey = stag;
+ iwmr->ibmr.lkey = stag;
+ iwmr->ibmr.pd = pd;
+ iwmr->ibmr.device = pd->device;
+ iwpbl = &iwmr->iwpbl;
+ iwpbl->iwmr = iwmr;
+ iwmr->type = IW_MEMREG_TYPE_MEM;
+ palloc = &iwpbl->pble_alloc;
+ iwmr->page_cnt = max_num_sg;
+ status = irdma_get_pble(iwdev->rf->pble_rsrc, palloc, iwmr->page_cnt,
+ true);
+ if (status)
+ goto err1;
+
+ err_code = irdma_hw_alloc_stag(iwdev, iwmr);
+ if (err_code)
+ goto err2;
+
+ iwpbl->pbl_allocated = true;
+ irdma_add_pdusecount(iwpd);
+
+ return &iwmr->ibmr;
+err2:
+ irdma_free_pble(iwdev->rf->pble_rsrc, palloc);
+err1:
+ irdma_free_stag(iwdev, stag);
+err:
+ kfree(iwmr);
+
+ return ERR_PTR(err_code);
+}
+
+/**
+ * irdma_set_page - populate pbl list for fmr
+ * @ibmr: ib mem to access iwarp mr pointer
+ * @addr: page dma address fro pbl list
+ */
+static int irdma_set_page(struct ib_mr *ibmr,
+ u64 addr)
+{
+ struct irdma_mr *iwmr = to_iwmr(ibmr);
+ struct irdma_pbl *iwpbl = &iwmr->iwpbl;
+ struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc;
+ u64 *pbl;
+
+ if (unlikely(iwmr->npages == iwmr->page_cnt))
+ return -ENOMEM;
+
+ pbl = (u64 *)palloc->level1.addr;
+ pbl[iwmr->npages++] = addr;
+
+ return 0;
+}
+
+/**
+ * irdma_map_mr_sg - map of sg list for fmr
+ * @ibmr: ib mem to access iwarp mr pointer
+ * @sg: scatter gather list for fmr
+ * @sg_nents: number of sg pages
+ */
+static int irdma_map_mr_sg(struct ib_mr *ibmr,
+ struct scatterlist *sg,
+ int sg_nents,
+ unsigned int *sg_offset)
+{
+ struct irdma_mr *iwmr = to_iwmr(ibmr);
+
+ iwmr->npages = 0;
+
+ return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, irdma_set_page);
+}
+
+/**
+ * irdma_drain_sq - drain the send queue
+ * @ibqp: ib qp pointer
+ */
+static void irdma_drain_sq(struct ib_qp *ibqp)
+{
+ struct irdma_qp *iwqp = to_iwqp(ibqp);
+ struct irdma_sc_qp *qp = &iwqp->sc_qp;
+
+ if (IRDMA_RING_MORE_WORK(qp->qp_uk.sq_ring))
+ wait_for_completion(&iwqp->sq_drained);
+}
+
+/**
+ * irdma_drain_rq - drain the receive queue
+ * @ibqp: ib qp pointer
+ */
+static void irdma_drain_rq(struct ib_qp *ibqp)
+{
+ struct irdma_qp *iwqp = to_iwqp(ibqp);
+ struct irdma_sc_qp *qp = &iwqp->sc_qp;
+
+ if (IRDMA_RING_MORE_WORK(qp->qp_uk.rq_ring))
+ wait_for_completion(&iwqp->rq_drained);
+}
+
+/**
+ * irdma_hwreg_mr - send cqp command for memory registration
+ * @iwdev: iwarp device
+ * @iwmr: iwarp mr pointer
+ * @access: access for MR
+ */
+static int irdma_hwreg_mr(struct irdma_device *iwdev,
+ struct irdma_mr *iwmr,
+ u16 access)
+{
+ struct irdma_pbl *iwpbl = &iwmr->iwpbl;
+ struct irdma_reg_ns_stag_info *stag_info;
+ struct irdma_pd *iwpd = to_iwpd(iwmr->ibmr.pd);
+ struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc;
+ enum irdma_status_code status;
+ int err = 0;
+ struct irdma_cqp_request *cqp_request;
+ struct cqp_cmds_info *cqp_info;
+
+ cqp_request = irdma_get_cqp_request(&iwdev->rf->cqp, true);
+ if (!cqp_request)
+ return -ENOMEM;
+
+ cqp_info = &cqp_request->info;
+ stag_info = &cqp_info->in.u.mr_reg_non_shared.info;
+ memset(stag_info, 0, sizeof(*stag_info));
+ stag_info->va = (void *)(unsigned long)iwpbl->user_base;
+ stag_info->stag_idx = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S;
+ stag_info->stag_key = (u8)iwmr->stag;
+ stag_info->total_len = iwmr->len;
+ stag_info->access_rights = access;
+ stag_info->pd_id = iwpd->sc_pd.pd_id;
+ stag_info->addr_type = IRDMA_ADDR_TYPE_VA_BASED;
+ stag_info->page_size = iwmr->page_size;
+
+ if (iwpbl->pbl_allocated) {
+ if (palloc->level == PBLE_LEVEL_1) {
+ stag_info->first_pm_pbl_index = palloc->level1.idx;
+ stag_info->chunk_size = 1;
+ } else {
+ stag_info->first_pm_pbl_index = palloc->level2.root.idx;
+ stag_info->chunk_size = 3;
+ }
+ } else {
+ stag_info->reg_addr_pa = iwmr->pgaddrmem[0];
+ }
+
+ cqp_info->cqp_cmd = IRDMA_OP_MR_REG_NON_SHARED;
+ cqp_info->post_sq = 1;
+ cqp_info->in.u.mr_reg_non_shared.dev = &iwdev->rf->sc_dev;
+ cqp_info->in.u.mr_reg_non_shared.scratch = (uintptr_t)cqp_request;
+ status = irdma_handle_cqp_op(iwdev->rf, cqp_request);
+ if (status) {
+ err = -ENOMEM;
+ irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_ERR,
+ "CQP-OP MR Reg fail");
+ }
+
+ return err;
+}
+
+/**
+ * irdma_reg_user_mr - Register a user memory region
+ * @pd: ptr of pd
+ * @start: virtual start address
+ * @length: length of mr
+ * @virt: virtual address
+ * @acc: access of mr
+ * @udata: user data
+ */
+static struct ib_mr *irdma_reg_user_mr(struct ib_pd *pd,
+ u64 start,
+ u64 len,
+ u64 virt,
+ int acc,
+ struct ib_udata *udata)
+{
+ struct irdma_pd *iwpd = to_iwpd(pd);
+ struct irdma_device *iwdev = to_iwdev(pd->device);
+ struct irdma_ucontext *ucontext;
+ struct irdma_pble_alloc *palloc;
+ struct irdma_pbl *iwpbl;
+ struct irdma_mr *iwmr;
+ struct ib_umem *region;
+ struct irdma_mem_reg_req req;
+ u64 pbl_depth = 0;
+ u32 stag = 0;
+ u16 access;
+ u64 region_len;
+ bool use_pbles = false;
+ unsigned long flags;
+ int err = -ENOSYS;
+ int ret, pg_shift;
+
+ if (iwdev->closing)
+ return ERR_PTR(-ENODEV);
+
+ if (len > IRDMA_MAX_MR_SIZE)
+ return ERR_PTR(-EINVAL);
+
+ region = ib_umem_get(udata, start, len, acc, 0);
+ if (IS_ERR(region))
+ return (struct ib_mr *)region;
+
+ if (ib_copy_from_udata(&req, udata, sizeof(req))) {
+ ib_umem_release(region);
+ return ERR_PTR(-EFAULT);
+ }
+
+ iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);
+ if (!iwmr) {
+ ib_umem_release(region);
+ return ERR_PTR(-ENOMEM);
+ }
+
+ iwpbl = &iwmr->iwpbl;
+ iwpbl->iwmr = iwmr;
+ iwmr->region = region;
+ iwmr->ibmr.pd = pd;
+ iwmr->ibmr.device = pd->device;
+ ucontext = to_ucontext(pd->uobject->context);
+ iwmr->page_size = PAGE_SIZE;
+ iwmr->page_msk = PAGE_MASK;
+
+ region_len = region->length + (start & (iwmr->page_size - 1));
+ pg_shift = ffs(iwmr->page_size) - 1;
+ pbl_depth = region_len >> pg_shift;
+ pbl_depth += (region_len & (iwmr->page_size - 1)) ? 1 : 0;
+ iwmr->len = region->length;
+ iwpbl->user_base = virt;
+ palloc = &iwpbl->pble_alloc;
+ iwmr->type = req.reg_type;
+ iwmr->page_cnt = (u32)pbl_depth;
+
+ switch (req.reg_type) {
+ case IW_MEMREG_TYPE_QP:
+ use_pbles = ((req.sq_pages + req.rq_pages) > 2);
+ err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles);
+ if (err)
+ goto error;
+
+ spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags);
+ list_add_tail(&iwpbl->list, &ucontext->qp_reg_mem_list);
+ iwpbl->on_list = true;
+ spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags);
+ break;
+ case IW_MEMREG_TYPE_CQ:
+ use_pbles = (req.cq_pages > 1);
+ err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles);
+ if (err)
+ goto error;
+
+ spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
+ list_add_tail(&iwpbl->list, &ucontext->cq_reg_mem_list);
+ iwpbl->on_list = true;
+ spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags);
+ break;
+ case IW_MEMREG_TYPE_MEM:
+ use_pbles = (iwmr->page_cnt != 1);
+ access = IRDMA_ACCESS_FLAGS_LOCALREAD;
+
+ err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles);
+ if (err)
+ goto error;
+
+ if (use_pbles) {
+ ret = irdma_check_mr_contiguous(palloc,
+ iwmr->page_size);
+ if (ret) {
+ irdma_free_pble(iwdev->rf->pble_rsrc, palloc);
+ iwpbl->pbl_allocated = false;
+ }
+ }
+
+ access |= irdma_get_user_access(acc);
+ stag = irdma_create_stag(iwdev);
+ if (!stag) {
+ err = -ENOMEM;
+ goto error;
+ }
+
+ iwmr->stag = stag;
+ iwmr->ibmr.rkey = stag;
+ iwmr->ibmr.lkey = stag;
+ err = irdma_hwreg_mr(iwdev, iwmr, access);
+ if (err) {
+ irdma_free_stag(iwdev, stag);
+ goto error;
+ }
+
+ break;
+ default:
+ goto error;
+ }
+
+ iwmr->type = req.reg_type;
+ if (req.reg_type == IW_MEMREG_TYPE_MEM)
+ irdma_add_pdusecount(iwpd);
+
+ return &iwmr->ibmr;
+
+error:
+ if (palloc->level != PBLE_LEVEL_0 && iwpbl->pbl_allocated)
+ irdma_free_pble(iwdev->rf->pble_rsrc, palloc);
+ ib_umem_release(region);
+ kfree(iwmr);
+
+ return ERR_PTR(err);
+}
+
+/**
+ * irdma_reg_phys_mr - register kernel physical memory
+ * @pd: ibpd pointer
+ * @addr: physical address of memory to register
+ * @size: size of memory to register
+ * @acc: Access rights
+ * @iova_start: start of virtual address for physical buffers
+ */
+struct ib_mr *irdma_reg_phys_mr(struct ib_pd *pd,
+ u64 addr,
+ u64 size,
+ int acc,
+ u64 *iova_start)
+{
+ struct irdma_pd *iwpd = to_iwpd(pd);
+ struct irdma_device *iwdev = to_iwdev(pd->device);
+ struct irdma_pbl *iwpbl;
+ struct irdma_mr *iwmr;
+ enum irdma_status_code status;
+ u32 stag;
+ u16 access = IRDMA_ACCESS_FLAGS_LOCALREAD;
+ int ret;
+
+ iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);
+ if (!iwmr)
+ return ERR_PTR(-ENOMEM);
+
+ iwmr->ibmr.pd = pd;
+ iwmr->ibmr.device = pd->device;
+ iwpbl = &iwmr->iwpbl;
+ iwpbl->iwmr = iwmr;
+ iwmr->type = IW_MEMREG_TYPE_MEM;
+ iwpbl->user_base = *iova_start;
+ stag = irdma_create_stag(iwdev);
+ if (!stag) {
+ ret = -ENOMEM;
+ goto err;
+ }
+
+ access |= irdma_get_user_access(acc);
+ iwmr->stag = stag;
+ iwmr->ibmr.rkey = stag;
+ iwmr->ibmr.lkey = stag;
+ iwmr->page_cnt = 1;
+ iwmr->pgaddrmem[0] = addr;
+ iwmr->len = size;
+ status = irdma_hwreg_mr(iwdev, iwmr, access);
+ if (status) {
+ irdma_free_stag(iwdev, stag);
+ ret = -ENOMEM;
+ goto err;
+ }
+
+ irdma_add_pdusecount(iwpd);
+ return &iwmr->ibmr;
+
+ err:
+ kfree(iwmr);
+
+ return ERR_PTR(ret);
+}
+
+/**
+ * irdma_get_dma_mr - register physical mem
+ * @pd: ptr of pd
+ * @acc: access for memory
+ */
+static struct ib_mr *irdma_get_dma_mr(struct ib_pd *pd, int acc)
+{
+ u64 kva = 0;
+
+ return irdma_reg_phys_mr(pd, 0, 0, acc, &kva);
+}
+
+/**
+ * irdma_del_mem_list - Deleting pbl list entries for CQ/QP
+ * @iwmr: iwmr for IB's user page addresses
+ * @ucontext: ptr to user context
+ */
+static void irdma_del_memlist(struct irdma_mr *iwmr,
+ struct irdma_ucontext *ucontext)
+{
+ struct irdma_pbl *iwpbl = &iwmr->iwpbl;
+ unsigned long flags;
+
+ switch (iwmr->type) {
+ case IW_MEMREG_TYPE_CQ:
+ spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
+ if (iwpbl->on_list) {
+ iwpbl->on_list = false;
+ list_del(&iwpbl->list);
+ }
+ spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags);
+ break;
+ case IW_MEMREG_TYPE_QP:
+ spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags);
+ if (iwpbl->on_list) {
+ iwpbl->on_list = false;
+ list_del(&iwpbl->list);
+ }
+ spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags);
+ break;
+ default:
+ break;
+ }
+}
+
+/**
+ * irdma_dereg_mr - deregister mr
+ * @ib_mr: mr ptr for dereg
+ */
+static int irdma_dereg_mr(struct ib_mr *ib_mr)
+{
+ struct ib_pd *ibpd = ib_mr->pd;
+ struct irdma_pd *iwpd = to_iwpd(ibpd);
+ struct irdma_mr *iwmr = to_iwmr(ib_mr);
+ struct irdma_device *iwdev = to_iwdev(ib_mr->device);
+ enum irdma_status_code status;
+ struct irdma_dealloc_stag_info *info;
+ struct irdma_pbl *iwpbl = &iwmr->iwpbl;
+ struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc;
+ struct irdma_cqp_request *cqp_request;
+ struct cqp_cmds_info *cqp_info;
+ u32 stag_idx;
+
+ if (iwmr->type != IW_MEMREG_TYPE_MEM) {
+ if (iwmr->region) {
+ struct irdma_ucontext *ucontext;
+
+ ucontext = to_ucontext(ibpd->uobject->context);
+ irdma_del_memlist(iwmr, ucontext);
+ }
+ goto done;
+ }
+
+ cqp_request = irdma_get_cqp_request(&iwdev->rf->cqp, true);
+ if (!cqp_request)
+ return -ENOMEM;
+
+ cqp_info = &cqp_request->info;
+ info = &cqp_info->in.u.dealloc_stag.info;
+ memset(info, 0, sizeof(*info));
+ info->pd_id = iwpd->sc_pd.pd_id & 0x00007fff;
+ info->stag_idx = RS_64_1(ib_mr->rkey, IRDMA_CQPSQ_STAG_IDX_S);
+ stag_idx = info->stag_idx;
+ info->mr = true;
+ if (iwpbl->pbl_allocated)
+ info->dealloc_pbl = true;
+
+ cqp_info->cqp_cmd = IRDMA_OP_DEALLOC_STAG;
+ cqp_info->post_sq = 1;
+ cqp_info->in.u.dealloc_stag.dev = &iwdev->rf->sc_dev;
+ cqp_info->in.u.dealloc_stag.scratch = (uintptr_t)cqp_request;
+ status = irdma_handle_cqp_op(iwdev->rf, cqp_request);
+ if (status)
+ irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_ERR,
+ "CQP-OP dealloc failed for stag_idx = 0x%x\n",
+ stag_idx);
+ irdma_rem_pdusecount(iwpd, iwdev);
+ irdma_free_stag(iwdev, iwmr->stag);
+done:
+ if (iwpbl->pbl_allocated)
+ irdma_free_pble(iwdev->rf->pble_rsrc, palloc);
+ if (iwmr->region)
+ ib_umem_release(iwmr->region);
+ kfree(iwmr);
+
+ return 0;
+}
+
+/**
+ * hw_rev_show
+ */
+static ssize_t hw_rev_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct irdma_ib_device *iwibdev = container_of(dev,
+ struct irdma_ib_device,
+ ibdev.dev);
+ u32 hw_rev = iwibdev->iwdev->rf->sc_dev.pci_rev;
+
+ return sprintf(buf, "%x\n", hw_rev);
+}
+
+/**
+ * hca_type_show
+ */
+static ssize_t hca_type_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ return sprintf(buf, "IRDMA\n");
+}
+
+/**
+ * board_id_show
+ */
+static ssize_t board_id_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ return sprintf(buf, "%.*s\n", 32, "IRDMA Board ID");
+}
+
+static DEVICE_ATTR_RO(hw_rev);
+static DEVICE_ATTR_RO(hca_type);
+static DEVICE_ATTR_RO(board_id);
+
+static struct attribute *irdma_dev_attributes[] = {
+ &dev_attr_hw_rev.attr,
+ &dev_attr_hca_type.attr,
+ &dev_attr_board_id.attr,
+ NULL
+};
+
+static const struct attribute_group irdma_attr_group = {
+ .attrs = irdma_dev_attributes,
+};
+
+/**
+ * irdma_copy_sg_list - copy sg list for qp
+ * @sg_list: copied into sg_list
+ * @sgl: copy from sgl
+ * @num_sges: count of sg entries
+ */
+static void irdma_copy_sg_list(struct irdma_sge *sg_list,
+ struct ib_sge *sgl,
+ int num_sges)
+{
+ unsigned int i;
+
+ for (i = 0; (i < num_sges) && (i < IRDMA_MAX_WQ_FRAGMENT_COUNT); i++) {
+ sg_list[i].tag_off = sgl[i].addr;
+ sg_list[i].len = sgl[i].length;
+ sg_list[i].stag = sgl[i].lkey;
+ }
+}
+
+/**
+ * irdma_post_send - kernel application wr
+ * @ibqp: qp ptr for wr
+ * @ib_wr: work request ptr
+ * @bad_wr: return of bad wr if err
+ */
+static int irdma_post_send(struct ib_qp *ibqp,
+ const struct ib_send_wr *ib_wr,
+ const struct ib_send_wr **bad_wr)
+{
+ struct irdma_qp *iwqp;
+ struct irdma_qp_uk *ukqp;
+ struct irdma_sc_dev *dev;
+ struct irdma_post_sq_info info;
+ enum irdma_status_code ret;
+ int err = 0;
+ unsigned long flags;
+ bool inv_stag;
+ struct irdma_ah *ah;
+ bool reflush = false;
+
+ iwqp = (struct irdma_qp *)ibqp;
+ ukqp = &iwqp->sc_qp.qp_uk;
+ dev = &iwqp->iwdev->rf->sc_dev;
+
+ spin_lock_irqsave(&iwqp->lock, flags);
+ if (iwqp->flush_issued && ukqp->sq_flush_complete)
+ reflush = true;
+
+ while (ib_wr) {
+ memset(&info, 0, sizeof(info));
+ inv_stag = false;
+ info.wr_id = (u64)(ib_wr->wr_id);
+ if ((ib_wr->send_flags & IB_SEND_SIGNALED) || iwqp->sig_all)
+ info.signaled = true;
+ if (ib_wr->send_flags & IB_SEND_FENCE)
+ info.read_fence = true;
+ switch (ib_wr->opcode) {
+ case IB_WR_SEND_WITH_IMM:
+ if (ukqp->qp_caps & IRDMA_SEND_WITH_IMM) {
+ info.imm_data_valid = true;
+ info.imm_data = ntohl(ib_wr->ex.imm_data);
+ } else {
+ err = -EINVAL;
+ break;
+ }
+ /* fall-through */
+ case IB_WR_SEND:
+ /* fall-through */
+ case IB_WR_SEND_WITH_INV:
+ if (ib_wr->opcode == IB_WR_SEND ||
+ ib_wr->opcode == IB_WR_SEND_WITH_IMM) {
+ if (ib_wr->send_flags & IB_SEND_SOLICITED)
+ info.op_type = IRDMA_OP_TYPE_SEND_SOL;
+ else
+ info.op_type = IRDMA_OP_TYPE_SEND;
+ } else {
+ if (ib_wr->send_flags & IB_SEND_SOLICITED)
+ info.op_type = IRDMA_OP_TYPE_SEND_SOL_INV;
+ else
+ info.op_type = IRDMA_OP_TYPE_SEND_INV;
+ info.stag_to_inv = ib_wr->ex.invalidate_rkey;
+ }
+
+ if (ib_wr->send_flags & IB_SEND_INLINE) {
+ info.op.inline_send.data = (void *)(unsigned long)
+ ib_wr->sg_list[0].addr;
+ info.op.inline_send.len = ib_wr->sg_list[0].length;
+ if (iwqp->ibqp.qp_type == IB_QPT_UD ||
+ iwqp->ibqp.qp_type == IB_QPT_GSI) {
+ ah = to_iwah(ud_wr(ib_wr)->ah);
+ info.op.inline_send.ah_id = ah->sc_ah.ah_info.ah_idx;
+ info.op.inline_send.qkey = ud_wr(ib_wr)->remote_qkey;
+ info.op.inline_send.dest_qp = ud_wr(ib_wr)->remote_qpn;
+ }
+ ret = ukqp->qp_ops.iw_inline_send(ukqp, &info, false);
+ } else {
+ info.op.send.num_sges = ib_wr->num_sge;
+ info.op.send.sg_list = (struct irdma_sge *)
+ ib_wr->sg_list;
+ if (iwqp->ibqp.qp_type == IB_QPT_UD ||
+ iwqp->ibqp.qp_type == IB_QPT_GSI) {
+ ah = to_iwah(ud_wr(ib_wr)->ah);
+ info.op.send.ah_id = ah->sc_ah.ah_info.ah_idx;
+ info.op.send.qkey = ud_wr(ib_wr)->remote_qkey;
+ info.op.send.dest_qp = ud_wr(ib_wr)->remote_qpn;
+ }
+ ret = ukqp->qp_ops.iw_send(ukqp, &info, false);
+ }
+
+ if (ret) {
+ if (ret == IRDMA_ERR_QP_TOOMANY_WRS_POSTED)
+ err = -ENOMEM;
+ else
+ err = -EINVAL;
+ }
+ break;
+ case IB_WR_RDMA_WRITE_WITH_IMM:
+ if (ukqp->qp_caps & IRDMA_WRITE_WITH_IMM) {
+ info.imm_data_valid = true;
+ info.imm_data = ntohl(ib_wr->ex.imm_data);
+ } else {
+ err = -EINVAL;
+ break;
+ }
+ /* fall-through */
+ case IB_WR_RDMA_WRITE:
+ if (ib_wr->send_flags & IB_SEND_SOLICITED)
+ info.op_type = IRDMA_OP_TYPE_RDMA_WRITE_SOL;
+ else
+ info.op_type = IRDMA_OP_TYPE_RDMA_WRITE;
+
+ if (ib_wr->send_flags & IB_SEND_INLINE) {
+ info.op.inline_rdma_write.data = (void *)ib_wr->sg_list[0].addr;
+ info.op.inline_rdma_write.len = ib_wr->sg_list[0].length;
+ info.op.inline_rdma_write.rem_addr.tag_off = rdma_wr(ib_wr)->remote_addr;
+ info.op.inline_rdma_write.rem_addr.stag = rdma_wr(ib_wr)->rkey;
+ ret = ukqp->qp_ops.iw_inline_rdma_write(ukqp, &info, false);
+ } else {
+ info.op.rdma_write.lo_sg_list = (void *)ib_wr->sg_list;
+ info.op.rdma_write.num_lo_sges = ib_wr->num_sge;
+ info.op.rdma_write.rem_addr.tag_off = rdma_wr(ib_wr)->remote_addr;
+ info.op.rdma_write.rem_addr.stag = rdma_wr(ib_wr)->rkey;
+ ret = ukqp->qp_ops.iw_rdma_write(ukqp, &info, false);
+ }
+
+ if (ret) {
+ if (ret == IRDMA_ERR_QP_TOOMANY_WRS_POSTED)
+ err = -ENOMEM;
+ else
+ err = -EINVAL;
+ }
+ break;
+ case IB_WR_RDMA_READ_WITH_INV:
+ inv_stag = true;
+ /* fall-through*/
+ case IB_WR_RDMA_READ:
+ if (ib_wr->num_sge > dev->hw_attrs.max_hw_read_sges) {
+ err = -EINVAL;
+ break;
+ }
+ info.op_type = IRDMA_OP_TYPE_RDMA_READ;
+ info.op.rdma_read.rem_addr.tag_off = rdma_wr(ib_wr)->remote_addr;
+ info.op.rdma_read.rem_addr.stag = rdma_wr(ib_wr)->rkey;
+ info.op.rdma_read.lo_sg_list = (void *)ib_wr->sg_list;
+ info.op.rdma_read.num_lo_sges = ib_wr->num_sge;
+
+ ret = ukqp->qp_ops.iw_rdma_read(ukqp, &info, inv_stag,
+ false);
+ if (ret) {
+ if (ret == IRDMA_ERR_QP_TOOMANY_WRS_POSTED)
+ err = -ENOMEM;
+ else
+ err = -EINVAL;
+ }
+ break;
+ case IB_WR_LOCAL_INV:
+ info.op_type = IRDMA_OP_TYPE_INV_STAG;
+ info.op.inv_local_stag.target_stag = ib_wr->ex.invalidate_rkey;
+ ret = ukqp->qp_ops.iw_stag_local_invalidate(ukqp, &info, true);
+ if (ret)
+ err = -ENOMEM;
+ break;
+ case IB_WR_REG_MR:
+ {
+ struct irdma_mr *iwmr = to_iwmr(reg_wr(ib_wr)->mr);
+ int flags = reg_wr(ib_wr)->access;
+ struct irdma_pble_alloc *palloc = &iwmr->iwpbl.pble_alloc;
+ struct irdma_fast_reg_stag_info info = {};
+
+ info.access_rights = IRDMA_ACCESS_FLAGS_LOCALREAD;
+ info.access_rights |= irdma_get_user_access(flags);
+ info.stag_key = reg_wr(ib_wr)->key & 0xff;
+ info.stag_idx = reg_wr(ib_wr)->key >> 8;
+ info.page_size = reg_wr(ib_wr)->mr->page_size;
+ info.wr_id = ib_wr->wr_id;
+ info.addr_type = IRDMA_ADDR_TYPE_VA_BASED;
+ info.va = (void *)(uintptr_t)iwmr->ibmr.iova;
+ info.total_len = iwmr->ibmr.length;
+ info.reg_addr_pa = *(u64 *)palloc->level1.addr;
+ info.first_pm_pbl_index = palloc->level1.idx;
+ info.local_fence = ib_wr->send_flags & IB_SEND_FENCE;
+ info.signaled = ib_wr->send_flags & IB_SEND_SIGNALED;
+ if (iwmr->npages > IRDMA_MIN_PAGES_PER_FMR)
+ info.chunk_size = 1;
+ ret = dev->iw_priv_qp_ops->iw_mr_fast_register(&iwqp->sc_qp,
+ &info,
+ true);
+ if (ret)
+ err = -ENOMEM;
+ break;
+ }
+ default:
+ err = -EINVAL;
+ irdma_debug(dev, IRDMA_DEBUG_ERR,
+ " upost_send bad opcode = 0x%x\n",
+ ib_wr->opcode);
+ break;
+ }
+
+ if (err)
+ break;
+ ib_wr = ib_wr->next;
+ }
+
+ if (!iwqp->flush_issued &&
+ iwqp->hw_iwarp_state <= (u32)IRDMA_QP_STATE_RTS) {
+ ukqp->qp_ops.iw_qp_post_wr(ukqp);
+ spin_unlock_irqrestore(&iwqp->lock, flags);
+ } else if (reflush) {
+ struct irdma_qp_flush_info flush_info = {};
+ struct irdma_pci_f *rf = iwqp->iwdev->rf;
+
+ iwqp->sc_qp.flush_sq = false;
+ iwqp->sc_qp.term_flags = 0;
+ spin_unlock_irqrestore(&iwqp->lock, flags);
+ ukqp->sq_flush_complete = false;
+ flush_info.sq = true;
+ flush_info.sq_major_code = IRDMA_FLUSH_MAJOR_ERR;
+ flush_info.sq_minor_code = IRDMA_FLUSH_MAJOR_ERR;
+ irdma_hw_flush_wqes(rf, &iwqp->sc_qp, &flush_info, false);
+ } else {
+ spin_unlock_irqrestore(&iwqp->lock, flags);
+ }
+ if (err)
+ *bad_wr = ib_wr;
+
+ return err;
+}
+
+/**
+ * irdma_post_recv - post receive wr for kernel application
+ * @ibqp: ib qp pointer
+ * @ib_wr: work request for receive
+ * @bad_wr: bad wr caused an error
+ */
+static int irdma_post_recv(struct ib_qp *ibqp,
+ const struct ib_recv_wr *ib_wr,
+ const struct ib_recv_wr **bad_wr)
+{
+ struct irdma_qp *iwqp;
+ struct irdma_qp_uk *ukqp;
+ struct irdma_post_rq_info post_recv = {};
+ struct irdma_sge sg_list[IRDMA_MAX_WQ_FRAGMENT_COUNT];
+ enum irdma_status_code ret = 0;
+ unsigned long flags;
+ int err = 0;
+ bool reflush = false;
+
+ iwqp = (struct irdma_qp *)ibqp;
+ ukqp = &iwqp->sc_qp.qp_uk;
+
+ spin_lock_irqsave(&iwqp->lock, flags);
+ if (iwqp->flush_issued && ukqp->rq_flush_complete)
+ reflush = true;
+
+ while (ib_wr) {
+ post_recv.num_sges = ib_wr->num_sge;
+ post_recv.wr_id = ib_wr->wr_id;
+ irdma_copy_sg_list(sg_list, ib_wr->sg_list, ib_wr->num_sge);
+ post_recv.sg_list = sg_list;
+ ret = ukqp->qp_ops.iw_post_receive(ukqp, &post_recv);
+ if (ret) {
+ irdma_debug(&iwqp->iwdev->rf->sc_dev, IRDMA_DEBUG_ERR,
+ "post_recv err %d\n", ret);
+ if (ret == IRDMA_ERR_QP_TOOMANY_WRS_POSTED)
+ err = -ENOMEM;
+ else
+ err = -EINVAL;
+ goto out;
+ }
+
+ ib_wr = ib_wr->next;
+ }
+
+out:
+ if (reflush) {
+ struct irdma_qp_flush_info flush_info = { };
+ struct irdma_pci_f *rf = iwqp->iwdev->rf;
+
+ iwqp->sc_qp.flush_rq = false;
+ spin_unlock_irqrestore(&iwqp->lock, flags);
+ ukqp->rq_flush_complete = false;
+ flush_info.rq = true;
+ irdma_hw_flush_wqes(rf, &iwqp->sc_qp, &flush_info, false);
+ } else {
+ spin_unlock_irqrestore(&iwqp->lock, flags);
+ }
+
+ if (err)
+ *bad_wr = ib_wr;
+
+ return err;
+}
+
+/**
+ * irdma_poll_cq - poll cq for completion (kernel apps)
+ * @ibcq: cq to poll
+ * @num_entries: number of entries to poll
+ * @entry: wr of entry completed
+ */
+static int irdma_poll_cq(struct ib_cq *ibcq,
+ int num_entries,
+ struct ib_wc *entry)
+{
+ struct irdma_cq *iwcq;
+ int cqe_count = 0;
+ struct irdma_cq_poll_info cq_poll_info;
+ enum irdma_status_code ret;
+ struct irdma_cq_uk *ukcq;
+ struct irdma_sc_qp *qp;
+ struct irdma_qp *iwqp;
+ unsigned long flags;
+
+ iwcq = (struct irdma_cq *)ibcq;
+ ukcq = &iwcq->sc_cq.cq_uk;
+
+ spin_lock_irqsave(&iwcq->lock, flags);
+ while (cqe_count < num_entries) {
+ ret = ukcq->ops.iw_cq_poll_cmpl(ukcq, &cq_poll_info);
+ if (ret == IRDMA_ERR_Q_EMPTY) {
+ break;
+ } else if (ret == IRDMA_ERR_Q_DESTROYED) {
+ continue;
+ } else if (ret) {
+ if (!cqe_count)
+ cqe_count = -1;
+ break;
+ }
+
+ entry->wc_flags = 0;
+ entry->wr_id = cq_poll_info.wr_id;
+ if (cq_poll_info.error) {
+ if (cq_poll_info.comp_status == IRDMA_COMPL_STATUS_FLUSHED)
+ entry->status = IB_WC_WR_FLUSH_ERR;
+ else if (cq_poll_info.comp_status == IRDMA_COMPL_STATUS_INVALID_LEN)
+ entry->status = IB_WC_LOC_LEN_ERR;
+ else
+ entry->status = IB_WC_GENERAL_ERR;
+ entry->vendor_err = cq_poll_info.major_err << 16 |
+ cq_poll_info.minor_err;
+ } else {
+ entry->status = IB_WC_SUCCESS;
+ if (cq_poll_info.imm_valid) {
+ entry->ex.imm_data =
+ htonl(cq_poll_info.imm_data);
+ entry->wc_flags |= IB_WC_WITH_IMM;
+ }
+ if (cq_poll_info.ud_smac_valid) {
+ ether_addr_copy(entry->smac, cq_poll_info.ud_smac);
+ entry->wc_flags |= IB_WC_WITH_SMAC;
+ }
+
+ if (cq_poll_info.ud_vlan_valid) {
+ entry->vlan_id = cq_poll_info.ud_vlan;
+ entry->wc_flags |= IB_WC_WITH_VLAN;
+ entry->sl = (cq_poll_info.ud_vlan >> 13) & 0x07;
+ }
+ }
+
+ switch (cq_poll_info.op_type) {
+ case IRDMA_OP_TYPE_RDMA_WRITE:
+ entry->opcode = IB_WC_RDMA_WRITE;
+ break;
+ case IRDMA_OP_TYPE_RDMA_READ_INV_STAG:
+ case IRDMA_OP_TYPE_RDMA_READ:
+ entry->opcode = IB_WC_RDMA_READ;
+ break;
+ case IRDMA_OP_TYPE_SEND_INV:
+ case IRDMA_OP_TYPE_SEND_SOL:
+ case IRDMA_OP_TYPE_SEND_SOL_INV:
+ case IRDMA_OP_TYPE_SEND:
+ entry->opcode = IB_WC_SEND;
+ if (cq_poll_info.stag_invalid_set)
+ entry->ex.invalidate_rkey = cq_poll_info.inv_stag;
+ break;
+ case IRDMA_OP_TYPE_REC:
+ entry->opcode = IB_WC_RECV;
+ break;
+ case IRDMA_OP_TYPE_REC_IMM:
+ entry->opcode = IB_WC_RECV_RDMA_WITH_IMM;
+ break;
+ default:
+ entry->opcode = IB_WC_RECV;
+ break;
+ }
+
+ qp = (struct irdma_sc_qp *)cq_poll_info.qp_handle;
+ entry->qp = (struct ib_qp *)qp->back_qp;
+ if (qp->qp_type == IRDMA_QP_TYPE_ROCE_UD) {
+ entry->src_qp = cq_poll_info.ud_src_qpn;
+ entry->wc_flags |=
+ (IB_WC_GRH | IB_WC_WITH_NETWORK_HDR_TYPE);
+ entry->network_hdr_type =
+ cq_poll_info.ipv4 ?
+ RDMA_NETWORK_IPV4 : RDMA_NETWORK_IPV6;
+ } else {
+ entry->src_qp = cq_poll_info.qp_id;
+ }
+ iwqp = (struct irdma_qp *)qp->back_qp;
+ if (iwqp->iwarp_state > IRDMA_QP_STATE_RTS) {
+ if (!IRDMA_RING_MORE_WORK(qp->qp_uk.sq_ring))
+ complete(&iwqp->sq_drained);
+ if (!IRDMA_RING_MORE_WORK(qp->qp_uk.rq_ring))
+ complete(&iwqp->rq_drained);
+ }
+ entry->byte_len = cq_poll_info.bytes_xfered;
+ entry++;
+ cqe_count++;
+ }
+ spin_unlock_irqrestore(&iwcq->lock, flags);
+
+ return cqe_count;
+}
+
+/**
+ * irdma_req_notify_cq - arm cq kernel application
+ * @ibcq: cq to arm
+ * @notify_flags: notofication flags
+ */
+static int irdma_req_notify_cq(struct ib_cq *ibcq,
+ enum ib_cq_notify_flags notify_flags)
+{
+ struct irdma_cq *iwcq;
+ struct irdma_cq_uk *ukcq;
+ unsigned long flags;
+ enum irdma_cmpl_notify cq_notify = IRDMA_CQ_COMPL_EVENT;
+
+ iwcq = (struct irdma_cq *)ibcq;
+ ukcq = &iwcq->sc_cq.cq_uk;
+ if (notify_flags == IB_CQ_SOLICITED)
+ cq_notify = IRDMA_CQ_COMPL_SOLICITED;
+ spin_lock_irqsave(&iwcq->lock, flags);
+ ukcq->ops.iw_cq_request_notification(ukcq, cq_notify);
+ spin_unlock_irqrestore(&iwcq->lock, flags);
+
+ return 0;
+}
+
+/**
+ * irdma_port_immutable - return port's immutable data
+ * @ibdev: ib dev struct
+ * @port_num: port number
+ * @immutable: immutable data for the port return
+ */
+static int irdma_port_immutable(struct ib_device *ibdev,
+ u8 port_num,
+ struct ib_port_immutable *immutable)
+{
+ struct ib_port_attr attr;
+ int err;
+ struct irdma_device *iwdev = to_iwdev(ibdev);
+
+ if (iwdev->roce_mode) {
+ immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
+ immutable->max_mad_size = IB_MGMT_MAD_SIZE;
+ } else {
+ immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
+ }
+ err = irdma_query_port(ibdev, port_num, &attr);
+ if (err)
+ return err;
+
+ immutable->pkey_tbl_len = attr.pkey_tbl_len;
+ immutable->gid_tbl_len = attr.gid_tbl_len;
+
+ return 0;
+}
+
+static const char * const irdma_hw_stat_names[] = {
+ /* 32bit names */
+ [IRDMA_HW_STAT_INDEX_RXVLANERR] = "rxVlanErrors",
+ [IRDMA_HW_STAT_INDEX_IP4RXDISCARD] = "ip4InDiscards",
+ [IRDMA_HW_STAT_INDEX_IP4RXTRUNC] = "ip4InTruncatedPkts",
+ [IRDMA_HW_STAT_INDEX_IP4TXNOROUTE] = "ip4OutNoRoutes",
+ [IRDMA_HW_STAT_INDEX_IP6RXDISCARD] = "ip6InDiscards",
+ [IRDMA_HW_STAT_INDEX_IP6RXTRUNC] = "ip6InTruncatedPkts",
+ [IRDMA_HW_STAT_INDEX_IP6TXNOROUTE] = "ip6OutNoRoutes",
+ [IRDMA_HW_STAT_INDEX_TCPRTXSEG] = "tcpRetransSegs",
+ [IRDMA_HW_STAT_INDEX_TCPRXOPTERR] = "tcpInOptErrors",
+ [IRDMA_HW_STAT_INDEX_TCPRXPROTOERR] = "tcpInProtoErrors",
+ [IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED] = "cnpHandled",
+ [IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED] = "cnpIgnored",
+ [IRDMA_HW_STAT_INDEX_TXNPCNPSENT] = "cnpSent",
+
+ /* 64bit names */
+ [IRDMA_HW_STAT_INDEX_IP4RXOCTS + IRDMA_HW_STAT_INDEX_MAX_32] =
+ "ip4InOctets",
+ [IRDMA_HW_STAT_INDEX_IP4RXPKTS + IRDMA_HW_STAT_INDEX_MAX_32] =
+ "ip4InPkts",
+ [IRDMA_HW_STAT_INDEX_IP4RXFRAGS + IRDMA_HW_STAT_INDEX_MAX_32] =
+ "ip4InReasmRqd",
+ [IRDMA_HW_STAT_INDEX_IP4RXMCOCTS + IRDMA_HW_STAT_INDEX_MAX_32] =
+ "ip4InMcastOctets",
+ [IRDMA_HW_STAT_INDEX_IP4RXMCPKTS + IRDMA_HW_STAT_INDEX_MAX_32] =
+ "ip4InMcastPkts",
+ [IRDMA_HW_STAT_INDEX_IP4TXOCTS + IRDMA_HW_STAT_INDEX_MAX_32] =
+ "ip4OutOctets",
+ [IRDMA_HW_STAT_INDEX_IP4TXPKTS + IRDMA_HW_STAT_INDEX_MAX_32] =
+ "ip4OutPkts",
+ [IRDMA_HW_STAT_INDEX_IP4TXFRAGS + IRDMA_HW_STAT_INDEX_MAX_32] =
+ "ip4OutSegRqd",
+ [IRDMA_HW_STAT_INDEX_IP4TXMCOCTS + IRDMA_HW_STAT_INDEX_MAX_32] =
+ "ip4OutMcastOctets",
+ [IRDMA_HW_STAT_INDEX_IP4TXMCPKTS + IRDMA_HW_STAT_INDEX_MAX_32] =
+ "ip4OutMcastPkts",
+ [IRDMA_HW_STAT_INDEX_IP6RXOCTS + IRDMA_HW_STAT_INDEX_MAX_32] =
+ "ip6InOctets",
+ [IRDMA_HW_STAT_INDEX_IP6RXPKTS + IRDMA_HW_STAT_INDEX_MAX_32] =
+ "ip6InPkts",
+ [IRDMA_HW_STAT_INDEX_IP6RXFRAGS + IRDMA_HW_STAT_INDEX_MAX_32] =
+ "ip6InReasmRqd",
+ [IRDMA_HW_STAT_INDEX_IP6RXMCOCTS + IRDMA_HW_STAT_INDEX_MAX_32] =
+ "ip6InMcastOctets",
+ [IRDMA_HW_STAT_INDEX_IP6RXMCPKTS + IRDMA_HW_STAT_INDEX_MAX_32] =
+ "ip6InMcastPkts",
+ [IRDMA_HW_STAT_INDEX_IP6TXOCTS + IRDMA_HW_STAT_INDEX_MAX_32] =
+ "ip6OutOctets",
+ [IRDMA_HW_STAT_INDEX_IP6TXPKTS + IRDMA_HW_STAT_INDEX_MAX_32] =
+ "ip6OutPkts",
+ [IRDMA_HW_STAT_INDEX_IP6TXFRAGS + IRDMA_HW_STAT_INDEX_MAX_32] =
+ "ip6OutSegRqd",
+ [IRDMA_HW_STAT_INDEX_IP6TXMCOCTS + IRDMA_HW_STAT_INDEX_MAX_32] =
+ "ip6OutMcastOctets",
+ [IRDMA_HW_STAT_INDEX_IP6TXMCPKTS + IRDMA_HW_STAT_INDEX_MAX_32] =
+ "ip6OutMcastPkts",
+ [IRDMA_HW_STAT_INDEX_TCPRXSEGS + IRDMA_HW_STAT_INDEX_MAX_32] =
+ "tcpInSegs",
+ [IRDMA_HW_STAT_INDEX_TCPTXSEG + IRDMA_HW_STAT_INDEX_MAX_32] =
+ "tcpOutSegs",
+ [IRDMA_HW_STAT_INDEX_RDMARXRDS + IRDMA_HW_STAT_INDEX_MAX_32] =
+ "iwInRdmaReads",
+ [IRDMA_HW_STAT_INDEX_RDMARXSNDS + IRDMA_HW_STAT_INDEX_MAX_32] =
+ "iwInRdmaSends",
+ [IRDMA_HW_STAT_INDEX_RDMARXWRS + IRDMA_HW_STAT_INDEX_MAX_32] =
+ "iwInRdmaWrites",
+ [IRDMA_HW_STAT_INDEX_RDMATXRDS + IRDMA_HW_STAT_INDEX_MAX_32] =
+ "iwOutRdmaReads",
+ [IRDMA_HW_STAT_INDEX_RDMATXSNDS + IRDMA_HW_STAT_INDEX_MAX_32] =
+ "iwOutRdmaSends",
+ [IRDMA_HW_STAT_INDEX_RDMATXWRS + IRDMA_HW_STAT_INDEX_MAX_32] =
+ "iwOutRdmaWrites",
+ [IRDMA_HW_STAT_INDEX_RDMAVBND + IRDMA_HW_STAT_INDEX_MAX_32] =
+ "iwRdmaBnd",
+ [IRDMA_HW_STAT_INDEX_RDMAVINV + IRDMA_HW_STAT_INDEX_MAX_32] =
+ "iwRdmaInv",
+ [IRDMA_HW_STAT_INDEX_UDPRXPKTS + IRDMA_HW_STAT_INDEX_MAX_32] =
+ "RxUDP",
+ [IRDMA_HW_STAT_INDEX_UDPTXPKTS + IRDMA_HW_STAT_INDEX_MAX_32] =
+ "TxUDP",
+ [IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS + IRDMA_HW_STAT_INDEX_MAX_32] =
+ "RxECNMrkd",
+};
+
+static void irdma_get_dev_fw_str(struct ib_device *dev,
+ char *str)
+{
+ struct irdma_device *iwdev = to_iwdev(dev);
+
+ snprintf(str, IB_FW_VERSION_NAME_MAX, "%u.%u",
+ FW_MAJOR_VER(&iwdev->rf->sc_dev),
+ FW_MINOR_VER(&iwdev->rf->sc_dev));
+}
+
+/**
+ * irdma_alloc_hw_stats - Allocate a hw stats structure
+ * @ibdev: device pointer from stack
+ * @port_num: port number
+ */
+static struct rdma_hw_stats *irdma_alloc_hw_stats(struct ib_device *ibdev,
+ u8 port_num)
+{
+ struct irdma_device *iwdev = to_iwdev(ibdev);
+ struct irdma_sc_dev *dev = &iwdev->rf->sc_dev;
+ int num_counters = IRDMA_HW_STAT_INDEX_MAX_32 +
+ IRDMA_HW_STAT_INDEX_MAX_64;
+ unsigned long lifespan = RDMA_HW_STATS_DEFAULT_LIFESPAN;
+
+ BUILD_BUG_ON(ARRAY_SIZE(irdma_hw_stat_names) != (IRDMA_HW_STAT_INDEX_MAX_32 +
+ IRDMA_HW_STAT_INDEX_MAX_64));
+
+ /*
+ * PFs get the default update lifespan, but VFs only update once
+ * per second
+ */
+ if (!dev->is_pf)
+ lifespan = 1000;
+
+ return rdma_alloc_hw_stats_struct(irdma_hw_stat_names, num_counters,
+ lifespan);
+}
+
+/**
+ * irdma_get_hw_stats - Populates the rdma_hw_stats structure
+ * @ibdev: device pointer from stack
+ * @stats: stats pointer from stack
+ * @port_num: port number
+ * @index: which hw counter the stack is requesting we update
+ */
+static int irdma_get_hw_stats(struct ib_device *ibdev,
+ struct rdma_hw_stats *stats,
+ u8 port_num,
+ int index)
+{
+ struct irdma_device *iwdev = to_iwdev(ibdev);
+ struct irdma_dev_hw_stats *hw_stats = &iwdev->vsi.pestat->hw_stats;
+
+ irdma_cqp_gather_stats_cmd(&iwdev->rf->sc_dev,
+ iwdev->vsi.pestat, true);
+ memcpy(&stats->value[0], hw_stats, sizeof(*hw_stats));
+
+ return stats->num_counters;
+}
+
+/**
+ * irdma_query_gid - Query port GID
+ * @ibdev: device pointer from stack
+ * @port: port number
+ * @index: Entry index
+ * @gid: Global ID
+ */
+static int irdma_query_gid(struct ib_device *ibdev,
+ u8 port,
+ int index,
+ union ib_gid *gid)
+{
+ struct irdma_device *iwdev = to_iwdev(ibdev);
+
+ memset(gid->raw, 0, sizeof(gid->raw));
+ ether_addr_copy(gid->raw, iwdev->netdev->dev_addr);
+
+ return 0;
+}
+
+/**
+ * irdma_modify_port Modify port properties
+ * @ibdev: device pointer from stack
+ * @port: port number
+ * @port_modify_mask: mask for port modifications
+ * @props: port properties
+ */
+static int irdma_modify_port(struct ib_device *ibdev,
+ u8 port,
+ int port_modify_mask,
+ struct ib_port_modify *props)
+{
+ return 0;
+}
+
+/**
+ * irdma_query_gid_roce - Query port GID for Roce
+ * @ibdev: device pointer from stack
+ * @port: port number
+ * @index: Entry index
+ * @gid: Global ID
+ */
+static int irdma_query_gid_roce(struct ib_device *ibdev,
+ u8 port,
+ int index,
+ union ib_gid *gid)
+{
+ int ret;
+
+ ret = rdma_query_gid(ibdev, port, index, gid);
+ if (ret == -EAGAIN) {
+ memcpy(gid, &zgid, sizeof(*gid));
+ return 0;
+ }
+
+ return ret;
+}
+
+/**
+ * mcast_list_add - Add a new mcast item to list
+ * @rf: RDMA PCI function
+ * @new_elem: pointer to element to add
+ */
+static void mcast_list_add(struct irdma_pci_f *rf,
+ struct mc_table_list *new_elem)
+{
+ list_add(&new_elem->list, &rf->mc_qht_list.list);
+}
+
+/**
+ * mcast_list_del - Remove an mcast item from list
+ * @ip_mcast: pointer to mcast IP address
+ */
+static void mcast_list_del(struct mc_table_list *mc_qht_elem)
+{
+ if (mc_qht_elem)
+ list_del(&mc_qht_elem->list);
+}
+
+/**
+ * irdma_mcast_list_lookup_ip - Search mcast list for address
+ * @rf: RDMA PCI function
+ * @ip_mcast: pointer to mcast IP address
+ */
+static struct mc_table_list *mcast_list_lookup_ip(struct irdma_pci_f *rf,
+ u32 *ip_mcast)
+{
+ struct mc_table_list *mc_qht_el;
+ struct list_head *pos, *q;
+
+ list_for_each_safe(pos, q, &rf->mc_qht_list.list) {
+ mc_qht_el = list_entry(pos, struct mc_table_list, list);
+ if (!memcmp(mc_qht_el->mc_info.dest_ip,
+ ip_mcast,
+ sizeof(mc_qht_el->mc_info.dest_ip))) {
+ return mc_qht_el;
+ }
+ }
+
+ return NULL;
+}
+
+/**
+ * irdma_mcast_cqp_op - perform a mcast cqp operation
+ * @iwdev: device
+ * @mc_grp_ctx: mcast group info
+ * @op: operation
+ *
+ * returns error status
+ */
+static int irdma_mcast_cqp_op(struct irdma_device *iwdev,
+ struct irdma_mcast_grp_info *mc_grp_ctx,
+ u8 op)
+{
+ struct cqp_cmds_info *cqp_info;
+ struct irdma_cqp_request *cqp_request;
+ enum irdma_status_code status;
+
+ cqp_request = irdma_get_cqp_request(&iwdev->rf->cqp, true);
+ if (!cqp_request)
+ return -ENOMEM;
+
+ cqp_request->info.in.u.mc_create.info = *mc_grp_ctx;
+ cqp_info = &cqp_request->info;
+ cqp_info->cqp_cmd = op;
+ cqp_info->post_sq = 1;
+ cqp_info->in.u.mc_create.scratch = (uintptr_t)cqp_request;
+ cqp_info->in.u.mc_create.cqp = &iwdev->rf->cqp.sc_cqp;
+ status = irdma_handle_cqp_op(iwdev->rf, cqp_request);
+ if (status) {
+ irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_ERR,
+ "CQP-OP_%s failed\n",
+ (op == IRDMA_OP_MC_MODIFY) ? "MODIFY" : "CREATE");
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+/**
+ * irdma_mcast_mac - Get the multicast MAC for an IP address
+ * @ipaddress: IPv4 or IPv6 address
+ * @mac: pointer to result MAC address
+ * @ipv4: flag indicating IPv4 or IPv6
+ *
+ */
+void irdma_mcast_mac(u32 *ip_addr, u8 *mac, bool ipv4)
+{
+ u8 *ip = (u8 *)ip_addr;
+
+ if (ipv4) {
+ unsigned char mac4[ETH_ALEN] = {0x01, 0x00, 0x5E, 0x00,
+ 0x00, 0x00};
+
+ mac4[3] = ip[2] & 0x7F;
+ mac4[4] = ip[1];
+ mac4[5] = ip[0];
+ ether_addr_copy(mac, mac4);
+ } else {
+ unsigned char mac6[ETH_ALEN] = {0x33, 0x33, 0x00, 0x00,
+ 0x00, 0x00};
+
+ mac6[2] = ip[3];
+ mac6[3] = ip[2];
+ mac6[4] = ip[1];
+ mac6[5] = ip[0];
+ ether_addr_copy(mac, mac6);
+ }
+}
+
+/**
+ * irdma_attach_mcast - attach a qp to a multicast group
+ * @ibqp: ptr to qp
+ * @ibgid: pointer to global ID
+ * @lid: local ID
+ *
+ * returns error status
+ */
+static int irdma_attach_mcast(struct ib_qp *ibqp,
+ union ib_gid *ibgid,
+ u16 lid)
+{
+ struct irdma_qp *iwqp = to_iwqp(ibqp);
+ struct irdma_device *iwdev = iwqp->iwdev;
+ struct irdma_pci_f *rf = iwdev->rf;
+ struct mc_table_list *mc_qht_elem;
+ struct irdma_mcast_grp_ctx_entry_info mcg_info = {};
+ enum irdma_status_code status;
+ bool allocated_mgn = false;
+ unsigned long flags;
+ u32 ip_addr[4] = {};
+ u32 mgn;
+ u32 no_mgs;
+ int ret = 0;
+ bool ipv4;
+ u16 vlan_id;
+ union {
+ struct sockaddr saddr;
+ struct sockaddr_in saddr_in;
+ struct sockaddr_in6 saddr_in6;
+ } sgid_addr;
+ unsigned char dmac[ETH_ALEN];
+
+ rdma_gid2ip(&sgid_addr.saddr, ibgid);
+ if (rdma_gid_attr_network_type(ibqp->av_sgid_attr) == RDMA_NETWORK_IPV6) {
+ irdma_copy_ip_ntohl(ip_addr,
+ sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32);
+ irdma_netdev_vlan_ipv6(ip_addr, &vlan_id, NULL);
+ ipv4 = false;
+ irdma_dev_info(&rf->sc_dev,
+ "qp_id=%d, IP6address=%pI6\n",
+ ibqp->qp_num, ip_addr);
+ irdma_mcast_mac(ip_addr, dmac, false);
+ } else {
+ ip_addr[0] = ntohl(sgid_addr.saddr_in.sin_addr.s_addr);
+ ipv4 = true;
+ vlan_id = irdma_get_vlan_ipv4(ip_addr);
+ irdma_mcast_mac(ip_addr, dmac, true);
+ irdma_dev_info(&rf->sc_dev,
+ "qp_id=%d, IP4address=%pI4, MAC=%pM\n",
+ ibqp->qp_num, ip_addr, dmac);
+ }
+
+ spin_lock_irqsave(&rf->qh_list_lock, flags);
+ mc_qht_elem = mcast_list_lookup_ip(rf, ip_addr);
+ if (!mc_qht_elem) {
+ spin_unlock_irqrestore(&rf->qh_list_lock, flags);
+ mc_qht_elem = kzalloc(sizeof(*mc_qht_elem), GFP_KERNEL);
+ if (!mc_qht_elem)
+ return -ENOMEM;
+
+ mc_qht_elem->mc_info.ipv4_valid = ipv4;
+ memcpy(mc_qht_elem->mc_info.dest_ip, ip_addr,
+ sizeof(mc_qht_elem->mc_info.dest_ip));
+ ret = irdma_alloc_rsrc(rf, rf->allocated_mcgs,
+ rf->max_mcg,
+ &mgn, &rf->next_mcg);
+ if (ret) {
+ kfree(mc_qht_elem);
+ return -ENOMEM;
+ }
+
+ allocated_mgn = true;
+ mc_qht_elem->mc_info.mgn = mgn;
+ status = irdma_allocate_dma_mem(&rf->hw,
+ &mc_qht_elem->mc_grp_ctx.dma_mem_mc,
+ sizeof(u64) * IRDMA_MAX_MGS_PER_CTX,
+ 4096);
+ if (status) {
+ irdma_free_rsrc(rf, rf->allocated_mcgs, mgn);
+ kfree(mc_qht_elem);
+ return -ENOMEM;
+ }
+
+ mc_qht_elem->mc_grp_ctx.mg_id = (u16)mgn;
+ memcpy(mc_qht_elem->mc_grp_ctx.dest_ip_addr, ip_addr,
+ sizeof(mc_qht_elem->mc_grp_ctx.dest_ip_addr));
+ mc_qht_elem->mc_grp_ctx.ipv4_valid = ipv4;
+ mc_qht_elem->mc_grp_ctx.vlan_id = vlan_id;
+ if (vlan_id != IRDMA_NO_VLAN)
+ mc_qht_elem->mc_grp_ctx.vlan_valid = true;
+ mc_qht_elem->mc_grp_ctx.hmc_fcn_id = iwdev->vsi.fcn_id;
+ ether_addr_copy(mc_qht_elem->mc_grp_ctx.dest_mac_addr, dmac);
+
+ spin_lock_irqsave(&rf->qh_list_lock, flags);
+ mcast_list_add(rf, mc_qht_elem);
+ } else {
+ if (mc_qht_elem->mc_grp_ctx.no_of_mgs == IRDMA_MAX_MGS_PER_CTX) {
+ spin_unlock_irqrestore(&rf->qh_list_lock, flags);
+ return -ENOMEM;
+ }
+ }
+
+ mcg_info.qp_id = iwqp->ibqp.qp_num;
+ no_mgs = mc_qht_elem->mc_grp_ctx.no_of_mgs;
+ rf->sc_dev.iw_uda_ops->mcast_grp_add(&mc_qht_elem->mc_grp_ctx,
+ &mcg_info);
+ spin_unlock_irqrestore(&rf->qh_list_lock, flags);
+
+ /* Only if there is a change do we need to modify or create */
+ if (!no_mgs) {
+ ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx,
+ IRDMA_OP_MC_CREATE);
+ } else if (no_mgs != mc_qht_elem->mc_grp_ctx.no_of_mgs) {
+ ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx,
+ IRDMA_OP_MC_MODIFY);
+ } else {
+ return 0;
+ }
+
+ if (ret)
+ goto error;
+
+ atomic_inc(&iwqp->mcast_ref_cnt);
+ return 0;
+
+error:
+ rf->sc_dev.iw_uda_ops->mcast_grp_del(&mc_qht_elem->mc_grp_ctx,
+ &mcg_info);
+ if (!mc_qht_elem->mc_grp_ctx.no_of_mgs) {
+ mcast_list_del(mc_qht_elem);
+ irdma_free_dma_mem(&rf->hw,
+ &mc_qht_elem->mc_grp_ctx.dma_mem_mc);
+ irdma_free_rsrc(rf, rf->allocated_mcgs,
+ mc_qht_elem->mc_grp_ctx.mg_id);
+ kfree(mc_qht_elem);
+ }
+
+ return ret;
+}
+
+/**
+ * irdma_detach_mcast - detach a qp from a multicast group
+ * @ibqp: ptr to qp
+ * @ibgid: pointer to global ID
+ * @lid: local ID
+ *
+ * returns error status
+ */
+static int irdma_detach_mcast(struct ib_qp *ibqp,
+ union ib_gid *ibgid,
+ u16 lid)
+{
+ struct irdma_qp *iwqp = to_iwqp(ibqp);
+ struct irdma_device *iwdev = iwqp->iwdev;
+ struct irdma_pci_f *rf = iwdev->rf;
+ u32 ip_addr[4] = {};
+ struct mc_table_list *mc_qht_elem;
+ struct irdma_mcast_grp_ctx_entry_info mcg_info = {};
+ int ret;
+ unsigned long flags;
+ union {
+ struct sockaddr saddr;
+ struct sockaddr_in saddr_in;
+ struct sockaddr_in6 saddr_in6;
+ } sgid_addr;
+
+ rdma_gid2ip(&sgid_addr.saddr, ibgid);
+ if (rdma_gid_attr_network_type(ibqp->av_sgid_attr) == RDMA_NETWORK_IPV6) {
+ irdma_copy_ip_ntohl(ip_addr,
+ sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32);
+ irdma_dev_info(&rf->sc_dev,
+ "qp_id=%d, IP6address=%pI6\n",
+ ibqp->qp_num, ip_addr);
+ } else {
+ ip_addr[0] = ntohl(sgid_addr.saddr_in.sin_addr.s_addr);
+ irdma_dev_info(&rf->sc_dev,
+ "qp_id=%d, IP4address=%pI4\n",
+ ibqp->qp_num, ip_addr);
+ }
+
+ spin_lock_irqsave(&rf->qh_list_lock, flags);
+ mc_qht_elem = mcast_list_lookup_ip(rf, ip_addr);
+ if (!mc_qht_elem) {
+ spin_unlock_irqrestore(&rf->qh_list_lock, flags);
+ irdma_pr_info("address not found MCG\n");
+ return 0; /* OK to remove group already removed */
+ }
+
+ mcg_info.qp_id = iwqp->ibqp.qp_num;
+ rf->sc_dev.iw_uda_ops->mcast_grp_del(&mc_qht_elem->mc_grp_ctx,
+ &mcg_info);
+ if (!mc_qht_elem->mc_grp_ctx.no_of_mgs) {
+ mcast_list_del(mc_qht_elem);
+ spin_unlock_irqrestore(&rf->qh_list_lock, flags);
+ ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx,
+ IRDMA_OP_MC_DESTROY);
+ if (ret) {
+ irdma_debug(&rf->sc_dev, IRDMA_DEBUG_ERR,
+ "failed MC_DESTROY MCG\n");
+ spin_lock_irqsave(&rf->qh_list_lock, flags);
+ mcast_list_add(rf, mc_qht_elem);
+ spin_unlock_irqrestore(&rf->qh_list_lock, flags);
+ return -EAGAIN;
+ }
+
+ irdma_free_dma_mem(&rf->hw,
+ &mc_qht_elem->mc_grp_ctx.dma_mem_mc);
+ irdma_free_rsrc(rf, rf->allocated_mcgs,
+ mc_qht_elem->mc_grp_ctx.mg_id);
+ kfree(mc_qht_elem);
+ } else {
+ spin_unlock_irqrestore(&rf->qh_list_lock, flags);
+ ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx,
+ IRDMA_OP_MC_MODIFY);
+ if (ret) {
+ irdma_debug(&rf->sc_dev, IRDMA_DEBUG_ERR,
+ "failed Modify MCG\n");
+ return ret;
+ }
+ }
+ atomic_dec(&iwqp->mcast_ref_cnt);
+
+ return 0;
+}
+
+/**
+ * irdma_create_ah - create address handle
+ * @ibpd: ptr to protection domain
+ * @ah_attr: address handle attributes
+ *
+ * returns a pointer to an address handle
+ */
+static struct ib_ah *irdma_create_ah(struct ib_pd *ibpd,
+ struct rdma_ah_attr *attr,
+ u32 flags,
+ struct ib_udata *udata)
+{
+ struct irdma_pd *pd = to_iwpd(ibpd);
+ struct irdma_ah *ah;
+ const struct ib_gid_attr *sgid_attr;
+ struct irdma_device *iwdev = to_iwdev(ibpd->device);
+ struct irdma_pci_f *rf = iwdev->rf;
+ struct irdma_sc_ah *sc_ah;
+ u32 ah_id = 0;
+ struct irdma_ah_info *ah_info;
+ struct irdma_create_ah_resp uresp;
+ union {
+ struct sockaddr saddr;
+ struct sockaddr_in saddr_in;
+ struct sockaddr_in6 saddr_in6;
+ } sgid_addr, dgid_addr;
+ int err;
+ u8 dmac[ETH_ALEN];
+
+ err = irdma_alloc_rsrc(rf, rf->allocated_ahs,
+ rf->max_ah, &ah_id, &rf->next_ah);
+ if (err)
+ return ERR_PTR(err);
+
+ ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
+ if (!ah) {
+ irdma_free_rsrc(rf, rf->allocated_ahs, ah_id);
+ return ERR_PTR(-ENOMEM);
+ }
+
+ ah->pd = pd;
+ sc_ah = &ah->sc_ah;
+ sc_ah->ah_info.ah_idx = ah_id;
+ sc_ah->ah_info.vsi = &iwdev->vsi;
+ iwdev->rf->sc_dev.iw_uda_ops->init_ah(&rf->sc_dev, sc_ah);
+ ah->sgid_index = attr->grh.sgid_index;
+ sgid_attr = attr->grh.sgid_attr;
+ memcpy(&ah->dgid, &attr->grh.dgid, sizeof(ah->dgid));
+ rdma_gid2ip(&sgid_addr.saddr, &sgid_attr->gid);
+ rdma_gid2ip(&dgid_addr.saddr, &attr->grh.dgid);
+ ah->av.attrs = *attr;
+ ah->av.net_type = rdma_gid_attr_network_type(sgid_attr);
+ ah->av.sgid_addr.saddr = sgid_addr.saddr;
+ ah->av.dgid_addr.saddr = dgid_addr.saddr;
+ ah_info = &sc_ah->ah_info;
+ ah_info->ah = sc_ah;
+ ah_info->ah_idx = ah_id;
+ ah_info->pd_idx = pd->sc_pd.pd_id;
+ ether_addr_copy(ah_info->mac_addr, iwdev->netdev->dev_addr);
+ if (attr->ah_flags & IB_AH_GRH) {
+ ah_info->flow_label = attr->grh.flow_label;
+ ah_info->hop_ttl = attr->grh.hop_limit;
+ ah_info->tc_tos = attr->grh.traffic_class;
+ }
+
+ ether_addr_copy(dmac, attr->roce.dmac);
+ if (rdma_gid_attr_network_type(sgid_attr) == RDMA_NETWORK_IPV4) {
+ ah_info->ipv4_valid = true;
+ ah_info->dest_ip_addr[0] =
+ ntohl(dgid_addr.saddr_in.sin_addr.s_addr);
+ ah_info->src_ip_addr[0] =
+ ntohl(sgid_addr.saddr_in.sin_addr.s_addr);
+ ah_info->do_lpbk = irdma_ipv4_is_lpb(ah_info->src_ip_addr[0],
+ ah_info->dest_ip_addr[0]);
+ if (ipv4_is_multicast(dgid_addr.saddr_in.sin_addr.s_addr))
+ irdma_mcast_mac(ah_info->dest_ip_addr, dmac, true);
+ } else {
+ irdma_copy_ip_ntohl(ah_info->dest_ip_addr,
+ dgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32);
+ irdma_copy_ip_ntohl(ah_info->src_ip_addr,
+ sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32);
+ ah_info->do_lpbk = irdma_ipv6_is_lpb(ah_info->src_ip_addr,
+ ah_info->dest_ip_addr);
+ if (rdma_is_multicast_addr(&dgid_addr.saddr_in6.sin6_addr))
+ irdma_mcast_mac(ah_info->dest_ip_addr, dmac, false);
+ }
+ if (sgid_attr->ndev && is_vlan_dev(sgid_attr->ndev))
+ ah_info->vlan_tag = vlan_dev_vlan_id(sgid_attr->ndev);
+ else
+ ah_info->vlan_tag = IRDMA_NO_VLAN;
+
+ ah_info->dst_arpindex = irdma_add_arp(iwdev->rf, ah_info->dest_ip_addr,
+ ah_info->ipv4_valid, dmac);
+
+ if (ah_info->dst_arpindex == -1) {
+ err = -EINVAL;
+ goto error;
+ }
+
+ if (ah_info->vlan_tag != 0xFFFF)
+ ah_info->insert_vlan_tag = true;
+
+ err = irdma_ah_cqp_op(iwdev->rf, sc_ah, IRDMA_OP_AH_CREATE,
+ flags & RDMA_CREATE_AH_SLEEPABLE,
+ irdma_gsi_ud_qp_ah_cb, sc_ah);
+ if (err) {
+ irdma_debug(&rf->sc_dev, IRDMA_DEBUG_ERR,
+ "CQP-OP Create AH fail");
+ goto error;
+ }
+
+ if (!(flags & RDMA_CREATE_AH_SLEEPABLE)) {
+ int cnt = CQP_COMPL_WAIT_TIME_MS * CQP_TIMEOUT_THRESHOLD;
+
+ do {
+ irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq);
+ mdelay(1);
+ } while (!sc_ah->ah_info.ah_valid && --cnt);
+
+ if (!cnt) {
+ irdma_debug(&rf->sc_dev, IRDMA_DEBUG_ERR,
+ "CQP create AH timed out");
+ err = -ETIMEDOUT;
+ goto error;
+ }
+ }
+
+ irdma_add_pdusecount(pd);
+ if (udata) {
+ uresp.ah_id = ah->sc_ah.ah_info.ah_idx;
+ err = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
+ }
+ return &ah->ibah;
+
+error:
+ kfree(ah);
+ irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs, ah_id);
+
+ return ERR_PTR(err);
+}
+
+/**
+ * irdma_destroy_ah - Destroy address handle
+ * @ah: pointer to address handle
+ */
+static int irdma_destroy_ah(struct ib_ah *ibah, u32 flags)
+{
+ struct irdma_device *iwdev = to_iwdev(ibah->device);
+ struct irdma_ah *ah = to_iwah(ibah);
+ int err;
+
+ if (!ah->sc_ah.ah_info.ah_valid)
+ return -EINVAL;
+
+ err = irdma_ah_cqp_op(iwdev->rf, &ah->sc_ah, IRDMA_OP_AH_DESTROY,
+ flags & RDMA_DESTROY_AH_SLEEPABLE,
+ irdma_destroy_ah_cb, ah);
+ if (!err)
+ return 0;
+
+ irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs,
+ ah->sc_ah.ah_info.ah_idx);
+ irdma_rem_pdusecount(ah->pd, iwdev);
+ kfree(ah);
+
+ return 0;
+}
+
+/**
+ * irdma_query_ah - Query address handle
+ * @ib_ah: pointer to address handle
+ */
+static int irdma_query_ah(struct ib_ah *ibah,
+ struct rdma_ah_attr *ah_attr)
+{
+ struct irdma_ah *ah = to_iwah(ibah);
+
+ memset(ah_attr, 0, sizeof(*ah_attr));
+ if (ah->av.attrs.ah_flags & IB_AH_GRH) {
+ ah_attr->ah_flags = IB_AH_GRH;
+ ah_attr->grh.flow_label = ah->sc_ah.ah_info.flow_label;
+ ah_attr->grh.traffic_class = ah->sc_ah.ah_info.tc_tos;
+ ah_attr->grh.hop_limit = ah->sc_ah.ah_info.hop_ttl;
+ ah_attr->grh.sgid_index = ah->sgid_index;
+ ah_attr->grh.sgid_index = ah->sgid_index;
+ memcpy(&ah_attr->grh.dgid, &ah->dgid, sizeof(ah_attr->grh.dgid));
+ }
+
+ return 0;
+}
+
+static enum rdma_link_layer irdma_get_link_layer(struct ib_device *ibdev,
+ u8 port_num)
+{
+ return IB_LINK_LAYER_ETHERNET;
+}
+
+static __be64 irdma_mac_to_guid(struct net_device *ndev)
+{
+ unsigned char *mac = ndev->dev_addr;
+ __be64 guid;
+ unsigned char *dst = (unsigned char *)&guid;
+
+ dst[0] = mac[0] ^ 2;
+ dst[1] = mac[1];
+ dst[2] = mac[2];
+ dst[3] = 0xff;
+ dst[4] = 0xfe;
+ dst[5] = mac[3];
+ dst[6] = mac[4];
+ dst[7] = mac[5];
+
+ return guid;
+}
+
+static struct net_device *irdma_get_netdev(struct ib_device *ibdev,
+ u8 port_num)
+{
+ struct irdma_device *iwdev = to_iwdev(ibdev);
+
+ if (iwdev->netdev) {
+ dev_hold(iwdev->netdev);
+ return iwdev->netdev;
+ }
+
+ return NULL;
+}
+
+static const struct ib_device_ops irdma_roce_dev_ops = {
+ .get_link_layer = irdma_get_link_layer,
+ .query_ah = irdma_query_ah,
+ .attach_mcast = irdma_attach_mcast,
+ .detach_mcast = irdma_detach_mcast,
+ .query_gid = irdma_query_gid_roce,
+ .modify_qp = irdma_modify_qp_roce,
+};
+
+static const struct ib_device_ops irdma_iw_dev_ops = {
+ .query_gid = irdma_query_gid,
+ .modify_qp = irdma_modify_qp,
+};
+
+static const struct ib_device_ops irdma_dev_ops = {
+ .get_port_immutable = irdma_port_immutable,
+ .get_netdev = irdma_get_netdev,
+ .query_port = irdma_query_port,
+ .modify_port = irdma_modify_port,
+ .query_pkey = irdma_query_pkey,
+ .alloc_ucontext = irdma_alloc_ucontext,
+ .dealloc_ucontext = irdma_dealloc_ucontext,
+ .mmap = irdma_mmap,
+ .alloc_pd = irdma_alloc_pd,
+ .dealloc_pd = irdma_dealloc_pd,
+ .create_qp = irdma_create_qp,
+ .query_qp = irdma_query_qp,
+ .destroy_qp = irdma_destroy_qp,
+ .create_cq = irdma_create_cq,
+ .destroy_cq = irdma_destroy_cq,
+ .get_dma_mr = irdma_get_dma_mr,
+ .reg_user_mr = irdma_reg_user_mr,
+ .dereg_mr = irdma_dereg_mr,
+ .alloc_mw = irdma_alloc_mw,
+ .dealloc_mw = irdma_dealloc_mw,
+ .alloc_hw_stats = irdma_alloc_hw_stats,
+ .get_hw_stats = irdma_get_hw_stats,
+ .query_device = irdma_query_device,
+ .create_ah = irdma_create_ah,
+ .destroy_ah = irdma_destroy_ah,
+ .drain_sq = irdma_drain_sq,
+ .drain_rq = irdma_drain_rq,
+ .alloc_mr = irdma_alloc_mr,
+ .map_mr_sg = irdma_map_mr_sg,
+ .get_dev_fw_str = irdma_get_dev_fw_str,
+ .poll_cq = irdma_poll_cq,
+ .req_notify_cq = irdma_req_notify_cq,
+ .post_send = irdma_post_send,
+ .post_recv = irdma_post_recv,
+ .disassociate_ucontext = irdma_disassociate_ucontext,
+};
+
+/**
+ * irdma_init_roce_device - initialization of roce rdma device
+ * @iwibdev: irdma ib device
+ */
+static void irdma_init_roce_device(struct irdma_ib_device *iwibdev)
+{
+ iwibdev->ibdev.uverbs_cmd_mask |=
+ (1ull << IB_USER_VERBS_CMD_ATTACH_MCAST) |
+ (1ull << IB_USER_VERBS_CMD_DETACH_MCAST);
+
+ iwibdev->ibdev.node_type = RDMA_NODE_IB_CA;
+ iwibdev->ibdev.node_guid = irdma_mac_to_guid(iwibdev->iwdev->netdev);
+ iwibdev->ibdev.uverbs_abi_ver = 1;
+ ib_set_device_ops(&iwibdev->ibdev, &irdma_roce_dev_ops);
+}
+
+/**
+ * irdma_init_roce_device - initialization of iwarp rdma device
+ * @iwibdev: irdma ib device
+ */
+static int irdma_init_iw_device(struct irdma_ib_device *iwibdev)
+{
+ struct net_device *netdev = iwibdev->iwdev->netdev;
+
+ iwibdev->ibdev.node_type = RDMA_NODE_RNIC;
+ ether_addr_copy((u8 *)&iwibdev->ibdev.node_guid, netdev->dev_addr);
+ iwibdev->ibdev.iwcm = kzalloc(sizeof(*iwibdev->ibdev.iwcm), GFP_KERNEL);
+ if (!iwibdev->ibdev.iwcm)
+ return -ENOMEM;
+
+ iwibdev->ibdev.iwcm->add_ref = irdma_add_ref;
+ iwibdev->ibdev.iwcm->rem_ref = irdma_rem_ref;
+ iwibdev->ibdev.iwcm->get_qp = irdma_get_qp;
+ iwibdev->ibdev.iwcm->connect = irdma_connect;
+ iwibdev->ibdev.iwcm->accept = irdma_accept;
+ iwibdev->ibdev.iwcm->reject = irdma_reject;
+ iwibdev->ibdev.iwcm->create_listen = irdma_create_listen;
+ iwibdev->ibdev.iwcm->destroy_listen = irdma_destroy_listen;
+ memcpy(iwibdev->ibdev.iwcm->ifname, netdev->name,
+ sizeof(iwibdev->ibdev.iwcm->ifname));
+ ib_set_device_ops(&iwibdev->ibdev, &irdma_iw_dev_ops);
+
+ return 0;
+}
+
+/**
+ * irdma_init_rdma_device - initialization of rdma device
+ * @iwdev: irdma device
+ */
+static int irdma_init_rdma_device(struct irdma_device *iwdev)
+{
+ struct irdma_ib_device *iwibdev;
+ struct pci_dev *pcidev = (struct pci_dev *)iwdev->rf->hw.dev_context;
+ int ret;
+
+ iwibdev = ib_alloc_device(irdma_ib_device, ibdev);
+ if (!iwibdev)
+ return -ENOMEM;
+
+ iwibdev->ibdev.owner = THIS_MODULE;
+ iwdev->iwibdev = iwibdev;
+ iwibdev->iwdev = iwdev;
+
+ iwibdev->ibdev.uverbs_abi_ver = IRDMA_ABI_VER;
+ iwibdev->ibdev.uverbs_cmd_mask =
+ (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
+ (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
+ (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
+ (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
+ (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
+ (1ull << IB_USER_VERBS_CMD_REG_MR) |
+ (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
+ (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
+ (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
+ (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
+ (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) |
+ (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
+ (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
+ (1ull << IB_USER_VERBS_CMD_QUERY_QP) |
+ (1ull << IB_USER_VERBS_CMD_POLL_CQ) |
+ (1ull << IB_USER_VERBS_CMD_CREATE_AH) |
+ (1ull << IB_USER_VERBS_CMD_DESTROY_AH) |
+ (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
+ (1ull << IB_USER_VERBS_CMD_ALLOC_MW) |
+ (1ull << IB_USER_VERBS_CMD_BIND_MW) |
+ (1ull << IB_USER_VERBS_CMD_DEALLOC_MW) |
+ (1ull << IB_USER_VERBS_CMD_POST_RECV) |
+ (1ull << IB_USER_VERBS_CMD_POST_SEND);
+
+ if (iwdev->roce_mode) {
+ irdma_init_roce_device(iwibdev);
+ } else {
+ ret = irdma_init_iw_device(iwibdev);
+ if (ret) {
+ ib_dealloc_device(&iwibdev->ibdev);
+ return ret;
+ }
+ }
+ iwibdev->ibdev.phys_port_cnt = 1;
+ iwibdev->ibdev.num_comp_vectors = iwdev->rf->ceqs_count;
+ iwibdev->ibdev.dev.parent = &pcidev->dev;
+ ib_set_device_ops(&iwibdev->ibdev, &irdma_dev_ops);
+
+ return 0;
+}
+
+/**
+ * irdma_port_ibevent - indicate port event
+ * @iwdev: iwarp device
+ */
+void irdma_port_ibevent(struct irdma_device *iwdev)
+{
+ struct irdma_ib_device *iwibdev = iwdev->iwibdev;
+ struct ib_event event;
+
+ event.device = &iwibdev->ibdev;
+ event.element.port_num = 1;
+ event.event = iwdev->iw_status ?
+ IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;
+ ib_dispatch_event(&event);
+}
+
+/**
+ * irdma_destroy_rdma_device - destroy rdma device and free resources
+ * @iwibdev: IB device ptr
+ */
+void irdma_destroy_rdma_device(struct irdma_ib_device *iwibdev)
+{
+ if (!iwibdev)
+ return;
+
+ ib_unregister_device(&iwibdev->ibdev);
+ kfree(iwibdev->ibdev.iwcm);
+ iwibdev->ibdev.iwcm = NULL;
+ wait_event_timeout(iwibdev->iwdev->close_wq,
+ !atomic64_read(&iwibdev->iwdev->use_count),
+ IRDMA_EVENT_TIMEOUT);
+
+ ib_dealloc_device(&iwibdev->ibdev);
+}
+
+/**
+ * irdma_register_rdma_device - register iwarp device to IB
+ * @iwdev: iwarp device
+ */
+int irdma_register_rdma_device(struct irdma_device *iwdev)
+{
+ int ret;
+ struct irdma_ib_device *iwibdev;
+
+ ret = irdma_init_rdma_device(iwdev);
+ if (ret)
+ return ret;
+
+ iwibdev = iwdev->iwibdev;
+ rdma_set_device_sysfs_group(&iwibdev->ibdev, &irdma_attr_group);
+ if (iwdev->rf->sc_dev.hw_attrs.hw_rev == IRDMA_GEN_1)
+ /* backward compat with old user-space libi40iw */
+ iwibdev->ibdev.driver_id = RDMA_DRIVER_I40IW;
+ else
+ iwibdev->ibdev.driver_id = RDMA_DRIVER_IRDMA;
+
+ ret = ib_register_device(&iwibdev->ibdev, "irdma%d");
+ if (ret)
+ goto error;
+
+ return 0;
+
+error:
+ kfree(iwdev->iwibdev->ibdev.iwcm);
+ iwdev->iwibdev->ibdev.iwcm = NULL;
+ ib_dealloc_device(&iwdev->iwibdev->ibdev);
+ if (ret)
+ irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_ERR,
+ "Register RDMA device fail\n");
+
+ return ret;
+}
diff --git a/drivers/infiniband/hw/irdma/verbs.h b/drivers/infiniband/hw/irdma/verbs.h
new file mode 100644
index 0000000..5a9534d
--- /dev/null
+++ b/drivers/infiniband/hw/irdma/verbs.h
@@ -0,0 +1,191 @@
+/* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB */
+/* Copyright (c) 2019, Intel Corporation. */
+
+#ifndef IRDMA_VERBS_H
+#define IRDMA_VERBS_H
+
+#define IRDMA_MAX_SAVED_PHY_PGADDR 4
+
+struct irdma_vma_data {
+ struct list_head list;
+ struct vm_area_struct *vma;
+ struct mutex *vma_list_mutex; /* protect the vma_list */
+};
+
+struct irdma_ucontext {
+ struct ib_ucontext ibucontext;
+ struct irdma_device *iwdev;
+ struct list_head cq_reg_mem_list;
+ spinlock_t cq_reg_mem_list_lock; /* protect CQ memory list */
+ struct list_head qp_reg_mem_list;
+ spinlock_t qp_reg_mem_list_lock; /* protect QP memory list */
+ struct list_head vma_list;
+ struct mutex vma_list_mutex; /* protect the vma_list */
+ int abi_ver;
+};
+
+struct irdma_pd {
+ struct ib_pd ibpd;
+ struct irdma_sc_pd sc_pd;
+ atomic_t usecount;
+};
+
+struct irdma_av {
+ u8 macaddr[16];
+ struct rdma_ah_attr attrs;
+ union {
+ struct sockaddr saddr;
+ struct sockaddr_in saddr_in;
+ struct sockaddr_in6 saddr_in6;
+ } sgid_addr, dgid_addr;
+ u8 net_type;
+};
+
+struct irdma_ah {
+ struct ib_ah ibah;
+ struct irdma_sc_ah sc_ah;
+ atomic_t usecount;
+ struct irdma_pd *pd;
+ struct irdma_av av;
+ u8 sgid_index;
+ union ib_gid dgid;
+};
+
+struct irdma_hmc_pble {
+ union {
+ u32 idx;
+ dma_addr_t addr;
+ };
+};
+
+struct irdma_cq_mr {
+ struct irdma_hmc_pble cq_pbl;
+ dma_addr_t shadow;
+};
+
+struct irdma_qp_mr {
+ struct irdma_hmc_pble sq_pbl;
+ struct irdma_hmc_pble rq_pbl;
+ dma_addr_t shadow;
+ struct page *sq_page;
+};
+
+struct irdma_pbl {
+ struct list_head list;
+ union {
+ struct irdma_qp_mr qp_mr;
+ struct irdma_cq_mr cq_mr;
+ };
+
+ bool pbl_allocated;
+ bool on_list;
+ u64 user_base;
+ struct irdma_pble_alloc pble_alloc;
+ struct irdma_mr *iwmr;
+};
+
+struct irdma_mr {
+ union {
+ struct ib_mr ibmr;
+ struct ib_mw ibmw;
+ struct ib_fmr ibfmr;
+ };
+ struct ib_umem *region;
+ u16 type;
+ u32 page_cnt;
+ u32 page_size;
+ u64 page_msk;
+ u32 npages;
+ u32 stag;
+ u64 len;
+ u64 pgaddrmem[IRDMA_MAX_SAVED_PHY_PGADDR];
+ struct irdma_pbl iwpbl;
+};
+
+struct irdma_cq {
+ struct ib_cq ibcq;
+ struct irdma_sc_cq sc_cq;
+ u16 cq_head;
+ u16 cq_size;
+ u16 cq_num;
+ bool user_mode;
+ u32 polled_cmpls;
+ u32 cq_mem_size;
+ struct irdma_dma_mem kmem;
+ spinlock_t lock; /* for poll cq */
+ struct irdma_pbl *iwpbl;
+};
+
+struct disconn_work {
+ struct work_struct work;
+ struct irdma_qp *iwqp;
+};
+
+struct iw_cm_id;
+
+struct irdma_qp_kmode {
+ struct irdma_dma_mem dma_mem;
+ u64 *wrid_mem;
+};
+
+struct irdma_qp {
+ struct ib_qp ibqp;
+ struct irdma_sc_qp sc_qp;
+ struct irdma_device *iwdev;
+ struct irdma_cq *iwscq;
+ struct irdma_cq *iwrcq;
+ struct irdma_pd *iwpd;
+ struct irdma_qp_host_ctx_info ctx_info;
+ union {
+ struct irdma_iwarp_offload_info iwarp_info;
+ struct irdma_roce_offload_info roce_info;
+ };
+
+ struct irdma_udp_offload_info udp_info;
+ struct irdma_ah roce_ah;
+ struct list_head teardown_entry;
+ void *allocated_buf;
+ atomic_t refcount;
+ struct iw_cm_id *cm_id;
+ void *cm_node;
+ struct ib_mr *lsmm_mr;
+ struct work_struct work;
+ atomic_t hw_mod_qp_pend;
+ enum ib_qp_state ibqp_state;
+ u32 iwarp_state;
+ u32 qp_mem_size;
+ u32 last_aeq;
+ atomic_t close_timer_started;
+ spinlock_t lock; /* serialize posting WRs to SQ/RQ */
+ struct irdma_qp_context *iwqp_context;
+ void *pbl_vbase;
+ dma_addr_t pbl_pbase;
+ struct page *page;
+ u8 active_conn:1;
+ u8 user_mode:1;
+ u8 hte_added:1;
+ u8 flush_issued:1;
+ u8 destroyed:1;
+ u8 sig_all:1;
+ u8 pau_mode:1;
+ u8 rsvd:1;
+ u16 term_sq_flush_code;
+ u16 term_rq_flush_code;
+ u8 hw_iwarp_state;
+ u8 hw_tcp_state;
+ struct irdma_qp_kmode kqp;
+ struct irdma_dma_mem host_ctx;
+ struct timer_list terminate_timer;
+ struct irdma_pbl *iwpbl;
+ struct irdma_dma_mem q2_ctx_mem;
+ struct irdma_dma_mem ietf_mem;
+ struct completion sq_drained;
+ struct completion rq_drained;
+ atomic_t mcast_ref_cnt;
+ wait_queue_head_t waitq;
+ wait_queue_head_t mod_qp_waitq;
+ u8 rts_ae_rcvd;
+};
+
+void irdma_mcast_mac(u32 *ip_addr, u8 *mac, bool ipv4);
+#endif /* IRDMA_VERBS_H */
diff --git a/include/uapi/rdma/rdma_user_ioctl_cmds.h b/include/uapi/rdma/rdma_user_ioctl_cmds.h
index 06c34d9..02bc8db 100644
--- a/include/uapi/rdma/rdma_user_ioctl_cmds.h
+++ b/include/uapi/rdma/rdma_user_ioctl_cmds.h
@@ -102,6 +102,7 @@ enum rdma_driver_id {
RDMA_DRIVER_RXE,
RDMA_DRIVER_HFI1,
RDMA_DRIVER_QIB,
+ RDMA_DRIVER_IRDMA,
};
#endif
--
1.8.3.1
^ permalink raw reply related
* [RFC 15/19] RDMA/irdma: Add miscellaneous utility definitions
From: Shiraz Saleem @ 2019-02-12 21:43 UTC (permalink / raw)
To: dledford, jgg, davem
Cc: linux-rdma, netdev, mustafa.ismail, jeffrey.t.kirsher,
Shiraz Saleem
In-Reply-To: <20190212214402.23284-1-shiraz.saleem@intel.com>
From: Mustafa Ismail <mustafa.ismail@intel.com>
Add miscellaneous utility functions and headers.
Signed-off-by: Mustafa Ismail <mustafa.ismail@intel.com>
Signed-off-by: Shiraz Saleem <shiraz.saleem@intel.com>
---
drivers/infiniband/hw/irdma/osdep.h | 153 ++
drivers/infiniband/hw/irdma/protos.h | 118 ++
drivers/infiniband/hw/irdma/status.h | 70 +
drivers/infiniband/hw/irdma/utils.c | 2566 ++++++++++++++++++++++++++++++++++
4 files changed, 2907 insertions(+)
create mode 100644 drivers/infiniband/hw/irdma/osdep.h
create mode 100644 drivers/infiniband/hw/irdma/protos.h
create mode 100644 drivers/infiniband/hw/irdma/status.h
create mode 100644 drivers/infiniband/hw/irdma/utils.c
diff --git a/drivers/infiniband/hw/irdma/osdep.h b/drivers/infiniband/hw/irdma/osdep.h
new file mode 100644
index 0000000..ade5536
--- /dev/null
+++ b/drivers/infiniband/hw/irdma/osdep.h
@@ -0,0 +1,153 @@
+/* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB */
+/* Copyright (c) 2019, Intel Corporation. */
+
+#ifndef IRDMA_OSDEP_H
+#define IRDMA_OSDEP_H
+
+#include <linux/version.h>
+#include <linux/kernel.h>
+#include <linux/vmalloc.h>
+#include <linux/string.h>
+#include <linux/bitops.h>
+#include <linux/pci.h>
+#include <net/tcp.h>
+#include <crypto/hash.h>
+/* get readq/writeq support for 32 bit kernels, use the low-first version */
+#include <linux/io-64-nonatomic-lo-hi.h>
+
+#define STATS_TIMER_DELAY 60000
+#define MAKEMASK(m, s) ((m) << (s))
+
+#define irdma_pr_err(fmt, args ...) \
+ pr_err("%s: "fmt, __func__, ## args)
+
+#define irdma_pr_info(fmt, args ...) \
+ pr_info("%s: " fmt, __func__, ## args)
+
+#define irdma_pr_warn(fmt, args ...) \
+ pr_warn("%s: " fmt, __func__, ## args)
+
+#define irdma_dev_err(dev, fmt, args ...) \
+ dev_err(to_device(dev), "%s: "fmt, __func__, ## args)
+
+#define irdma_dev_info(dev, fmt, args ...) \
+ dev_info(to_device(dev), "%s: "fmt, __func__, ## args)
+
+#define irdma_dev_warn(dev, fmt, args ...) \
+ dev_warn(to_device(dev), "%s: "fmt, __func__, ## args)
+
+#define to_device(ptr) \
+ (&((struct pci_dev *)((ptr)->hw->dev_context))->dev)
+
+#ifdef CONFIG_DYNAMIC_DEBUG
+#define irdma_debug(dev, prefix, ...) \
+ dev_dbg(to_device(dev), prefix ": " __VA_ARGS__)
+#define irdma_debug_buf(dev, prefix, desc, buf, size) \
+ print_hex_dump_debug(prefix ": " desc " ", \
+ DUMP_PREFIX_OFFSET, \
+ 16, 8, buf, size, false)
+#else
+#define irdma_debug(dev, mask, fmt, ...) \
+do { \
+ if (((mask) & (dev)->debug_mask)) \
+ dev_info(to_device(dev), \
+ "%s: " fmt, __func__, \
+ ##__VA_ARGS__); \
+} while (0)
+#define irdma_debug_buf(dev, mask, desc, buf, size) \
+do { \
+ if (((mask) & (dev)->debug_mask)) \
+ print_hex_dump_debug(desc " ", \
+ DUMP_PREFIX_OFFSET, \
+ 16, 8, buf, size, false); \
+} while (0)
+#endif
+
+struct irdma_dma_info {
+ dma_addr_t *dmaaddrs;
+};
+
+struct irdma_dma_mem {
+ void *va;
+ dma_addr_t pa;
+ u32 size;
+} __packed;
+
+struct irdma_virt_mem {
+ void *va;
+ u32 size;
+} __packed;
+
+struct irdma_sc_vsi;
+struct irdma_sc_dev;
+struct irdma_sc_qp;
+struct irdma_puda_buf;
+struct irdma_puda_cmpl_info;
+struct irdma_update_sds_info;
+struct irdma_hmc_fcn_info;
+struct irdma_virtchnl_work_info;
+struct irdma_manage_vf_pble_info;
+struct irdma_hw;
+struct irdma_pci_f;
+
+u8 __iomem *irdma_get_hw_addr(void *dev);
+void irdma_ieq_mpa_crc_ae(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp);
+enum irdma_status_code irdma_vf_wait_vchnl_resp(struct irdma_sc_dev *dev);
+bool irdma_vf_clear_to_send(struct irdma_sc_dev *dev);
+void irdma_add_dev_ref(struct irdma_sc_dev *dev);
+void irdma_put_dev_ref(struct irdma_sc_dev *dev);
+enum irdma_status_code irdma_ieq_check_mpacrc(struct shash_desc *desc,
+ void *addr,
+ u32 len, u32 val);
+struct irdma_sc_qp *irdma_ieq_get_qp(struct irdma_sc_dev *dev,
+ struct irdma_puda_buf *buf);
+void irdma_send_ieq_ack(struct irdma_sc_qp *qp);
+void irdma_ieq_update_tcpip_info(struct irdma_puda_buf *buf,
+ u16 len,
+ u32 seqnum);
+void irdma_free_hash_desc(struct shash_desc *hash_desc);
+enum irdma_status_code irdma_init_hash_desc(struct shash_desc **hash_desc);
+enum irdma_status_code
+irdma_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,
+ struct irdma_puda_buf *buf);
+enum irdma_status_code irdma_cqp_sds_cmd(struct irdma_sc_dev *dev,
+ struct irdma_update_sds_info *info);
+enum irdma_status_code
+irdma_cqp_manage_hmc_fcn_cmd(struct irdma_sc_dev *dev,
+ struct irdma_hmc_fcn_info *hmcfcninfo);
+enum irdma_status_code
+irdma_cqp_query_fpm_val_cmd(struct irdma_sc_dev *dev,
+ struct irdma_dma_mem *val_mem,
+ u8 hmc_fn_id);
+enum irdma_status_code
+irdma_cqp_commit_fpm_val_cmd(struct irdma_sc_dev *dev,
+ struct irdma_dma_mem *val_mem,
+ u8 hmc_fn_id);
+enum irdma_status_code irdma_alloc_query_fpm_buf(struct irdma_sc_dev *dev,
+ struct irdma_dma_mem *mem);
+enum irdma_status_code
+irdma_cqp_manage_vf_pble_bp(struct irdma_sc_dev *dev,
+ struct irdma_manage_vf_pble_info *info);
+void irdma_cqp_spawn_worker(struct irdma_sc_dev *dev,
+ struct irdma_virtchnl_work_info *work_info,
+ u32 iw_vf_idx);
+void *irdma_remove_head(struct list_head *list);
+enum irdma_status_code
+irdma_qp_suspend_resume(struct irdma_sc_qp *qp, bool suspend);
+void irdma_term_modify_qp(struct irdma_sc_qp *qp,
+ u8 next_state,
+ u8 term,
+ u8 term_len);
+void irdma_terminate_done(struct irdma_sc_qp *qp, int timeout_occurred);
+void irdma_terminate_start_timer(struct irdma_sc_qp *qp);
+void irdma_terminate_del_timer(struct irdma_sc_qp *qp);
+enum irdma_status_code
+irdma_hw_manage_vf_pble_bp(struct irdma_pci_f *rf,
+ struct irdma_manage_vf_pble_info *info,
+ bool wait);
+void irdma_hw_stats_start_timer(struct irdma_sc_vsi *vsi);
+void irdma_hw_stats_stop_timer(struct irdma_sc_vsi *vsi);
+void wr32(struct irdma_hw *hw, u32 reg, u32 val);
+u32 rd32(struct irdma_hw *hw, u32 reg);
+u64 rd64(struct irdma_hw *hw, u32 reg);
+#endif /* _IRDMA_OSDEP_H_ */
diff --git a/drivers/infiniband/hw/irdma/protos.h b/drivers/infiniband/hw/irdma/protos.h
new file mode 100644
index 0000000..d65004b
--- /dev/null
+++ b/drivers/infiniband/hw/irdma/protos.h
@@ -0,0 +1,118 @@
+/* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB */
+/* Copyright (c) 2019, Intel Corporation. */
+
+#ifndef IRDMA_PROTOS_H
+#define IRDMA_PROTOS_H
+
+#define PAUSE_TIMER_VAL 0xffff
+#define REFRESH_THRESHOLD 0x7fff
+#define HIGH_THRESHOLD 0x800
+#define LOW_THRESHOLD 0x200
+#define ALL_TC2PFC 0xff
+#define CQP_COMPL_WAIT_TIME_MS 10
+#define CQP_TIMEOUT_THRESHOLD 500
+
+/**
+ * irdma_insert_wqe_hdr - write wqe header
+ * @wqe: cqp wqe for header
+ * @header: header for the cqp wqe
+ */
+static inline void irdma_insert_wqe_hdr(__le64 *wqe, u64 hdr)
+{
+ wmb(); /* make sure WQE is populated before polarity is set */
+ set_64bit_val(wqe, 24, hdr);
+}
+
+/* init operations */
+enum irdma_status_code irdma_sc_ctrl_init(enum irdma_vers ver,
+ struct irdma_sc_dev *dev,
+ struct irdma_device_init_info *info);
+void irdma_sc_rt_init(struct irdma_sc_dev *dev);
+void irdma_sc_cqp_post_sq(struct irdma_sc_cqp *cqp);
+__le64 *irdma_sc_cqp_get_next_send_wqe(struct irdma_sc_cqp *cqp, u64 scratch);
+enum irdma_status_code irdma_sc_mr_fast_register(struct irdma_sc_qp *qp,
+ struct irdma_fast_reg_stag_info *info,
+ bool post_sq);
+/* HMC/FPM functions */
+enum irdma_status_code irdma_sc_init_iw_hmc(struct irdma_sc_dev *dev,
+ u8 hmc_fn_id);
+/* stats misc */
+enum irdma_status_code irdma_cqp_gather_stats_cmd(struct irdma_sc_dev *dev,
+ struct irdma_vsi_pestat *pestat,
+ bool wait);
+enum irdma_status_code irdma_cqp_ws_node_cmd(struct irdma_sc_dev *dev,
+ u8 cmd,
+ struct irdma_ws_node_info *node_info);
+enum irdma_status_code irdma_cqp_up_map_cmd(struct irdma_sc_dev *dev,
+ u8 cmd,
+ struct irdma_up_info *map_info);
+enum irdma_status_code irdma_cqp_ceq_cmd(struct irdma_sc_dev *dev,
+ struct irdma_sc_ceq *sc_ceq,
+ u8 op);
+enum irdma_status_code
+irdma_cqp_stats_inst_cmd(struct irdma_sc_vsi *vsi,
+ u8 cmd,
+ struct irdma_stats_inst_info *stats_info);
+u16 irdma_alloc_ws_node_id(struct irdma_sc_dev *dev);
+void irdma_free_ws_node_id(struct irdma_sc_dev *dev, u16 node_id);
+void irdma_update_stats(struct irdma_dev_hw_stats *hw_stats,
+ struct irdma_gather_stats *gather_stats,
+ struct irdma_gather_stats *last_gather_stats);
+/* vsi functions */
+enum irdma_status_code irdma_vsi_stats_init(struct irdma_sc_vsi *vsi,
+ struct irdma_vsi_stats_info *info);
+void irdma_vsi_stats_free(struct irdma_sc_vsi *vsi);
+void irdma_sc_vsi_init(struct irdma_sc_vsi *vsi,
+ struct irdma_vsi_init_info *info);
+enum irdma_status_code irdma_sc_add_cq_ctx(struct irdma_sc_ceq *ceq, struct irdma_sc_cq *cq);
+void irdma_sc_remove_cq_ctx(struct irdma_sc_ceq *ceq, struct irdma_sc_cq *cq);
+/* misc L2 param change functions */
+void irdma_change_l2params(struct irdma_sc_vsi *vsi,
+ struct irdma_l2params *l2params);
+void irdma_suspend_qps(struct irdma_sc_vsi *vsi);
+void irdma_qp_add_qos(struct irdma_sc_qp *qp);
+void irdma_qp_rem_qos(struct irdma_sc_qp *qp);
+struct irdma_sc_qp *irdma_get_qp_from_list(struct list_head *head,
+ struct irdma_sc_qp *qp);
+void irdma_reinitialize_ieq(struct irdma_sc_vsi *vsi);
+u16 irdma_alloc_ws_node_id(struct irdma_sc_dev *dev);
+void irdma_free_ws_node_id(struct irdma_sc_dev *dev, u16 node_id);
+enum irdma_status_code irdma_lan_register_qset(struct irdma_sc_vsi *vsi,
+ struct irdma_ws_node *tc_node);
+void irdma_lan_unregister_qset(struct irdma_sc_vsi *vsi,
+ struct irdma_ws_node *tc_node);
+/* terminate functions*/
+void irdma_terminate_send_fin(struct irdma_sc_qp *qp);
+
+void irdma_terminate_connection(struct irdma_sc_qp *qp,
+ struct irdma_aeqe_info *info);
+
+void irdma_terminate_received(struct irdma_sc_qp *qp,
+ struct irdma_aeqe_info *info);
+/* dynamic memory allocation */
+enum irdma_status_code irdma_allocate_dma_mem(struct irdma_hw *hw,
+ struct irdma_dma_mem *mem,
+ u64 size,
+ u32 alignment);
+void irdma_free_dma_mem(struct irdma_hw *hw, struct irdma_dma_mem *mem);
+enum irdma_status_code irdma_allocate_virt_mem(struct irdma_hw *hw,
+ struct irdma_virt_mem *mem,
+ u32 size);
+enum irdma_status_code irdma_free_virt_mem(struct irdma_hw *hw,
+ struct irdma_virt_mem *mem);
+/* misc */
+u8 irdma_get_encoded_wqe_size(u32 wqsize, bool cqpsq);
+void irdma_modify_qp_to_err(struct irdma_sc_qp *sc_qp);
+enum irdma_status_code irdma_sc_static_hmc_pages_allocated(struct irdma_sc_cqp *cqp,
+ u64 scratch,
+ u8 hmc_fn_id,
+ bool post_sq,
+ bool poll_registers);
+enum irdma_status_code irdma_cfg_fpm_val(struct irdma_sc_dev *dev,
+ u32 qp_count);
+enum irdma_status_code irdma_get_rdma_features(struct irdma_sc_dev *dev);
+void free_sd_mem(struct irdma_sc_dev *dev);
+enum irdma_status_code irdma_process_cqp_cmd(struct irdma_sc_dev *dev,
+ struct cqp_cmds_info *pcmdinfo);
+enum irdma_status_code irdma_process_bh(struct irdma_sc_dev *dev);
+#endif /* IRDMA_PROTOS_H */
diff --git a/drivers/infiniband/hw/irdma/status.h b/drivers/infiniband/hw/irdma/status.h
new file mode 100644
index 0000000..a23e42c
--- /dev/null
+++ b/drivers/infiniband/hw/irdma/status.h
@@ -0,0 +1,70 @@
+/* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB */
+/* Copyright (c) 2019, Intel Corporation. */
+
+#ifndef IRDMA_STATUS_H
+#define IRDMA_STATUS_H
+
+/* Error Codes */
+enum irdma_status_code {
+ IRDMA_SUCCESS = 0,
+ IRDMA_ERR_NVM = -1,
+ IRDMA_ERR_NVM_CHECKSUM = -2,
+ IRDMA_ERR_CFG = -4,
+ IRDMA_ERR_PARAM = -5,
+ IRDMA_ERR_DEVICE_NOT_SUPPORTED = -6,
+ IRDMA_ERR_RESET_FAILED = -7,
+ IRDMA_ERR_SWFW_SYNC = -8,
+ IRDMA_ERR_NO_MEMORY = -9,
+ IRDMA_ERR_BAD_PTR = -10,
+ IRDMA_ERR_INVALID_PD_ID = -11,
+ IRDMA_ERR_INVALID_QP_ID = -12,
+ IRDMA_ERR_INVALID_CQ_ID = -13,
+ IRDMA_ERR_INVALID_CEQ_ID = -14,
+ IRDMA_ERR_INVALID_AEQ_ID = -15,
+ IRDMA_ERR_INVALID_SIZE = -16,
+ IRDMA_ERR_INVALID_ARP_INDEX = -17,
+ IRDMA_ERR_INVALID_FPM_FUNC_ID = -18,
+ IRDMA_ERR_QP_INVALID_MSG_SIZE = -19,
+ IRDMA_ERR_QP_TOOMANY_WRS_POSTED = -20,
+ IRDMA_ERR_INVALID_FRAG_COUNT = -21,
+ IRDMA_ERR_Q_EMPTY = -22,
+ IRDMA_ERR_INVALID_ALIGNMENT = -23,
+ IRDMA_ERR_FLUSHED_Q = -24,
+ IRDMA_ERR_INVALID_PUSH_PAGE_INDEX = -25,
+ IRDMA_ERR_INVALID_INLINE_DATA_SIZE = -26,
+ IRDMA_ERR_TIMEOUT = -27,
+ IRDMA_ERR_OPCODE_MISMATCH = -28,
+ IRDMA_ERR_CQP_COMPL_ERROR = -29,
+ IRDMA_ERR_INVALID_VF_ID = -30,
+ IRDMA_ERR_INVALID_HMCFN_ID = -31,
+ IRDMA_ERR_BACKING_PAGE_ERROR = -32,
+ IRDMA_ERR_NO_PBLCHUNKS_AVAILABLE = -33,
+ IRDMA_ERR_INVALID_PBLE_INDEX = -34,
+ IRDMA_ERR_INVALID_SD_INDEX = -35,
+ IRDMA_ERR_INVALID_PAGE_DESC_INDEX = -36,
+ IRDMA_ERR_INVALID_SD_TYPE = -37,
+ IRDMA_ERR_MEMCPY_FAILED = -38,
+ IRDMA_ERR_INVALID_HMC_OBJ_INDEX = -39,
+ IRDMA_ERR_INVALID_HMC_OBJ_COUNT = -40,
+ IRDMA_ERR_BUF_TOO_SHORT = -43,
+ IRDMA_ERR_BAD_IWARP_CQE = -44,
+ IRDMA_ERR_NVM_BLANK_MODE = -45,
+ IRDMA_ERR_NOT_IMPL = -46,
+ IRDMA_ERR_PE_DOORBELL_NOT_ENA = -47,
+ IRDMA_ERR_NOT_READY = -48,
+ IRDMA_NOT_SUPPORTED = -49,
+ IRDMA_ERR_FIRMWARE_API_VER = -50,
+ IRDMA_ERR_RING_FULL = -51,
+ IRDMA_ERR_MPA_CRC = -61,
+ IRDMA_ERR_NO_TXBUFS = -62,
+ IRDMA_ERR_SEQ_NUM = -63,
+ IRDMA_ERR_list_empty = -64,
+ IRDMA_ERR_INVALID_MAC_ADDR = -65,
+ IRDMA_ERR_BAD_STAG = -66,
+ IRDMA_ERR_CQ_COMPL_ERROR = -67,
+ IRDMA_ERR_Q_DESTROYED = -68,
+ IRDMA_ERR_INVALID_FEAT_CNT = -69,
+ IRDMA_ERR_REG_CQ_FULL = -70,
+ IRDMA_ERR_VF_MSG_ERROR = -71,
+};
+#endif /* IRDMA_STATUS_H */
diff --git a/drivers/infiniband/hw/irdma/utils.c b/drivers/infiniband/hw/irdma/utils.c
new file mode 100644
index 0000000..d3d93fa
--- /dev/null
+++ b/drivers/infiniband/hw/irdma/utils.c
@@ -0,0 +1,2566 @@
+// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
+/* Copyright (c) 2019, Intel Corporation. */
+
+#include <linux/mii.h>
+#include <linux/in.h>
+#include <linux/init.h>
+#include <asm/irq.h>
+#include <asm/byteorder.h>
+#include <net/neighbour.h>
+#include "main.h"
+
+/**
+ * irdma_arp_table -manage arp table
+ * @rf: RDMA PCI function
+ * @ip_addr: ip address for device
+ * @mac_addr: mac address ptr
+ * @action: modify, delete or add
+ */
+int irdma_arp_table(struct irdma_pci_f *rf,
+ u32 *ip_addr,
+ bool ipv4,
+ u8 *mac_addr,
+ u32 action)
+{
+ int arp_index;
+ int err;
+ u32 ip[4] = {};
+
+ if (ipv4)
+ ip[0] = *ip_addr;
+ else
+ memcpy(ip, ip_addr, sizeof(ip));
+
+ for (arp_index = 0; (u32)arp_index < rf->arp_table_size; arp_index++)
+ if (!memcmp(rf->arp_table[arp_index].ip_addr, ip, sizeof(ip)))
+ break;
+
+ switch (action) {
+ case IRDMA_ARP_ADD:
+ if (arp_index != rf->arp_table_size)
+ return -1;
+
+ arp_index = 0;
+ err = irdma_alloc_rsrc(rf, rf->allocated_arps,
+ rf->arp_table_size,
+ (u32 *)&arp_index,
+ &rf->next_arp_index);
+ if (err)
+ return err;
+
+ memcpy(rf->arp_table[arp_index].ip_addr, ip,
+ sizeof(rf->arp_table[arp_index].ip_addr));
+ ether_addr_copy(rf->arp_table[arp_index].mac_addr, mac_addr);
+ break;
+ case IRDMA_ARP_RESOLVE:
+ if (arp_index == rf->arp_table_size)
+ return -1;
+
+ break;
+ case IRDMA_ARP_DELETE:
+ if (arp_index == rf->arp_table_size)
+ return -1;
+
+ memset(rf->arp_table[arp_index].ip_addr, 0,
+ sizeof(rf->arp_table[arp_index].ip_addr));
+ eth_zero_addr(rf->arp_table[arp_index].mac_addr);
+ irdma_free_rsrc(rf, rf->allocated_arps, arp_index);
+ break;
+ default:
+ return -1;
+ }
+
+ return arp_index;
+}
+
+/**
+ * irdma_add_arp - add a new arp entry if needed
+ * @rf: RDMA function
+ * @ip: IP address
+ * @mac: MAC address
+ */
+int irdma_add_arp(struct irdma_pci_f *rf, u32 *ip, bool ipv4, u8 *mac)
+{
+ int arpidx;
+
+ arpidx = irdma_arp_table(rf, &ip[0], ipv4, NULL, IRDMA_ARP_RESOLVE);
+ if (arpidx >= 0) {
+ if (ether_addr_equal(rf->arp_table[arpidx].mac_addr, mac))
+ return arpidx;
+
+ irdma_manage_arp_cache(rf,
+ rf->arp_table[arpidx].mac_addr,
+ ip,
+ ipv4,
+ IRDMA_ARP_DELETE);
+ }
+
+ irdma_manage_arp_cache(rf, mac, ip, ipv4, IRDMA_ARP_ADD);
+
+ return irdma_arp_table(rf, ip, ipv4, NULL, IRDMA_ARP_RESOLVE);
+}
+
+/**
+ * wr32 - write 32 bits to hw register
+ * @hw: hardware information including registers
+ * @reg: register offset
+ * @value: value to write to register
+ */
+inline void wr32(struct irdma_hw *hw, u32 reg, u32 val)
+{
+ writel(val, hw->hw_addr + reg);
+}
+
+/**
+ * rd32 - read a 32 bit hw register
+ * @hw: hardware information including registers
+ * @reg: register offset
+ *
+ * Return value of register content
+ */
+inline u32 rd32(struct irdma_hw *hw, u32 reg)
+{
+ return readl(hw->hw_addr + reg);
+}
+
+/**
+ * rd64 - read a 64 bit hw register
+ * @hw: hardware information including registers
+ * @reg: register offset
+ *
+ * Return value of register content
+ */
+inline u64 rd64(struct irdma_hw *hw, u32 reg)
+{
+ return readq(hw->hw_addr + reg);
+}
+
+/**
+ * irdma_inetaddr_event - system notifier for ipv4 addr events
+ * @notfier: not used
+ * @event: event for notifier
+ * @ptr: if address
+ */
+int irdma_inetaddr_event(struct notifier_block *notifier,
+ unsigned long event,
+ void *ptr)
+{
+ struct in_ifaddr *ifa = ptr;
+ struct net_device *event_netdev = ifa->ifa_dev->dev;
+ struct net_device *netdev;
+ struct net_device *upper_dev;
+ struct irdma_device *iwdev;
+ u32 local_ipaddr;
+
+ iwdev = irdma_find_netdev(event_netdev);
+ if (!iwdev)
+ return NOTIFY_DONE;
+
+ if (iwdev->init_state < IP_ADDR_REGISTERED || iwdev->closing)
+ return NOTIFY_DONE;
+
+ netdev = iwdev->netdev;
+ upper_dev = netdev_master_upper_dev_get(netdev);
+ if (netdev != event_netdev)
+ return NOTIFY_DONE;
+
+ if (upper_dev) {
+ struct in_device *in;
+
+ rcu_read_lock();
+ in = __in_dev_get_rcu(upper_dev);
+ local_ipaddr = ntohl(in->ifa_list->ifa_address);
+ rcu_read_unlock();
+ } else {
+ local_ipaddr = ntohl(ifa->ifa_address);
+ }
+
+ switch (event) {
+ case NETDEV_DOWN:
+ irdma_manage_arp_cache(iwdev->rf,
+ netdev->dev_addr,
+ &local_ipaddr,
+ true,
+ IRDMA_ARP_DELETE);
+ irdma_if_notify(iwdev, netdev, &local_ipaddr, true, false);
+ break;
+ case NETDEV_UP:
+ /* Fall through */
+ case NETDEV_CHANGEADDR:
+ irdma_add_arp(iwdev->rf, &local_ipaddr, true, netdev->dev_addr);
+ irdma_if_notify(iwdev, netdev, &local_ipaddr, true, true);
+ break;
+ default:
+ break;
+ }
+
+ return NOTIFY_DONE;
+}
+
+/**
+ * irdma_inet6addr_event - system notifier for ipv6 addr events
+ * @notfier: not used
+ * @event: event for notifier
+ * @ptr: if address
+ */
+int irdma_inet6addr_event(struct notifier_block *notifier,
+ unsigned long event,
+ void *ptr)
+{
+ struct inet6_ifaddr *ifa = (struct inet6_ifaddr *)ptr;
+ struct net_device *event_netdev = ifa->idev->dev;
+ struct net_device *netdev;
+ struct irdma_device *iwdev;
+ u32 local_ipaddr6[4];
+
+ iwdev = irdma_find_netdev(event_netdev);
+ if (!iwdev)
+ return NOTIFY_DONE;
+
+ if (iwdev->init_state < IP_ADDR_REGISTERED || iwdev->closing)
+ return NOTIFY_DONE;
+
+ netdev = iwdev->netdev;
+ if (netdev != event_netdev)
+ return NOTIFY_DONE;
+
+ irdma_copy_ip_ntohl(local_ipaddr6, ifa->addr.in6_u.u6_addr32);
+ switch (event) {
+ case NETDEV_DOWN:
+ irdma_manage_arp_cache(iwdev->rf,
+ netdev->dev_addr,
+ local_ipaddr6,
+ false,
+ IRDMA_ARP_DELETE);
+ irdma_if_notify(iwdev, netdev, local_ipaddr6, false, false);
+ break;
+ case NETDEV_UP:
+ /* Fall through */
+ case NETDEV_CHANGEADDR:
+ irdma_add_arp(iwdev->rf, local_ipaddr6, false, netdev->dev_addr);
+ irdma_if_notify(iwdev, netdev, local_ipaddr6, false, true);
+ break;
+ default:
+ break;
+ }
+
+ return NOTIFY_DONE;
+}
+
+/**
+ * irdma_net_event - system notifier for net events
+ * @notfier: not used
+ * @event: event for notifier
+ * @ptr: neighbor
+ */
+int irdma_net_event(struct notifier_block *notifier,
+ unsigned long event,
+ void *ptr)
+{
+ struct neighbour *neigh = ptr;
+ struct irdma_device *iwdev;
+ __be32 *p;
+ u32 local_ipaddr[4];
+
+ switch (event) {
+ case NETEVENT_NEIGH_UPDATE:
+ iwdev = irdma_find_netdev((struct net_device *)neigh->dev);
+ if (!iwdev)
+ return NOTIFY_DONE;
+
+ if (iwdev->init_state < IP_ADDR_REGISTERED || iwdev->closing)
+ return NOTIFY_DONE;
+
+ p = (__be32 *)neigh->primary_key;
+ irdma_copy_ip_ntohl(local_ipaddr, p);
+ if (neigh->nud_state & NUD_VALID)
+ irdma_add_arp(iwdev->rf, local_ipaddr, false, neigh->ha);
+
+ else
+ irdma_manage_arp_cache(iwdev->rf,
+ neigh->ha,
+ local_ipaddr,
+ false,
+ IRDMA_ARP_DELETE);
+ break;
+ default:
+ break;
+ }
+
+ return NOTIFY_DONE;
+}
+
+/**
+ * irdma_netdevice_event - system notifier for netdev events
+ * @notfier: not used
+ * @event: event for notifier
+ * @ptr: netdev
+ */
+int irdma_netdevice_event(struct notifier_block *notifier,
+ unsigned long event,
+ void *ptr)
+{
+ struct net_device *event_netdev;
+ struct net_device *netdev;
+ struct irdma_device *iwdev;
+
+ event_netdev = netdev_notifier_info_to_dev(ptr);
+
+ iwdev = irdma_find_netdev(event_netdev);
+ if (!iwdev) {
+ irdma_handle_netdev(event_netdev);
+ return NOTIFY_DONE;
+ }
+
+ if (iwdev->init_state < RDMA_DEV_REGISTERED || iwdev->closing)
+ return NOTIFY_DONE;
+
+ netdev = (struct net_device *)iwdev->netdev;
+ if (netdev != event_netdev)
+ return NOTIFY_DONE;
+
+ iwdev->iw_status = 1;
+
+ switch (event) {
+ case NETDEV_DOWN:
+ iwdev->iw_status = 0;
+ /* Fall through */
+ case NETDEV_UP:
+ irdma_port_ibevent(iwdev);
+ break;
+ default:
+ break;
+ }
+
+ return NOTIFY_DONE;
+}
+
+/**
+ * irdma_get_cqp_request - get cqp struct
+ * @cqp: device cqp ptr
+ * @wait: cqp to be used in wait mode
+ */
+struct irdma_cqp_request *irdma_get_cqp_request(struct irdma_cqp *cqp,
+ bool wait)
+{
+ struct irdma_cqp_request *cqp_request = NULL;
+ unsigned long flags;
+
+ spin_lock_irqsave(&cqp->req_lock, flags);
+ if (!list_empty(&cqp->cqp_avail_reqs)) {
+ cqp_request = list_entry(cqp->cqp_avail_reqs.next,
+ struct irdma_cqp_request, list);
+ list_del_init(&cqp_request->list);
+ }
+ spin_unlock_irqrestore(&cqp->req_lock, flags);
+ if (!cqp_request) {
+ cqp_request = kzalloc(sizeof(*cqp_request), GFP_ATOMIC);
+ if (cqp_request) {
+ cqp_request->dynamic = true;
+ if (wait)
+ init_waitqueue_head(&cqp_request->waitq);
+ }
+ }
+ if (!cqp_request) {
+ irdma_debug(cqp->sc_cqp.dev, IRDMA_DEBUG_ERR,
+ "CQP Request Fail: No Memory");
+ return NULL;
+ }
+
+ if (wait) {
+ atomic_set(&cqp_request->refcount, 2);
+ cqp_request->waiting = true;
+ } else {
+ atomic_set(&cqp_request->refcount, 1);
+ }
+ memset(&cqp_request->compl_info, 0, sizeof(cqp_request->compl_info));
+
+ return cqp_request;
+}
+
+/**
+ * irdma_free_cqp_request - free cqp request
+ * @cqp: cqp ptr
+ * @cqp_request: to be put back in cqp list
+ */
+void irdma_free_cqp_request(struct irdma_cqp *cqp,
+ struct irdma_cqp_request *cqp_request)
+{
+ unsigned long flags;
+
+ if (cqp_request->dynamic) {
+ kfree(cqp_request);
+ } else {
+ cqp_request->request_done = false;
+ cqp_request->callback_fcn = NULL;
+ cqp_request->waiting = false;
+
+ spin_lock_irqsave(&cqp->req_lock, flags);
+ list_add_tail(&cqp_request->list, &cqp->cqp_avail_reqs);
+ spin_unlock_irqrestore(&cqp->req_lock, flags);
+ }
+ wake_up(&cqp->remove_wq);
+}
+
+/**
+ * irdma_put_cqp_request - dec ref count and free if 0
+ * @cqp: cqp ptr
+ * @cqp_request: to be put back in cqp list
+ */
+void irdma_put_cqp_request(struct irdma_cqp *cqp,
+ struct irdma_cqp_request *cqp_request)
+{
+ if (atomic_dec_and_test(&cqp_request->refcount))
+ irdma_free_cqp_request(cqp, cqp_request);
+}
+
+/**
+ * irdma_free_pending_cqp_request -free pending cqp request objs
+ * @cqp: cqp ptr
+ * @cqp_request: to be put back in cqp list
+ */
+static void
+irdma_free_pending_cqp_request(struct irdma_cqp *cqp,
+ struct irdma_cqp_request *cqp_request)
+{
+ if (cqp_request->waiting) {
+ cqp_request->compl_info.error = true;
+ cqp_request->request_done = true;
+ wake_up(&cqp_request->waitq);
+ }
+ wait_event_timeout(cqp->remove_wq,
+ atomic_read(&cqp_request->refcount) == 1,
+ 1000);
+ irdma_put_cqp_request(cqp, cqp_request);
+}
+
+/**
+ * irdma_cleanup_pending_cqp_op - clean-up cqp with no
+ * completions
+ * @rf: RDMA PCI function
+ */
+void irdma_cleanup_pending_cqp_op(struct irdma_pci_f *rf)
+{
+ struct irdma_sc_dev *dev = &rf->sc_dev;
+ struct irdma_cqp *cqp = &rf->cqp;
+ struct irdma_cqp_request *cqp_request = NULL;
+ struct cqp_cmds_info *pcmdinfo = NULL;
+ u32 i, pending_work, wqe_idx;
+
+ pending_work = IRDMA_RING_USED_QUANTA(cqp->sc_cqp.sq_ring);
+ wqe_idx = IRDMA_RING_CURRENT_TAIL(cqp->sc_cqp.sq_ring);
+ for (i = 0; i < pending_work; i++) {
+ cqp_request = (struct irdma_cqp_request *)
+ (unsigned long)cqp->scratch_array[wqe_idx];
+ if (cqp_request)
+ irdma_free_pending_cqp_request(cqp, cqp_request);
+ wqe_idx = (wqe_idx + 1) % IRDMA_RING_SIZE(cqp->sc_cqp.sq_ring);
+ }
+
+ while (!list_empty(&dev->cqp_cmd_head)) {
+ pcmdinfo = (struct cqp_cmds_info *)
+ irdma_remove_head(&dev->cqp_cmd_head);
+ cqp_request =
+ container_of(pcmdinfo, struct irdma_cqp_request, info);
+ if (cqp_request)
+ irdma_free_pending_cqp_request(cqp, cqp_request);
+ }
+}
+
+/**
+ * irdma_free_qp_worker - worker for freeing QP resources
+ * @work: ptr to work struct
+ */
+static void irdma_free_qp_worker(struct work_struct *work)
+{
+ struct irdma_qp *iwqp = container_of(work, struct irdma_qp, work);
+
+ irdma_rem_pdusecount(iwqp->iwpd, iwqp->iwdev);
+ irdma_rem_devusecount(iwqp->iwdev);
+ irdma_free_qp_rsrc(iwqp->iwdev, iwqp, iwqp->ibqp.qp_num);
+}
+
+/**
+ * irdma_free_qp - callback after destroy cqp completes
+ * @cqp_request: cqp request for destroy qp
+ * @num: not used
+ */
+static void irdma_free_qp(struct irdma_cqp_request *cqp_request,
+ u32 num)
+{
+ struct irdma_sc_qp *qp = (struct irdma_sc_qp *)cqp_request->param;
+ struct irdma_qp *iwqp = (struct irdma_qp *)qp->back_qp;
+ struct irdma_device *iwdev = iwqp->iwdev;
+
+ INIT_WORK(&iwqp->work, irdma_free_qp_worker);
+ queue_work(iwdev->rf->free_qp_wq, &iwqp->work);
+}
+
+/**
+ * irdma_wait_event - wait for completion
+ * @rf: RDMA PCI function
+ * @cqp_request: cqp request to wait
+ */
+static int irdma_wait_event(struct irdma_pci_f *rf,
+ struct irdma_cqp_request *cqp_request)
+{
+ struct cqp_cmds_info *info = &cqp_request->info;
+ struct irdma_cqp *iwcqp = &rf->cqp;
+ struct irdma_cqp_timeout cqp_timeout = {};
+ bool cqp_error = false;
+ int err_code = 0;
+
+ cqp_timeout.compl_cqp_cmds =
+ rf->sc_dev.cqp_cmd_stats[IRDMA_OP_CMPL_CMDS];
+ do {
+ up(&iwcqp->cqp_compl_sem);
+ if (wait_event_timeout(cqp_request->waitq,
+ cqp_request->request_done,
+ msecs_to_jiffies(CQP_COMPL_WAIT_TIME_MS)))
+ break;
+
+ rf->sc_dev.cqp_ops->check_cqp_progress(&cqp_timeout,
+ &rf->sc_dev);
+
+ if (cqp_timeout.count < CQP_TIMEOUT_THRESHOLD)
+ continue;
+
+ irdma_debug(&rf->sc_dev, IRDMA_DEBUG_ERR,
+ "error cqp command 0x%x timed out",
+ info->cqp_cmd);
+ err_code = -ETIME;
+ if (!rf->reset) {
+ rf->reset = true;
+ irdma_request_reset(rf);
+ }
+ goto done;
+ } while (1);
+
+ cqp_error = cqp_request->compl_info.error;
+ if (cqp_error) {
+ irdma_debug(&rf->sc_dev, IRDMA_DEBUG_ERR,
+ "error cqp command 0x%x completion maj = 0x%x min=0x%x\n",
+ info->cqp_cmd,
+ cqp_request->compl_info.maj_err_code,
+ cqp_request->compl_info.min_err_code);
+ err_code = -EPROTO;
+ goto done;
+ }
+
+done:
+ irdma_put_cqp_request(iwcqp, cqp_request);
+
+ return err_code;
+}
+
+/**
+ * irdma_handle_cqp_op - process cqp command
+ * @rf: RDMA PCI function
+ * @cqp_request: cqp request to process
+ */
+enum irdma_status_code irdma_handle_cqp_op(struct irdma_pci_f *rf,
+ struct irdma_cqp_request
+ *cqp_request)
+{
+ struct irdma_sc_dev *dev = &rf->sc_dev;
+ enum irdma_status_code status;
+ struct cqp_cmds_info *info = &cqp_request->info;
+ int err_code = 0;
+
+ if (rf->reset) {
+ irdma_free_cqp_request(&rf->cqp, cqp_request);
+ return IRDMA_ERR_CQP_COMPL_ERROR;
+ }
+
+ status = irdma_process_cqp_cmd(dev, info);
+ if (status) {
+ irdma_debug(&rf->sc_dev, IRDMA_DEBUG_ERR,
+ "error cqp command 0x%x failed\n", info->cqp_cmd);
+ irdma_free_cqp_request(&rf->cqp, cqp_request);
+ return status;
+ }
+
+ if (cqp_request->waiting)
+ err_code = irdma_wait_event(rf, cqp_request);
+ if (err_code)
+ status = IRDMA_ERR_CQP_COMPL_ERROR;
+
+ return status;
+}
+
+/**
+ * irdma_add_devusecount - add dev refcount
+ * @iwdev: dev for refcount
+ */
+void irdma_add_devusecount(struct irdma_device *iwdev)
+{
+ atomic64_inc(&iwdev->use_count);
+}
+
+/**
+ * irdma_rem_devusecount - decrement refcount for dev
+ * @iwdev: device
+ */
+void irdma_rem_devusecount(struct irdma_device *iwdev)
+{
+ if (!atomic64_dec_and_test(&iwdev->use_count))
+ return;
+ wake_up(&iwdev->close_wq);
+}
+
+/**
+ * irdma_add_pdusecount - add pd refcount
+ * @iwpd: pd for refcount
+ */
+void irdma_add_pdusecount(struct irdma_pd *iwpd)
+{
+ atomic_inc(&iwpd->usecount);
+}
+
+/**
+ * irdma_rem_pdusecount - decrement refcount for pd and free if 0
+ * @iwpd: pd for refcount
+ * @iwdev: iwarp device
+ */
+void irdma_rem_pdusecount(struct irdma_pd *iwpd,
+ struct irdma_device *iwdev)
+{
+ if (!atomic_dec_and_test(&iwpd->usecount))
+ return;
+ irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_pds, iwpd->sc_pd.pd_id);
+ kfree(iwpd);
+}
+
+/**
+ * irdma_add_ref - add refcount for qp
+ * @ibqp: iqarp qp
+ */
+void irdma_add_ref(struct ib_qp *ibqp)
+{
+ struct irdma_qp *iwqp = (struct irdma_qp *)ibqp;
+
+ atomic_inc(&iwqp->refcount);
+}
+
+/**
+ * irdma_rem_ref - rem refcount for qp and free if 0
+ * @ibqp: iqarp qp
+ */
+void irdma_rem_ref(struct ib_qp *ibqp)
+{
+ struct irdma_qp *iwqp;
+ enum irdma_status_code status;
+ struct irdma_cqp_request *cqp_request;
+ struct cqp_cmds_info *cqp_info;
+ struct irdma_device *iwdev;
+ u32 qp_num;
+ unsigned long flags;
+
+ iwqp = to_iwqp(ibqp);
+ iwdev = iwqp->iwdev;
+ spin_lock_irqsave(&iwdev->rf->qptable_lock, flags);
+ if (!atomic_dec_and_test(&iwqp->refcount)) {
+ spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags);
+ return;
+ }
+
+ qp_num = iwqp->ibqp.qp_num;
+ iwdev->rf->qp_table[qp_num] = NULL;
+ spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags);
+ cqp_request = irdma_get_cqp_request(&iwdev->rf->cqp, false);
+ if (!cqp_request)
+ return;
+
+ cqp_request->callback_fcn = irdma_free_qp;
+ cqp_request->param = (void *)&iwqp->sc_qp;
+ cqp_info = &cqp_request->info;
+ cqp_info->cqp_cmd = IRDMA_OP_QP_DESTROY;
+ cqp_info->post_sq = 1;
+ cqp_info->in.u.qp_destroy.qp = &iwqp->sc_qp;
+ cqp_info->in.u.qp_destroy.scratch = (uintptr_t)cqp_request;
+ cqp_info->in.u.qp_destroy.remove_hash_idx = true;
+ status = irdma_handle_cqp_op(iwdev->rf, cqp_request);
+ if (!status)
+ return;
+
+ irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_ERR,
+ "CQP-OP Destroy QP fail");
+ irdma_rem_pdusecount(iwqp->iwpd, iwdev);
+ irdma_free_qp_rsrc(iwdev, iwqp, qp_num);
+ irdma_rem_devusecount(iwdev);
+}
+
+/**
+ * irdma_get_qp - get qp address
+ * @device: iwarp device
+ * @qpn: qp number
+ */
+struct ib_qp *irdma_get_qp(struct ib_device *device,
+ int qpn)
+{
+ struct irdma_device *iwdev = to_iwdev(device);
+
+ if (qpn < IW_FIRST_QPN || qpn >= iwdev->rf->max_qp)
+ return NULL;
+
+ return &iwdev->rf->qp_table[qpn]->ibqp;
+}
+
+/**
+ * irdma_get_hw_addr - return hw addr
+ * @par: points to shared dev
+ */
+u8 __iomem *irdma_get_hw_addr(void *par)
+{
+ struct irdma_sc_dev *dev = (struct irdma_sc_dev *)par;
+
+ return dev->hw->hw_addr;
+}
+
+/**
+ * irdma_remove_head - return head entry and remove from list
+ * @list: list for entry
+ */
+void *irdma_remove_head(struct list_head *list)
+{
+ struct list_head *entry;
+
+ if (list_empty(list))
+ return NULL;
+
+ entry = (void *)list->next;
+ list_del(entry);
+
+ return (void *)entry;
+}
+
+/**
+ * irdma_allocate_dma_mem - Memory alloc helper fn
+ * @hw: pointer to the HW structure
+ * @mem: ptr to mem struct to fill out
+ * @size: size of memory requested
+ * @alignment: what to align the allocation to
+ */
+enum irdma_status_code irdma_allocate_dma_mem(struct irdma_hw *hw,
+ struct irdma_dma_mem *mem,
+ u64 size,
+ u32 alignment)
+{
+ struct pci_dev *pcidev = (struct pci_dev *)hw->dev_context;
+
+ if (!mem)
+ return IRDMA_ERR_PARAM;
+
+ mem->size = ALIGN(size, alignment);
+ mem->va = dma_alloc_coherent(&pcidev->dev, mem->size,
+ (dma_addr_t *)&mem->pa, GFP_KERNEL);
+ if (!mem->va)
+ return IRDMA_ERR_NO_MEMORY;
+
+ return 0;
+}
+
+/**
+ * irdma_free_dma_mem - Memory free helper fn
+ * @hw: pointer to the HW structure
+ * @mem: ptr to mem struct to free
+ */
+void irdma_free_dma_mem(struct irdma_hw *hw,
+ struct irdma_dma_mem *mem)
+{
+ struct pci_dev *pcidev = (struct pci_dev *)hw->dev_context;
+
+ if (!mem || !mem->va)
+ return;
+
+ dma_free_coherent(&pcidev->dev, mem->size,
+ mem->va, (dma_addr_t)mem->pa);
+
+ mem->va = NULL;
+}
+
+/**
+ * irdma_allocate_virt_mem - virtual memory alloc helper fn
+ * @hw: pointer to the HW structure
+ * @mem: ptr to mem struct to fill out
+ * @size: size of memory requested
+ */
+enum irdma_status_code irdma_allocate_virt_mem(struct irdma_hw *hw,
+ struct irdma_virt_mem *mem,
+ u32 size)
+{
+ if (!mem)
+ return IRDMA_ERR_PARAM;
+
+ mem->size = size;
+ mem->va = kzalloc(size, GFP_KERNEL);
+
+ if (mem->va)
+ return 0;
+ else
+ return IRDMA_ERR_NO_MEMORY;
+}
+
+/**
+ * irdma_free_virt_mem - virtual memory free helper fn
+ * @hw: pointer to the HW structure
+ * @mem: ptr to mem struct to free
+ */
+enum irdma_status_code irdma_free_virt_mem(struct irdma_hw *hw,
+ struct irdma_virt_mem *mem)
+{
+ if (!mem)
+ return IRDMA_ERR_PARAM;
+ kfree(mem->va);
+ return 0;
+}
+
+/**
+ * irdma_cqp_sds_cmd - create cqp command for sd
+ * @dev: hardware control device structure
+ * @sd_info: information for sd cqp
+ *
+ */
+enum irdma_status_code irdma_cqp_sds_cmd(struct irdma_sc_dev *dev,
+ struct irdma_update_sds_info *sdinfo)
+{
+ enum irdma_status_code status;
+ struct irdma_cqp_request *cqp_request;
+ struct cqp_cmds_info *cqp_info;
+ struct irdma_pci_f *rf = dev->back_dev;
+
+ cqp_request = irdma_get_cqp_request(&rf->cqp, false);
+ if (!cqp_request)
+ return IRDMA_ERR_NO_MEMORY;
+
+ cqp_info = &cqp_request->info;
+ memcpy(&cqp_info->in.u.update_pe_sds.info, sdinfo,
+ sizeof(cqp_info->in.u.update_pe_sds.info));
+ cqp_info->cqp_cmd = IRDMA_OP_UPDATE_PE_SDS;
+ cqp_info->post_sq = 1;
+ cqp_info->in.u.update_pe_sds.dev = dev;
+ cqp_info->in.u.update_pe_sds.scratch = (uintptr_t)cqp_request;
+ status = irdma_handle_cqp_op(rf, cqp_request);
+ if (status)
+ irdma_debug(dev, IRDMA_DEBUG_ERR, "CQP-OP Update SD's fail");
+
+ return status;
+}
+
+/**
+ * irdma_qp_suspend_resume - cqp command for suspend/resume
+ * @qp: hardware control qp
+ * @suspend: flag if suspend or resume
+ */
+enum irdma_status_code irdma_qp_suspend_resume(struct irdma_sc_qp *qp,
+ bool suspend)
+{
+ struct irdma_sc_dev *dev = qp->dev;
+ enum irdma_status_code status;
+ struct irdma_cqp_request *cqp_request;
+ struct irdma_sc_cqp *cqp = dev->cqp;
+ struct cqp_cmds_info *cqp_info;
+ struct irdma_pci_f *rf = dev->back_dev;
+
+ cqp_request = irdma_get_cqp_request(&rf->cqp, false);
+ if (!cqp_request)
+ return IRDMA_ERR_NO_MEMORY;
+
+ cqp_info = &cqp_request->info;
+ cqp_info->cqp_cmd = (suspend) ? IRDMA_OP_SUSPEND : IRDMA_OP_RESUME;
+ cqp_info->in.u.suspend_resume.cqp = cqp;
+ cqp_info->in.u.suspend_resume.qp = qp;
+ cqp_info->in.u.suspend_resume.scratch = (uintptr_t)cqp_request;
+ status = irdma_handle_cqp_op(rf, cqp_request);
+ if (status)
+ irdma_debug(dev, IRDMA_DEBUG_ERR,
+ "CQP-OP QP Suspend/Resume fail");
+
+ return status;
+}
+
+/**
+ * irdma_term_modify_qp - modify qp for term message
+ * @qp: hardware control qp
+ * @next_state: qp's next state
+ * @term: terminate code
+ * @term_len: length
+ */
+void irdma_term_modify_qp(struct irdma_sc_qp *qp,
+ u8 next_state,
+ u8 term,
+ u8 term_len)
+{
+ struct irdma_qp *iwqp;
+
+ iwqp = (struct irdma_qp *)qp->back_qp;
+ irdma_next_iw_state(iwqp, next_state, 0, term, term_len);
+};
+
+/**
+ * irdma_terminate_done - after terminate is completed
+ * @qp: hardware control qp
+ * @timeout_occurred: indicates if terminate timer expired
+ */
+void irdma_terminate_done(struct irdma_sc_qp *qp,
+ int timeout_occurred)
+{
+ struct irdma_qp *iwqp;
+ u32 next_iwarp_state = IRDMA_QP_STATE_ERROR;
+ u8 hte = 0;
+ bool first_time;
+ unsigned long flags;
+
+ iwqp = (struct irdma_qp *)qp->back_qp;
+ spin_lock_irqsave(&iwqp->lock, flags);
+ if (iwqp->hte_added) {
+ iwqp->hte_added = 0;
+ hte = 1;
+ }
+ first_time = !(qp->term_flags & IRDMA_TERM_DONE);
+ qp->term_flags |= IRDMA_TERM_DONE;
+ spin_unlock_irqrestore(&iwqp->lock, flags);
+ if (first_time) {
+ if (!timeout_occurred)
+ irdma_terminate_del_timer(qp);
+ else
+ next_iwarp_state = IRDMA_QP_STATE_CLOSING;
+
+ irdma_next_iw_state(iwqp, next_iwarp_state, hte, 0, 0);
+ irdma_cm_disconn(iwqp);
+ }
+}
+
+static void irdma_terminate_timeout(struct timer_list *t)
+{
+ struct irdma_qp *iwqp = from_timer(iwqp, t, terminate_timer);
+ struct irdma_sc_qp *qp = (struct irdma_sc_qp *)&iwqp->sc_qp;
+
+ irdma_terminate_done(qp, 1);
+ irdma_rem_ref(&iwqp->ibqp);
+}
+
+/**
+ * irdma_terminate_start_timer - start terminate timeout
+ * @qp: hardware control qp
+ */
+void irdma_terminate_start_timer(struct irdma_sc_qp *qp)
+{
+ struct irdma_qp *iwqp;
+
+ iwqp = (struct irdma_qp *)qp->back_qp;
+ irdma_add_ref(&iwqp->ibqp);
+ timer_setup(&iwqp->terminate_timer, irdma_terminate_timeout, 0);
+ iwqp->terminate_timer.expires = jiffies + HZ;
+
+ add_timer(&iwqp->terminate_timer);
+}
+
+/**
+ * irdma_terminate_del_timer - delete terminate timeout
+ * @qp: hardware control qp
+ */
+void irdma_terminate_del_timer(struct irdma_sc_qp *qp)
+{
+ struct irdma_qp *iwqp;
+ int ret;
+
+ iwqp = (struct irdma_qp *)qp->back_qp;
+ ret = del_timer(&iwqp->terminate_timer);
+ if (ret)
+ irdma_rem_ref(&iwqp->ibqp);
+}
+
+/**
+ * irdma_cqp_query_fpm_values_cmd - send cqp command for fpm
+ * @iwdev: function device struct
+ * @val_mem: buffer for fpm
+ * @hmc_fn_id: function id for fpm
+ */
+enum irdma_status_code
+irdma_cqp_query_fpm_val_cmd(struct irdma_sc_dev *dev,
+ struct irdma_dma_mem *val_mem,
+ u8 hmc_fn_id)
+{
+ enum irdma_status_code status;
+ struct irdma_cqp_request *cqp_request;
+ struct cqp_cmds_info *cqp_info;
+ struct irdma_pci_f *rf = dev->back_dev;
+
+ cqp_request = irdma_get_cqp_request(&rf->cqp, true);
+ if (!cqp_request)
+ return IRDMA_ERR_NO_MEMORY;
+
+ cqp_info = &cqp_request->info;
+ cqp_request->param = NULL;
+ cqp_info->in.u.query_fpm_val.cqp = dev->cqp;
+ cqp_info->in.u.query_fpm_val.fpm_val_pa = val_mem->pa;
+ cqp_info->in.u.query_fpm_val.fpm_val_va = val_mem->va;
+ cqp_info->in.u.query_fpm_val.hmc_fn_id = hmc_fn_id;
+ cqp_info->cqp_cmd = IRDMA_OP_QUERY_FPM_VAL;
+ cqp_info->post_sq = 1;
+ cqp_info->in.u.query_fpm_val.scratch = (uintptr_t)cqp_request;
+ status = irdma_handle_cqp_op(rf, cqp_request);
+ if (status)
+ irdma_debug(dev, IRDMA_DEBUG_ERR, "CQP-OP Query FPM fail");
+
+ return status;
+}
+
+/**
+ * irdma_cqp_commit_fpm_values_cmd - commit fpm values in hw
+ * @dev: hardware control device structure
+ * @val_mem: buffer with fpm values
+ * @hmc_fn_id: function id for fpm
+ */
+enum irdma_status_code
+irdma_cqp_commit_fpm_val_cmd(struct irdma_sc_dev *dev,
+ struct irdma_dma_mem *val_mem,
+ u8 hmc_fn_id)
+{
+ enum irdma_status_code status;
+ struct irdma_cqp_request *cqp_request;
+ struct cqp_cmds_info *cqp_info;
+ struct irdma_pci_f *rf = dev->back_dev;
+
+ cqp_request = irdma_get_cqp_request(&rf->cqp, true);
+ if (!cqp_request)
+ return IRDMA_ERR_NO_MEMORY;
+
+ cqp_info = &cqp_request->info;
+ cqp_request->param = NULL;
+ cqp_info->in.u.commit_fpm_val.cqp = dev->cqp;
+ cqp_info->in.u.commit_fpm_val.fpm_val_pa = val_mem->pa;
+ cqp_info->in.u.commit_fpm_val.fpm_val_va = val_mem->va;
+ cqp_info->in.u.commit_fpm_val.hmc_fn_id = hmc_fn_id;
+ cqp_info->cqp_cmd = IRDMA_OP_COMMIT_FPM_VAL;
+ cqp_info->post_sq = 1;
+ cqp_info->in.u.commit_fpm_val.scratch = (uintptr_t)cqp_request;
+ status = irdma_handle_cqp_op(rf, cqp_request);
+ if (status)
+ irdma_debug(dev, IRDMA_DEBUG_ERR, "CQP-OP Commit FPM fail");
+
+ return status;
+}
+
+/**
+ * irdma_cqp_cq_create_cmd - create a cq for the cqp
+ * @dev: device pointer
+ * @cq: pointer to created cq
+ */
+enum irdma_status_code
+irdma_cqp_cq_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq)
+{
+ struct irdma_pci_f *rf = dev->back_dev;
+ struct irdma_cqp *iwcqp = &rf->cqp;
+ struct irdma_cqp_request *cqp_request;
+ struct cqp_cmds_info *cqp_info;
+ enum irdma_status_code status;
+
+ cqp_request = irdma_get_cqp_request(iwcqp, true);
+ if (!cqp_request)
+ return IRDMA_ERR_NO_MEMORY;
+
+ cqp_info = &cqp_request->info;
+ cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE;
+ cqp_info->post_sq = 1;
+ cqp_info->in.u.cq_create.cq = cq;
+ cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request;
+ status = irdma_handle_cqp_op(rf, cqp_request);
+ if (status)
+ irdma_debug(dev, IRDMA_DEBUG_ERR, "CQP-OP Create CQ fail");
+
+ return status;
+}
+
+/**
+ * irdma_cqp_qp_create_cmd - create a qp for the cqp
+ * @dev: device pointer
+ * @qp: pointer to created qp
+ */
+enum irdma_status_code irdma_cqp_qp_create_cmd(struct irdma_sc_dev *dev,
+ struct irdma_sc_qp *qp)
+{
+ struct irdma_pci_f *rf = dev->back_dev;
+ struct irdma_cqp *iwcqp = &rf->cqp;
+ struct irdma_cqp_request *cqp_request;
+ struct cqp_cmds_info *cqp_info;
+ struct irdma_create_qp_info *qp_info;
+ enum irdma_status_code status;
+
+ cqp_request = irdma_get_cqp_request(iwcqp, true);
+ if (!cqp_request)
+ return IRDMA_ERR_NO_MEMORY;
+
+ cqp_info = &cqp_request->info;
+ qp_info = &cqp_request->info.in.u.qp_create.info;
+ memset(qp_info, 0, sizeof(*qp_info));
+ qp_info->cq_num_valid = true;
+ qp_info->next_iwarp_state = IRDMA_QP_STATE_RTS;
+ cqp_info->cqp_cmd = IRDMA_OP_QP_CREATE;
+ cqp_info->post_sq = 1;
+ cqp_info->in.u.qp_create.qp = qp;
+ cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request;
+ status = irdma_handle_cqp_op(rf, cqp_request);
+ if (status)
+ irdma_debug(dev, IRDMA_DEBUG_ERR, "CQP-OP QP create fail");
+
+ return status;
+}
+
+/**
+ * irdma_dealloc_push_page - free a push page for qp
+ * @rf: RDMA PCI function
+ * @qp: hardware control qp
+ */
+static void irdma_dealloc_push_page(struct irdma_pci_f *rf,
+ struct irdma_sc_qp *qp)
+{
+ struct irdma_cqp_request *cqp_request;
+ struct cqp_cmds_info *cqp_info;
+ enum irdma_status_code status;
+
+ if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX)
+ return;
+
+ cqp_request = irdma_get_cqp_request(&rf->cqp, false);
+ if (!cqp_request)
+ return;
+
+ cqp_info = &cqp_request->info;
+ cqp_info->cqp_cmd = IRDMA_OP_MANAGE_PUSH_PAGE;
+ cqp_info->post_sq = 1;
+ cqp_info->in.u.manage_push_page.info.push_idx = qp->push_idx;
+ cqp_info->in.u.manage_push_page.info.qs_handle = qp->qs_handle;
+ cqp_info->in.u.manage_push_page.info.free_page = 1;
+ cqp_info->in.u.manage_push_page.cqp = &rf->cqp.sc_cqp;
+ cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request;
+ status = irdma_handle_cqp_op(rf, cqp_request);
+ if (!status)
+ qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX;
+ else
+ irdma_debug(&rf->sc_dev, IRDMA_DEBUG_ERR,
+ "CQP-OP dealloc Push page fail");
+}
+
+/**
+ * irdma_free_qp_resources - free up memory resources for qp
+ * @iwdev: iwarp device
+ * @iwqp: qp ptr (user or kernel)
+ * @qp_num: qp number assigned
+ */
+void irdma_free_qp_rsrc(struct irdma_device *iwdev,
+ struct irdma_qp *iwqp,
+ u32 qp_num)
+{
+ struct irdma_pci_f *rf = iwdev->rf;
+
+ irdma_ieq_cleanup_qp(iwdev->vsi.ieq, &iwqp->sc_qp);
+ irdma_dealloc_push_page(rf, &iwqp->sc_qp);
+ if (iwqp->sc_qp.vsi)
+ iwqp->sc_qp.dev->ws_remove(iwqp->sc_qp.vsi, iwqp->sc_qp.user_pri);
+
+ if (qp_num > 2)
+ irdma_free_rsrc(rf, rf->allocated_qps, qp_num);
+ irdma_free_dma_mem(rf->sc_dev.hw, &iwqp->q2_ctx_mem);
+ irdma_free_dma_mem(rf->sc_dev.hw, &iwqp->kqp.dma_mem);
+ kfree(iwqp->kqp.wrid_mem);
+ iwqp->kqp.wrid_mem = NULL;
+ kfree(iwqp->allocated_buf);
+}
+
+/**
+ * irdma_cq_wq_destroy - send cq destroy cqp
+ * @rf: RDMA PCI function
+ * @cq: hardware control cq
+ */
+void irdma_cq_wq_destroy(struct irdma_pci_f *rf, struct irdma_sc_cq *cq)
+{
+ enum irdma_status_code status;
+ struct irdma_cqp_request *cqp_request;
+ struct cqp_cmds_info *cqp_info;
+
+ cqp_request = irdma_get_cqp_request(&rf->cqp, true);
+ if (!cqp_request)
+ return;
+
+ cqp_info = &cqp_request->info;
+ cqp_info->cqp_cmd = IRDMA_OP_CQ_DESTROY;
+ cqp_info->post_sq = 1;
+ cqp_info->in.u.cq_destroy.cq = cq;
+ cqp_info->in.u.cq_destroy.scratch = (uintptr_t)cqp_request;
+ status = irdma_handle_cqp_op(rf, cqp_request);
+ if (status)
+ irdma_debug(&rf->sc_dev, IRDMA_DEBUG_ERR,
+ "CQP-OP Destroy CQ fail");
+}
+
+/**
+ * irdma_hw_modify_qp_callback - handle state for modifyQPs that don't wait
+ * @cqp_request: modify QP completion
+ * @num: not used
+ */
+static void irdma_hw_modify_qp_callback(struct irdma_cqp_request *cqp_request,
+ u32 num)
+{
+ struct cqp_cmds_info *cqp_info;
+ struct irdma_qp *iwqp;
+
+ cqp_info = &cqp_request->info;
+ iwqp = cqp_info->in.u.qp_modify.qp->back_qp;
+ atomic_dec(&iwqp->hw_mod_qp_pend);
+ wake_up(&iwqp->mod_qp_waitq);
+}
+
+/**
+ * irdma_hw_modify_qp - setup cqp for modify qp
+ * @rf: RDMA PCI function
+ * @iwqp: qp ptr (user or kernel)
+ * @info: info for modify qp
+ * @wait: flag to wait or not for modify qp completion
+ */
+enum irdma_status_code irdma_hw_modify_qp(struct irdma_device *iwdev,
+ struct irdma_qp *iwqp,
+ struct irdma_modify_qp_info *info,
+ bool wait)
+{
+ enum irdma_status_code status;
+ struct irdma_pci_f *rf = iwdev->rf;
+ struct irdma_cqp_request *cqp_request;
+ struct cqp_cmds_info *cqp_info;
+ struct irdma_modify_qp_info *m_info;
+
+ cqp_request = irdma_get_cqp_request(&rf->cqp, wait);
+ if (!cqp_request)
+ return IRDMA_ERR_NO_MEMORY;
+
+ if (!wait) {
+ cqp_request->callback_fcn = irdma_hw_modify_qp_callback;
+ atomic_inc(&iwqp->hw_mod_qp_pend);
+ }
+ cqp_info = &cqp_request->info;
+ m_info = &cqp_info->in.u.qp_modify.info;
+ memcpy(m_info, info, sizeof(*m_info));
+ cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY;
+ cqp_info->post_sq = 1;
+ cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp;
+ cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request;
+ status = irdma_handle_cqp_op(rf, cqp_request);
+ if (status) {
+ if (rdma_protocol_roce(&iwdev->iwibdev->ibdev, 1))
+ return status;
+ switch (m_info->next_iwarp_state) {
+ struct irdma_gen_ae_info ae_info;
+
+ case IRDMA_QP_STATE_RTS:
+ case IRDMA_QP_STATE_IDLE:
+ case IRDMA_QP_STATE_TERMINATE:
+ case IRDMA_QP_STATE_CLOSING:
+ if (info->curr_iwarp_state == IRDMA_QP_STATE_IDLE)
+ irdma_send_reset(iwqp->cm_node);
+ else
+ iwqp->sc_qp.term_flags = IRDMA_TERM_DONE;
+ if (!wait) {
+ ae_info.ae_code = IRDMA_AE_BAD_CLOSE;
+ ae_info.ae_src = 0;
+ irdma_gen_ae(rf, &iwqp->sc_qp, &ae_info, false);
+ } else {
+ cqp_request = irdma_get_cqp_request(&rf->cqp, wait);
+ if (!cqp_request)
+ return IRDMA_ERR_NO_MEMORY;
+
+ cqp_info = &cqp_request->info;
+ m_info = &cqp_info->in.u.qp_modify.info;
+ memcpy(m_info, info, sizeof(*m_info));
+ cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY;
+ cqp_info->post_sq = 1;
+ cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp;
+ cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request;
+ m_info->next_iwarp_state = IRDMA_QP_STATE_ERROR;
+ m_info->reset_tcp_conn = true;
+ irdma_handle_cqp_op(rf, cqp_request);
+ }
+ break;
+ case IRDMA_QP_STATE_ERROR:
+ default:
+ break;
+ }
+ }
+
+ return status;
+}
+
+/**
+ * irdma_cqp_cq_destroy_cmd - destroy the cqp cq
+ * @dev: device pointer
+ * @cq: pointer to cq
+ */
+void irdma_cqp_cq_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq)
+{
+ struct irdma_pci_f *rf = dev->back_dev;
+
+ irdma_cq_wq_destroy(rf, cq);
+}
+
+/**
+ * irdma_cqp_qp_destroy_cmd - destroy the cqp
+ * @dev: device pointer
+ * @qp: pointer to qp
+ */
+void irdma_cqp_qp_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)
+{
+ struct irdma_pci_f *rf = dev->back_dev;
+ struct irdma_cqp *iwcqp = &rf->cqp;
+ struct irdma_cqp_request *cqp_request;
+ struct cqp_cmds_info *cqp_info;
+ enum irdma_status_code status;
+
+ cqp_request = irdma_get_cqp_request(iwcqp, true);
+ if (!cqp_request)
+ return;
+
+ cqp_info = &cqp_request->info;
+ memset(cqp_info, 0, sizeof(*cqp_info));
+ cqp_info->cqp_cmd = IRDMA_OP_QP_DESTROY;
+ cqp_info->post_sq = 1;
+ cqp_info->in.u.qp_destroy.qp = qp;
+ cqp_info->in.u.qp_destroy.scratch = (uintptr_t)cqp_request;
+ cqp_info->in.u.qp_destroy.remove_hash_idx = true;
+ status = irdma_handle_cqp_op(rf, cqp_request);
+ if (status)
+ irdma_debug(dev, IRDMA_DEBUG_ERR, "CQP QP_DESTROY fail");
+}
+
+/**
+ * irdma_ieq_mpa_crc_ae - generate AE for crc error
+ * @dev: hardware control device structure
+ * @qp: hardware control qp
+ */
+void irdma_ieq_mpa_crc_ae(struct irdma_sc_dev *dev,
+ struct irdma_sc_qp *qp)
+{
+ struct irdma_gen_ae_info info = {};
+ struct irdma_pci_f *rf = dev->back_dev;
+
+ irdma_debug(dev, IRDMA_DEBUG_AEQ, "Generate MPA CRC AE\n");
+ info.ae_code = IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR;
+ info.ae_src = IRDMA_AE_SOURCE_RQ;
+ irdma_gen_ae(rf, qp, &info, false);
+}
+
+/**
+ * irdma_init_hash_desc - initialize hash for crc calculation
+ * @desc: cryption type
+ */
+enum irdma_status_code irdma_init_hash_desc(struct shash_desc **desc)
+{
+ struct crypto_shash *tfm;
+ struct shash_desc *tdesc;
+
+ tfm = crypto_alloc_shash("crc32c", 0, 0);
+ if (IS_ERR(tfm))
+ return IRDMA_ERR_MPA_CRC;
+
+ tdesc = kzalloc(sizeof(*tdesc) + crypto_shash_descsize(tfm), GFP_KERNEL);
+ if (!tdesc) {
+ crypto_free_shash(tfm);
+ return IRDMA_ERR_MPA_CRC;
+ }
+
+ tdesc->tfm = tfm;
+ *desc = tdesc;
+
+ return 0;
+}
+
+/**
+ * irdma_free_hash_desc - free hash desc
+ * @desc: to be freed
+ */
+void irdma_free_hash_desc(struct shash_desc *desc)
+{
+ if (desc) {
+ crypto_free_shash(desc->tfm);
+ kfree(desc);
+ }
+}
+
+/**
+ * irdma_ieq_check_mpacrc - check if mpa crc is OK
+ * @desc: desc for hash
+ * @addr: address of buffer for crc
+ * @length: length of buffer
+ * @value: value to be compared
+ */
+enum irdma_status_code irdma_ieq_check_mpacrc(struct shash_desc *desc,
+ void *addr,
+ u32 len,
+ u32 val)
+{
+ u32 crc = 0;
+ int ret;
+ enum irdma_status_code ret_code = 0;
+
+ crypto_shash_init(desc);
+ ret = crypto_shash_update(desc, addr, len);
+ if (!ret)
+ crypto_shash_final(desc, (u8 *)&crc);
+ if (crc != val) {
+ irdma_pr_err("mpa crc check fail");
+ ret_code = IRDMA_ERR_MPA_CRC;
+ }
+
+ return ret_code;
+}
+
+/**
+ * irdma_ieq_get_qp - get qp based on quad in puda buffer
+ * @dev: hardware control device structure
+ * @buf: receive puda buffer on exception q
+ */
+struct irdma_sc_qp *irdma_ieq_get_qp(struct irdma_sc_dev *dev,
+ struct irdma_puda_buf *buf)
+{
+ struct irdma_qp *iwqp;
+ struct irdma_cm_node *cm_node;
+ struct irdma_device *iwdev = buf->vsi->back_vsi;
+ u32 loc_addr[4] = {};
+ u32 rem_addr[4] = {};
+ u16 loc_port, rem_port;
+ struct ipv6hdr *ip6h;
+ struct iphdr *iph = (struct iphdr *)buf->iph;
+ struct tcphdr *tcph = (struct tcphdr *)buf->tcph;
+ struct irdma_pci_f *rf = dev->back_dev;
+ struct udphdr *udph;
+ struct irdma_bth *bth;
+
+ if (iph->protocol == IPPROTO_UDP) {
+ udph = (struct udphdr *)tcph;
+ bth = (struct irdma_bth *)udph + sizeof(*udph);
+ iwqp = rf->qp_table[be32_to_cpu(bth->qpn)];
+ return &iwqp->sc_qp;
+ }
+
+ if (iph->version == 4) {
+ loc_addr[0] = ntohl(iph->daddr);
+ rem_addr[0] = ntohl(iph->saddr);
+ } else {
+ ip6h = (struct ipv6hdr *)buf->iph;
+ irdma_copy_ip_ntohl(loc_addr, ip6h->daddr.in6_u.u6_addr32);
+ irdma_copy_ip_ntohl(rem_addr, ip6h->saddr.in6_u.u6_addr32);
+ }
+ loc_port = ntohs(tcph->dest);
+ rem_port = ntohs(tcph->source);
+ cm_node = irdma_find_node(&iwdev->cm_core, rem_port, rem_addr, loc_port,
+ loc_addr, false, true);
+ if (!cm_node)
+ return NULL;
+
+ iwqp = cm_node->iwqp;
+
+ return &iwqp->sc_qp;
+}
+
+/**
+ * irdma_send_ieq_ack - ACKs for duplicate or OOO partials FPDUs
+ * @qp: qp ptr
+ */
+void irdma_send_ieq_ack(struct irdma_sc_qp *qp)
+{
+ struct irdma_cm_node *cm_node = ((struct irdma_qp *)qp->back_qp)->cm_node;
+ struct irdma_puda_buf *buf = qp->pfpdu.lastrcv_buf;
+ struct tcphdr *tcph = (struct tcphdr *)buf->tcph;
+
+ cm_node->tcp_cntxt.rcv_nxt = qp->pfpdu.nextseqnum;
+ cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->ack_seq);
+
+ irdma_send_ack(cm_node);
+}
+
+/**
+ * irdma_puda_ieq_get_ah_info - get AH info from IEQ buffer
+ * @qp: qp pointer
+ * @ah_info: AH info pointer
+ */
+void irdma_puda_ieq_get_ah_info(struct irdma_sc_qp *qp,
+ struct irdma_ah_info *ah_info)
+{
+ struct irdma_puda_buf *buf = qp->pfpdu.ah_buf;
+ struct iphdr *iph;
+ struct ipv6hdr *ip6h;
+
+ memset(ah_info, 0, sizeof(*ah_info));
+ ah_info->do_lpbk = true;
+ ah_info->vlan_tag = buf->vlan_id;
+ ah_info->insert_vlan_tag = buf->vlan_valid;
+ ah_info->ipv4_valid = buf->ipv4;
+ ah_info->vsi = qp->vsi;
+
+ if (buf->smac_valid)
+ ether_addr_copy(ah_info->mac_addr, buf->smac);
+
+ if (buf->ipv4) {
+ ah_info->ipv4_valid = true;
+ iph = (struct iphdr *)buf->iph;
+ ah_info->hop_ttl = iph->ttl;
+ ah_info->tc_tos = iph->tos;
+ ah_info->dest_ip_addr[0] = ntohl(iph->daddr);
+ ah_info->src_ip_addr[0] = ntohl(iph->saddr);
+ } else {
+ ip6h = (struct ipv6hdr *)buf->iph;
+ ah_info->hop_ttl = ip6h->hop_limit;
+ ah_info->tc_tos = ip6h->priority;
+ irdma_copy_ip_ntohl(ah_info->dest_ip_addr,
+ ip6h->daddr.in6_u.u6_addr32);
+ irdma_copy_ip_ntohl(ah_info->src_ip_addr,
+ ip6h->saddr.in6_u.u6_addr32);
+ }
+
+ ah_info->dst_arpindex = irdma_arp_table(qp->dev->back_dev,
+ ah_info->dest_ip_addr,
+ ah_info->ipv4_valid,
+ NULL, IRDMA_ARP_RESOLVE);
+}
+
+/**
+ * irdma_gen1_ieq_update_tcpip_info - update tcpip in the buffer
+ * @buf: puda to update
+ * @length: length of buffer
+ * @seqnum: seq number for tcp
+ */
+static void irdma_gen1_ieq_update_tcpip_info(struct irdma_puda_buf *buf,
+ u16 len,
+ u32 seqnum)
+{
+ struct tcphdr *tcph;
+ struct iphdr *iph;
+ u16 iphlen;
+ u16 pktsize;
+ u8 *addr = (u8 *)buf->mem.va;
+
+ iphlen = (buf->ipv4) ? 20 : 40;
+ iph = (struct iphdr *)(addr + buf->maclen);
+ tcph = (struct tcphdr *)(addr + buf->maclen + iphlen);
+ pktsize = len + buf->tcphlen + iphlen;
+ iph->tot_len = htons(pktsize);
+ tcph->seq = htonl(seqnum);
+}
+
+/**
+ * irdma_ieq_update_tcpip_info - update tcpip in the buffer
+ * @buf: puda to update
+ * @length: length of buffer
+ * @seqnum: seq number for tcp
+ */
+void irdma_ieq_update_tcpip_info(struct irdma_puda_buf *buf,
+ u16 len,
+ u32 seqnum)
+{
+ struct tcphdr *tcph;
+ u16 pktsize;
+ u8 *addr;
+
+ if (buf->vsi->dev->hw_attrs.hw_rev == IRDMA_GEN_1)
+ return irdma_gen1_ieq_update_tcpip_info(buf, len, seqnum);
+
+ addr = (u8 *)buf->mem.va;
+ tcph = (struct tcphdr *)addr;
+ pktsize = len + buf->tcphlen;
+ tcph->seq = htonl(seqnum);
+}
+
+/**
+ * irdma_gen1_puda_get_tcpip_info - get tcpip info from puda
+ * buffer
+ * @info: to get information
+ * @buf: puda buffer
+ */
+static enum irdma_status_code
+irdma_gen1_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,
+ struct irdma_puda_buf *buf)
+{
+ struct iphdr *iph;
+ struct ipv6hdr *ip6h;
+ struct tcphdr *tcph;
+ u16 iphlen;
+ u16 pkt_len;
+ u8 *mem = (u8 *)buf->mem.va;
+ struct ethhdr *ethh = (struct ethhdr *)buf->mem.va;
+
+ if (ethh->h_proto == htons(0x8100)) {
+ info->vlan_valid = true;
+ buf->vlan_id = ntohs(((struct vlan_ethhdr *)ethh)->h_vlan_TCI) &
+ VLAN_VID_MASK;
+ }
+
+ buf->maclen = (info->vlan_valid) ? 18 : 14;
+ iphlen = (info->l3proto) ? 40 : 20;
+ buf->ipv4 = (info->l3proto) ? false : true;
+ buf->iph = mem + buf->maclen;
+ iph = (struct iphdr *)buf->iph;
+ buf->tcph = buf->iph + iphlen;
+ tcph = (struct tcphdr *)buf->tcph;
+
+ if (buf->ipv4) {
+ pkt_len = ntohs(iph->tot_len);
+ } else {
+ ip6h = (struct ipv6hdr *)buf->iph;
+ pkt_len = ntohs(ip6h->payload_len) + iphlen;
+ }
+
+ buf->totallen = pkt_len + buf->maclen;
+
+ if (info->payload_len < buf->totallen) {
+ irdma_debug(buf->vsi->dev, IRDMA_DEBUG_ERR,
+ "payload_len = 0x%x totallen expected0x%x\n",
+ info->payload_len, buf->totallen);
+ return IRDMA_ERR_INVALID_SIZE;
+ }
+
+ buf->tcphlen = (tcph->doff) << 2;
+ buf->datalen = pkt_len - iphlen - buf->tcphlen;
+ buf->data = (buf->datalen) ? buf->tcph + buf->tcphlen : NULL;
+ buf->hdrlen = buf->maclen + iphlen + buf->tcphlen;
+ buf->seqnum = ntohl(tcph->seq);
+
+ return 0;
+}
+
+/**
+ * irdma_puda_get_tcpip_info - get tcpip info from puda buffer
+ * @info: to get information
+ * @buf: puda buffer
+ */
+enum irdma_status_code
+irdma_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,
+ struct irdma_puda_buf *buf)
+{
+ struct tcphdr *tcph;
+ u32 pkt_len;
+ u8 *mem;
+
+ if (buf->vsi->dev->hw_attrs.hw_rev == IRDMA_GEN_1)
+ return irdma_gen1_puda_get_tcpip_info(info, buf);
+
+ mem = (u8 *)buf->mem.va;
+ buf->vlan_valid = info->vlan_valid;
+ if (info->vlan_valid)
+ buf->vlan_id = info->vlan;
+
+ buf->ipv4 = info->ipv4;
+ if (buf->ipv4)
+ buf->iph = mem + 20;
+ else
+ buf->iph = mem;
+
+ buf->tcph = mem + IRDMA_TCP_OFFSET;
+ tcph = (struct tcphdr *)buf->tcph;
+ pkt_len = info->payload_len;
+ buf->totallen = pkt_len;
+ buf->tcphlen = (tcph->doff) << 2;
+ buf->datalen = pkt_len - IRDMA_TCP_OFFSET - buf->tcphlen;
+ buf->data = (buf->datalen) ? buf->tcph + buf->tcphlen : NULL;
+ buf->hdrlen = IRDMA_TCP_OFFSET + buf->tcphlen;
+ buf->seqnum = ntohl(tcph->seq);
+
+ if (info->smac_valid) {
+ ether_addr_copy(buf->smac, info->smac);
+ buf->smac_valid = true;
+ }
+
+ return 0;
+}
+
+/**
+ * irdma_process_stats - Checking for wrap and update stats
+ *
+ * @pestat: stats structure pointer
+ */
+static void irdma_process_stats(struct irdma_vsi_pestat *pestat)
+{
+ struct irdma_gather_stats *gather_stats =
+ pestat->gather_info.gather_stats;
+ struct irdma_gather_stats *last_gather_stats =
+ pestat->gather_info.last_gather_stats;
+ irdma_update_stats(&pestat->hw_stats, gather_stats, last_gather_stats);
+}
+
+/**
+ * irdma_cqp_gather_stats_gen1 - Gather stats
+ * @vsi: pointer to vsi structure
+ * @gather_info: pointer to gather stats info
+ */
+static void irdma_cqp_gather_stats_gen1(struct irdma_sc_dev *dev,
+ struct irdma_vsi_pestat *pestat)
+{
+ struct irdma_gather_stats *gather_stats = pestat->gather_info.gather_stats;
+ u32 stats_inst_offset_32;
+ u32 stats_inst_offset_64;
+
+ stats_inst_offset_32 = (pestat->gather_info.use_stats_inst) ?
+ pestat->gather_info.stats_inst_index : pestat->hw->hmc.hmc_fn_id;
+ stats_inst_offset_32 *= 4;
+ stats_inst_offset_64 = stats_inst_offset_32 * 2;
+
+ gather_stats->rxvlanerr =
+ rd32(dev->hw,
+ dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_RXVLANERR]
+ + stats_inst_offset_32);
+ gather_stats->ip4rxdiscard =
+ rd32(dev->hw,
+ dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP4RXDISCARD]
+ + stats_inst_offset_32);
+ gather_stats->ip4rxtrunc =
+ rd32(dev->hw,
+ dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP4RXTRUNC]
+ + stats_inst_offset_32);
+ gather_stats->ip4txnoroute =
+ rd32(dev->hw,
+ dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP4TXNOROUTE]
+ + stats_inst_offset_32);
+ gather_stats->ip6rxdiscard =
+ rd32(dev->hw,
+ dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP6RXDISCARD]
+ + stats_inst_offset_32);
+ gather_stats->ip6rxtrunc =
+ rd32(dev->hw,
+ dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP6RXTRUNC]
+ + stats_inst_offset_32);
+ gather_stats->ip6txnoroute =
+ rd32(dev->hw,
+ dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP6TXNOROUTE]
+ + stats_inst_offset_32);
+ gather_stats->tcprtxseg =
+ rd32(dev->hw,
+ dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_TCPRTXSEG]
+ + stats_inst_offset_32);
+ gather_stats->tcprxopterr =
+ rd32(dev->hw,
+ dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_TCPRXOPTERR]
+ + stats_inst_offset_32);
+
+ gather_stats->ip4rxocts =
+ rd64(dev->hw,
+ dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4RXOCTS]
+ + stats_inst_offset_64);
+ gather_stats->ip4rxpkts =
+ rd64(dev->hw,
+ dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4RXPKTS]
+ + stats_inst_offset_64);
+ gather_stats->ip4txfrag =
+ rd64(dev->hw,
+ dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4RXFRAGS]
+ + stats_inst_offset_64);
+ gather_stats->ip4rxmcpkts =
+ rd64(dev->hw,
+ dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4RXMCPKTS]
+ + stats_inst_offset_64);
+ gather_stats->ip4txocts =
+ rd64(dev->hw,
+ dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4TXOCTS]
+ + stats_inst_offset_64);
+ gather_stats->ip4txpkts =
+ rd64(dev->hw,
+ dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4TXPKTS]
+ + stats_inst_offset_64);
+ gather_stats->ip4txfrag =
+ rd64(dev->hw,
+ dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4TXFRAGS]
+ + stats_inst_offset_64);
+ gather_stats->ip4txmcpkts =
+ rd64(dev->hw,
+ dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4TXMCPKTS]
+ + stats_inst_offset_64);
+ gather_stats->ip6rxocts =
+ rd64(dev->hw,
+ dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6RXOCTS]
+ + stats_inst_offset_64);
+ gather_stats->ip6rxpkts =
+ rd64(dev->hw,
+ dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6RXPKTS]
+ + stats_inst_offset_64);
+ gather_stats->ip6txfrags =
+ rd64(dev->hw,
+ dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6RXFRAGS]
+ + stats_inst_offset_64);
+ gather_stats->ip6rxmcpkts =
+ rd64(dev->hw,
+ dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6RXMCPKTS]
+ + stats_inst_offset_64);
+ gather_stats->ip6txocts =
+ rd64(dev->hw,
+ dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6TXOCTS]
+ + stats_inst_offset_64);
+ gather_stats->ip6txpkts =
+ rd64(dev->hw,
+ dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6TXPKTS]
+ + stats_inst_offset_64);
+ gather_stats->ip6txfrags =
+ rd64(dev->hw,
+ dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6TXFRAGS]
+ + stats_inst_offset_64);
+ gather_stats->ip6txmcpkts =
+ rd64(dev->hw,
+ dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6TXMCPKTS]
+ + stats_inst_offset_64);
+ gather_stats->tcprxsegs =
+ rd64(dev->hw,
+ dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_TCPRXSEGS]
+ + stats_inst_offset_64);
+ gather_stats->tcptxsegs =
+ rd64(dev->hw,
+ dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_TCPTXSEG]
+ + stats_inst_offset_64);
+ gather_stats->rdmarxrds =
+ rd64(dev->hw,
+ dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMARXRDS]
+ + stats_inst_offset_64);
+ gather_stats->rdmarxsnds =
+ rd64(dev->hw,
+ dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMARXSNDS]
+ + stats_inst_offset_64);
+ gather_stats->rdmarxwrs =
+ rd64(dev->hw,
+ dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMARXWRS]
+ + stats_inst_offset_64);
+ gather_stats->rdmatxrds =
+ rd64(dev->hw,
+ dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMATXRDS]
+ + stats_inst_offset_64);
+ gather_stats->rdmatxsnds =
+ rd64(dev->hw,
+ dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMATXSNDS]
+ + stats_inst_offset_64);
+ gather_stats->rdmatxwrs =
+ rd64(dev->hw,
+ dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMATXWRS]
+ + stats_inst_offset_64);
+ gather_stats->rdmavbn =
+ rd64(dev->hw,
+ dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMAVBND]
+ + stats_inst_offset_64);
+ gather_stats->rdmavinv =
+ rd64(dev->hw,
+ dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMAVINV]
+ + stats_inst_offset_64);
+ gather_stats->udprxpkts =
+ rd64(dev->hw,
+ dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_UDPRXPKTS]
+ + stats_inst_offset_64);
+ gather_stats->udptxpkts =
+ rd64(dev->hw,
+ dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_UDPTXPKTS]
+ + stats_inst_offset_64);
+
+ irdma_process_stats(pestat);
+}
+
+/**
+ * irdma_process_cqp_stats - Checking for wrap and update stats
+ * @cqp_request: cqp_request structure pointer
+ * @unused param of callback
+ */
+static void irdma_process_cqp_stats(struct irdma_cqp_request *cqp_request,
+ u32 unused)
+{
+ struct irdma_vsi_pestat *pestat = cqp_request->param;
+
+ irdma_process_stats(pestat);
+}
+
+/**
+ * irdma_cqp_gather_stats_cmd - Gather stats
+ * @vsi: pointer to vsi structure
+ * @gather_info: pointer to gather stats info
+ */
+enum irdma_status_code irdma_cqp_gather_stats_cmd(struct irdma_sc_dev *dev,
+ struct irdma_vsi_pestat *pestat,
+ bool wait)
+
+{
+ struct irdma_pci_f *rf = dev->back_dev;
+ struct irdma_cqp *iwcqp = &rf->cqp;
+ struct irdma_cqp_request *cqp_request;
+ struct cqp_cmds_info *cqp_info;
+ enum irdma_status_code status;
+
+ cqp_request = irdma_get_cqp_request(iwcqp, wait);
+ if (!cqp_request)
+ return IRDMA_ERR_NO_MEMORY;
+
+ cqp_info = &cqp_request->info;
+ memset(cqp_info, 0, sizeof(*cqp_info));
+ cqp_info->cqp_cmd = IRDMA_OP_STATS_GATHER;
+ cqp_info->post_sq = 1;
+ cqp_info->in.u.stats_gather.info = pestat->gather_info;
+ cqp_info->in.u.stats_gather.scratch = (uintptr_t)cqp_request;
+ cqp_info->in.u.stats_gather.cqp = &rf->cqp.sc_cqp;
+ cqp_request->param = pestat;
+ if (!wait)
+ cqp_request->callback_fcn = irdma_process_cqp_stats;
+ status = irdma_handle_cqp_op(rf, cqp_request);
+ if (status)
+ irdma_debug(dev, IRDMA_DEBUG_ERR, "CQP STATS_GATHER fail");
+ else if (wait)
+ irdma_process_stats(pestat);
+
+ return status;
+}
+
+/**
+ * irdma_hw_stats_timeout - Stats timer-handler which updates all HW stats
+ * @t: timer_list pointer
+ */
+static void irdma_hw_stats_timeout(struct timer_list *t)
+{
+ struct irdma_vsi_pestat *pf_devstat =
+ from_timer(pf_devstat, t, stats_timer);
+ struct irdma_sc_vsi *sc_vsi = pf_devstat->vsi;
+ struct irdma_device *iwdev = sc_vsi->back_vsi;
+
+ if (iwdev->closing || iwdev->init_state != RDMA_DEV_REGISTERED)
+ goto exit;
+ if (sc_vsi->dev->hw_attrs.hw_rev == IRDMA_GEN_1)
+ irdma_cqp_gather_stats_gen1(sc_vsi->dev, sc_vsi->pestat);
+ else
+ irdma_cqp_gather_stats_cmd(sc_vsi->dev, sc_vsi->pestat, false);
+
+exit:
+ mod_timer(&pf_devstat->stats_timer,
+ jiffies + msecs_to_jiffies(STATS_TIMER_DELAY));
+}
+
+/**
+ * irdma_hw_stats_start_timer - Start periodic stats timer
+ * @vsi: vsi structure pointer
+ */
+void irdma_hw_stats_start_timer(struct irdma_sc_vsi *vsi)
+{
+ struct irdma_vsi_pestat *devstat = vsi->pestat;
+
+ timer_setup(&devstat->stats_timer, irdma_hw_stats_timeout, 0);
+ mod_timer(&devstat->stats_timer,
+ jiffies + msecs_to_jiffies(STATS_TIMER_DELAY));
+}
+
+/**
+ * irdma_hw_stats_del_timer - Delete periodic stats timer
+ * @vsi: pointer to vsi structure
+ */
+void irdma_hw_stats_stop_timer(struct irdma_sc_vsi *vsi)
+{
+ struct irdma_vsi_pestat *devstat = vsi->pestat;
+
+ del_timer_sync(&devstat->stats_timer);
+}
+
+/**
+ * irdma_cqp_stats_inst_cmd - Allocate/free stats instance
+ * @vsi: pointer to vsi structure
+ * @stats_info: pointer to allocate stats info
+ */
+enum irdma_status_code
+irdma_cqp_stats_inst_cmd(struct irdma_sc_vsi *vsi,
+ u8 cmd,
+ struct irdma_stats_inst_info *stats_info)
+{
+ struct irdma_pci_f *rf = vsi->dev->back_dev;
+ struct irdma_cqp *iwcqp = &rf->cqp;
+ struct irdma_cqp_request *cqp_request;
+ struct cqp_cmds_info *cqp_info;
+ enum irdma_status_code status;
+ bool wait = false;
+
+ if (cmd == IRDMA_OP_STATS_ALLOCATE)
+ wait = true;
+ cqp_request = irdma_get_cqp_request(iwcqp, wait);
+ if (!cqp_request)
+ return IRDMA_ERR_NO_MEMORY;
+
+ cqp_info = &cqp_request->info;
+ memset(cqp_info, 0, sizeof(*cqp_info));
+ cqp_info->cqp_cmd = cmd;
+ cqp_info->post_sq = 1;
+ cqp_info->in.u.stats_manage.info = *stats_info;
+ cqp_info->in.u.stats_manage.scratch = (uintptr_t)cqp_request;
+ cqp_info->in.u.stats_manage.cqp = &rf->cqp.sc_cqp;
+ status = irdma_handle_cqp_op(rf, cqp_request);
+ if (status)
+ irdma_debug(&rf->sc_dev, IRDMA_DEBUG_ERR,
+ "CQP MANAGE_STATS fail");
+ else if (wait)
+ stats_info->stats_idx = cqp_request->compl_info.op_ret_val;
+
+ return status;
+}
+
+/**
+ * irdma_cqp_ceq_cmd - Create/Destroy CEQ's after CEQ 0
+ * @iwdev: pointer to device info
+ * @sc_ceq: pointer to ceq structure
+ * @op: Create or Destroy
+ */
+enum irdma_status_code irdma_cqp_ceq_cmd(struct irdma_sc_dev *dev,
+ struct irdma_sc_ceq *sc_ceq,
+ u8 op)
+{
+ struct irdma_cqp_request *cqp_request;
+ struct cqp_cmds_info *cqp_info;
+ struct irdma_pci_f *rf = dev->back_dev;
+
+ cqp_request = irdma_get_cqp_request(&rf->cqp, true);
+ if (!cqp_request)
+ return IRDMA_ERR_NO_MEMORY;
+
+ cqp_info = &cqp_request->info;
+ cqp_info->post_sq = 1;
+ cqp_info->cqp_cmd = op;
+ cqp_info->in.u.ceq_create.ceq = sc_ceq;
+ cqp_info->in.u.ceq_create.scratch = (uintptr_t)cqp_request;
+
+ return irdma_handle_cqp_op(rf, cqp_request);
+}
+
+/**
+ * irdma_cqp_ws_node_cmd - Add/modify/delete ws node
+ * @vsi: pointer to vsi structure
+ * @command: Add, modify or delete
+ * @node_info: pointer to ws node info
+ */
+enum irdma_status_code
+irdma_cqp_ws_node_cmd(struct irdma_sc_dev *dev,
+ u8 cmd,
+ struct irdma_ws_node_info *node_info)
+{
+ struct irdma_pci_f *rf = dev->back_dev;
+ struct irdma_cqp *iwcqp = &rf->cqp;
+ struct irdma_sc_cqp *cqp = &iwcqp->sc_cqp;
+ struct irdma_cqp_request *cqp_request;
+ struct cqp_cmds_info *cqp_info;
+ enum irdma_status_code status;
+ bool poll;
+
+ if (!rf->sc_dev.ceq_valid)
+ poll = true;
+ else
+ poll = false;
+
+ cqp_request = irdma_get_cqp_request(iwcqp, !poll);
+ if (!cqp_request)
+ return IRDMA_ERR_NO_MEMORY;
+
+ cqp_info = &cqp_request->info;
+ memset(cqp_info, 0, sizeof(*cqp_info));
+ cqp_info->cqp_cmd = cmd;
+ cqp_info->post_sq = 1;
+ cqp_info->in.u.ws_node.info = *node_info;
+ cqp_info->in.u.ws_node.cqp = cqp;
+ cqp_info->in.u.ws_node.scratch = (uintptr_t)cqp_request;
+ status = irdma_handle_cqp_op(rf, cqp_request);
+ if (status) {
+ irdma_debug(dev, IRDMA_DEBUG_ERR, "CQP WS_NODE fail\n");
+ return status;
+ }
+
+ if (poll) {
+ struct irdma_ccq_cqe_info compl_info;
+
+ status = cqp->dev->cqp_ops->poll_for_cqp_op_done(cqp,
+ IRDMA_CQP_OP_WORK_SCHED_NODE,
+ &compl_info);
+ node_info->qs_handle = compl_info.op_ret_val;
+ irdma_debug(cqp->dev, IRDMA_DEBUG_DCB,
+ "opcode=%d, compl_info.retval=%d\n",
+ compl_info.op_code, compl_info.op_ret_val);
+ } else {
+ node_info->qs_handle = cqp_request->compl_info.op_ret_val;
+ }
+
+ return status;
+}
+
+/**
+ * irdma_cqp_up_map_cmd - Set the up-up mapping
+ * @vsi: pointer to vsi structure
+ * @map_info: pointer to up map info
+ */
+enum irdma_status_code irdma_cqp_up_map_cmd(struct irdma_sc_dev *dev,
+ u8 cmd,
+ struct irdma_up_info *map_info)
+{
+ struct irdma_pci_f *rf = dev->back_dev;
+ struct irdma_cqp *iwcqp = &rf->cqp;
+ struct irdma_sc_cqp *cqp = &iwcqp->sc_cqp;
+ struct irdma_cqp_request *cqp_request;
+ struct cqp_cmds_info *cqp_info;
+ enum irdma_status_code status;
+
+ cqp_request = irdma_get_cqp_request(iwcqp, false);
+ if (!cqp_request)
+ return IRDMA_ERR_NO_MEMORY;
+
+ cqp_info = &cqp_request->info;
+ memset(cqp_info, 0, sizeof(*cqp_info));
+ cqp_info->cqp_cmd = cmd;
+ cqp_info->post_sq = 1;
+ cqp_info->in.u.up_map.info = *map_info;
+ cqp_info->in.u.up_map.cqp = cqp;
+ cqp_info->in.u.up_map.scratch = (uintptr_t)cqp_request;
+ status = irdma_handle_cqp_op(rf, cqp_request);
+ if (status) {
+ irdma_debug(dev, IRDMA_DEBUG_ERR, "CQP UP MAP fail\n");
+ return status;
+ }
+
+ return status;
+}
+
+/**
+ * irdma_ah_cqp_op - perform an AH cqp operation
+ * @rf: RDMA PCI function
+ * @sc_ah: address handle
+ * @cmd: AH operation
+ * @wait: wait if true
+ * @callback_fcn: Callback function on CQP op completion
+ * @cb_param: parameter for callback function
+ *
+ * returns errno
+ */
+int irdma_ah_cqp_op(struct irdma_pci_f *rf,
+ struct irdma_sc_ah *sc_ah,
+ u8 cmd,
+ bool wait,
+ void (*callback_fcn)(struct irdma_cqp_request*, u32),
+ void *cb_param)
+{
+ struct irdma_cqp_request *cqp_request;
+ struct cqp_cmds_info *cqp_info;
+ struct irdma_device *iwdev;
+ enum irdma_status_code status;
+
+ cqp_request = irdma_get_cqp_request(&rf->cqp, wait);
+ if (!cqp_request)
+ return -ENOMEM;
+
+ cqp_info = &cqp_request->info;
+ cqp_info->cqp_cmd = cmd;
+ cqp_info->post_sq = 1;
+ if (cmd == IRDMA_OP_AH_CREATE) {
+ cqp_info->in.u.ah_create.info = sc_ah->ah_info;
+ cqp_info->in.u.ah_create.scratch = (uintptr_t)cqp_request;
+ cqp_info->in.u.ah_create.cqp = &rf->cqp.sc_cqp;
+ } else if (cmd == IRDMA_OP_AH_DESTROY) {
+ cqp_info->in.u.ah_destroy.info = sc_ah->ah_info;
+ cqp_info->in.u.ah_destroy.scratch = (uintptr_t)cqp_request;
+ cqp_info->in.u.ah_destroy.cqp = &rf->cqp.sc_cqp;
+ } else {
+ return -EINVAL;
+ }
+
+ if (!wait) {
+ cqp_request->callback_fcn = callback_fcn;
+ cqp_request->param = cb_param;
+ }
+ status = irdma_handle_cqp_op(rf, cqp_request);
+ if (!status) {
+ if (wait) {
+ iwdev = sc_ah->ah_info.vsi->back_vsi;
+ if (cmd == IRDMA_OP_AH_CREATE) {
+ irdma_add_devusecount(iwdev);
+ sc_ah->ah_info.ah_valid = true;
+
+ } else {
+ irdma_rem_devusecount(iwdev);
+ sc_ah->ah_info.ah_valid = false;
+ }
+ }
+ } else {
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+/**
+ * irdma_ieq_ah_cb - callback after creation of AH for IEQ
+ * @cqp_request: pointer to cqp_request of create AH
+ * @unused: unused param
+ */
+static void irdma_ieq_ah_cb(struct irdma_cqp_request *cqp_request,
+ u32 unused)
+{
+ struct irdma_sc_qp *qp = cqp_request->param;
+ struct irdma_sc_ah *sc_ah = qp->pfpdu.ah;
+ struct irdma_device *iwdev = qp->vsi->back_vsi;
+ unsigned long flags;
+
+ spin_lock_irqsave(&qp->pfpdu.lock, flags);
+ if (!cqp_request->compl_info.op_ret_val) {
+ sc_ah->ah_info.ah_valid = true;
+ irdma_add_devusecount(iwdev);
+ irdma_ieq_process_fpdus(qp, qp->vsi->ieq);
+ } else {
+ sc_ah->ah_info.ah_valid = false;
+ irdma_ieq_cleanup_qp(qp->vsi->ieq, qp);
+ }
+ spin_unlock_irqrestore(&qp->pfpdu.lock, flags);
+}
+
+/**
+ * irdma_ilq_ah_cb - callback after creation of AH for ILQ
+ * @cqp_request: pointer to cqp_request of create AH
+ * @unused: unused param
+ */
+static void irdma_ilq_ah_cb(struct irdma_cqp_request *cqp_request,
+ u32 unused)
+{
+ struct irdma_cm_node *cm_node = cqp_request->param;
+ struct irdma_sc_ah *sc_ah = cm_node->ah;
+ struct irdma_device *iwdev = sc_ah->ah_info.vsi->back_vsi;
+
+ if (!cqp_request->compl_info.op_ret_val) {
+ sc_ah->ah_info.ah_valid = true;
+ irdma_add_devusecount(iwdev);
+ } else {
+ sc_ah->ah_info.ah_valid = false;
+ }
+}
+
+/**
+ * irdma_puda_create_ah - create AH for ILQ/IEQ qp's
+ * @dev: device pointer
+ * @ah_info: Address handle info
+ * @wait: When true will wait for operation to complete
+ * @type: ILQ/IEQ
+ * @cb_param: Callback param when not waiting
+ * @ah_ret: Returned pointer to address handle if created
+ *
+ */
+enum irdma_status_code irdma_puda_create_ah(struct irdma_sc_dev *dev,
+ struct irdma_ah_info *ah_info,
+ bool wait,
+ enum puda_rsrc_type type,
+ void *cb_param,
+ struct irdma_sc_ah **ah_ret)
+{
+ struct irdma_sc_ah *ah;
+ struct irdma_pci_f *rf = dev->back_dev;
+ int err;
+
+ ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
+ *ah_ret = ah;
+ if (!ah)
+ return IRDMA_ERR_NO_MEMORY;
+
+ err = irdma_alloc_rsrc(rf, rf->allocated_ahs,
+ rf->max_ah, &ah_info->ah_idx, &rf->next_ah);
+ if (err)
+ goto err_free;
+
+ ah->dev = dev;
+ ah->ah_info = *ah_info;
+
+ if (type == IRDMA_PUDA_RSRC_TYPE_ILQ)
+ err = irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_CREATE, wait,
+ irdma_ilq_ah_cb, cb_param);
+ else
+ err = irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_CREATE, wait,
+ irdma_ieq_ah_cb, cb_param);
+
+ if (err)
+ goto error;
+ return 0;
+
+error:
+ irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx);
+err_free:
+ kfree(ah);
+ *ah_ret = NULL;
+ return IRDMA_ERR_NO_MEMORY;
+}
+
+/**
+ * irdma_puda_free_ah - free a puda address handle
+ * @dev: device pointer
+ * @ah: The address handle to free
+ */
+void irdma_puda_free_ah(struct irdma_sc_dev *dev, struct irdma_sc_ah *ah)
+{
+ struct irdma_pci_f *rf = dev->back_dev;
+
+ if (!ah)
+ return;
+
+ if (ah->ah_info.ah_valid) {
+ irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_DESTROY, false, NULL, NULL);
+ irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx);
+ irdma_rem_devusecount(ah->ah_info.vsi->back_vsi);
+ }
+
+ kfree(ah);
+}
+
+/**
+ * irdma_gsi_ud_qp_ah_cb - callback after creation of AH for GSI/ID QP
+ * @cqp_request: pointer to cqp_request of create AH
+ * @unused: unused param
+ */
+void irdma_gsi_ud_qp_ah_cb(struct irdma_cqp_request *cqp_request, u32 unused)
+{
+ struct irdma_sc_ah *sc_ah = cqp_request->param;
+
+ if (!cqp_request->compl_info.op_ret_val)
+ sc_ah->ah_info.ah_valid = true;
+ else
+ sc_ah->ah_info.ah_valid = false;
+}
+
+/**
+ * irdma_destroy_ah_cb - callback after destroy of AH
+ * @cqp_request: pointer to cqp_request of destroy AH
+ * @unused: unused param
+ */
+void irdma_destroy_ah_cb(struct irdma_cqp_request *cqp_request, u32 unused)
+{
+ struct irdma_ah *ah = cqp_request->param;
+ struct irdma_device *iwdev = ah->sc_ah.ah_info.vsi->back_vsi;
+
+ if (!cqp_request->compl_info.op_ret_val)
+ irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs,
+ ah->sc_ah.ah_info.ah_idx);
+ irdma_rem_pdusecount(ah->pd, iwdev);
+ kfree(ah);
+}
+
+/**
+ * irdma_prm_add_pble_mem - add moemory to pble resources
+ * @pprm: pble resource manager
+ * @pchunk: chunk of memory to add
+ */
+enum irdma_status_code irdma_prm_add_pble_mem(struct irdma_pble_prm *pprm,
+ struct irdma_chunk *pchunk)
+{
+ u64 sizeofbitmap;
+ enum irdma_status_code ret_code = 0;
+
+ if (pchunk->size & 0xfff)
+ return IRDMA_ERR_PARAM;
+
+ sizeofbitmap = (u64)pchunk->size >> pprm->pble_shift;
+
+ ret_code = irdma_allocate_virt_mem(pchunk->dev->hw,
+ &pchunk->bitmapmem,
+ sizeofbitmap >> 3);
+
+ if (ret_code)
+ return ret_code;
+
+ pchunk->bitmapbuf = pchunk->bitmapmem.va;
+ bitmap_zero(pchunk->bitmapbuf, sizeofbitmap);
+
+ pchunk->sizeofbitmap = sizeofbitmap;
+ /* each pble is 8 bytes hence shift by 3 */
+ pprm->total_pble_alloc += pchunk->size >> 3;
+ pprm->free_pble_cnt += pchunk->size >> 3;
+
+ return 0;
+}
+
+/**
+ * irdma_prm_add_pble_mem - get pble's from prm
+ * @pprm: pble resource manager
+ * @pchunkinfo: return information about chunk where pble's were acquired
+ * @memsize: size of pble memory needed
+ * @vaddr: returns virtual address of pble memory
+ * @fpm_addr: returns fpm address of pble memory
+ */
+enum irdma_status_code irdma_prm_get_pbles(struct irdma_pble_prm *pprm,
+ struct irdma_pble_chunkinfo *chunkinfo,
+ u32 mem_size,
+ u64 *vaddr,
+ u64 *fpm_addr)
+{
+ u64 bits_needed;
+ u64 bit_idx = PBLE_INVALID_IDX;
+ struct irdma_chunk *pchunk = NULL;
+ struct list_head *chunk_entry = pprm->clist.next;
+ u32 offset;
+ unsigned long flags;
+ *vaddr = 0;
+ *fpm_addr = 0;
+
+ bits_needed = (mem_size + (1 << pprm->pble_shift) - 1) >> pprm->pble_shift;
+
+ spin_lock_irqsave(&pprm->prm_lock, flags);
+ while (chunk_entry != &pprm->clist) {
+ pchunk = (struct irdma_chunk *)chunk_entry;
+ bit_idx = bitmap_find_next_zero_area(pchunk->bitmapbuf,
+ pchunk->sizeofbitmap,
+ 0,
+ bits_needed,
+ 0);
+ if (bit_idx < pchunk->sizeofbitmap)
+ break;
+
+ /* list.next used macro */
+ chunk_entry = pchunk->list.next;
+ }
+
+ if (!pchunk || bit_idx >= pchunk->sizeofbitmap) {
+ spin_unlock_irqrestore(&pprm->prm_lock, flags);
+ return IRDMA_ERR_NO_MEMORY;
+ }
+
+ bitmap_set(pchunk->bitmapbuf, bit_idx, bits_needed);
+ offset = bit_idx << pprm->pble_shift;
+ *vaddr = (u64)((u8 *)pchunk->vaddr + offset);
+ *fpm_addr = pchunk->fpm_addr + offset;
+
+ chunkinfo->pchunk = pchunk;
+ chunkinfo->bit_idx = bit_idx;
+ chunkinfo->bits_used = bits_needed;
+ /* 3 is sizeof pble divide */
+ pprm->free_pble_cnt -= chunkinfo->bits_used << (pprm->pble_shift - 3);
+ spin_unlock_irqrestore(&pprm->prm_lock, flags);
+
+ return 0;
+}
+
+/**
+ * irdma_prm_return pbles - return pbles back to prm
+ * @pprm: pble resource manager
+ * @pchunkinfo: chunk where pble's were acquired and to be freed
+ */
+void irdma_prm_return_pbles(struct irdma_pble_prm *pprm,
+ struct irdma_pble_chunkinfo *chunkinfo)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&pprm->prm_lock, flags);
+ pprm->free_pble_cnt += chunkinfo->bits_used << (pprm->pble_shift - 3);
+ bitmap_clear(chunkinfo->pchunk->bitmapbuf,
+ chunkinfo->bit_idx,
+ chunkinfo->bits_used);
+ spin_unlock_irqrestore(&pprm->prm_lock, flags);
+}
+
+/**
+ * irdma_free_paged_mem - free virtual paged memory back to system
+ * @chunk: chunk to free with paged memory
+ */
+void irdma_pble_free_paged_mem(struct irdma_chunk *chunk)
+{
+ struct pci_dev *pcidev = (struct pci_dev *)chunk->dev->hw->dev_context;
+ int i;
+
+ if (!chunk->pg_cnt)
+ goto done;
+
+ for (i = 0; i < chunk->pg_cnt; i++)
+ dma_unmap_page(&pcidev->dev,
+ chunk->dmainfo.dmaaddrs[i],
+ PAGE_SIZE,
+ DMA_BIDIRECTIONAL);
+
+done:
+ kfree(chunk->dmainfo.dmaaddrs);
+ chunk->dmainfo.dmaaddrs = NULL;
+ vfree((void *)chunk->vaddr);
+ chunk->vaddr = 0;
+ chunk->type = 0;
+}
+
+/**
+ * irdma_pble_get_paged_mem -allocate paged memory for pbles
+ * @chunk: chunk to add for paged memory
+ * @pg_cnt: number of pages needed
+ */
+enum irdma_status_code irdma_pble_get_paged_mem(struct irdma_chunk *chunk, int pg_cnt)
+{
+ struct pci_dev *pcidev = (struct pci_dev *)chunk->dev->hw->dev_context;
+ struct page *page;
+ u8 *addr;
+ u32 size;
+ int i;
+
+ chunk->dmainfo.dmaaddrs = kzalloc(pg_cnt << 3, GFP_KERNEL);
+ if (!chunk->dmainfo.dmaaddrs)
+ return IRDMA_ERR_NO_MEMORY;
+
+ size = PAGE_SIZE * pg_cnt;
+ chunk->vaddr = (u64)vmalloc(size);
+ if (!chunk->vaddr) {
+ kfree(chunk->dmainfo.dmaaddrs);
+ chunk->dmainfo.dmaaddrs = NULL;
+ return IRDMA_ERR_NO_MEMORY;
+ }
+
+ chunk->size = size;
+ addr = (u8 *)chunk->vaddr;
+
+ for (i = 0; i < pg_cnt; i++) {
+ page = vmalloc_to_page((void *)addr);
+ if (!page)
+ break;
+
+ chunk->dmainfo.dmaaddrs[i] = dma_map_page(&pcidev->dev,
+ page,
+ 0,
+ PAGE_SIZE,
+ DMA_BIDIRECTIONAL);
+ if (dma_mapping_error(&pcidev->dev, chunk->dmainfo.dmaaddrs[i]))
+ break;
+
+ addr += PAGE_SIZE;
+ }
+
+ chunk->pg_cnt = i;
+ chunk->type = PBLE_SD_PAGED;
+ if (i == pg_cnt)
+ return 0;
+
+ irdma_pble_free_paged_mem(chunk);
+
+ return IRDMA_ERR_NO_MEMORY;
+}
+
+/**
+ * irdma_alloc_ws_node_id - Allocate a tx scheduler node ID
+ * @dev: device pointer
+ */
+u16 irdma_alloc_ws_node_id(struct irdma_sc_dev *dev)
+{
+ struct irdma_pci_f *rf = dev->back_dev;
+ u32 node_id;
+
+ if (irdma_alloc_rsrc(rf, rf->allocated_ws_nodes,
+ rf->max_ws_node_id,
+ &node_id, &rf->next_ws_node_id))
+ return IRDMA_WS_NODE_INVALID;
+
+ return (u16)node_id;
+}
+
+/**
+ * irdma_free_ws_node_id - Free a tx scheduler node ID
+ * @dev: device pointer
+ */
+void irdma_free_ws_node_id(struct irdma_sc_dev *dev, u16 node_id)
+{
+ struct irdma_pci_f *rf = dev->back_dev;
+
+ irdma_free_rsrc(rf, rf->allocated_ws_nodes, (u32)node_id);
+}
+
+/**
+ * irdma_modify_qp_to_err - Modify a QP to error
+ * @sc_qp: qp structure
+ */
+void irdma_modify_qp_to_err(struct irdma_sc_qp *sc_qp)
+{
+ struct irdma_qp *qp = sc_qp->back_qp;
+ struct ib_qp_attr attr;
+
+ attr.qp_state = IB_QPS_ERR;
+ irdma_modify_qp(&qp->ibqp, &attr, IB_QP_STATE, NULL);
+}
--
1.8.3.1
^ permalink raw reply related
* [RFC 02/19] net/ice: Create framework for VSI queue context
From: Shiraz Saleem @ 2019-02-12 21:43 UTC (permalink / raw)
To: dledford, jgg, davem
Cc: linux-rdma, netdev, mustafa.ismail, jeffrey.t.kirsher,
Anirudh Venkataramanan
In-Reply-To: <20190212214402.23284-1-shiraz.saleem@intel.com>
From: Anirudh Venkataramanan <anirudh.venkataramanan@intel.com>
This patch introduces a framework to store queue specific information
in VSI queue contexts. Currently VSI queue context (represented by
struct ice_q_ctx) only has q_handle as a member. In future patches,
this structure will be updated to hold queue specific information.
This will be submitted as a patch to the ice driver in the very near
future. It was necessary to make this patch a part of this RFC to
enable functional testing of the i40iw driver.
Signed-off-by: Anirudh Venkataramanan <anirudh.venkataramanan@intel.com>
Signed-off-by: Jeff Kirsher <jeffrey.t.kirsher@intel.com>
---
drivers/net/ethernet/intel/ice/ice_sched.c | 177 ++++++++++++++++++----------
drivers/net/ethernet/intel/ice/ice_switch.h | 9 ++
drivers/net/ethernet/intel/ice/ice_type.h | 1 +
3 files changed, 124 insertions(+), 63 deletions(-)
diff --git a/drivers/net/ethernet/intel/ice/ice_sched.c b/drivers/net/ethernet/intel/ice/ice_sched.c
index a168185..e95bed5 100644
--- a/drivers/net/ethernet/intel/ice/ice_sched.c
+++ b/drivers/net/ethernet/intel/ice/ice_sched.c
@@ -591,6 +591,94 @@ void ice_free_sched_node(struct ice_port_info *pi, struct ice_sched_node *node)
}
/**
+ * ice_alloc_lan_q_ctx - allocate LAN queue contexts for the given VSI and TC
+ * @hw: pointer to the HW struct
+ * @vsi_handle: VSI handle
+ * @tc: TC number
+ * @new_numqs: number of queues
+ */
+static enum ice_status
+ice_alloc_lan_q_ctx(struct ice_hw *hw, u16 vsi_handle, u8 tc, u16 new_numqs)
+{
+ struct ice_vsi_ctx *vsi_ctx;
+ struct ice_q_ctx *q_ctx;
+
+ vsi_ctx = ice_get_vsi_ctx(hw, vsi_handle);
+ if (!vsi_ctx)
+ return ICE_ERR_PARAM;
+ /* allocate LAN queue contexts */
+ if (!vsi_ctx->lan_q_ctx[tc]) {
+ vsi_ctx->lan_q_ctx[tc] = devm_kcalloc(ice_hw_to_dev(hw),
+ new_numqs,
+ sizeof(*q_ctx),
+ GFP_KERNEL);
+ if (!vsi_ctx->lan_q_ctx[tc])
+ return ICE_ERR_NO_MEMORY;
+ vsi_ctx->num_lan_q_entries[tc] = new_numqs;
+ return 0;
+ }
+ /* num queues are increased, update the queue contexts */
+ if (new_numqs > vsi_ctx->num_lan_q_entries[tc]) {
+ u16 prev_num = vsi_ctx->num_lan_q_entries[tc];
+
+ q_ctx = devm_kcalloc(ice_hw_to_dev(hw), new_numqs,
+ sizeof(*q_ctx), GFP_KERNEL);
+ if (!q_ctx)
+ return ICE_ERR_NO_MEMORY;
+ memcpy(q_ctx, vsi_ctx->lan_q_ctx[tc],
+ prev_num * sizeof(*q_ctx));
+ devm_kfree(ice_hw_to_dev(hw), vsi_ctx->lan_q_ctx[tc]);
+ vsi_ctx->lan_q_ctx[tc] = q_ctx;
+ vsi_ctx->num_lan_q_entries[tc] = new_numqs;
+ }
+ return 0;
+}
+
+/**
+ * ice_alloc_rdma_q_ctx - allocate RDMA queue contexts for the given VSI and TC
+ * @hw: pointer to the HW struct
+ * @vsi_handle: VSI handle
+ * @tc: TC number
+ * @new_numqs: number of queues
+ */
+static enum ice_status
+ice_alloc_rdma_q_ctx(struct ice_hw *hw, u16 vsi_handle, u8 tc, u16 new_numqs)
+{
+ struct ice_vsi_ctx *vsi_ctx;
+ struct ice_q_ctx *q_ctx;
+
+ vsi_ctx = ice_get_vsi_ctx(hw, vsi_handle);
+ if (!vsi_ctx)
+ return ICE_ERR_PARAM;
+ /* allocate RDMA queue contexts */
+ if (!vsi_ctx->rdma_q_ctx[tc]) {
+ vsi_ctx->rdma_q_ctx[tc] = devm_kcalloc(ice_hw_to_dev(hw),
+ new_numqs,
+ sizeof(*q_ctx),
+ GFP_KERNEL);
+ if (!vsi_ctx->rdma_q_ctx[tc])
+ return ICE_ERR_NO_MEMORY;
+ vsi_ctx->num_rdma_q_entries[tc] = new_numqs;
+ return 0;
+ }
+ /* num queues are increased, update the queue contexts */
+ if (new_numqs > vsi_ctx->num_rdma_q_entries[tc]) {
+ u16 prev_num = vsi_ctx->num_rdma_q_entries[tc];
+
+ q_ctx = devm_kcalloc(ice_hw_to_dev(hw), new_numqs,
+ sizeof(*q_ctx), GFP_KERNEL);
+ if (!q_ctx)
+ return ICE_ERR_NO_MEMORY;
+ memcpy(q_ctx, vsi_ctx->rdma_q_ctx[tc],
+ prev_num * sizeof(*q_ctx));
+ devm_kfree(ice_hw_to_dev(hw), vsi_ctx->rdma_q_ctx[tc]);
+ vsi_ctx->rdma_q_ctx[tc] = q_ctx;
+ vsi_ctx->num_rdma_q_entries[tc] = new_numqs;
+ }
+ return 0;
+}
+
+/**
* ice_sched_clear_tx_topo - clears the schduler tree nodes
* @pi: port information structure
*
@@ -1295,42 +1383,6 @@ struct ice_sched_node *
}
/**
- * ice_sched_rm_vsi_child_nodes - remove VSI child nodes from the tree
- * @pi: port information structure
- * @vsi_node: pointer to the VSI node
- * @num_nodes: pointer to the num nodes that needs to be removed per layer
- * @owner: node owner (lan or rdma)
- *
- * This function removes the VSI child nodes from the tree. It gets called for
- * lan and rdma separately.
- */
-static void
-ice_sched_rm_vsi_child_nodes(struct ice_port_info *pi,
- struct ice_sched_node *vsi_node, u16 *num_nodes,
- u8 owner)
-{
- struct ice_sched_node *node, *next;
- u8 i, qgl, vsil;
- u16 num;
-
- qgl = ice_sched_get_qgrp_layer(pi->hw);
- vsil = ice_sched_get_vsi_layer(pi->hw);
-
- for (i = qgl; i > vsil; i--) {
- num = num_nodes[i];
- node = ice_sched_get_first_node(pi->hw, vsi_node, i);
- while (node && num) {
- next = node->sibling;
- if (node->owner == owner && !node->num_children) {
- ice_free_sched_node(pi, node);
- num--;
- }
- node = next;
- }
- }
-}
-
-/**
* ice_sched_calc_vsi_support_nodes - calculate number of VSI support nodes
* @hw: pointer to the hw struct
* @tc_node: pointer to TC node
@@ -1465,7 +1517,6 @@ struct ice_sched_node *
ice_sched_update_vsi_child_nodes(struct ice_port_info *pi, u16 vsi_handle,
u8 tc, u16 new_numqs, u8 owner)
{
- u16 prev_num_nodes[ICE_AQC_TOPO_MAX_LEVEL_NUM] = { 0 };
u16 new_num_nodes[ICE_AQC_TOPO_MAX_LEVEL_NUM] = { 0 };
struct ice_sched_node *vsi_node;
struct ice_sched_node *tc_node;
@@ -1473,7 +1524,6 @@ struct ice_sched_node *
enum ice_status status = 0;
struct ice_hw *hw = pi->hw;
u16 prev_numqs;
- u8 i;
tc_node = ice_sched_get_tc_node(pi, tc);
if (!tc_node)
@@ -1490,38 +1540,39 @@ struct ice_sched_node *
if (owner == ICE_SCHED_NODE_OWNER_LAN)
prev_numqs = vsi_ctx->sched.max_lanq[tc];
else
- return ICE_ERR_PARAM;
-
- /* num queues are not changed */
- if (prev_numqs == new_numqs)
+ prev_numqs = vsi_ctx->sched.max_rdmaq[tc];
+ /* num queues are not changed or less than the previous number */
+ if (new_numqs <= prev_numqs)
return status;
-
- /* calculate number of nodes based on prev/new number of qs */
- if (prev_numqs)
- ice_sched_calc_vsi_child_nodes(hw, prev_numqs, prev_num_nodes);
-
- if (new_numqs)
- ice_sched_calc_vsi_child_nodes(hw, new_numqs, new_num_nodes);
-
- if (prev_numqs > new_numqs) {
- for (i = 0; i < ICE_AQC_TOPO_MAX_LEVEL_NUM; i++)
- new_num_nodes[i] = prev_num_nodes[i] - new_num_nodes[i];
-
- ice_sched_rm_vsi_child_nodes(pi, vsi_node, new_num_nodes,
- owner);
+ if (owner == ICE_SCHED_NODE_OWNER_LAN) {
+ status = ice_alloc_lan_q_ctx(hw, vsi_handle, tc, new_numqs);
+ if (status)
+ return status;
} else {
- for (i = 0; i < ICE_AQC_TOPO_MAX_LEVEL_NUM; i++)
- new_num_nodes[i] -= prev_num_nodes[i];
-
- status = ice_sched_add_vsi_child_nodes(pi, vsi_handle, tc_node,
- new_num_nodes, owner);
+ status = ice_alloc_rdma_q_ctx(hw, vsi_handle, tc, new_numqs);
if (status)
return status;
}
- vsi_ctx->sched.max_lanq[tc] = new_numqs;
+ if (new_numqs)
+ ice_sched_calc_vsi_child_nodes(hw, new_numqs, new_num_nodes);
+ /* Keep the max number of queue configuration all the time. Update the
+ * tree only if number of queues > previous number of queues. This may
+ * leave some extra nodes in the tree if number of queues < previous
+ * number but that wouldn't harm anything. Removing those extra nodes
+ * may complicate the code if those nodes are part of SRL or
+ * individually rate limited.
+ */
+ status = ice_sched_add_vsi_child_nodes(pi, vsi_handle, tc_node,
+ new_num_nodes, owner);
+ if (status)
+ return status;
+ if (owner == ICE_SCHED_NODE_OWNER_LAN)
+ vsi_ctx->sched.max_lanq[tc] = new_numqs;
+ else
+ vsi_ctx->sched.max_rdmaq[tc] = new_numqs;
- return status;
+ return 0;
}
/**
diff --git a/drivers/net/ethernet/intel/ice/ice_switch.h b/drivers/net/ethernet/intel/ice/ice_switch.h
index d5ef0bd..78040bb 100644
--- a/drivers/net/ethernet/intel/ice/ice_switch.h
+++ b/drivers/net/ethernet/intel/ice/ice_switch.h
@@ -10,6 +10,11 @@
#define ICE_DFLT_VSI_INVAL 0xff
#define ICE_VSI_INVAL_ID 0xffff
+/* VSI queue context structure */
+struct ice_q_ctx {
+ u16 q_handle;
+};
+
/* VSI context structure for add/get/update/free operations */
struct ice_vsi_ctx {
u16 vsi_num;
@@ -20,6 +25,10 @@ struct ice_vsi_ctx {
struct ice_sched_vsi_info sched;
u8 alloc_from_pool;
u8 vf_num;
+ u16 num_lan_q_entries[ICE_MAX_TRAFFIC_CLASS];
+ struct ice_q_ctx *lan_q_ctx[ICE_MAX_TRAFFIC_CLASS];
+ u16 num_rdma_q_entries[ICE_MAX_TRAFFIC_CLASS];
+ struct ice_q_ctx *rdma_q_ctx[ICE_MAX_TRAFFIC_CLASS];
};
enum ice_sw_fwd_act_type {
diff --git a/drivers/net/ethernet/intel/ice/ice_type.h b/drivers/net/ethernet/intel/ice/ice_type.h
index 0ea4281..6209edc 100644
--- a/drivers/net/ethernet/intel/ice/ice_type.h
+++ b/drivers/net/ethernet/intel/ice/ice_type.h
@@ -245,6 +245,7 @@ struct ice_sched_vsi_info {
struct ice_sched_node *ag_node[ICE_MAX_TRAFFIC_CLASS];
struct list_head list_entry;
u16 max_lanq[ICE_MAX_TRAFFIC_CLASS];
+ u16 max_rdmaq[ICE_MAX_TRAFFIC_CLASS];
u16 vsi_id;
};
--
1.8.3.1
^ permalink raw reply related
* [RFC 18/19] RDMA/irdma: Add Kconfig and Makefile
From: Shiraz Saleem @ 2019-02-12 21:44 UTC (permalink / raw)
To: dledford, jgg, davem
Cc: linux-rdma, netdev, mustafa.ismail, jeffrey.t.kirsher,
Shiraz Saleem
In-Reply-To: <20190212214402.23284-1-shiraz.saleem@intel.com>
Add Kconfig and Makefile to build irdma driver
Signed-off-by: Shiraz Saleem <shiraz.saleem@intel.com>
---
drivers/infiniband/Kconfig | 1 +
drivers/infiniband/hw/Makefile | 1 +
drivers/infiniband/hw/irdma/Kconfig | 11 +++++++++++
drivers/infiniband/hw/irdma/Makefile | 31 +++++++++++++++++++++++++++++++
4 files changed, 44 insertions(+)
create mode 100644 drivers/infiniband/hw/irdma/Kconfig
create mode 100644 drivers/infiniband/hw/irdma/Makefile
diff --git a/drivers/infiniband/Kconfig b/drivers/infiniband/Kconfig
index a1fb840d..a841a07 100644
--- a/drivers/infiniband/Kconfig
+++ b/drivers/infiniband/Kconfig
@@ -95,6 +95,7 @@ source "drivers/infiniband/hw/qib/Kconfig"
source "drivers/infiniband/hw/cxgb3/Kconfig"
source "drivers/infiniband/hw/cxgb4/Kconfig"
source "drivers/infiniband/hw/i40iw/Kconfig"
+source "drivers/infiniband/hw/irdma/Kconfig"
source "drivers/infiniband/hw/mlx4/Kconfig"
source "drivers/infiniband/hw/mlx5/Kconfig"
source "drivers/infiniband/hw/nes/Kconfig"
diff --git a/drivers/infiniband/hw/Makefile b/drivers/infiniband/hw/Makefile
index e4f31c1..bc79ef3 100644
--- a/drivers/infiniband/hw/Makefile
+++ b/drivers/infiniband/hw/Makefile
@@ -4,6 +4,7 @@ obj-$(CONFIG_INFINIBAND_QIB) += qib/
obj-$(CONFIG_INFINIBAND_CXGB3) += cxgb3/
obj-$(CONFIG_INFINIBAND_CXGB4) += cxgb4/
obj-$(CONFIG_INFINIBAND_I40IW) += i40iw/
+obj-$(CONFIG_INFINIBAND_IRDMA) += irdma/
obj-$(CONFIG_MLX4_INFINIBAND) += mlx4/
obj-$(CONFIG_MLX5_INFINIBAND) += mlx5/
obj-$(CONFIG_INFINIBAND_NES) += nes/
diff --git a/drivers/infiniband/hw/irdma/Kconfig b/drivers/infiniband/hw/irdma/Kconfig
new file mode 100644
index 0000000..652f5f9
--- /dev/null
+++ b/drivers/infiniband/hw/irdma/Kconfig
@@ -0,0 +1,11 @@
+config INFINIBAND_IRDMA
+ tristate "Intel(R) Ethernet Connection RDMA Driver"
+ depends on INET && (I40E || ICE)
+ depends on IPV6 || !IPV6
+ depends on PCI
+ select GENERIC_ALLOCATOR
+ ---help---
+ This is an Ethernet RDMA driver that supports E810 (iWARP/RoCE)
+ and X722 (iWARP) network devices.
+ To compile this driver as a module, choose M here. The module
+ will be called irdma.
diff --git a/drivers/infiniband/hw/irdma/Makefile b/drivers/infiniband/hw/irdma/Makefile
new file mode 100644
index 0000000..cfc8126
--- /dev/null
+++ b/drivers/infiniband/hw/irdma/Makefile
@@ -0,0 +1,31 @@
+# SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+# Copyright (c) 2019, Intel Corporation.
+
+#
+# Makefile for the Intel(R) Ethernet Connection RDMA Linux Driver
+#
+
+ccflags-y := -I $(srctree)/drivers/net/ethernet/intel/i40e
+ccflags-y += -I $(srctree)/drivers/net/ethernet/intel/ice
+
+obj-$(CONFIG_INFINIBAND_IRDMA) += irdma.o
+
+irdma-objs := main.o \
+ i40iw_if.o \
+ i40iw_hw.o \
+ irdma_if.o \
+ hw.o \
+ icrdma_hw.o \
+ cm.o \
+ ctrl.o \
+ hmc.o \
+ pble.o \
+ puda.o \
+ uk.o \
+ utils.o \
+ verbs.o \
+ uda.o \
+ ws.o \
+ trace.o \
+
+CFLAGS_trace.o = -I$(src)
--
1.8.3.1
^ permalink raw reply related
* [RFC 17/19] RDMA/irdma: Add ABI definitions
From: Shiraz Saleem @ 2019-02-12 21:44 UTC (permalink / raw)
To: dledford, jgg, davem
Cc: linux-rdma, netdev, mustafa.ismail, jeffrey.t.kirsher,
Shiraz Saleem
In-Reply-To: <20190212214402.23284-1-shiraz.saleem@intel.com>
From: Mustafa Ismail <mustafa.ismail@intel.com>
Add ABI definitions for irdma.
Signed-off-by: Mustafa Ismail <mustafa.ismail@intel.com>
Signed-off-by: Shiraz Saleem <shiraz.saleem@intel.com>
---
include/uapi/rdma/irdma-abi.h | 140 ++++++++++++++++++++++++++++++++++++++++++
1 file changed, 140 insertions(+)
create mode 100644 include/uapi/rdma/irdma-abi.h
diff --git a/include/uapi/rdma/irdma-abi.h b/include/uapi/rdma/irdma-abi.h
new file mode 100644
index 0000000..04ce5c2
--- /dev/null
+++ b/include/uapi/rdma/irdma-abi.h
@@ -0,0 +1,140 @@
+/*
+ * Copyright (c) 2006 - 2019 Intel Corporation. All rights reserved.
+ * Copyright (c) 2005 Topspin Communications. All rights reserved.
+ * Copyright (c) 2005 Cisco Systems. All rights reserved.
+ * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+#ifndef IRDMA_ABI_H
+#define IRDMA_ABI_H
+
+#include <linux/types.h>
+
+#define IRDMA_ABI_VER 6
+
+enum irdma_memreg_type {
+ IW_MEMREG_TYPE_MEM = 0,
+ IW_MEMREG_TYPE_QP = 1,
+ IW_MEMREG_TYPE_CQ = 2,
+ IW_MEMREG_TYPE_RSVD = 3,
+ IW_MEMREG_TYPE_MW = 4,
+};
+
+struct irdma_alloc_ucontext_req {
+ __u32 rsvd32;
+ __u8 userspace_ver;
+ __u8 rsvd8[3];
+};
+
+struct irdma_alloc_ucontext_resp {
+ __u8 kernel_ver;
+ __u8 rsvd[7];
+ struct irdma_hw_attrs hw_attrs;
+};
+
+struct i40iw_alloc_ucontext_resp {
+ __u32 max_pds; /* maximum pds allowed for this user process */
+ __u32 max_qps; /* maximum qps allowed for this user process */
+ __u32 wq_size; /* size of the WQs (sq+rq) allocated to the mmaped area */
+ __u8 kernel_ver;
+ __u8 reserved[3];
+};
+
+struct irdma_alloc_pd_resp {
+ __u32 pd_id;
+ __u8 rsvd[4];
+};
+
+struct irdma_create_cq_req {
+ __aligned_u64 user_cq_buf;
+ __aligned_u64 user_shadow_area;
+};
+
+struct irdma_create_qp_req {
+ __u64 user_wqe_bufs;
+ __u64 user_compl_ctx;
+};
+
+struct i40iw_create_qp_req {
+ __aligned_u64 user_wqe_buffers;
+ __aligned_u64 user_compl_ctx;
+
+ /* UDA QP PHB */
+ __aligned_u64 user_sq_phb; /* place for VA of the sq phb buff */
+ __aligned_u64 user_rq_phb; /* place for VA of the rq phb buff */
+};
+
+struct irdma_mem_reg_req {
+ __u16 reg_type; /* Memory, QP or CQ */
+ __u16 cq_pages;
+ __u16 rq_pages;
+ __u16 sq_pages;
+};
+
+struct irdma_create_cq_resp {
+ __u32 cq_id;
+ __u32 cq_size;
+};
+
+struct i40iw_create_cq_resp {
+ __u32 cq_id;
+ __u32 cq_size;
+ __u32 mmap_db_index;
+ __u32 reserved;
+};
+
+struct irdma_create_qp_resp {
+ __u32 qp_id;
+ __u32 actual_sq_size;
+ __u32 actual_rq_size;
+ __u32 irdma_drv_opt;
+ __u16 push_idx;
+ __u8 lsmm;
+ __u8 rsvd;
+ __u32 qp_caps;
+};
+
+struct i40iw_create_qp_resp {
+ __u32 qp_id;
+ __u32 actual_sq_size;
+ __u32 actual_rq_size;
+ __u32 i40iw_drv_opt;
+ __u16 push_idx;
+ __u8 lsmm;
+ __u8 rsvd2;
+};
+
+struct irdma_create_ah_resp {
+ __u32 ah_id;
+ __u32 rsvd[4];
+};
+#endif /* IRDMA_ABI_H */
--
1.8.3.1
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox