* [PATCH 05/25 v2] mlx4_core: add slave resource allocation
From: Yevgeny Petrilin @ 2009-11-06 3:08 UTC (permalink / raw)
To: rdreier-FYB4Gu1CFyUAvxtiuMwx3w
Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA, netdev-u79uwXL29TY76Z2rM5mHXA,
liranl-VPRAkNaXOzVS1MOuV/RT9w, tziporet-VPRAkNaXOzVS1MOuV/RT9w,
yevgenyp-VPRAkNaXOzVS1MOuV/RT9w
From: Liran Liss <liranl-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
All QPs/CQs/SRQs/MPTs/MTTs are allocated from shared pools, which are owned by
the master. In addition, all backing ICM memory for these objects is managed by
the master.
To allow slaves to allocate resources, ICM allocation is separated from the rest
of the object state, which is held at the slave.
Slaves can then reserve resource ranges and allocate ICM over the comm channel.
Signed-off-by: Liran Liss <liranl-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
Signed-off-by: Yevgeny Petrilin <yevgenyp-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
---
drivers/net/mlx4/cmd.c | 110 +++++++++++++++++++++++++++++++++
drivers/net/mlx4/cq.c | 91 +++++++++++++++++++++-------
drivers/net/mlx4/mlx4.h | 27 ++++++++
drivers/net/mlx4/mr.c | 125 ++++++++++++++++++++++++++++++++++----
drivers/net/mlx4/qp.c | 151 +++++++++++++++++++++++++++++++++-------------
drivers/net/mlx4/srq.c | 88 ++++++++++++++++++++-------
include/linux/mlx4/cmd.h | 2 +
7 files changed, 496 insertions(+), 98 deletions(-)
diff --git a/drivers/net/mlx4/cmd.c b/drivers/net/mlx4/cmd.c
index 717dd50..fe23415 100644
--- a/drivers/net/mlx4/cmd.c
+++ b/drivers/net/mlx4/cmd.c
@@ -418,6 +418,100 @@ static int mlx4_ACCESS_MEM(struct mlx4_dev *dev, u64 master_addr,
MLX4_CMD_TIME_CLASS_A);
}
+static int mlx4_RESOURCE_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr,
+ struct mlx4_cmd_mailbox *inbox,
+ struct mlx4_cmd_mailbox *outbox)
+{
+ u32 param1 = *((u32 *) &vhcr->in_param);
+ u32 param2 = *(((u32 *) &vhcr->in_param) + 1);
+ int ret;
+
+#if 0
+ char *res[] = {"QP", "CQ", "SRQ", "MPT", "MTT"};
+ mlx4_warn(dev, "resource wrapper - %s (mode: %s) type:%s param1:%d param2:%d\n",
+ vhcr->op == MLX4_CMD_ALLOC_RES ? "allocate" : "free",
+ vhcr->op_modifier == ICM_RESERVE ? "reserve" :
+ (vhcr->op_modifier == ICM_ALLOC ? "alloc" : "reserve+alloc"),
+ res[vhcr->in_modifier], param1, param2);
+#endif
+
+ vhcr->errno = 0;
+ switch (vhcr->in_modifier) {
+ case RES_QP:
+ switch (vhcr->op_modifier) {
+ case ICM_RESERVE:
+ if (vhcr->op == MLX4_CMD_ALLOC_RES) {
+ vhcr->errno = mlx4_qp_reserve_range(dev, param1, param2, &ret);
+ if (!vhcr->errno)
+ vhcr->out_param = ret;
+ } else {
+ mlx4_qp_release_range(dev, param1, param2);
+ }
+ break;
+ case ICM_ALLOC:
+ if (vhcr->op == MLX4_CMD_ALLOC_RES)
+ vhcr->errno = mlx4_qp_alloc_icm(dev, param1);
+ else
+ mlx4_qp_free_icm(dev, param1);
+ break;
+ default:
+ vhcr->errno = -EINVAL;
+ }
+ break;
+ case RES_CQ:
+ if (vhcr->op == MLX4_CMD_ALLOC_RES) {
+ vhcr->errno = mlx4_cq_alloc_icm(dev, &ret);
+ if (!vhcr->errno)
+ vhcr->out_param = ret;
+ } else
+ mlx4_cq_free_icm(dev, param1);
+ break;
+ case RES_SRQ:
+ if (vhcr->op == MLX4_CMD_ALLOC_RES) {
+ vhcr->errno = mlx4_srq_alloc_icm(dev, &ret);
+ if (!vhcr->errno)
+ vhcr->out_param = ret;
+ } else
+ mlx4_srq_free_icm(dev, param1);
+ break;
+ case RES_MPT:
+ switch (vhcr->op_modifier) {
+ case ICM_RESERVE:
+ if (vhcr->op == MLX4_CMD_ALLOC_RES) {
+ ret = mlx4_mr_reserve(dev);
+ if (ret == -1)
+ vhcr->errno = -ENOMEM;
+ else
+ vhcr->out_param = ret;
+ } else
+ mlx4_mr_release(dev, param1);
+ break;
+ case ICM_ALLOC:
+ if (vhcr->op == MLX4_CMD_ALLOC_RES)
+ vhcr->errno = mlx4_mr_alloc_icm(dev, param1);
+ else
+ mlx4_mr_free_icm(dev, param1);
+ break;
+ default:
+ vhcr->errno = -EINVAL;
+ }
+ break;
+ case RES_MTT:
+ if (vhcr->op == MLX4_CMD_ALLOC_RES) {
+ ret = mlx4_alloc_mtt_range(dev, param1 /* order */);
+ if (ret == -1)
+ vhcr->errno = -ENOMEM;
+ else
+ vhcr->out_param = ret;
+ } else
+ mlx4_free_mtt_range(dev, param1 /* first */, param2 /* order */);
+ break;
+ default:
+ vhcr->errno = -EINVAL;
+ }
+ return 0;
+}
+
static struct mlx4_cmd_info {
u8 opcode;
bool has_inbox;
@@ -462,6 +556,22 @@ static struct mlx4_cmd_info {
.verify = NULL,
.wrapper = NULL
},
+ {
+ .opcode = MLX4_CMD_ALLOC_RES,
+ .has_inbox = false,
+ .has_outbox = false,
+ .out_is_imm = true,
+ .verify = NULL,
+ .wrapper = mlx4_RESOURCE_wrapper
+ },
+ {
+ .opcode = MLX4_CMD_FREE_RES,
+ .has_inbox = false,
+ .has_outbox = false,
+ .out_is_imm = false,
+ .verify = NULL,
+ .wrapper = mlx4_RESOURCE_wrapper
+ },
{
.opcode = MLX4_CMD_SW2HW_MPT,
diff --git a/drivers/net/mlx4/cq.c b/drivers/net/mlx4/cq.c
index ccfe276..3fb9f7f 100644
--- a/drivers/net/mlx4/cq.c
+++ b/drivers/net/mlx4/cq.c
@@ -186,6 +186,70 @@ int mlx4_cq_resize(struct mlx4_dev *dev, struct mlx4_cq *cq,
}
EXPORT_SYMBOL_GPL(mlx4_cq_resize);
+int mlx4_cq_alloc_icm(struct mlx4_dev *dev, int *cqn)
+{
+ struct mlx4_priv *priv = mlx4_priv(dev);
+ struct mlx4_cq_table *cq_table = &priv->cq_table;
+ u64 out_param;
+ int err;
+
+ if (mlx4_is_slave(dev)) {
+ err = mlx4_cmd_imm(dev, 0, &out_param, RES_CQ,
+ ICM_RESERVE_AND_ALLOC,
+ MLX4_CMD_ALLOC_RES,
+ MLX4_CMD_TIME_CLASS_A);
+ if (err) {
+ *cqn = -1;
+ return err;
+ } else {
+ *cqn = out_param;
+ return 0;
+ }
+ }
+
+ *cqn = mlx4_bitmap_alloc(&cq_table->bitmap);
+ if (*cqn == -1)
+ return -ENOMEM;
+
+ err = mlx4_table_get(dev, &cq_table->table, *cqn);
+ if (err)
+ goto err_out;
+
+ err = mlx4_table_get(dev, &cq_table->cmpt_table, *cqn);
+ if (err)
+ goto err_put;
+ return 0;
+
+err_put:
+ mlx4_table_put(dev, &cq_table->table, *cqn);
+
+err_out:
+ mlx4_bitmap_free(&cq_table->bitmap, *cqn);
+ return err;
+}
+
+void mlx4_cq_free_icm(struct mlx4_dev *dev, int cqn)
+{
+ struct mlx4_priv *priv = mlx4_priv(dev);
+ struct mlx4_cq_table *cq_table = &priv->cq_table;
+ u64 in_param;
+ int err;
+
+ if (mlx4_is_slave(dev)) {
+ *((u32 *) &in_param) = cqn;
+ *(((u32 *) &in_param) + 1) = 0;
+ err = mlx4_cmd(dev, in_param, RES_CQ, ICM_RESERVE_AND_ALLOC,
+ MLX4_CMD_FREE_RES,
+ MLX4_CMD_TIME_CLASS_A);
+ if (err)
+ mlx4_warn(dev, "Failed freeing cq:%d\n", cqn);
+ } else {
+ mlx4_table_put(dev, &cq_table->cmpt_table, cqn);
+ mlx4_table_put(dev, &cq_table->table, cqn);
+ mlx4_bitmap_free(&cq_table->bitmap, cqn);
+ }
+}
+
int mlx4_cq_alloc(struct mlx4_dev *dev, int nent, struct mlx4_mtt *mtt,
struct mlx4_uar *uar, u64 db_rec, struct mlx4_cq *cq,
unsigned vector, int collapsed)
@@ -202,23 +266,15 @@ int mlx4_cq_alloc(struct mlx4_dev *dev, int nent, struct mlx4_mtt *mtt,
cq->vector = vector;
- cq->cqn = mlx4_bitmap_alloc(&cq_table->bitmap);
- if (cq->cqn == -1)
- return -ENOMEM;
-
- err = mlx4_table_get(dev, &cq_table->table, cq->cqn);
+ err = mlx4_cq_alloc_icm(dev, &cq->cqn);
if (err)
- goto err_out;
-
- err = mlx4_table_get(dev, &cq_table->cmpt_table, cq->cqn);
- if (err)
- goto err_put;
+ return err;
spin_lock_irq(&cq_table->lock);
err = radix_tree_insert(&cq_table->tree, cq->cqn, cq);
spin_unlock_irq(&cq_table->lock);
if (err)
- goto err_cmpt_put;
+ goto err_icm;
mailbox = mlx4_alloc_cmd_mailbox(dev);
if (IS_ERR(mailbox)) {
@@ -257,14 +313,8 @@ err_radix:
radix_tree_delete(&cq_table->tree, cq->cqn);
spin_unlock_irq(&cq_table->lock);
-err_cmpt_put:
- mlx4_table_put(dev, &cq_table->cmpt_table, cq->cqn);
-
-err_put:
- mlx4_table_put(dev, &cq_table->table, cq->cqn);
-
-err_out:
- mlx4_bitmap_free(&cq_table->bitmap, cq->cqn);
+err_icm:
+ mlx4_cq_free_icm(dev, cq->cqn);
return err;
}
@@ -290,8 +340,7 @@ void mlx4_cq_free(struct mlx4_dev *dev, struct mlx4_cq *cq)
complete(&cq->free);
wait_for_completion(&cq->free);
- mlx4_table_put(dev, &cq_table->table, cq->cqn);
- mlx4_bitmap_free(&cq_table->bitmap, cq->cqn);
+ mlx4_cq_free_icm(dev, cq->cqn);
}
EXPORT_SYMBOL_GPL(mlx4_cq_free);
diff --git a/drivers/net/mlx4/mlx4.h b/drivers/net/mlx4/mlx4.h
index 7516730..ace0bce 100644
--- a/drivers/net/mlx4/mlx4.h
+++ b/drivers/net/mlx4/mlx4.h
@@ -94,6 +94,21 @@ enum {
MLX4_COMM_CMD_VHCR_POST
};
+enum mlx4_resource {
+ RES_QP,
+ RES_CQ,
+ RES_SRQ,
+ RES_MPT,
+ RES_MTT
+};
+
+enum mlx4_alloc_mode {
+ ICM_RESERVE_AND_ALLOC,
+ ICM_RESERVE,
+ ICM_ALLOC,
+ ICM_MAC_VLAN,
+};
+
enum {
MLX4_MFUNC_MAX_EQES = 8,
MLX4_MFUNC_EQE_MASK = (MLX4_MFUNC_MAX_EQES - 1)
@@ -422,6 +437,18 @@ void mlx4_cleanup_qp_table(struct mlx4_dev *dev);
void mlx4_cleanup_srq_table(struct mlx4_dev *dev);
void mlx4_cleanup_mcg_table(struct mlx4_dev *dev);
+int mlx4_qp_alloc_icm(struct mlx4_dev *dev, int qpn);
+void mlx4_qp_free_icm(struct mlx4_dev *dev, int qpn);
+int mlx4_cq_alloc_icm(struct mlx4_dev *dev, int *cqn);
+void mlx4_cq_free_icm(struct mlx4_dev *dev, int cqn);
+int mlx4_srq_alloc_icm(struct mlx4_dev *dev, int *srqn);
+void mlx4_srq_free_icm(struct mlx4_dev *dev, int srqn);
+int mlx4_mr_reserve(struct mlx4_dev *dev);
+void mlx4_mr_release(struct mlx4_dev *dev, u32 index);
+int mlx4_mr_alloc_icm(struct mlx4_dev *dev, u32 index);
+void mlx4_mr_free_icm(struct mlx4_dev *dev, u32 index);
+u32 mlx4_alloc_mtt_range(struct mlx4_dev *dev, int order);
+void mlx4_free_mtt_range(struct mlx4_dev *dev, u32 first_seg, int order);
int mlx4_WRITE_MTT_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr,
struct mlx4_cmd_mailbox *inbox,
struct mlx4_cmd_mailbox *outbox);
diff --git a/drivers/net/mlx4/mr.c b/drivers/net/mlx4/mr.c
index 11a3d26..aa1eba6 100644
--- a/drivers/net/mlx4/mr.c
+++ b/drivers/net/mlx4/mr.c
@@ -178,10 +178,26 @@ static void mlx4_buddy_cleanup(struct mlx4_buddy *buddy)
kfree(buddy->num_free);
}
-static u32 mlx4_alloc_mtt_range(struct mlx4_dev *dev, int order)
+u32 mlx4_alloc_mtt_range(struct mlx4_dev *dev, int order)
{
struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table;
+ u64 in_param;
+ u64 out_param;
u32 seg;
+ int err;
+
+ if (mlx4_is_slave(dev)) {
+ *((u32 *) &in_param) = order;
+ *(((u32 *) &in_param) + 1) = 0;
+ err = mlx4_cmd_imm(dev, in_param, &out_param, RES_MTT,
+ ICM_RESERVE_AND_ALLOC,
+ MLX4_CMD_ALLOC_RES,
+ MLX4_CMD_TIME_CLASS_A);
+ if (err)
+ return -1;
+ else
+ return out_param;
+ }
seg = mlx4_buddy_alloc(&mr_table->mtt_buddy, order);
if (seg == -1)
@@ -219,16 +235,33 @@ int mlx4_mtt_init(struct mlx4_dev *dev, int npages, int page_shift,
}
EXPORT_SYMBOL_GPL(mlx4_mtt_init);
-void mlx4_mtt_cleanup(struct mlx4_dev *dev, struct mlx4_mtt *mtt)
+void mlx4_free_mtt_range(struct mlx4_dev *dev, u32 first_seg, int order)
{
struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table;
+ u64 in_param;
+ int err;
+
+ if (mlx4_is_slave(dev)) {
+ *((u32 *) &in_param) = first_seg;
+ *(((u32 *) &in_param) + 1) = order;
+ err = mlx4_cmd(dev, in_param, RES_MTT, ICM_RESERVE_AND_ALLOC,
+ MLX4_CMD_FREE_RES,
+ MLX4_CMD_TIME_CLASS_A);
+ if (err)
+ mlx4_warn(dev, "Failed to free mtt range at:%d order:%d\n", first_seg, order);
+ } else {
+ mlx4_buddy_free(&mr_table->mtt_buddy, first_seg, order);
+ mlx4_table_put_range(dev, &mr_table->mtt_table, first_seg,
+ first_seg + (1 << order) - 1);
+ }
+}
+void mlx4_mtt_cleanup(struct mlx4_dev *dev, struct mlx4_mtt *mtt)
+{
if (mtt->order < 0)
return;
- mlx4_buddy_free(&mr_table->mtt_buddy, mtt->first_seg, mtt->order);
- mlx4_table_put_range(dev, &mr_table->mtt_table, mtt->first_seg,
- mtt->first_seg + (1 << mtt->order) - 1);
+ mlx4_free_mtt_range(dev, mtt->first_seg, mtt->order);
}
EXPORT_SYMBOL_GPL(mlx4_mtt_cleanup);
@@ -291,14 +324,81 @@ static int mlx4_WRITE_MTT(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox
MLX4_CMD_TIME_CLASS_A);
}
+int mlx4_mr_reserve(struct mlx4_dev *dev)
+{
+ struct mlx4_priv *priv = mlx4_priv(dev);
+ u64 out_param;
+ int err;
+
+ if (mlx4_is_slave(dev)) {
+ err = mlx4_cmd_imm(dev, 0, &out_param, RES_MPT, ICM_RESERVE,
+ MLX4_CMD_ALLOC_RES,
+ MLX4_CMD_TIME_CLASS_A);
+ if (err)
+ return -1;
+ return out_param;
+ }
+ return mlx4_bitmap_alloc(&priv->mr_table.mpt_bitmap);
+}
+
+void mlx4_mr_release(struct mlx4_dev *dev, u32 index)
+{
+ struct mlx4_priv *priv = mlx4_priv(dev);
+ u64 in_param;
+ int err;
+
+ if (mlx4_is_slave(dev)) {
+ *((u32 *) &in_param) = index;
+ *(((u32 *) &in_param) + 1) = 0;
+ err = mlx4_cmd(dev, in_param, RES_MPT, ICM_RESERVE,
+ MLX4_CMD_FREE_RES,
+ MLX4_CMD_TIME_CLASS_A);
+ if (err)
+ mlx4_warn(dev, "Failed to release mr index:%d\n", index);
+ } else
+ mlx4_bitmap_free(&priv->mr_table.mpt_bitmap, index);
+}
+
+int mlx4_mr_alloc_icm(struct mlx4_dev *dev, u32 index)
+{
+ struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table;
+ u64 param;
+
+ if (mlx4_is_slave(dev)) {
+ *((u32 *) ¶m) = index;
+ *(((u32 *) ¶m) + 1) = 0;
+ return mlx4_cmd_imm(dev, param, ¶m, RES_MPT, ICM_ALLOC,
+ MLX4_CMD_ALLOC_RES,
+ MLX4_CMD_TIME_CLASS_A);
+ } else
+ return mlx4_table_get(dev, &mr_table->dmpt_table, index);
+}
+
+void mlx4_mr_free_icm(struct mlx4_dev *dev, u32 index)
+{
+ struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table;
+ u64 in_param;
+ int err;
+
+ if (mlx4_is_slave(dev)) {
+ *((u32 *) &in_param) = index;
+ *(((u32 *) &in_param) + 1) = 0;
+ err = mlx4_cmd(dev, in_param, RES_MPT, ICM_ALLOC,
+ MLX4_CMD_FREE_RES,
+ MLX4_CMD_TIME_CLASS_A);
+ if (err)
+ mlx4_warn(dev, "Failed to free icm of mr index:%d\n", index);
+ } else
+ mlx4_table_put(dev, &mr_table->dmpt_table, index);
+}
+
int mlx4_mr_alloc(struct mlx4_dev *dev, u32 pd, u64 iova, u64 size, u32 access,
int npages, int page_shift, struct mlx4_mr *mr)
{
- struct mlx4_priv *priv = mlx4_priv(dev);
u32 index;
int err;
- index = mlx4_bitmap_alloc(&priv->mr_table.mpt_bitmap);
+ index = mlx4_mr_reserve(dev);
if (index == -1)
return -ENOMEM;
@@ -311,7 +411,7 @@ int mlx4_mr_alloc(struct mlx4_dev *dev, u32 pd, u64 iova, u64 size, u32 access,
err = mlx4_mtt_init(dev, npages, page_shift, &mr->mtt);
if (err)
- mlx4_bitmap_free(&priv->mr_table.mpt_bitmap, index);
+ mlx4_mr_release(dev, index);
return err;
}
@@ -319,7 +419,6 @@ EXPORT_SYMBOL_GPL(mlx4_mr_alloc);
void mlx4_mr_free(struct mlx4_dev *dev, struct mlx4_mr *mr)
{
- struct mlx4_priv *priv = mlx4_priv(dev);
int err;
if (mr->enabled) {
@@ -331,18 +430,18 @@ void mlx4_mr_free(struct mlx4_dev *dev, struct mlx4_mr *mr)
}
mlx4_mtt_cleanup(dev, &mr->mtt);
- mlx4_bitmap_free(&priv->mr_table.mpt_bitmap, key_to_hw_index(mr->key));
+ mlx4_mr_release(dev, key_to_hw_index(mr->key));
+ mlx4_mr_free_icm(dev, key_to_hw_index(mr->key));
}
EXPORT_SYMBOL_GPL(mlx4_mr_free);
int mlx4_mr_enable(struct mlx4_dev *dev, struct mlx4_mr *mr)
{
- struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table;
struct mlx4_cmd_mailbox *mailbox;
struct mlx4_mpt_entry *mpt_entry;
int err;
- err = mlx4_table_get(dev, &mr_table->dmpt_table, key_to_hw_index(mr->key));
+ err = mlx4_mr_alloc_icm(dev, key_to_hw_index(mr->key));
if (err)
return err;
@@ -400,7 +499,7 @@ err_cmd:
mlx4_free_cmd_mailbox(dev, mailbox);
err_table:
- mlx4_table_put(dev, &mr_table->dmpt_table, key_to_hw_index(mr->key));
+ mlx4_mr_free_icm(dev, key_to_hw_index(mr->key));
return err;
}
EXPORT_SYMBOL_GPL(mlx4_mr_enable);
diff --git a/drivers/net/mlx4/qp.c b/drivers/net/mlx4/qp.c
index 42ab9fc..065c7fc 100644
--- a/drivers/net/mlx4/qp.c
+++ b/drivers/net/mlx4/qp.c
@@ -149,13 +149,24 @@ int mlx4_qp_reserve_range(struct mlx4_dev *dev, int cnt, int align, int *base)
{
struct mlx4_priv *priv = mlx4_priv(dev);
struct mlx4_qp_table *qp_table = &priv->qp_table;
- int qpn;
-
- qpn = mlx4_bitmap_alloc_range(&qp_table->bitmap, cnt, align);
- if (qpn == -1)
- return -ENOMEM;
+ u64 in_param;
+ u64 out_param;
+ int err;
- *base = qpn;
+ if (mlx4_is_slave(dev)) {
+ *((u32 *) &in_param) = cnt;
+ *(((u32 *) &in_param) + 1) = align;
+ err = mlx4_cmd_imm(dev, in_param, &out_param, RES_QP, ICM_RESERVE,
+ MLX4_CMD_ALLOC_RES,
+ MLX4_CMD_TIME_CLASS_A);
+ if (err)
+ return err;
+ *base = out_param;
+ } else {
+ *base = mlx4_bitmap_alloc_range(&qp_table->bitmap, cnt, align);
+ if (*base == -1)
+ return -ENOMEM;
+ }
return 0;
}
EXPORT_SYMBOL_GPL(mlx4_qp_reserve_range);
@@ -164,73 +175,133 @@ void mlx4_qp_release_range(struct mlx4_dev *dev, int base_qpn, int cnt)
{
struct mlx4_priv *priv = mlx4_priv(dev);
struct mlx4_qp_table *qp_table = &priv->qp_table;
- if (base_qpn < dev->caps.sqp_start + 8)
- return;
+ u64 in_param;
+ int err;
- mlx4_bitmap_free_range(&qp_table->bitmap, base_qpn, cnt);
+ if (mlx4_is_slave(dev)) {
+ *((u32 *) &in_param) = base_qpn;
+ *(((u32 *) &in_param) + 1) = cnt;
+ err = mlx4_cmd(dev, in_param, RES_QP, ICM_RESERVE,
+ MLX4_CMD_FREE_RES,
+ MLX4_CMD_TIME_CLASS_A);
+ if (err) {
+ mlx4_warn(dev, "Failed to release qp range base:%d cnt:%d\n",
+ base_qpn, cnt);
+ }
+ } else {
+ if (base_qpn < dev->caps.sqp_start + 8)
+ return;
+ mlx4_bitmap_free_range(&qp_table->bitmap, base_qpn, cnt);
+ }
}
EXPORT_SYMBOL_GPL(mlx4_qp_release_range);
-int mlx4_qp_alloc(struct mlx4_dev *dev, int qpn, struct mlx4_qp *qp)
+int mlx4_qp_alloc_icm(struct mlx4_dev *dev, int qpn)
{
struct mlx4_priv *priv = mlx4_priv(dev);
struct mlx4_qp_table *qp_table = &priv->qp_table;
+ u64 param;
int err;
- if (!qpn)
- return -EINVAL;
-
- qp->qpn = qpn;
-
- err = mlx4_table_get(dev, &qp_table->qp_table, qp->qpn);
+ if (mlx4_is_slave(dev)) {
+ *((u32 *) ¶m) = qpn;
+ *(((u32 *) ¶m) + 1) = 0;
+ return mlx4_cmd_imm(dev, param, ¶m, RES_QP, ICM_ALLOC,
+ MLX4_CMD_ALLOC_RES,
+ MLX4_CMD_TIME_CLASS_A);
+ }
+ err = mlx4_table_get(dev, &qp_table->qp_table, qpn);
if (err)
goto err_out;
- err = mlx4_table_get(dev, &qp_table->auxc_table, qp->qpn);
+ err = mlx4_table_get(dev, &qp_table->auxc_table, qpn);
if (err)
goto err_put_qp;
- err = mlx4_table_get(dev, &qp_table->altc_table, qp->qpn);
+ err = mlx4_table_get(dev, &qp_table->altc_table, qpn);
if (err)
goto err_put_auxc;
- err = mlx4_table_get(dev, &qp_table->rdmarc_table, qp->qpn);
+ err = mlx4_table_get(dev, &qp_table->rdmarc_table, qpn);
if (err)
goto err_put_altc;
- err = mlx4_table_get(dev, &qp_table->cmpt_table, qp->qpn);
+ err = mlx4_table_get(dev, &qp_table->cmpt_table, qpn);
if (err)
goto err_put_rdmarc;
- spin_lock_irq(&qp_table->lock);
- err = radix_tree_insert(&dev->qp_table_tree, qp->qpn & (dev->caps.num_qps - 1), qp);
- spin_unlock_irq(&qp_table->lock);
- if (err)
- goto err_put_cmpt;
-
- atomic_set(&qp->refcount, 1);
- init_completion(&qp->free);
-
return 0;
-err_put_cmpt:
- mlx4_table_put(dev, &qp_table->cmpt_table, qp->qpn);
-
err_put_rdmarc:
- mlx4_table_put(dev, &qp_table->rdmarc_table, qp->qpn);
+ mlx4_table_put(dev, &qp_table->rdmarc_table, qpn);
err_put_altc:
- mlx4_table_put(dev, &qp_table->altc_table, qp->qpn);
+ mlx4_table_put(dev, &qp_table->altc_table, qpn);
err_put_auxc:
- mlx4_table_put(dev, &qp_table->auxc_table, qp->qpn);
+ mlx4_table_put(dev, &qp_table->auxc_table, qpn);
err_put_qp:
- mlx4_table_put(dev, &qp_table->qp_table, qp->qpn);
+ mlx4_table_put(dev, &qp_table->qp_table, qpn);
err_out:
return err;
}
+
+void mlx4_qp_free_icm(struct mlx4_dev *dev, int qpn)
+{
+ struct mlx4_priv *priv = mlx4_priv(dev);
+ struct mlx4_qp_table *qp_table = &priv->qp_table;
+ u64 in_param;
+ int err;
+
+ if (mlx4_is_slave(dev)) {
+ *((u32 *) &in_param) = qpn;
+ *(((u32 *) &in_param) + 1) = 0;
+ err = mlx4_cmd(dev, in_param, RES_QP, ICM_ALLOC,
+ MLX4_CMD_FREE_RES,
+ MLX4_CMD_TIME_CLASS_A);
+ if (err)
+ mlx4_warn(dev, "Failed to free icm of qp:%d\n", qpn);
+ } else {
+ mlx4_table_put(dev, &qp_table->cmpt_table, qpn);
+ mlx4_table_put(dev, &qp_table->rdmarc_table, qpn);
+ mlx4_table_put(dev, &qp_table->altc_table, qpn);
+ mlx4_table_put(dev, &qp_table->auxc_table, qpn);
+ mlx4_table_put(dev, &qp_table->qp_table, qpn);
+ }
+}
+
+int mlx4_qp_alloc(struct mlx4_dev *dev, int qpn, struct mlx4_qp *qp)
+{
+ struct mlx4_priv *priv = mlx4_priv(dev);
+ struct mlx4_qp_table *qp_table = &priv->qp_table;
+ int err;
+
+ if (!qpn)
+ return -EINVAL;
+
+ qp->qpn = qpn;
+
+ err = mlx4_qp_alloc_icm(dev, qpn);
+ if (err)
+ return err;
+
+ spin_lock_irq(&qp_table->lock);
+ err = radix_tree_insert(&dev->qp_table_tree, qp->qpn & (dev->caps.num_qps - 1), qp);
+ spin_unlock_irq(&qp_table->lock);
+ if (err)
+ goto err_icm;
+
+ atomic_set(&qp->refcount, 1);
+ init_completion(&qp->free);
+
+ return 0;
+
+err_icm:
+ mlx4_qp_free_icm(dev, qpn);
+ return err;
+}
EXPORT_SYMBOL_GPL(mlx4_qp_alloc);
void mlx4_qp_remove(struct mlx4_dev *dev, struct mlx4_qp *qp)
@@ -246,17 +317,11 @@ EXPORT_SYMBOL_GPL(mlx4_qp_remove);
void mlx4_qp_free(struct mlx4_dev *dev, struct mlx4_qp *qp)
{
- struct mlx4_qp_table *qp_table = &mlx4_priv(dev)->qp_table;
-
if (atomic_dec_and_test(&qp->refcount))
complete(&qp->free);
wait_for_completion(&qp->free);
- mlx4_table_put(dev, &qp_table->cmpt_table, qp->qpn);
- mlx4_table_put(dev, &qp_table->rdmarc_table, qp->qpn);
- mlx4_table_put(dev, &qp_table->altc_table, qp->qpn);
- mlx4_table_put(dev, &qp_table->auxc_table, qp->qpn);
- mlx4_table_put(dev, &qp_table->qp_table, qp->qpn);
+ mlx4_qp_free_icm(dev, qp->qpn);
}
EXPORT_SYMBOL_GPL(mlx4_qp_free);
diff --git a/drivers/net/mlx4/srq.c b/drivers/net/mlx4/srq.c
index 1377d0d..ed11f18 100644
--- a/drivers/net/mlx4/srq.c
+++ b/drivers/net/mlx4/srq.c
@@ -108,32 +108,86 @@ static int mlx4_QUERY_SRQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox
MLX4_CMD_TIME_CLASS_A);
}
-int mlx4_srq_alloc(struct mlx4_dev *dev, u32 pdn, struct mlx4_mtt *mtt,
- u64 db_rec, struct mlx4_srq *srq)
+int mlx4_srq_alloc_icm(struct mlx4_dev *dev, int *srqn)
{
struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table;
- struct mlx4_cmd_mailbox *mailbox;
- struct mlx4_srq_context *srq_context;
- u64 mtt_addr;
+ u64 out_param;
int err;
- srq->srqn = mlx4_bitmap_alloc(&srq_table->bitmap);
- if (srq->srqn == -1)
+ if (mlx4_is_slave(dev)) {
+ err = mlx4_cmd_imm(dev, 0, &out_param, RES_SRQ,
+ ICM_RESERVE_AND_ALLOC,
+ MLX4_CMD_ALLOC_RES,
+ MLX4_CMD_TIME_CLASS_A);
+ if (err) {
+ *srqn = -1;
+ return err;
+ } else {
+ *srqn = out_param;
+ return 0;
+ }
+ }
+
+ *srqn = mlx4_bitmap_alloc(&srq_table->bitmap);
+ if (*srqn == -1)
return -ENOMEM;
- err = mlx4_table_get(dev, &srq_table->table, srq->srqn);
+ err = mlx4_table_get(dev, &srq_table->table, *srqn);
if (err)
goto err_out;
- err = mlx4_table_get(dev, &srq_table->cmpt_table, srq->srqn);
+ err = mlx4_table_get(dev, &srq_table->cmpt_table, *srqn);
if (err)
goto err_put;
+ return 0;
+
+err_put:
+ mlx4_table_put(dev, &srq_table->table, *srqn);
+
+err_out:
+ mlx4_bitmap_free(&srq_table->bitmap, *srqn);
+ return err;
+}
+
+void mlx4_srq_free_icm(struct mlx4_dev *dev, int srqn)
+{
+ struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table;
+ u64 in_param;
+ int err;
+
+ if (mlx4_is_slave(dev)) {
+ *((u32 *) &in_param) = srqn;
+ *(((u32 *) &in_param) + 1) = 0;
+ err = mlx4_cmd(dev, in_param, RES_SRQ, ICM_RESERVE_AND_ALLOC,
+ MLX4_CMD_FREE_RES,
+ MLX4_CMD_TIME_CLASS_A);
+ if (err)
+ mlx4_warn(dev, "Failed freeing cq:%d\n", srqn);
+ } else {
+ mlx4_table_put(dev, &srq_table->cmpt_table, srqn);
+ mlx4_table_put(dev, &srq_table->table, srqn);
+ mlx4_bitmap_free(&srq_table->bitmap, srqn);
+ }
+}
+
+int mlx4_srq_alloc(struct mlx4_dev *dev, u32 pdn, struct mlx4_mtt *mtt,
+ u64 db_rec, struct mlx4_srq *srq)
+{
+ struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table;
+ struct mlx4_cmd_mailbox *mailbox;
+ struct mlx4_srq_context *srq_context;
+ u64 mtt_addr;
+ int err;
+
+ err = mlx4_srq_alloc_icm(dev, &srq->srqn);
+ if (err)
+ return err;
spin_lock_irq(&srq_table->lock);
err = radix_tree_insert(&srq_table->tree, srq->srqn, srq);
spin_unlock_irq(&srq_table->lock);
if (err)
- goto err_cmpt_put;
+ goto err_icm;
mailbox = mlx4_alloc_cmd_mailbox(dev);
if (IS_ERR(mailbox)) {
@@ -170,15 +224,8 @@ err_radix:
radix_tree_delete(&srq_table->tree, srq->srqn);
spin_unlock_irq(&srq_table->lock);
-err_cmpt_put:
- mlx4_table_put(dev, &srq_table->cmpt_table, srq->srqn);
-
-err_put:
- mlx4_table_put(dev, &srq_table->table, srq->srqn);
-
-err_out:
- mlx4_bitmap_free(&srq_table->bitmap, srq->srqn);
-
+err_icm:
+ mlx4_srq_free_icm(dev, srq->srqn);
return err;
}
EXPORT_SYMBOL_GPL(mlx4_srq_alloc);
@@ -200,8 +247,7 @@ void mlx4_srq_free(struct mlx4_dev *dev, struct mlx4_srq *srq)
complete(&srq->free);
wait_for_completion(&srq->free);
- mlx4_table_put(dev, &srq_table->table, srq->srqn);
- mlx4_bitmap_free(&srq_table->bitmap, srq->srqn);
+ mlx4_srq_free_icm(dev, srq->srqn);
}
EXPORT_SYMBOL_GPL(mlx4_srq_free);
diff --git a/include/linux/mlx4/cmd.h b/include/linux/mlx4/cmd.h
index b84ff08..be2a184 100644
--- a/include/linux/mlx4/cmd.h
+++ b/include/linux/mlx4/cmd.h
@@ -125,6 +125,8 @@ enum {
MLX4_CMD_DUMP_ETH_STATS = 0x49,
/* virtual commands */
+ MLX4_CMD_ALLOC_RES = 0x50,
+ MLX4_CMD_FREE_RES = 0x51,
MLX4_CMD_GET_EVENT = 0x52,
/* debug commands */
--
1.5.3.7
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 06/25 v2] mlx4_core: slave multicast support
From: Yevgeny Petrilin @ 2009-11-06 3:08 UTC (permalink / raw)
To: rdreier-FYB4Gu1CFyUAvxtiuMwx3w
Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA, netdev-u79uwXL29TY76Z2rM5mHXA,
liranl-VPRAkNaXOzVS1MOuV/RT9w, tziporet-VPRAkNaXOzVS1MOuV/RT9w,
yevgenyp-VPRAkNaXOzVS1MOuV/RT9w
From: Liran Liss <liranl-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
Multicast table processing requires multiple related commands.
To keep things simple, low-level multicast handling is done only by the master;
a new virtual command is added to allow slaves to attach/detach QPs to mulitcast
groups at a higher abstraction level.
Signed-off-by: Liran Liss <liranl-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
Signed-off-by: Yevgeny Petrilin <yevgenyp-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
---
drivers/net/mlx4/cmd.c | 8 ++++++++
drivers/net/mlx4/mcg.c | 45 +++++++++++++++++++++++++++++++++++++++++++++
drivers/net/mlx4/mlx4.h | 4 ++++
include/linux/mlx4/cmd.h | 1 +
4 files changed, 58 insertions(+), 0 deletions(-)
diff --git a/drivers/net/mlx4/cmd.c b/drivers/net/mlx4/cmd.c
index fe23415..02c4d7a 100644
--- a/drivers/net/mlx4/cmd.c
+++ b/drivers/net/mlx4/cmd.c
@@ -825,6 +825,14 @@ static struct mlx4_cmd_info {
/* Native multicast commands are not available for guests */
{
+ .opcode = MLX4_CMD_MCAST_ATTACH,
+ .has_inbox = true,
+ .has_outbox = false,
+ .out_is_imm = false,
+ .verify = NULL,
+ .wrapper = mlx4_MCAST_wrapper
+ },
+ {
.opcode = MLX4_CMD_DIAG_RPRT,
.has_inbox = false,
.has_outbox = true,
diff --git a/drivers/net/mlx4/mcg.c b/drivers/net/mlx4/mcg.c
index 5ccbce9..96f09ed 100644
--- a/drivers/net/mlx4/mcg.c
+++ b/drivers/net/mlx4/mcg.c
@@ -146,6 +146,45 @@ static int find_mgm(struct mlx4_dev *dev,
return err;
}
+int mlx4_MCAST_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr,
+ struct mlx4_cmd_mailbox *inbox,
+ struct mlx4_cmd_mailbox *outbox)
+{
+ struct mlx4_qp qp; /* dummy for calling attach/detach */
+
+ qp.qpn = vhcr->in_modifier & 0xffffff;
+ if (vhcr->op_modifier)
+ return mlx4_multicast_attach(dev, &qp, inbox->buf, vhcr->in_modifier >> 31);
+ else
+ return mlx4_multicast_detach(dev, &qp, inbox->buf);
+}
+
+static int mlx4_MCAST(struct mlx4_dev *dev, struct mlx4_qp *qp,
+ u8 gid[16], u8 attach,
+ u8 block_loopback)
+{
+ struct mlx4_cmd_mailbox *mailbox;
+ int err;
+ int qpn;
+
+ if (!mlx4_is_slave(dev))
+ return -EBADF;
+
+ mailbox = mlx4_alloc_cmd_mailbox(dev);
+ if (IS_ERR(mailbox))
+ return PTR_ERR(mailbox);
+
+ memcpy(mailbox->buf, gid, 16);
+ qpn = qp->qpn;
+ if (attach && block_loopback)
+ qpn |= (1 << 31);
+
+ err = mlx4_cmd(dev, mailbox->dma, qpn, attach, MLX4_CMD_MCAST_ATTACH,
+ MLX4_CMD_TIME_CLASS_A);
+ mlx4_free_cmd_mailbox(dev, mailbox);
+ return err;
+}
+
int mlx4_multicast_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16],
int block_mcast_loopback)
{
@@ -159,6 +198,9 @@ int mlx4_multicast_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16],
int i;
int err;
+ if (mlx4_is_slave(dev))
+ return mlx4_MCAST(dev, qp, gid, 1, block_mcast_loopback);
+
mailbox = mlx4_alloc_cmd_mailbox(dev);
if (IS_ERR(mailbox))
return PTR_ERR(mailbox);
@@ -254,6 +296,9 @@ int mlx4_multicast_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16])
int i, loc;
int err;
+ if (mlx4_is_slave(dev))
+ return mlx4_MCAST(dev, qp, gid, 0, 0);
+
mailbox = mlx4_alloc_cmd_mailbox(dev);
if (IS_ERR(mailbox))
return PTR_ERR(mailbox);
diff --git a/drivers/net/mlx4/mlx4.h b/drivers/net/mlx4/mlx4.h
index ace0bce..b5700a6 100644
--- a/drivers/net/mlx4/mlx4.h
+++ b/drivers/net/mlx4/mlx4.h
@@ -507,4 +507,8 @@ void mlx4_init_vlan_table(struct mlx4_dev *dev, struct mlx4_vlan_table *table);
int mlx4_SET_PORT(struct mlx4_dev *dev, u8 port);
int mlx4_get_port_ib_caps(struct mlx4_dev *dev, u8 port, __be32 *caps);
+int mlx4_MCAST_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr,
+ struct mlx4_cmd_mailbox *inbox,
+ struct mlx4_cmd_mailbox *outbox);
+
#endif /* MLX4_H */
diff --git a/include/linux/mlx4/cmd.h b/include/linux/mlx4/cmd.h
index be2a184..04255cd 100644
--- a/include/linux/mlx4/cmd.h
+++ b/include/linux/mlx4/cmd.h
@@ -128,6 +128,7 @@ enum {
MLX4_CMD_ALLOC_RES = 0x50,
MLX4_CMD_FREE_RES = 0x51,
MLX4_CMD_GET_EVENT = 0x52,
+ MLX4_CMD_MCAST_ATTACH = 0x54,
/* debug commands */
MLX4_CMD_QUERY_DEBUG_MSG = 0x2a,
--
1.5.3.7
From: Liran Liss <liranl-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 07/25 v2] mlx4_core: add port para-virtualization
From: Yevgeny Petrilin @ 2009-11-06 3:08 UTC (permalink / raw)
To: rdreier-FYB4Gu1CFyUAvxtiuMwx3w
Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA, netdev-u79uwXL29TY76Z2rM5mHXA,
liranl-VPRAkNaXOzVS1MOuV/RT9w, tziporet-VPRAkNaXOzVS1MOuV/RT9w,
yevgenyp-VPRAkNaXOzVS1MOuV/RT9w
From: Liran Liss <liranl-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
Ports are a shared resource among functions, so special behavior is needed here:
- Bring up ports if at least one function has done so.
- Bring down ports if all functions have done so.
- Aggregate IB port capabilities
Signed-off-by: Liran Liss <liranl-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
Signed-off-by: Yevgeny Petrilin <yevgenyp-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
---
drivers/net/mlx4/cmd.c | 32 ++++++++++++++++++++
drivers/net/mlx4/fw.c | 55 ++++++++++++++++++++++++++++++++++
drivers/net/mlx4/mlx4.h | 16 ++++++++++
drivers/net/mlx4/port.c | 76 +++++++++++++++++++++++++++++++++++++++++++++++
4 files changed, 179 insertions(+), 0 deletions(-)
diff --git a/drivers/net/mlx4/cmd.c b/drivers/net/mlx4/cmd.c
index 02c4d7a..b0fd998 100644
--- a/drivers/net/mlx4/cmd.c
+++ b/drivers/net/mlx4/cmd.c
@@ -541,6 +541,38 @@ static struct mlx4_cmd_info {
},
{
+ .opcode = MLX4_CMD_INIT_PORT,
+ .has_inbox = false,
+ .has_outbox = false,
+ .out_is_imm = false,
+ .verify = NULL,
+ .wrapper = mlx4_INIT_PORT_wrapper},
+ {
+ .opcode = MLX4_CMD_CLOSE_PORT,
+ .has_inbox = false,
+ .has_outbox = false,
+ .out_is_imm = false,
+ .verify = NULL,
+ .wrapper = mlx4_CLOSE_PORT_wrapper
+ },
+ {
+ .opcode = MLX4_CMD_QUERY_PORT,
+ .has_inbox = false,
+ .has_outbox = true,
+ .out_is_imm = false,
+ .verify = NULL,
+ .wrapper = mlx4_QUERY_PORT_wrapper
+ },
+ {
+ .opcode = MLX4_CMD_SET_PORT,
+ .has_inbox = true,
+ .has_outbox = false,
+ .out_is_imm = false,
+ .verify = NULL,
+ .wrapper = mlx4_SET_PORT_wrapper
+ },
+
+ {
.opcode = MLX4_CMD_SW2HW_EQ,
.has_inbox = true,
.has_outbox = false,
diff --git a/drivers/net/mlx4/fw.c b/drivers/net/mlx4/fw.c
index 4ca8060..a63f84a 100644
--- a/drivers/net/mlx4/fw.c
+++ b/drivers/net/mlx4/fw.c
@@ -135,6 +135,14 @@ int mlx4_MOD_STAT_CFG(struct mlx4_dev *dev, struct mlx4_mod_stat_cfg *cfg)
return err;
}
+int mlx4_QUERY_PORT_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr,
+ struct mlx4_cmd_mailbox *inbox,
+ struct mlx4_cmd_mailbox *outbox)
+{
+ return mlx4_cmd_box(dev, 0, outbox->dma, vhcr->in_modifier, 0, MLX4_CMD_QUERY_PORT,
+ MLX4_CMD_TIME_CLASS_B);
+}
+
int mlx4_QUERY_DEV_CAP(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
{
struct mlx4_cmd_mailbox *mailbox;
@@ -809,6 +817,29 @@ int mlx4_INIT_HCA(struct mlx4_dev *dev, struct mlx4_init_hca_param *param)
return err;
}
+int mlx4_INIT_PORT_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr,
+ struct mlx4_cmd_mailbox *inbox,
+ struct mlx4_cmd_mailbox *outbox)
+{
+ struct mlx4_priv *priv = mlx4_priv(dev);
+ int port;
+ int err;
+
+ port = vhcr->in_modifier;
+ if (priv->mfunc.master.slave_state[slave].init_port_mask & (1 << port))
+ return 0;
+
+ /* Enable port only if it was previously disabled */
+ if (!priv->mfunc.master.init_port_ref[port]) {
+ err = mlx4_INIT_PORT(dev, port);
+ if (err)
+ return err;
+ }
+ ++priv->mfunc.master.init_port_ref[port];
+ priv->mfunc.master.slave_state[slave].init_port_mask |= (1 << port);
+ return 0;
+}
+
int mlx4_INIT_PORT(struct mlx4_dev *dev, int port)
{
struct mlx4_cmd_mailbox *mailbox;
@@ -863,6 +894,30 @@ int mlx4_INIT_PORT(struct mlx4_dev *dev, int port)
}
EXPORT_SYMBOL_GPL(mlx4_INIT_PORT);
+int mlx4_CLOSE_PORT_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr,
+ struct mlx4_cmd_mailbox *inbox,
+ struct mlx4_cmd_mailbox *outbox)
+{
+ struct mlx4_priv *priv = mlx4_priv(dev);
+ int port;
+ int err;
+
+ port = vhcr->in_modifier;
+ if (!(priv->mfunc.master.slave_state[slave].init_port_mask & (1 << port)))
+ return 0;
+
+ /* CX1: master doesn't have interfaces - close port if this slave is
+ * the last user */
+ if (priv->mfunc.master.init_port_ref[port] == 1) {
+ err = mlx4_CLOSE_PORT(dev, port);
+ if (err)
+ return err;
+ }
+ --priv->mfunc.master.init_port_ref[port];
+ priv->mfunc.master.slave_state[slave].init_port_mask &= ~(1 << port);
+ return 0;
+}
+
int mlx4_CLOSE_PORT(struct mlx4_dev *dev, int port)
{
return mlx4_cmd(dev, 0, port, 0, MLX4_CMD_CLOSE_PORT, 1000);
diff --git a/drivers/net/mlx4/mlx4.h b/drivers/net/mlx4/mlx4.h
index b5700a6..8565be5 100644
--- a/drivers/net/mlx4/mlx4.h
+++ b/drivers/net/mlx4/mlx4.h
@@ -46,6 +46,7 @@
#include <linux/mlx4/driver.h>
#include <linux/mlx4/doorbell.h>
#include <linux/mlx4/cmd.h>
+#include <rdma/ib_verbs.h>
#define DRV_NAME "mlx4_core"
#define PFX DRV_NAME ": "
@@ -212,11 +213,14 @@ struct mlx4_slave_eqe {
struct mlx4_slave_state {
u8 comm_toggle;
u8 last_cmd;
+ u8 init_port_mask;
dma_addr_t vhcr_dma;
+ __be32 ib_cap_mask[MLX4_MAX_PORTS + 1];
};
struct mlx4_mfunc_master_ctx {
struct mlx4_slave_state *slave_state;
+ int init_port_ref[MLX4_MAX_PORTS + 1];
};
struct mlx4_vhcr {
@@ -505,6 +509,18 @@ void mlx4_init_mac_table(struct mlx4_dev *dev, struct mlx4_mac_table *table);
void mlx4_init_vlan_table(struct mlx4_dev *dev, struct mlx4_vlan_table *table);
int mlx4_SET_PORT(struct mlx4_dev *dev, u8 port);
+int mlx4_SET_PORT_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr,
+ struct mlx4_cmd_mailbox *inbox,
+ struct mlx4_cmd_mailbox *outbox);
+int mlx4_INIT_PORT_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr,
+ struct mlx4_cmd_mailbox *inbox,
+ struct mlx4_cmd_mailbox *outbox);
+int mlx4_CLOSE_PORT_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr,
+ struct mlx4_cmd_mailbox *inbox,
+ struct mlx4_cmd_mailbox *outbox);
+int mlx4_QUERY_PORT_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr,
+ struct mlx4_cmd_mailbox *inbox,
+ struct mlx4_cmd_mailbox *outbox);
int mlx4_get_port_ib_caps(struct mlx4_dev *dev, u8 port, __be32 *caps);
int mlx4_MCAST_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr,
diff --git a/drivers/net/mlx4/port.c b/drivers/net/mlx4/port.c
index 606aa58..67f0751 100644
--- a/drivers/net/mlx4/port.c
+++ b/drivers/net/mlx4/port.c
@@ -294,6 +294,82 @@ int mlx4_get_port_ib_caps(struct mlx4_dev *dev, u8 port, __be32 *caps)
return err;
}
+int mlx4_SET_PORT_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr,
+ struct mlx4_cmd_mailbox *inbox,
+ struct mlx4_cmd_mailbox *outbox)
+{
+ struct mlx4_priv *priv = mlx4_priv(dev);
+ int reset_qkey_viols;
+ int port;
+ int is_eth;
+ int err;
+ int i;
+ __be32 agg_cap_mask;
+ __be32 slave_cap_mask;
+ __be32 new_cap_mask;
+
+ port = vhcr->in_modifier & 0xff;
+ is_eth = vhcr->op_modifier;
+
+ /* For Ethernet, we currently support only slave0.
+ * TODO: add multi-vf support */
+ if (is_eth) {
+ if (slave)
+ return -EINVAL;
+ return mlx4_cmd(dev, inbox->dma, vhcr->in_modifier,
+ vhcr->op_modifier,
+ MLX4_CMD_SET_PORT,
+ MLX4_CMD_TIME_CLASS_B);
+ }
+
+ /* For IB, we only consider:
+ * - The capability mask, which is set to the aggregate of all slave frunction
+ * capabilities
+ * - The QKey violatin counter - reset according to each request.
+ */
+
+ if (dev->flags & MLX4_FLAG_OLD_PORT_CMDS) {
+ reset_qkey_viols = (*(u8 *) inbox->buf) & 0x40;
+ new_cap_mask = ((__be32 *) inbox->buf)[2];
+ } else {
+ reset_qkey_viols = ((u8 *) inbox->buf)[3] & 0x1;
+ new_cap_mask = ((__be32 *) inbox->buf)[1];
+ }
+
+ /* CX1: only slave0 has access to qp0 */
+ if (slave && (new_cap_mask & cpu_to_be32(IB_PORT_SM))) {
+ mlx4_warn(dev, "denying sm port capability for slave:%d\n", slave);
+ return -EINVAL;
+ }
+
+ agg_cap_mask = 0;
+ slave_cap_mask = priv->mfunc.master.slave_state[slave].ib_cap_mask[port];
+ priv->mfunc.master.slave_state[slave].ib_cap_mask[port] = new_cap_mask;
+ for (i = 0; i < dev->num_slaves; i++)
+ agg_cap_mask |= priv->mfunc.master.slave_state[slave].ib_cap_mask[port];
+
+#if 0
+ mlx4_warn(dev, "old_slave_cap:0x%x slave_cap:0x%x cap:0x%x qkey_reset:%d\n",
+ slave_cap_mask, priv->mfunc.master.slave_state[slave].ib_cap_mask[port],
+ agg_cap_mask, reset_qkey_viols);
+#endif
+
+ memset(inbox->buf, 0, 256);
+ if (dev->flags & MLX4_FLAG_OLD_PORT_CMDS) {
+ *(u8 *) inbox->buf = !!reset_qkey_viols << 6;
+ ((__be32 *) inbox->buf)[2] = agg_cap_mask;
+ } else {
+ ((u8 *) inbox->buf)[3] = !!reset_qkey_viols;
+ ((__be32 *) inbox->buf)[1] = agg_cap_mask;
+ }
+
+ err = mlx4_cmd(dev, inbox->dma, port, is_eth, MLX4_CMD_SET_PORT,
+ MLX4_CMD_TIME_CLASS_B);
+ if (err)
+ priv->mfunc.master.slave_state[slave].ib_cap_mask[port] = slave_cap_mask;
+ return err;
+}
+
int mlx4_SET_PORT(struct mlx4_dev *dev, u8 port)
{
struct mlx4_cmd_mailbox *mailbox;
--
1.5.3.7
From: Liran Liss <liranl-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 09/25 v2] mlx4_core: dispatch slave asynch events
From: Yevgeny Petrilin @ 2009-11-06 3:08 UTC (permalink / raw)
To: rdreier-FYB4Gu1CFyUAvxtiuMwx3w
Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA, netdev-u79uwXL29TY76Z2rM5mHXA,
liranl-VPRAkNaXOzVS1MOuV/RT9w, tziporet-VPRAkNaXOzVS1MOuV/RT9w,
yevgenyp-VPRAkNaXOzVS1MOuV/RT9w
From: Liran Liss <liranl-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
Affiliated and unaffiliated asynch events are handled by a single EQ owned by
the master. A per-slave SW event queue is added to log and dispatch both slave-specific
events and events that apply to all slaves.
Signed-off-by: Liran Liss <liranl-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
Signed-off-by: Yevgeny Petrilin <yevgenyp-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
---
drivers/net/mlx4/cmd.c | 12 ++++++-
drivers/net/mlx4/eq.c | 92 +++++++++++++++++++++++++++++++++++++++++++---
drivers/net/mlx4/mlx4.h | 8 ++++
3 files changed, 105 insertions(+), 7 deletions(-)
diff --git a/drivers/net/mlx4/cmd.c b/drivers/net/mlx4/cmd.c
index b0fd998..634e5c9 100644
--- a/drivers/net/mlx4/cmd.c
+++ b/drivers/net/mlx4/cmd.c
@@ -604,6 +604,14 @@ static struct mlx4_cmd_info {
.verify = NULL,
.wrapper = mlx4_RESOURCE_wrapper
},
+ {
+ .opcode = MLX4_CMD_GET_EVENT,
+ .has_inbox = false,
+ .has_outbox = false,
+ .out_is_imm = true,
+ .verify = NULL,
+ .wrapper = mlx4_GET_EVENT_wrapper
+ },
{
.opcode = MLX4_CMD_SW2HW_MPT,
@@ -1150,8 +1158,10 @@ int mlx4_multi_func_init(struct mlx4_dev *dev)
if (!priv->mfunc.master.slave_state)
goto err_comm;
- for (i = 0; i < dev->num_slaves; ++i)
+ for (i = 0; i < dev->num_slaves; ++i) {
priv->mfunc.master.slave_state[i].last_cmd = MLX4_COMM_CMD_RESET;
+ spin_lock_init(&priv->mfunc.master.slave_state[i].lock);
+ }
INIT_DELAYED_WORK(&priv->mfunc.comm_work, mlx4_master_poll_comm);
priv->mfunc.comm_wq = create_singlethread_workqueue("mlx4_comm");
diff --git a/drivers/net/mlx4/eq.c b/drivers/net/mlx4/eq.c
index 70c16d4..1e8b62d 100644
--- a/drivers/net/mlx4/eq.c
+++ b/drivers/net/mlx4/eq.c
@@ -160,6 +160,61 @@ static struct mlx4_eqe *next_eqe_sw(struct mlx4_eq *eq)
return !!(eqe->owner & 0x80) ^ !!(eq->cons_index & eq->nent) ? NULL : eqe;
}
+void mlx4_slave_event(struct mlx4_dev *dev, int slave, u8 type, u8 port, u32 param)
+{
+ struct mlx4_priv *priv = mlx4_priv(dev);
+ struct mlx4_slave_state *ctx = &priv->mfunc.master.slave_state[slave];
+ unsigned long flags;
+
+ if (ctx->last_cmd != MLX4_COMM_CMD_VHCR_POST) {
+ mlx4_warn(dev, "received event for inactive slave:%d\n", slave);
+ return;
+ }
+
+ /* Unconditionally add the new event - during overflows, we drop the
+ * oldest events */
+ spin_lock_irqsave(&ctx->lock, flags);
+ ctx->eq[ctx->eq_pi & MLX4_MFUNC_EQE_MASK].type = type;
+ ctx->eq[ctx->eq_pi & MLX4_MFUNC_EQE_MASK].port = port;
+ ctx->eq[ctx->eq_pi & MLX4_MFUNC_EQE_MASK].param = param;
+ ++ctx->eq_pi;
+ spin_unlock_irqrestore(&ctx->lock, flags);
+}
+
+static void mlx4_slave_event_all(struct mlx4_dev *dev, u8 type, u8 port, u32 param)
+{
+ struct mlx4_priv *priv = mlx4_priv(dev);
+ int i;
+
+ for (i = 0; i < dev->num_slaves; ++i)
+ if (priv->mfunc.master.slave_state[i].last_cmd == MLX4_COMM_CMD_VHCR_POST)
+ mlx4_slave_event(dev, i, type, port, param);
+}
+
+int mlx4_GET_EVENT_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr,
+ struct mlx4_cmd_mailbox *inbox,
+ struct mlx4_cmd_mailbox *outbox)
+{
+ struct mlx4_priv *priv = mlx4_priv(dev);
+ struct mlx4_slave_state *ctx = &priv->mfunc.master.slave_state[slave];
+ unsigned long flags;
+
+ spin_lock_irqsave(&ctx->lock, flags);
+ if (ctx->eq_ci == ctx->eq_pi) {
+ vhcr->out_param = MLX4_EVENT_TYPE_NONE;
+ } else if ((u16) (ctx->eq_pi - ctx->eq_ci) > MLX4_MFUNC_MAX_EQES) {
+ ctx->eq_ci = ctx->eq_pi - MLX4_MFUNC_MAX_EQES;
+ vhcr->out_param = MLX4_EVENT_TYPE_EQ_OVERFLOW;
+ } else {
+ vhcr->out_param = ctx->eq[ctx->eq_ci & MLX4_MFUNC_EQE_MASK].type |
+ ((u64) ctx->eq[ctx->eq_ci & MLX4_MFUNC_EQE_MASK].port << 8) |
+ ((u64) ctx->eq[ctx->eq_ci & MLX4_MFUNC_EQE_MASK].param << 32);
+ ++ctx->eq_ci;
+ }
+ spin_unlock_irqrestore(&ctx->lock, flags);
+ return 0;
+}
+
static int mlx4_GET_EVENT(struct mlx4_dev *dev, struct mlx4_slave_eqe *eqe)
{
int ret;
@@ -205,14 +260,26 @@ static int mlx4_eq_int(struct mlx4_dev *dev, struct mlx4_eq *eq)
case MLX4_EVENT_TYPE_PATH_MIG_FAILED:
case MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
case MLX4_EVENT_TYPE_WQ_ACCESS_ERROR:
- mlx4_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff,
- eqe->type);
+ if (mlx4_is_master(dev)) {
+ /* TODO: forward only to slave owning the QP */
+ mlx4_slave_event(dev, 0, eqe->type, 0,
+ be32_to_cpu(eqe->event.qp.qpn) &
+ 0xffffff);
+ } else
+ mlx4_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) &
+ 0xffffff, eqe->type);
break;
case MLX4_EVENT_TYPE_SRQ_LIMIT:
case MLX4_EVENT_TYPE_SRQ_CATAS_ERROR:
- mlx4_srq_event(dev, be32_to_cpu(eqe->event.srq.srqn) & 0xffffff,
- eqe->type);
+ if (mlx4_is_master(dev)) {
+ /* TODO: forward only to slave owning the SRQ */
+ mlx4_slave_event(dev, 0, eqe->type, 0,
+ be32_to_cpu(eqe->event.srq.srqn) &
+ 0xffffff);
+ } else
+ mlx4_srq_event(dev, be32_to_cpu(eqe->event.srq.srqn) &
+ 0xffffff, eqe->type);
break;
case MLX4_EVENT_TYPE_CMD:
@@ -227,10 +294,18 @@ static int mlx4_eq_int(struct mlx4_dev *dev, struct mlx4_eq *eq)
if (eqe->subtype == MLX4_PORT_CHANGE_SUBTYPE_DOWN) {
mlx4_dispatch_event(dev, MLX4_DEV_EVENT_PORT_DOWN,
port);
+ if (mlx4_is_master(dev)) {
+ mlx4_slave_event_all(dev, MLX4_EVENT_TYPE_PORT_CHANGE,
+ port, MLX4_DEV_EVENT_PORT_DOWN);
+ }
mlx4_priv(dev)->sense.do_sense_port[port] = 1;
} else {
mlx4_dispatch_event(dev, MLX4_DEV_EVENT_PORT_UP,
port);
+ if (mlx4_is_master(dev)) {
+ mlx4_slave_event_all(dev, MLX4_EVENT_TYPE_PORT_CHANGE,
+ port, MLX4_DEV_EVENT_PORT_UP);
+ }
mlx4_priv(dev)->sense.do_sense_port[port] = 0;
}
break;
@@ -240,8 +315,13 @@ static int mlx4_eq_int(struct mlx4_dev *dev, struct mlx4_eq *eq)
eqe->event.cq_err.syndrome == 1 ?
"overrun" : "access violation",
be32_to_cpu(eqe->event.cq_err.cqn) & 0xffffff);
- mlx4_cq_event(dev, be32_to_cpu(eqe->event.cq_err.cqn),
- eqe->type);
+ if (mlx4_is_master(dev)) {
+ /* TODO: forward only to slave owning the CQ */
+ mlx4_slave_event(dev, 0, eqe->type, 0,
+ be32_to_cpu(eqe->event.cq_err.cqn));
+ } else
+ mlx4_cq_event(dev, be32_to_cpu(eqe->event.cq_err.cqn),
+ eqe->type);
break;
case MLX4_EVENT_TYPE_EQ_OVERFLOW:
diff --git a/drivers/net/mlx4/mlx4.h b/drivers/net/mlx4/mlx4.h
index 8565be5..f680940 100644
--- a/drivers/net/mlx4/mlx4.h
+++ b/drivers/net/mlx4/mlx4.h
@@ -216,6 +216,10 @@ struct mlx4_slave_state {
u8 init_port_mask;
dma_addr_t vhcr_dma;
__be32 ib_cap_mask[MLX4_MAX_PORTS + 1];
+ struct mlx4_slave_eqe eq[MLX4_MFUNC_MAX_EQES];
+ u16 eq_pi;
+ u16 eq_ci;
+ spinlock_t lock;
};
struct mlx4_mfunc_master_ctx {
@@ -422,6 +426,10 @@ int mlx4_reset(struct mlx4_dev *dev);
int mlx4_alloc_eq_table(struct mlx4_dev *dev);
void mlx4_free_eq_table(struct mlx4_dev *dev);
+void mlx4_slave_event(struct mlx4_dev *dev, int slave, u8 type, u8 port, u32 param);
+int mlx4_GET_EVENT_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr,
+ struct mlx4_cmd_mailbox *inbox,
+ struct mlx4_cmd_mailbox *outbox);
int mlx4_init_pd_table(struct mlx4_dev *dev);
int mlx4_init_uar_table(struct mlx4_dev *dev);
--
1.5.3.7
From: Liran Liss <liranl-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 10/25 v2] mlx4_core: track slave special qps
From: Yevgeny Petrilin @ 2009-11-06 3:08 UTC (permalink / raw)
To: rdreier-FYB4Gu1CFyUAvxtiuMwx3w
Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA, netdev-u79uwXL29TY76Z2rM5mHXA,
liranl-VPRAkNaXOzVS1MOuV/RT9w, tziporet-VPRAkNaXOzVS1MOuV/RT9w,
yevgenyp-VPRAkNaXOzVS1MOuV/RT9w
From: Liran Liss <liranl-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
A single function can own the actual Infiniband hw special qps.
This function demuxes inbound mads and protects outbound mads from spoofing.
To achieve this, it needs to know the special qps used by other functions.
This patch tracks special qp registration and notifies the owning function
on any changes.
Signed-off-by: Liran Liss <liranl-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
Signed-off-by: Yevgeny Petrilin <yevgenyp-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
---
drivers/net/mlx4/cmd.c | 16 ++++++++++
drivers/net/mlx4/eq.c | 15 +++++++++
drivers/net/mlx4/mlx4.h | 11 +++++++
drivers/net/mlx4/qp.c | 67 +++++++++++++++++++++++++++++++++++++++++++
include/linux/mlx4/cmd.h | 1 +
include/linux/mlx4/device.h | 3 ++
6 files changed, 113 insertions(+), 0 deletions(-)
diff --git a/drivers/net/mlx4/cmd.c b/drivers/net/mlx4/cmd.c
index 634e5c9..485e976 100644
--- a/drivers/net/mlx4/cmd.c
+++ b/drivers/net/mlx4/cmd.c
@@ -539,6 +539,14 @@ static struct mlx4_cmd_info {
.verify = NULL,
.wrapper = NULL
},
+ {
+ .opcode = MLX4_CMD_GET_SLAVE_SQP,
+ .has_inbox = false,
+ .has_outbox = true,
+ .out_is_imm = false,
+ .verify = NULL,
+ .wrapper = mlx4_GET_SLAVE_SQP_wrapper
+ },
{
.opcode = MLX4_CMD_INIT_PORT,
@@ -855,6 +863,14 @@ static struct mlx4_cmd_info {
.wrapper = NULL
},
{
+ .opcode = MLX4_CMD_CONF_SPECIAL_QP,
+ .has_inbox = false,
+ .has_outbox = false,
+ .out_is_imm = false,
+ .verify = NULL,
+ .wrapper = mlx4_CONF_SPECIAL_QP_wrapper
+ },
+ {
.opcode = MLX4_CMD_MAD_IFC,
.has_inbox = true,
.has_outbox = true,
diff --git a/drivers/net/mlx4/eq.c b/drivers/net/mlx4/eq.c
index 1e8b62d..a27e1c4 100644
--- a/drivers/net/mlx4/eq.c
+++ b/drivers/net/mlx4/eq.c
@@ -358,6 +358,17 @@ static int mlx4_eq_int(struct mlx4_dev *dev, struct mlx4_eq *eq)
return eqes_found;
}
+static void mlx4_update_sqp(struct mlx4_dev *dev)
+{
+ if (!dev->caps.sqp_demux) {
+ mlx4_warn(dev, "unexpected update_sqp event\n");
+ return;
+ }
+ if (mlx4_GET_SLAVE_SQP(dev, mlx4_priv(dev)->mfunc.demux_sqp,
+ dev->caps.sqp_demux))
+ mlx4_warn(dev, "couldn't update sqp\n");
+}
+
void mlx4_slave_async_eq_poll(struct work_struct *work)
{
struct delayed_work *delay = container_of(work, struct delayed_work, work);
@@ -402,6 +413,10 @@ void mlx4_slave_async_eq_poll(struct work_struct *work)
mlx4_warn(dev, "slave async EQ overrun\n");
break;
+ case MLX4_EVENT_TYPE_SQP_UPDATE:
+ mlx4_update_sqp(dev);
+ break;
+
default:
mlx4_warn(dev, "Unhandled event:%02x\n", eqe.type);
}
diff --git a/drivers/net/mlx4/mlx4.h b/drivers/net/mlx4/mlx4.h
index f680940..e2d80c6 100644
--- a/drivers/net/mlx4/mlx4.h
+++ b/drivers/net/mlx4/mlx4.h
@@ -111,6 +111,7 @@ enum mlx4_alloc_mode {
};
enum {
+ MLX4_MFUNC_MAX = 64,
MLX4_MFUNC_MAX_EQES = 8,
MLX4_MFUNC_EQE_MASK = (MLX4_MFUNC_MAX_EQES - 1)
};
@@ -219,6 +220,7 @@ struct mlx4_slave_state {
struct mlx4_slave_eqe eq[MLX4_MFUNC_MAX_EQES];
u16 eq_pi;
u16 eq_ci;
+ int sqp_start;
spinlock_t lock;
};
@@ -246,6 +248,7 @@ struct mlx4_mfunc {
dma_addr_t vhcr_dma;
struct mlx4_mfunc_master_ctx master;
+ u32 demux_sqp[MLX4_MFUNC_MAX];
};
struct mlx4_cmd {
@@ -531,6 +534,14 @@ int mlx4_QUERY_PORT_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *v
struct mlx4_cmd_mailbox *outbox);
int mlx4_get_port_ib_caps(struct mlx4_dev *dev, u8 port, __be32 *caps);
+int mlx4_CONF_SPECIAL_QP_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr,
+ struct mlx4_cmd_mailbox *inbox,
+ struct mlx4_cmd_mailbox *outbox);
+int mlx4_GET_SLAVE_SQP(struct mlx4_dev *dev, u32 *sqp, int num);
+int mlx4_GET_SLAVE_SQP_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr,
+ struct mlx4_cmd_mailbox *inbox,
+ struct mlx4_cmd_mailbox *outbox);
+
int mlx4_MCAST_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr,
struct mlx4_cmd_mailbox *inbox,
struct mlx4_cmd_mailbox *outbox);
diff --git a/drivers/net/mlx4/qp.c b/drivers/net/mlx4/qp.c
index 065c7fc..99b9ded 100644
--- a/drivers/net/mlx4/qp.c
+++ b/drivers/net/mlx4/qp.c
@@ -145,6 +145,54 @@ int mlx4_qp_modify(struct mlx4_dev *dev, struct mlx4_mtt *mtt,
}
EXPORT_SYMBOL_GPL(mlx4_qp_modify);
+u32 mlx4_get_slave_sqp(struct mlx4_dev *dev, int slave)
+{
+ if (mlx4_is_master(dev) && slave < dev->num_slaves) {
+ return mlx4_priv(dev)->mfunc.master.slave_state[slave].sqp_start;
+ }
+ if (mlx4_is_slave(dev) && slave < dev->caps.sqp_demux) {
+ return mlx4_priv(dev)->mfunc.demux_sqp[slave];
+ }
+ return 0;
+}
+EXPORT_SYMBOL_GPL(mlx4_get_slave_sqp);
+
+int mlx4_GET_SLAVE_SQP_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr,
+ struct mlx4_cmd_mailbox *inbox,
+ struct mlx4_cmd_mailbox *outbox)
+{
+ u32 *slave_sqp = outbox->buf;
+ int i;
+
+ /* CX1: special qp demultiplexing is done by slave0 */
+ if (slave) {
+ mlx4_warn(dev, "Denying slave_sqp request from slave:%d\n", slave);
+ return -EINVAL;
+ }
+ for (i = 0; i < 64; i++)
+ slave_sqp[i] = mlx4_get_slave_sqp(dev, i);
+ return 0;
+}
+
+int mlx4_GET_SLAVE_SQP(struct mlx4_dev *dev, u32 *sqp, int num)
+{
+ struct mlx4_cmd_mailbox *mailbox;
+ int err;
+
+ mailbox = mlx4_alloc_cmd_mailbox(dev);
+ if (IS_ERR(mailbox))
+ return PTR_ERR(mailbox);
+
+ err = mlx4_cmd_box(dev, 0, mailbox->dma, 0, 0, MLX4_CMD_GET_SLAVE_SQP,
+ MLX4_CMD_TIME_CLASS_A);
+ if (!err)
+ memcpy(sqp, mailbox->buf, sizeof (u32) * num);
+
+ mlx4_free_cmd_mailbox(dev, mailbox);
+ return err;
+}
+EXPORT_SYMBOL_GPL(mlx4_GET_SLAVE_SQP);
+
int mlx4_qp_reserve_range(struct mlx4_dev *dev, int cnt, int align, int *base)
{
struct mlx4_priv *priv = mlx4_priv(dev);
@@ -331,6 +379,25 @@ static int mlx4_CONF_SPECIAL_QP(struct mlx4_dev *dev, u32 base_qpn)
MLX4_CMD_TIME_CLASS_B);
}
+int mlx4_CONF_SPECIAL_QP_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr,
+ struct mlx4_cmd_mailbox *inbox,
+ struct mlx4_cmd_mailbox *outbox)
+{
+ struct mlx4_priv *priv = mlx4_priv(dev);
+ int ret;
+
+ priv->mfunc.master.slave_state[slave].sqp_start = vhcr->in_modifier & 0xffffff;
+ if (!slave) {
+ /* CX1: slave0 owns real special QPs */
+ ret = mlx4_CONF_SPECIAL_QP(dev, priv->mfunc.master.slave_state[slave].sqp_start);
+ if (ret)
+ return ret;
+ }
+ /* Notify slave0 that an SQP change occured */
+ mlx4_slave_event(dev, 0, MLX4_EVENT_TYPE_SQP_UPDATE, 0, 0);
+ return 0;
+}
+
int mlx4_init_qp_table(struct mlx4_dev *dev)
{
struct mlx4_qp_table *qp_table = &mlx4_priv(dev)->qp_table;
diff --git a/include/linux/mlx4/cmd.h b/include/linux/mlx4/cmd.h
index 04255cd..8444b89 100644
--- a/include/linux/mlx4/cmd.h
+++ b/include/linux/mlx4/cmd.h
@@ -129,6 +129,7 @@ enum {
MLX4_CMD_FREE_RES = 0x51,
MLX4_CMD_GET_EVENT = 0x52,
MLX4_CMD_MCAST_ATTACH = 0x54,
+ MLX4_CMD_GET_SLAVE_SQP = 0x55,
/* debug commands */
MLX4_CMD_QUERY_DEBUG_MSG = 0x2a,
diff --git a/include/linux/mlx4/device.h b/include/linux/mlx4/device.h
index 9735f40..b20c8d8 100644
--- a/include/linux/mlx4/device.h
+++ b/include/linux/mlx4/device.h
@@ -99,6 +99,7 @@ enum mlx4_event {
MLX4_EVENT_TYPE_EQ_OVERFLOW = 0x0f,
MLX4_EVENT_TYPE_ECC_DETECT = 0x0e,
MLX4_EVENT_TYPE_CMD = 0x0a,
+ MLX4_EVENT_TYPE_SQP_UPDATE = 0xfe,
MLX4_EVENT_TYPE_NONE = 0xff,
};
@@ -242,6 +243,7 @@ struct mlx4_caps {
int log_num_prios;
enum mlx4_port_type port_type[MLX4_MAX_PORTS + 1];
u8 supported_type[MLX4_MAX_PORTS + 1];
+ u8 sqp_demux;
u32 port_mask;
enum mlx4_port_type possible_type[MLX4_MAX_PORTS + 1];
};
@@ -470,6 +472,7 @@ void mlx4_qp_release_range(struct mlx4_dev *dev, int base_qpn, int cnt);
int mlx4_qp_alloc(struct mlx4_dev *dev, int qpn, struct mlx4_qp *qp);
void mlx4_qp_free(struct mlx4_dev *dev, struct mlx4_qp *qp);
+u32 mlx4_get_slave_sqp(struct mlx4_dev *dev, int vf);
int mlx4_srq_alloc(struct mlx4_dev *dev, u32 pdn, struct mlx4_mtt *mtt,
u64 db_rec, struct mlx4_srq *srq);
--
1.5.3.7
From: Liran Liss <liranl-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 11/25 v2] mlx4_core: per-function capabilities
From: Yevgeny Petrilin @ 2009-11-06 3:09 UTC (permalink / raw)
To: rdreier-FYB4Gu1CFyUAvxtiuMwx3w
Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA, netdev-u79uwXL29TY76Z2rM5mHXA,
liranl-VPRAkNaXOzVS1MOuV/RT9w, tziporet-VPRAkNaXOzVS1MOuV/RT9w,
yevgenyp-VPRAkNaXOzVS1MOuV/RT9w
From: Liran Liss <liranl-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
The master function builds a HW profile, and manages all resources.
Other functions query the master for function-specific capabilities.
EQs, MSI-X vectors, and UARs are statically divided among all functions,
while other resources are dynamically assigned later upon request.
Signed-off-by: Liran Liss <liranl-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
Signed-off-by: Yevgeny Petrilin <yevgenyp-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
---
drivers/net/mlx4/cmd.c | 9 +++++
drivers/net/mlx4/fw.c | 78 +++++++++++++++++++++++++++++++++++++++++-
drivers/net/mlx4/fw.h | 4 ++
drivers/net/mlx4/main.c | 70 ++++++++++++++++++++++++++++++++++++++-
drivers/net/mlx4/mlx4.h | 4 ++-
drivers/net/mlx4/profile.c | 26 ++++++++++++--
include/linux/mlx4/cmd.h | 1 +
include/linux/mlx4/device.h | 3 ++
8 files changed, 187 insertions(+), 8 deletions(-)
diff --git a/drivers/net/mlx4/cmd.c b/drivers/net/mlx4/cmd.c
index 485e976..d68d0e1 100644
--- a/drivers/net/mlx4/cmd.c
+++ b/drivers/net/mlx4/cmd.c
@@ -39,6 +39,7 @@
#include <asm/io.h>
#include "mlx4.h"
+#include "fw.h"
#define CMD_POLL_TOKEN 0xffff
@@ -532,6 +533,14 @@ static struct mlx4_cmd_info {
.wrapper = NULL
},
{
+ .opcode = MLX4_CMD_QUERY_SLAVE_CAP,
+ .has_inbox = false,
+ .has_outbox = true,
+ .out_is_imm = false,
+ .verify = NULL,
+ .wrapper = mlx4_QUERY_SLAVE_CAP_wrapper
+ },
+ {
.opcode = MLX4_CMD_QUERY_ADAPTER,
.has_inbox = false,
.has_outbox = true,
diff --git a/drivers/net/mlx4/fw.c b/drivers/net/mlx4/fw.c
index 87eb2d4..60889d3 100644
--- a/drivers/net/mlx4/fw.c
+++ b/drivers/net/mlx4/fw.c
@@ -143,6 +143,76 @@ int mlx4_QUERY_PORT_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *v
MLX4_CMD_TIME_CLASS_B);
}
+int mlx4_QUERY_SLAVE_CAP_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr,
+ struct mlx4_cmd_mailbox *inbox,
+ struct mlx4_cmd_mailbox *outbox)
+{
+ struct mlx4_caps *caps = outbox->buf;
+ int i;
+
+ memcpy(caps, &dev->caps, sizeof *caps);
+
+ /* CX1: blue-flame not supported in vfs */
+ caps->bf_reg_size = 0;
+ caps->bf_regs_per_page = 0;
+
+ /* CX1:
+ * On ingress, slave0 gets full ownership over qp1, and demultiplexes GSI mads
+ * on behalf of other slaves.
+ * On egress, slave>0 must tunnel their GSI mads for validation by the qp1 owner */
+ if (slave) {
+ caps->sqp_demux = 0;
+ for (i = 1; i <= dev->caps.num_ports; ++i) {
+ caps->gid_table_len[i] = 1;
+ caps->pkey_table_len[i] = 1;
+ }
+ } else {
+ caps->sqp_demux = dev->num_slaves;
+ for (i = 1; i <= dev->caps.num_ports; ++i) {
+ caps->gid_table_len[i] = dev->num_slaves;
+ caps->pkey_table_len[i] = 1;
+ }
+ }
+
+ /* Slave functions allocate themselves EQs, UARs, and PDs.
+ * - num is the maximum resource index
+ * - reserved is the minimum resource index */
+ caps->num_eqs = dev->caps.reserved_eqs + MLX4_MFUNC_EQ_NUM * (slave + 2);
+ caps->reserved_eqs = dev->caps.reserved_eqs + MLX4_MFUNC_EQ_NUM * (slave + 1);
+ caps->num_uars = dev->caps.num_uars * (slave + 1);
+ caps->reserved_uars = max_t(int, dev->caps.num_uars * slave, dev->caps.reserved_uars);
+
+ /* PDs have the same range in every guest; the distinction is in the msbs,
+ * which contains the guest ID (vf + 1) */
+ caps->pd_base = slave + 1;
+
+ /* All other resources are allocated by the master, but we still report
+ * 'num' and 'reserved' capabilities as follows:
+ * - num remains the maximum resource index
+ * - 'num - reserved' is the total available objects of a resource, but
+ * resource indices may be less than 'reserved'
+ * TODO: set per-resource quotas */
+ return 0;
+}
+
+int mlx4_QUERY_SLAVE_CAP(struct mlx4_dev *dev, struct mlx4_caps *caps)
+{
+ struct mlx4_cmd_mailbox *mailbox;
+ int err;
+
+ mailbox = mlx4_alloc_cmd_mailbox(dev);
+ if (IS_ERR(mailbox))
+ return PTR_ERR(mailbox);
+
+ err = mlx4_cmd_box(dev, 0, mailbox->dma, 0, 0, MLX4_CMD_QUERY_SLAVE_CAP,
+ MLX4_CMD_TIME_CLASS_A);
+ if (!err)
+ memcpy(caps, mailbox->buf, sizeof *caps);
+
+ mlx4_free_cmd_mailbox(dev, mailbox);
+ return err;
+}
+
int mlx4_QUERY_DEV_CAP(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
{
struct mlx4_cmd_mailbox *mailbox;
@@ -405,9 +475,13 @@ int mlx4_QUERY_DEV_CAP(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
* Each UAR has 4 EQ doorbells; so if a UAR is reserved, then
* we can't use any EQs whose doorbell falls on that page,
* even if the EQ itself isn't reserved.
+ * CX1: in multi-function mode, EQ doorbells are virtualized so the whole EQ
+ * range is available: the master and each of the slaves get 4 EQ doorbells
+ * starting from the first non-reserved EQ.
*/
- dev_cap->reserved_eqs = max(dev_cap->reserved_uars * 4,
- dev_cap->reserved_eqs);
+ if (!mlx4_is_mfunc(dev))
+ dev_cap->reserved_eqs = max(dev_cap->reserved_uars * 4,
+ dev_cap->reserved_eqs);
mlx4_dbg(dev, "Max ICM size %lld MB\n",
(unsigned long long) dev_cap->max_icm_sz >> 20);
diff --git a/drivers/net/mlx4/fw.h b/drivers/net/mlx4/fw.h
index 526d7f3..d066c69 100644
--- a/drivers/net/mlx4/fw.h
+++ b/drivers/net/mlx4/fw.h
@@ -160,6 +160,10 @@ struct mlx4_set_ib_param {
};
int mlx4_QUERY_DEV_CAP(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap);
+int mlx4_QUERY_SLAVE_CAP(struct mlx4_dev *dev, struct mlx4_caps *caps);
+int mlx4_QUERY_SLAVE_CAP_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr,
+ struct mlx4_cmd_mailbox *inbox,
+ struct mlx4_cmd_mailbox *outbox);
int mlx4_MAP_FA(struct mlx4_dev *dev, struct mlx4_icm *icm);
int mlx4_UNMAP_FA(struct mlx4_dev *dev);
int mlx4_RUN_FW(struct mlx4_dev *dev);
diff --git a/drivers/net/mlx4/main.c b/drivers/net/mlx4/main.c
index 291a505..4edc8d7 100644
--- a/drivers/net/mlx4/main.c
+++ b/drivers/net/mlx4/main.c
@@ -141,6 +141,7 @@ static void mlx4_set_port_mask(struct mlx4_dev *dev)
if (dev->caps.port_type[i] == MLX4_PORT_TYPE_IB)
dev->caps.port_mask |= 1 << (i - 1);
}
+
static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
{
int err;
@@ -185,6 +186,7 @@ static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
dev->caps.supported_type[i] = dev_cap->supported_port_types[i];
}
+ dev->caps.uar_page_size = PAGE_SIZE;
dev->caps.num_uars = dev_cap->uar_size / PAGE_SIZE;
dev->caps.local_ca_ack_delay = dev_cap->local_ca_ack_delay;
dev->caps.bf_reg_size = dev_cap->bf_reg_size;
@@ -211,7 +213,9 @@ static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
dev->caps.reserved_mtts = DIV_ROUND_UP(dev_cap->reserved_mtts,
dev->caps.mtts_per_seg);
dev->caps.reserved_mrws = dev_cap->reserved_mrws;
- dev->caps.reserved_uars = dev_cap->reserved_uars;
+
+ /* The first 128 UARs are used for EQ doorbells */
+ dev->caps.reserved_uars = max_t(int, 128, dev_cap->reserved_uars);
dev->caps.reserved_pds = dev_cap->reserved_pds;
dev->caps.mtt_entry_sz = dev->caps.mtts_per_seg * dev_cap->mtt_entry_sz;
dev->caps.max_msg_sz = dev_cap->max_msg_sz;
@@ -265,6 +269,70 @@ static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_ADDR] +
dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_EXCH];
+ /* CX1: vf0 takes role of GSI multiplexing instead of master */
+ dev->caps.sqp_demux = 0;
+ return 0;
+}
+
+static int mlx4_slave_cap(struct mlx4_dev *dev)
+{
+ int err;
+ u32 page_size;
+
+ err = mlx4_QUERY_SLAVE_CAP(dev, &dev->caps);
+ if (err)
+ return err;
+
+ page_size = ~dev->caps.page_size_cap + 1;
+ mlx4_warn(dev, "HCA minimum page size:%d\n", page_size);
+ if (page_size > PAGE_SIZE) {
+ mlx4_err(dev, "HCA minimum page size of %d bigger than "
+ "kernel PAGE_SIZE of %ld, aborting.\n",
+ page_size, PAGE_SIZE);
+ return -ENODEV;
+ }
+
+ /* TODO: relax this assumption */
+ if (dev->caps.uar_page_size != PAGE_SIZE) {
+ mlx4_err(dev, "UAR size:%d != kernel PAGE_SIZE of %ld\n",
+ dev->caps.uar_page_size, PAGE_SIZE);
+ return -ENODEV;
+ }
+
+ if (dev->caps.num_ports > MLX4_MAX_PORTS) {
+ mlx4_err(dev, "HCA has %d ports, but we only support %d, "
+ "aborting.\n", dev->caps.num_ports, MLX4_MAX_PORTS);
+ return -ENODEV;
+ }
+
+ if (dev->caps.uar_page_size * (dev->caps.num_uars -
+ dev->caps.reserved_uars) >
+ pci_resource_len(dev->pdev, 2)) {
+ mlx4_err(dev, "HCA reported UAR region size of 0x%x bigger than "
+ "PCI resource 2 size of 0x%llx, aborting.\n",
+ dev->caps.uar_page_size * dev->caps.num_uars,
+ (unsigned long long) pci_resource_len(dev->pdev, 2));
+ return -ENODEV;
+ }
+
+ /* Adjust eq number */
+ if (dev->caps.num_eqs - dev->caps.reserved_eqs > num_possible_cpus() + 1)
+ dev->caps.num_eqs = dev->caps.reserved_eqs + num_possible_cpus() + 1;
+
+#if 0
+ mlx4_warn(dev, "sqp_demux:%d\n", dev->caps.sqp_demux);
+ mlx4_warn(dev, "num_uars:%d reserved_uars:%d uar region:0x%x bar2:0x%llx\n",
+ dev->caps.num_uars, dev->caps.reserved_uars,
+ dev->caps.uar_page_size * dev->caps.num_uars,
+ pci_resource_len(dev->pdev, 2));
+ mlx4_warn(dev, "num_eqs:%d reserved_eqs:%d\n", dev->caps.num_eqs,
+ dev->caps.reserved_eqs);
+ mlx4_warn(dev, "num_pds:%d reserved_pds:%d slave_pd_shift:%d pd_base:%d\n",
+ dev->caps.num_pds,
+ dev->caps.reserved_pds,
+ dev->caps.slave_pd_shift,
+ dev->caps.pd_base);
+#endif
return 0;
}
diff --git a/drivers/net/mlx4/mlx4.h b/drivers/net/mlx4/mlx4.h
index e2d80c6..0686856 100644
--- a/drivers/net/mlx4/mlx4.h
+++ b/drivers/net/mlx4/mlx4.h
@@ -69,7 +69,8 @@ enum {
};
enum {
- MLX4_NUM_PDS = 1 << 15
+ MLX4_NUM_PDS = 1 << 15,
+ MLX4_SLAVE_PD_SHIFT = 17, /* the 7 msbs encode the slave id */
};
enum {
@@ -112,6 +113,7 @@ enum mlx4_alloc_mode {
enum {
MLX4_MFUNC_MAX = 64,
+ MLX4_MFUNC_EQ_NUM = 4,
MLX4_MFUNC_MAX_EQES = 8,
MLX4_MFUNC_EQE_MASK = (MLX4_MFUNC_MAX_EQES - 1)
};
diff --git a/drivers/net/mlx4/profile.c b/drivers/net/mlx4/profile.c
index ca25b9d..f6197e9 100644
--- a/drivers/net/mlx4/profile.c
+++ b/drivers/net/mlx4/profile.c
@@ -105,9 +105,19 @@ u64 mlx4_make_profile(struct mlx4_dev *dev,
profile[MLX4_RES_AUXC].num = request->num_qp;
profile[MLX4_RES_SRQ].num = request->num_srq;
profile[MLX4_RES_CQ].num = request->num_cq;
- profile[MLX4_RES_EQ].num = min_t(unsigned, dev_cap->max_eqs,
- dev_cap->reserved_eqs +
- num_possible_cpus() + 1);
+ if (mlx4_is_master(dev)) {
+ profile[MLX4_RES_EQ].num = dev_cap->reserved_eqs +
+ MLX4_MFUNC_EQ_NUM *
+ (dev->num_slaves + 1);
+ if (profile[MLX4_RES_EQ].num > dev_cap->max_eqs) {
+ mlx4_warn(dev, "Not enough eqs for:%ld slave functions\n", dev->num_slaves);
+ kfree(profile);
+ return -ENOMEM;
+ }
+ } else
+ profile[MLX4_RES_EQ].num = min_t(unsigned, dev_cap->max_eqs,
+ dev_cap->reserved_eqs +
+ num_possible_cpus() + 1);
profile[MLX4_RES_DMPT].num = request->num_mpt;
profile[MLX4_RES_CMPT].num = MLX4_NUM_CMPTS;
profile[MLX4_RES_MTT].num = request->num_mtt;
@@ -196,7 +206,13 @@ u64 mlx4_make_profile(struct mlx4_dev *dev,
init_hca->log_num_cqs = profile[i].log_num;
break;
case MLX4_RES_EQ:
- dev->caps.num_eqs = profile[i].num;
+ if (mlx4_is_master(dev)) {
+ dev->caps.num_eqs = dev_cap->reserved_eqs +
+ min_t(unsigned,
+ MLX4_MFUNC_EQ_NUM,
+ num_possible_cpus() + 1);
+ } else
+ dev->caps.num_eqs = profile[i].num;
init_hca->eqc_base = profile[i].start;
init_hca->log_num_eqs = profile[i].log_num;
break;
@@ -232,6 +248,8 @@ u64 mlx4_make_profile(struct mlx4_dev *dev,
* of the HCA profile anyway.
*/
dev->caps.num_pds = MLX4_NUM_PDS;
+ dev->caps.slave_pd_shift = MLX4_SLAVE_PD_SHIFT;
+ dev->caps.pd_base = 0;
kfree(profile);
return total_size;
diff --git a/include/linux/mlx4/cmd.h b/include/linux/mlx4/cmd.h
index 8444b89..99145d4 100644
--- a/include/linux/mlx4/cmd.h
+++ b/include/linux/mlx4/cmd.h
@@ -128,6 +128,7 @@ enum {
MLX4_CMD_ALLOC_RES = 0x50,
MLX4_CMD_FREE_RES = 0x51,
MLX4_CMD_GET_EVENT = 0x52,
+ MLX4_CMD_QUERY_SLAVE_CAP = 0x53,
MLX4_CMD_MCAST_ATTACH = 0x54,
MLX4_CMD_GET_SLAVE_SQP = 0x55,
diff --git a/include/linux/mlx4/device.h b/include/linux/mlx4/device.h
index b20c8d8..4b9091a 100644
--- a/include/linux/mlx4/device.h
+++ b/include/linux/mlx4/device.h
@@ -192,6 +192,7 @@ struct mlx4_caps {
int pkey_table_len[MLX4_MAX_PORTS + 1];
int local_ca_ack_delay;
int num_uars;
+ int uar_page_size;
int bf_reg_size;
int bf_regs_per_page;
int max_sq_sg;
@@ -226,6 +227,8 @@ struct mlx4_caps {
int num_qp_per_mgm;
int num_pds;
int reserved_pds;
+ int slave_pd_shift;
+ int pd_base;
int mtt_entry_sz;
u32 max_msg_sz;
u32 page_size_cap;
--
1.5.3.7
From: Liran Liss <liranl-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 12/25 v2] mlx4_core: associate resources with specific functions
From: Yevgeny Petrilin @ 2009-11-06 3:09 UTC (permalink / raw)
To: rdreier-FYB4Gu1CFyUAvxtiuMwx3w
Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA, netdev-u79uwXL29TY76Z2rM5mHXA,
liranl-VPRAkNaXOzVS1MOuV/RT9w, tziporet-VPRAkNaXOzVS1MOuV/RT9w,
yevgenyp-VPRAkNaXOzVS1MOuV/RT9w
From: Liran Liss <liranl-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
Resources are associated with functions by means of pds - the upper 7 msbs
specify the function id. Since cqs do not refer to a specific pd, the function
id is passed explicitly to hw in the cq context.
For uars, which are statically partitioned between slaves, we only need to adjust
their offsets with respect to the associated function bar.
Signed-off-by: Liran Liss <liranl-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
Signed-off-by: Yevgeny Petrilin <yevgenyp-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
---
drivers/net/mlx4/cq.c | 4 +++-
drivers/net/mlx4/eq.c | 13 ++++++++++++-
drivers/net/mlx4/pd.c | 12 +++++++++---
3 files changed, 24 insertions(+), 5 deletions(-)
diff --git a/drivers/net/mlx4/cq.c b/drivers/net/mlx4/cq.c
index 3fb9f7f..9f9f246 100644
--- a/drivers/net/mlx4/cq.c
+++ b/drivers/net/mlx4/cq.c
@@ -59,7 +59,8 @@ struct mlx4_cq_context {
__be32 solicit_producer_index;
__be32 consumer_index;
__be32 producer_index;
- u32 reserved4[2];
+ __be32 pd; /* for sriov guest id */
+ u32 reserved4;
__be64 db_rec_addr;
};
@@ -293,6 +294,7 @@ int mlx4_cq_alloc(struct mlx4_dev *dev, int nent, struct mlx4_mtt *mtt,
mtt_addr = mlx4_mtt_addr(dev, mtt);
cq_context->mtt_base_addr_h = mtt_addr >> 32;
cq_context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff);
+ cq_context->pd = cpu_to_be32(dev->caps.pd_base << dev->caps.slave_pd_shift);
cq_context->db_rec_addr = cpu_to_be64(db_rec);
err = mlx4_SW2HW_CQ(dev, mailbox, cq->cqn);
diff --git a/drivers/net/mlx4/eq.c b/drivers/net/mlx4/eq.c
index a27e1c4..46064cc 100644
--- a/drivers/net/mlx4/eq.c
+++ b/drivers/net/mlx4/eq.c
@@ -486,9 +486,20 @@ static void __iomem *mlx4_get_eq_uar(struct mlx4_dev *dev, struct mlx4_eq *eq)
{
struct mlx4_priv *priv = mlx4_priv(dev);
int index;
+ int offset;
- index = eq->eqn / 4 - dev->caps.reserved_eqs / 4;
+ /* CX1: slave EQ DBs are located in the comm channel page */
+ if (mlx4_is_slave(dev) || mlx4_is_master(dev)) {
+ if (eq->eqn - dev->caps.reserved_eqs >= MLX4_MFUNC_EQ_NUM) {
+ mlx4_err(dev, "eqn:%d doorbell out of range (reserved:%d)\n",
+ eq->eqn, dev->caps.reserved_eqs);
+ return NULL;
+ }
+ offset = 0x800 + (eq->eqn - dev->caps.reserved_eqs) * 8;
+ return ((void *) priv->mfunc.comm) + offset;
+ }
+ index = eq->eqn / 4 - dev->caps.reserved_eqs / 4;
if (!priv->eq_table.uar_map[index]) {
priv->eq_table.uar_map[index] =
ioremap(pci_resource_start(dev->pdev, 2) +
diff --git a/drivers/net/mlx4/pd.c b/drivers/net/mlx4/pd.c
index c4988d6..08eaf08 100644
--- a/drivers/net/mlx4/pd.c
+++ b/drivers/net/mlx4/pd.c
@@ -45,7 +45,7 @@ int mlx4_pd_alloc(struct mlx4_dev *dev, u32 *pdn)
*pdn = mlx4_bitmap_alloc(&priv->pd_bitmap);
if (*pdn == -1)
return -ENOMEM;
-
+ *pdn |= dev->caps.pd_base << dev->caps.slave_pd_shift;
return 0;
}
EXPORT_SYMBOL_GPL(mlx4_pd_alloc);
@@ -72,12 +72,18 @@ void mlx4_cleanup_pd_table(struct mlx4_dev *dev)
int mlx4_uar_alloc(struct mlx4_dev *dev, struct mlx4_uar *uar)
{
+ int offset;
+
uar->index = mlx4_bitmap_alloc(&mlx4_priv(dev)->uar_table.bitmap);
if (uar->index == -1)
return -ENOMEM;
- uar->pfn = (pci_resource_start(dev->pdev, 2) >> PAGE_SHIFT) + uar->index;
-
+ if (mlx4_is_slave(dev))
+ offset = uar->index % ((int) pci_resource_len(dev->pdev, 2) /
+ dev->caps.uar_page_size);
+ else
+ offset = uar->index;
+ uar->pfn = (pci_resource_start(dev->pdev, 2) >> PAGE_SHIFT) + offset;
return 0;
}
EXPORT_SYMBOL_GPL(mlx4_uar_alloc);
--
1.5.3.7
From: Liran Liss <liranl-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 13/25 v2] mlx4_core: remove unused driver/kernel uar (kar)
From: Yevgeny Petrilin @ 2009-11-06 3:09 UTC (permalink / raw)
To: rdreier-FYB4Gu1CFyUAvxtiuMwx3w
Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA, netdev-u79uwXL29TY76Z2rM5mHXA,
liranl-VPRAkNaXOzVS1MOuV/RT9w, tziporet-VPRAkNaXOzVS1MOuV/RT9w,
yevgenyp-VPRAkNaXOzVS1MOuV/RT9w
From: Liran Liss <liranl-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org
Signed-off-by: Liran Liss <liranl-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
Signed-off-by: Yevgeny Petrilin <yevgenyp-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
---
drivers/net/mlx4/main.c | 25 +------------------------
drivers/net/mlx4/mlx4.h | 2 --
2 files changed, 1 insertions(+), 26 deletions(-)
diff --git a/drivers/net/mlx4/main.c b/drivers/net/mlx4/main.c
index 4edc8d7..d8143c6 100644
--- a/drivers/net/mlx4/main.c
+++ b/drivers/net/mlx4/main.c
@@ -883,26 +883,11 @@ static int mlx4_setup_hca(struct mlx4_dev *dev)
return err;
}
- err = mlx4_uar_alloc(dev, &priv->driver_uar);
- if (err) {
- mlx4_err(dev, "Failed to allocate driver access region, "
- "aborting.\n");
- goto err_uar_table_free;
- }
-
- priv->kar = ioremap(priv->driver_uar.pfn << PAGE_SHIFT, PAGE_SIZE);
- if (!priv->kar) {
- mlx4_err(dev, "Couldn't map kernel access region, "
- "aborting.\n");
- err = -ENOMEM;
- goto err_uar_free;
- }
-
err = mlx4_init_pd_table(dev);
if (err) {
mlx4_err(dev, "Failed to initialize "
"protection domain table, aborting.\n");
- goto err_kar_unmap;
+ goto err_uar_table_free;
}
err = mlx4_init_mr_table(dev);
@@ -1015,12 +1000,6 @@ err_mr_table_free:
err_pd_table_free:
mlx4_cleanup_pd_table(dev);
-err_kar_unmap:
- iounmap(priv->kar);
-
-err_uar_free:
- mlx4_uar_free(dev, &priv->driver_uar);
-
err_uar_table_free:
mlx4_cleanup_uar_table(dev);
return err;
@@ -1316,8 +1295,6 @@ static void mlx4_remove_one(struct pci_dev *pdev)
mlx4_cleanup_mr_table(dev);
mlx4_cleanup_pd_table(dev);
- iounmap(priv->kar);
- mlx4_uar_free(dev, &priv->driver_uar);
mlx4_cleanup_uar_table(dev);
mlx4_free_eq_table(dev);
mlx4_close_hca(dev);
diff --git a/drivers/net/mlx4/mlx4.h b/drivers/net/mlx4/mlx4.h
index 0686856..f0afbb6 100644
--- a/drivers/net/mlx4/mlx4.h
+++ b/drivers/net/mlx4/mlx4.h
@@ -401,8 +401,6 @@ struct mlx4_priv {
void __iomem *clr_base;
- struct mlx4_uar driver_uar;
- void __iomem *kar;
struct mlx4_port_info port[MLX4_MAX_PORTS + 1];
struct mlx4_sense sense;
struct mutex port_mutex;
--
1.5.3.7
From: Liran Liss <liranl-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 16/25 v2] mlx4_core: boot sriov
From: Yevgeny Petrilin @ 2009-11-06 3:09 UTC (permalink / raw)
To: rdreier-FYB4Gu1CFyUAvxtiuMwx3w
Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA, netdev-u79uwXL29TY76Z2rM5mHXA,
liranl-VPRAkNaXOzVS1MOuV/RT9w, tziporet-VPRAkNaXOzVS1MOuV/RT9w,
yevgenyp-VPRAkNaXOzVS1MOuV/RT9w
From: Liran Liss <liranl-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
Add virtual function device ids. Setting VF flag to device private data
Establish comm channel when sriov is enabled, and boot vfs through it.
Each slave gets one msi-X for completions, the master also gets one for
async events.
Signed-off-by: Liran Liss <liranl-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
Signed-off-by: Yevgeny Petrilin <yevgenyp-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
---
drivers/net/mlx4/eq.c | 21 ++--
drivers/net/mlx4/main.c | 327 +++++++++++++++++++++++++++++++++++------------
drivers/net/mlx4/mlx4.h | 4 +
3 files changed, 264 insertions(+), 88 deletions(-)
diff --git a/drivers/net/mlx4/eq.c b/drivers/net/mlx4/eq.c
index b0c8022..29b2cd6 100644
--- a/drivers/net/mlx4/eq.c
+++ b/drivers/net/mlx4/eq.c
@@ -770,16 +770,18 @@ int mlx4_init_eq_table(struct mlx4_dev *dev)
goto err_out_unmap;
}
- err = mlx4_create_eq(dev, MLX4_NUM_ASYNC_EQE + MLX4_NUM_SPARE_EQE,
- (dev->flags & MLX4_FLAG_MSI_X) ? dev->caps.num_comp_vectors : 0,
- &priv->eq_table.eq[dev->caps.num_comp_vectors]);
- if (err)
- goto err_out_comp;
+ if (!mlx4_is_slave(dev)) {
+ err = mlx4_create_eq(dev, MLX4_NUM_ASYNC_EQE + MLX4_NUM_SPARE_EQE,
+ (dev->flags & MLX4_FLAG_MSI_X) ? dev->caps.num_comp_vectors : 0,
+ &priv->eq_table.eq[dev->caps.num_comp_vectors]);
+ if (err)
+ goto err_out_comp;
+ }
if (dev->flags & MLX4_FLAG_MSI_X) {
const char *eq_name;
- for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i) {
+ for (i = 0; i < dev->caps.num_comp_vectors + !(mlx4_is_slave(dev)); ++i) {
if (i < dev->caps.num_comp_vectors) {
snprintf(priv->eq_table.irq_names +
i * MLX4_IRQNAME_SIZE,
@@ -825,13 +827,14 @@ int mlx4_init_eq_table(struct mlx4_dev *dev)
priv->eq_table.eq[dev->caps.num_comp_vectors].eqn, err);
}
- for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i)
+ for (i = 0; i < dev->caps.num_comp_vectors + !(mlx4_is_slave(dev)); ++i)
eq_set_ci(&priv->eq_table.eq[i], 1);
return 0;
err_out_async:
- mlx4_free_eq(dev, &priv->eq_table.eq[dev->caps.num_comp_vectors]);
+ if (!mlx4_is_slave(dev))
+ mlx4_free_eq(dev, &priv->eq_table.eq[dev->caps.num_comp_vectors]);
err_out_comp:
i = dev->caps.num_comp_vectors;
@@ -866,7 +869,7 @@ void mlx4_cleanup_eq_table(struct mlx4_dev *dev)
mlx4_free_irqs(dev);
- for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i)
+ for (i = 0; i < dev->caps.num_comp_vectors + !mlx4_is_slave(dev); ++i)
mlx4_free_eq(dev, &priv->eq_table.eq[i]);
if (!mlx4_is_slave(dev))
diff --git a/drivers/net/mlx4/main.c b/drivers/net/mlx4/main.c
index 2852445..787bcde 100644
--- a/drivers/net/mlx4/main.c
+++ b/drivers/net/mlx4/main.c
@@ -73,6 +73,23 @@ MODULE_PARM_DESC(msi_x, "attempt to use MSI-X if nonzero");
#endif /* CONFIG_PCI_MSI */
+#ifdef CONFIG_PCI_IOV
+
+static int sr_iov;
+module_param(sr_iov, int, 0444);
+MODULE_PARM_DESC(sr_iov, "enable #sr_iov functions if sr_iov > 0");
+
+static int probe_vf;
+module_param(probe_vf, int, 0444);
+MODULE_PARM_DESC(probe_vf, "number of vfs to probe by pf driver (sr_iov > 0)");
+
+#else /* CONFIG_PCI_IOV */
+
+#define sr_iov 0
+#define probe_vf 0
+
+#endif /* CONFIG_PCI_IOV */
+
static char mlx4_version[] __devinitdata =
DRV_NAME ": Mellanox ConnectX core driver v"
DRV_VERSION " (" DRV_RELDATE ")\n";
@@ -779,12 +796,56 @@ static void mlx4_free_icms(struct mlx4_dev *dev)
mlx4_free_icm(dev, priv->fw.aux_icm, 0);
}
+static void mlx4_slave_exit(struct mlx4_dev *dev)
+{
+ struct mlx4_priv *priv = mlx4_priv(dev);
+
+ down(&priv->cmd.poll_sem);
+ if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_RESET, 0, MLX4_COMM_TIME))
+ mlx4_warn(dev, "Failed to close slave function.\n");
+ up(&priv->cmd.poll_sem);
+}
+
static void mlx4_close_hca(struct mlx4_dev *dev)
{
- mlx4_CLOSE_HCA(dev, 0);
- mlx4_free_icms(dev);
- mlx4_UNMAP_FA(dev);
- mlx4_free_icm(dev, mlx4_priv(dev)->fw.fw_icm, 0);
+ if (mlx4_is_slave(dev))
+ mlx4_slave_exit(dev);
+ else {
+ mlx4_CLOSE_HCA(dev, 0);
+ mlx4_free_icms(dev);
+ mlx4_UNMAP_FA(dev);
+ mlx4_free_icm(dev, mlx4_priv(dev)->fw.fw_icm, 0);
+ }
+}
+
+static int mlx4_init_slave(struct mlx4_dev *dev)
+{
+ struct mlx4_priv *priv = mlx4_priv(dev);
+ u64 dma = (u64) priv->mfunc.vhcr_dma;
+
+ down(&priv->cmd.poll_sem);
+ mlx4_warn(dev, "Sending reset\n");
+ if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_RESET, 0, MLX4_COMM_TIME))
+ goto err;
+ mlx4_warn(dev, "Sending vhcr0\n");
+ if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_VHCR0, dma >> 48,
+ MLX4_COMM_TIME))
+ goto err;
+ if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_VHCR1, dma >> 32,
+ MLX4_COMM_TIME))
+ goto err;
+ if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_VHCR2, dma >> 16,
+ MLX4_COMM_TIME))
+ goto err;
+ if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_VHCR_EN, dma, MLX4_COMM_TIME))
+ goto err;
+ up(&priv->cmd.poll_sem);
+ return 0;
+
+err:
+ mlx4_comm_cmd(dev, MLX4_COMM_CMD_RESET, 0, 0);
+ up(&priv->cmd.poll_sem);
+ return -EIO;
}
static int mlx4_init_hca(struct mlx4_dev *dev)
@@ -798,51 +859,67 @@ static int mlx4_init_hca(struct mlx4_dev *dev)
u64 icm_size;
int err;
- err = mlx4_QUERY_FW(dev);
- if (err) {
- if (err == -EACCES)
- mlx4_info(dev, "non-primary physical function, skipping.\n");
- else
- mlx4_err(dev, "QUERY_FW command failed, aborting.\n");
- return err;
- }
+ if (!mlx4_is_slave(dev)) {
+ err = mlx4_QUERY_FW(dev); /* TODO: verify FW version in slaves as well */
+ if (err) {
+ if (err == -EACCES)
+ mlx4_info(dev, "non-primary physical function, skipping.\n");
+ else {
+ mlx4_err(dev, "QUERY_FW command failed, aborting.\n");
+ if (mlx4_is_master(dev))
+ mlx4_err(dev, "Are you using SRIOV-enabled firmware?\n");
+ }
+ return err;
+ }
- err = mlx4_load_fw(dev);
- if (err) {
- mlx4_err(dev, "Failed to start FW, aborting.\n");
- return err;
- }
+ err = mlx4_load_fw(dev);
+ if (err) {
+ mlx4_err(dev, "Failed to start FW, aborting.\n");
+ return err;
+ }
- mlx4_cfg.log_pg_sz_m = 1;
- mlx4_cfg.log_pg_sz = 0;
- err = mlx4_MOD_STAT_CFG(dev, &mlx4_cfg);
- if (err)
- mlx4_warn(dev, "Failed to override log_pg_sz parameter\n");
+ mlx4_cfg.log_pg_sz_m = 1;
+ mlx4_cfg.log_pg_sz = 0;
+ err = mlx4_MOD_STAT_CFG(dev, &mlx4_cfg);
+ if (err)
+ mlx4_warn(dev, "Failed to override log_pg_sz parameter\n");
- err = mlx4_dev_cap(dev, &dev_cap);
- if (err) {
- mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting.\n");
- goto err_stop_fw;
- }
+ err = mlx4_dev_cap(dev, &dev_cap);
+ if (err) {
+ mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting.\n");
+ goto err_stop_fw;
+ }
- profile = default_profile;
+ profile = default_profile;
+ icm_size = mlx4_make_profile(dev, &profile, &dev_cap, &init_hca);
+ if ((long long) icm_size < 0) {
+ err = icm_size;
+ goto err_stop_fw;
+ }
- icm_size = mlx4_make_profile(dev, &profile, &dev_cap, &init_hca);
- if ((long long) icm_size < 0) {
- err = icm_size;
- goto err_stop_fw;
- }
+ init_hca.log_uar_sz = ilog2(dev->caps.num_uars);
- init_hca.log_uar_sz = ilog2(dev->caps.num_uars);
+ err = mlx4_init_icm(dev, &dev_cap, &init_hca, icm_size);
+ if (err)
+ goto err_stop_fw;
- err = mlx4_init_icm(dev, &dev_cap, &init_hca, icm_size);
- if (err)
- goto err_stop_fw;
+ err = mlx4_INIT_HCA(dev, &init_hca);
+ if (err) {
+ mlx4_err(dev, "INIT_HCA command failed, aborting.\n");
+ goto err_free_icm;
+ }
+ } else {
+ err = mlx4_init_slave(dev);
+ if (err) {
+ mlx4_err(dev, "Failed to initialize slave\n");
+ return err;
+ }
- err = mlx4_INIT_HCA(dev, &init_hca);
- if (err) {
- mlx4_err(dev, "INIT_HCA command failed, aborting.\n");
- goto err_free_icm;
+ err = mlx4_slave_cap(dev);
+ if (err) {
+ mlx4_err(dev, "Failed to obtain slave caps\n");
+ goto err_close;
+ }
}
err = mlx4_QUERY_ADAPTER(dev, &adapter);
@@ -857,15 +934,17 @@ static int mlx4_init_hca(struct mlx4_dev *dev)
return 0;
err_close:
- mlx4_CLOSE_HCA(dev, 0);
+ mlx4_close_hca(dev);
err_free_icm:
- mlx4_free_icms(dev);
+ if (!mlx4_is_slave(dev))
+ mlx4_free_icms(dev);
err_stop_fw:
- mlx4_UNMAP_FA(dev);
- mlx4_free_icm(dev, priv->fw.fw_icm, 0);
-
+ if (!mlx4_is_slave(dev)) {
+ mlx4_UNMAP_FA(dev);
+ mlx4_free_icm(dev, priv->fw.fw_icm, 0);
+ }
return err;
}
@@ -1019,8 +1098,13 @@ static void mlx4_enable_msi_x(struct mlx4_dev *dev)
int i;
if (msi_x) {
- nreq = min_t(int, dev->caps.num_eqs - dev->caps.reserved_eqs,
- num_possible_cpus() + 1);
+ /* The master only uses en event EQ,
+ * Each one of the slaves have 1 completion eq */
+ if (mlx4_is_mfunc(dev))
+ nreq = 1;
+ else
+ nreq = min_t(int, dev->caps.num_eqs - dev->caps.reserved_eqs,
+ num_possible_cpus() + 1);
entries = kcalloc(nreq, sizeof *entries, GFP_KERNEL);
if (!entries)
goto no_msi;
@@ -1029,8 +1113,10 @@ static void mlx4_enable_msi_x(struct mlx4_dev *dev)
entries[i].entry = i;
retry:
+ printk("Requesting %d MSIX vectors\n", nreq);
err = pci_enable_msix(dev->pdev, entries, nreq);
if (err) {
+ printk("pci_enable_msix failed\n");
/* Try again if at least 2 vectors are available */
if (err > 1) {
mlx4_info(dev, "Requested %d vectors, "
@@ -1043,7 +1129,7 @@ static void mlx4_enable_msi_x(struct mlx4_dev *dev)
goto no_msi;
}
- dev->caps.num_comp_vectors = nreq - 1;
+ dev->caps.num_comp_vectors = nreq - !mlx4_is_slave(dev);
for (i = 0; i < nreq; ++i)
priv->eq_table.eq[i].irq = entries[i].vector;
@@ -1054,7 +1140,9 @@ static void mlx4_enable_msi_x(struct mlx4_dev *dev)
}
no_msi:
- dev->caps.num_comp_vectors = 1;
+ /* CX1: no completion vectors needed for the master since it doesn't
+ * have interfaces */
+ dev->caps.num_comp_vectors = mlx4_is_master(dev) ? 0 : 1;
for (i = 0; i < 2; ++i)
priv->eq_table.eq[i].irq = dev->pdev->irq;
@@ -1111,10 +1199,9 @@ static int __mlx4_init_one(struct pci_dev *pdev, const struct pci_device_id *id)
}
/*
- * Check for BARs. We expect 0: 1MB
+ * Check for BARs.
*/
- if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM) ||
- pci_resource_len(pdev, 0) != 1 << 20) {
+ if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
dev_err(&pdev->dev, "Missing DCS, aborting.\n");
err = -ENODEV;
goto err_disable_pdev;
@@ -1172,34 +1259,84 @@ static int __mlx4_init_one(struct pci_dev *pdev, const struct pci_device_id *id)
INIT_LIST_HEAD(&priv->pgdir_list);
mutex_init(&priv->pgdir_mutex);
- /*
- * Now reset the HCA before we touch the PCI capabilities or
- * attempt a firmware command, since a boot ROM may have left
- * the HCA in an undefined state.
- */
- err = mlx4_reset(dev);
- if (err) {
- mlx4_err(dev, "Failed to reset HCA, aborting.\n");
- goto err_free_dev;
+ /* Detect if this device is a virtual function */
+ if (sr_iov && id->driver_data & MLX4_VF) {
+ /* When acting as pf, we normally skip vfs unless explicitly
+ * requested to probe them.
+ * TODO: add ARI support */
+ if (PCI_FUNC(pdev->devfn) > probe_vf) {
+ mlx4_warn(dev, "Skipping virtual function:%d\n",
+ PCI_FUNC(pdev->devfn));
+ err = -ENODEV;
+ goto err_free_dev;
+ }
+ mlx4_warn(dev, "Detected virtual function - running in slave mode\n");
+ dev->flags |= MLX4_FLAG_SLAVE;
+ }
+
+ /* We reset the device and enable SRIOV only for physical devices */
+ if (!mlx4_is_slave(dev)) {
+ /*
+ * Now reset the HCA before we touch the PCI capabilities or
+ * attempt a firmware command, since a boot ROM may have left
+ * the HCA in an undefined state.
+ */
+ err = mlx4_reset(dev);
+ if (err) {
+ mlx4_err(dev, "Failed to reset HCA, aborting.\n");
+ goto err_free_dev;
+ }
+ if (sr_iov) {
+ mlx4_warn(dev, "Enabling sriov with:%d vfs\n", sr_iov);
+ if (pci_enable_sriov(pdev, sr_iov)) {
+ mlx4_err(dev, "Failed to enable sriov, aborting.\n");
+ goto err_free_dev;
+ }
+ mlx4_warn(dev, "Running in master mode\n");
+ dev->flags |= MLX4_FLAG_SRIOV | MLX4_FLAG_MASTER;
+ dev->num_slaves = sr_iov;
+ }
}
if (mlx4_cmd_init(dev)) {
mlx4_err(dev, "Failed to init command interface, aborting.\n");
- goto err_free_dev;
+ goto err_sriov;
+ }
+
+ /* In slave functions, the communication channel must be initialized before
+ * posting commands */
+ if (mlx4_is_slave(dev)) {
+ if (mlx4_multi_func_init(dev)) {
+ mlx4_err(dev, "Failed to init slave mfunc interface, aborting.\n");
+ goto err_cmd;
+ }
}
err = mlx4_init_hca(dev);
if (err)
goto err_cmd;
+ /* In master functions, the communication channel must be initialized after obtaining
+ * its address from fw */
+ if (mlx4_is_master(dev)) {
+ if (mlx4_multi_func_init(dev)) {
+ mlx4_err(dev, "Failed to init master mfunc interface, aborting.\n");
+ goto err_close;
+ }
+ }
+
err = mlx4_alloc_eq_table(dev);
if (err)
goto err_close;
mlx4_enable_msi_x(dev);
+ if (mlx4_is_slave(dev) && !(dev->flags & MLX4_FLAG_MSI_X)) {
+ mlx4_err(dev, "INTx is not supported in slave mode, aborting.\n");
+ goto err_free_eq;
+ }
err = mlx4_setup_hca(dev);
- if (err == -EBUSY && (dev->flags & MLX4_FLAG_MSI_X)) {
+ if (err == -EBUSY && (dev->flags & MLX4_FLAG_MSI_X) && !mlx4_is_slave(dev)) {
dev->flags &= ~MLX4_FLAG_MSI_X;
pci_disable_msix(pdev);
err = mlx4_setup_hca(dev);
@@ -1223,6 +1360,12 @@ static int __mlx4_init_one(struct pci_dev *pdev, const struct pci_device_id *id)
pci_set_drvdata(pdev, dev);
+ /* Start serving comm channel:
+ * - In master function: poll for commands
+ * - in slave functions: poll for events
+ * TODO - enable comm channel interrupts */
+ if (mlx4_is_master(dev) || mlx4_is_slave(dev))
+ queue_delayed_work(priv->mfunc.comm_wq, &priv->mfunc.comm_work, 0);
return 0;
err_port:
@@ -1233,7 +1376,8 @@ err_port:
mlx4_cleanup_qp_table(dev);
mlx4_cleanup_srq_table(dev);
mlx4_cleanup_cq_table(dev);
- mlx4_cmd_use_polling(dev);
+ if (!mlx4_is_master(dev) && !mlx4_is_slave(dev))
+ mlx4_cmd_use_polling(dev);
mlx4_cleanup_eq_table(dev);
mlx4_cleanup_mr_table(dev);
mlx4_cleanup_pd_table(dev);
@@ -1251,6 +1395,12 @@ err_close:
err_cmd:
mlx4_cmd_cleanup(dev);
+err_sriov:
+ if (mlx4_is_master(dev) || mlx4_is_slave(dev))
+ mlx4_multi_func_cleanup(dev);
+ if (sr_iov && (dev->flags & MLX4_FLAG_SRIOV))
+ pci_disable_sriov(pdev);
+
err_free_dev:
kfree(priv);
@@ -1283,19 +1433,24 @@ static void mlx4_remove_one(struct pci_dev *pdev)
int p;
if (dev) {
+ /* Stop serving commands and events over comm channel */
+ if (mlx4_is_master(dev) || mlx4_is_slave(dev))
+ cancel_delayed_work_sync(&priv->mfunc.comm_work);
mlx4_stop_sense(dev);
mlx4_unregister_device(dev);
for (p = 1; p <= dev->caps.num_ports; p++) {
mlx4_cleanup_port_info(&priv->port[p]);
- mlx4_CLOSE_PORT(dev, p);
+ if (!mlx4_is_slave(dev))
+ mlx4_CLOSE_PORT(dev, p);
}
mlx4_cleanup_mcg_table(dev);
mlx4_cleanup_qp_table(dev);
mlx4_cleanup_srq_table(dev);
mlx4_cleanup_cq_table(dev);
- mlx4_cmd_use_polling(dev);
+ if (!mlx4_is_master(dev) && !mlx4_is_slave(dev))
+ mlx4_cmd_use_polling(dev);
mlx4_cleanup_eq_table(dev);
mlx4_cleanup_mr_table(dev);
mlx4_cleanup_pd_table(dev);
@@ -1303,10 +1458,16 @@ static void mlx4_remove_one(struct pci_dev *pdev)
mlx4_cleanup_uar_table(dev);
mlx4_free_eq_table(dev);
mlx4_close_hca(dev);
+ if (mlx4_is_master(dev) || mlx4_is_slave(dev))
+ mlx4_multi_func_cleanup(dev);
mlx4_cmd_cleanup(dev);
if (dev->flags & MLX4_FLAG_MSI_X)
pci_disable_msix(pdev);
+ if (sr_iov && (dev->flags & MLX4_FLAG_SRIOV)) {
+ mlx4_warn(dev, "Disabling sriov\n");
+ pci_disable_sriov(pdev);
+ }
kfree(priv);
pci_release_regions(pdev);
@@ -1321,19 +1482,27 @@ int mlx4_restart_one(struct pci_dev *pdev)
return __mlx4_init_one(pdev, NULL);
}
+
static struct pci_device_id mlx4_pci_table[] = {
- { PCI_VDEVICE(MELLANOX, 0x6340) }, /* MT25408 "Hermon" SDR */
- { PCI_VDEVICE(MELLANOX, 0x634a) }, /* MT25408 "Hermon" DDR */
- { PCI_VDEVICE(MELLANOX, 0x6354) }, /* MT25408 "Hermon" QDR */
- { PCI_VDEVICE(MELLANOX, 0x6732) }, /* MT25408 "Hermon" DDR PCIe gen2 */
- { PCI_VDEVICE(MELLANOX, 0x673c) }, /* MT25408 "Hermon" QDR PCIe gen2 */
- { PCI_VDEVICE(MELLANOX, 0x6368) }, /* MT25408 "Hermon" EN 10GigE */
- { PCI_VDEVICE(MELLANOX, 0x6750) }, /* MT25408 "Hermon" EN 10GigE PCIe gen2 */
- { PCI_VDEVICE(MELLANOX, 0x6372) }, /* MT25458 ConnectX EN 10GBASE-T 10GigE */
- { PCI_VDEVICE(MELLANOX, 0x675a) }, /* MT25458 ConnectX EN 10GBASE-T+Gen2 10GigE */
- { PCI_VDEVICE(MELLANOX, 0x6764) }, /* MT26468 ConnectX EN 10GigE PCIe gen2*/
- { PCI_VDEVICE(MELLANOX, 0x6746) }, /* MT26438 ConnectX EN 40GigE PCIe gen2 5GT/s */
- { PCI_VDEVICE(MELLANOX, 0x676e) }, /* MT26478 ConnectX2 40GigE PCIe gen2 */
+ { MLX4_VDEVICE(MELLANOX, 0x6340, 0) }, /* MT25408 "Hermon" SDR */
+ { MLX4_VDEVICE(MELLANOX, 0x6341, MLX4_VF) }, /* MT25408 "Hermon" SDR VF */
+ { MLX4_VDEVICE(MELLANOX, 0x634a, 0) }, /* MT25408 "Hermon" DDR */
+ { MLX4_VDEVICE(MELLANOX, 0x634b, MLX4_VF) }, /* MT25408 "Hermon" DDR VF */
+ { MLX4_VDEVICE(MELLANOX, 0x6354, 0) }, /* MT25408 "Hermon" QDR */
+ { MLX4_VDEVICE(MELLANOX, 0x6732, 0) }, /* MT25408 "Hermon" DDR PCIe gen2 */
+ { MLX4_VDEVICE(MELLANOX, 0x6733, MLX4_VF) }, /* MT25408 "Hermon" DDR PCIe gen2 VF */
+ { MLX4_VDEVICE(MELLANOX, 0x673c, 0) }, /* MT25408 "Hermon" QDR PCIe gen2 */
+ { MLX4_VDEVICE(MELLANOX, 0x673d, MLX4_VF) }, /* MT25408 "Hermon" QDR PCIe gen2 VF */
+ { MLX4_VDEVICE(MELLANOX, 0x6368, 0) }, /* MT25408 "Hermon" EN 10GigE */
+ { MLX4_VDEVICE(MELLANOX, 0x6369, MLX4_VF) }, /* MT25408 "Hermon" EN 10GigE VF */
+ { MLX4_VDEVICE(MELLANOX, 0x6750, 0) }, /* MT25408 "Hermon" EN 10GigE PCIe gen2 */
+ { MLX4_VDEVICE(MELLANOX, 0x6751, MLX4_VF) }, /* MT25408 "Hermon" EN 10GigE PCIe gen2 VF */
+ { MLX4_VDEVICE(MELLANOX, 0x6372, 0) }, /* MT25458 ConnectX EN 10GBASE-T 10GigE */
+ { MLX4_VDEVICE(MELLANOX, 0x675a, 0) }, /* MT25458 ConnectX EN 10GBASE-T+Gen2 10GigE */
+ { MLX4_VDEVICE(MELLANOX, 0x6764, MLX4_VF) }, /* MT26468 ConnectX EN 10GigE PCIe gen2*/
+ { MLX4_VDEVICE(MELLANOX, 0x6765, 0) }, /* MT26468 ConnectX EN 10GigE PCIe gen2 VF*/
+ { MLX4_VDEVICE(MELLANOX, 0x6746, 0) }, /* MT26438 ConnectX EN 40GigE PCIe gen2 5GT/s */
+ { MLX4_VDEVICE(MELLANOX, 0x676e, 0) }, /* MT26478 ConnectX2 40GigE PCIe gen2 */
{ 0, }
};
diff --git a/drivers/net/mlx4/mlx4.h b/drivers/net/mlx4/mlx4.h
index f0afbb6..b94c590 100644
--- a/drivers/net/mlx4/mlx4.h
+++ b/drivers/net/mlx4/mlx4.h
@@ -137,6 +137,10 @@ extern int mlx4_debug_level;
#define mlx4_warn(mdev, format, arg...) \
dev_warn(&mdev->pdev->dev, format, ## arg)
+#define MLX4_VF (1 << 0)
+#define MLX4_VDEVICE(vendor, device, flags) \
+ PCI_VDEVICE(vendor, device), (flags)
+
struct mlx4_bitmap {
u32 last;
u32 top;
--
1.5.3.7
From: Liran Liss <liranl-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 17/25 v2] mlx4: Changed Mac management
From: Yevgeny Petrilin @ 2009-11-06 3:09 UTC (permalink / raw)
To: rdreier-FYB4Gu1CFyUAvxtiuMwx3w
Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA, netdev-u79uwXL29TY76Z2rM5mHXA,
liranl-VPRAkNaXOzVS1MOuV/RT9w, tziporet-VPRAkNaXOzVS1MOuV/RT9w,
yevgenyp-VPRAkNaXOzVS1MOuV/RT9w
RX qp is given from a pool of previously reserved range of QPs.
Activating Mac steering now.
Giving an option to change mac without the need to change the RX QP.
All the mac management is done by the master, added wrappers for the
mac management.
Signed-off-by: Yevgeny Petrilin <yevgenyp-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
---
drivers/net/mlx4/cmd.c | 26 +++++++++++
drivers/net/mlx4/en_netdev.c | 32 +++++++-------
drivers/net/mlx4/en_port.c | 1 +
drivers/net/mlx4/en_port.h | 8 +++-
drivers/net/mlx4/en_rx.c | 11 +----
drivers/net/mlx4/main.c | 11 +++--
drivers/net/mlx4/mlx4.h | 7 ++-
drivers/net/mlx4/port.c | 96 ++++++++++++++++++++++++++++++++----------
include/linux/mlx4/cmd.h | 9 ++--
include/linux/mlx4/device.h | 5 +-
10 files changed, 144 insertions(+), 62 deletions(-)
diff --git a/drivers/net/mlx4/cmd.c b/drivers/net/mlx4/cmd.c
index d68d0e1..71d822a 100644
--- a/drivers/net/mlx4/cmd.c
+++ b/drivers/net/mlx4/cmd.c
@@ -507,6 +507,24 @@ static int mlx4_RESOURCE_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vh
} else
mlx4_free_mtt_range(dev, param1 /* first */, param2 /* order */);
break;
+ case RES_MAC:
+ switch (vhcr->op) {
+ case MLX4_CMD_ALLOC_RES:
+ ret = mlx4_register_mac(dev, vhcr->op_modifier,
+ vhcr->in_param, (int *) &vhcr->out_param);
+ vhcr->errno = ret;
+ break;
+ case MLX4_CMD_FREE_RES:
+ mlx4_unregister_mac(dev, vhcr->op_modifier, vhcr->in_param);
+ break;
+ case MLX4_CMD_REPLACE_RES:
+ ret = mlx4_replace_mac(dev, vhcr->op_modifier,
+ vhcr->out_param, vhcr->in_param);
+ vhcr->errno = ret;
+ break;
+ default:
+ vhcr->errno = -EINVAL;
+ }
default:
vhcr->errno = -EINVAL;
}
@@ -622,6 +640,14 @@ static struct mlx4_cmd_info {
.wrapper = mlx4_RESOURCE_wrapper
},
{
+ .opcode = MLX4_CMD_REPLACE_RES,
+ .has_inbox = false,
+ .has_outbox = false,
+ .out_is_imm = true,
+ .verify = NULL,
+ .wrapper = mlx4_RESOURCE_wrapper
+ },
+ {
.opcode = MLX4_CMD_GET_EVENT,
.has_inbox = false,
.has_outbox = false,
diff --git a/drivers/net/mlx4/en_netdev.c b/drivers/net/mlx4/en_netdev.c
index c48b0f4..fd96078 100644
--- a/drivers/net/mlx4/en_netdev.c
+++ b/drivers/net/mlx4/en_netdev.c
@@ -145,9 +145,8 @@ static void mlx4_en_do_set_mac(struct work_struct *work)
mutex_lock(&mdev->state_lock);
if (priv->port_up) {
/* Remove old MAC and insert the new one */
- mlx4_unregister_mac(mdev->dev, priv->port, priv->mac_index);
- err = mlx4_register_mac(mdev->dev, priv->port,
- priv->mac, &priv->mac_index);
+ err = mlx4_replace_mac(mdev->dev, priv->port,
+ priv->base_qpn, priv->mac);
if (err)
en_err(priv, "Failed changing HW MAC address\n");
} else
@@ -596,10 +595,19 @@ int mlx4_en_start_port(struct net_device *dev)
++rx_index;
}
+ /* Set port mac number */
+ en_dbg(DRV, priv, "Setting mac for port %d\n", priv->port);
+ err = mlx4_register_mac(mdev->dev, priv->port,
+ priv->mac, &priv->base_qpn);
+ if (err) {
+ en_err(priv, "Failed setting port mac\n");
+ goto cq_err;
+ }
+
err = mlx4_en_config_rss_steer(priv);
if (err) {
en_err(priv, "Failed configuring rss steering\n");
- goto cq_err;
+ goto mac_err;
}
/* Configure tx cq's and rings */
@@ -652,21 +660,13 @@ int mlx4_en_start_port(struct net_device *dev)
en_err(priv, "Failed setting default qp numbers\n");
goto tx_err;
}
- /* Set port mac number */
- en_dbg(DRV, priv, "Setting mac for port %d\n", priv->port);
- err = mlx4_register_mac(mdev->dev, priv->port,
- priv->mac, &priv->mac_index);
- if (err) {
- en_err(priv, "Failed setting port mac\n");
- goto tx_err;
- }
/* Init port */
en_dbg(HW, priv, "Initializing port\n");
err = mlx4_INIT_PORT(mdev->dev, priv->port);
if (err) {
en_err(priv, "Failed Initializing port\n");
- goto mac_err;
+ goto tx_err;
}
/* Schedule multicast task to populate multicast list */
@@ -676,8 +676,6 @@ int mlx4_en_start_port(struct net_device *dev)
netif_tx_start_all_queues(dev);
return 0;
-mac_err:
- mlx4_unregister_mac(mdev->dev, priv->port, priv->mac_index);
tx_err:
while (tx_index--) {
mlx4_en_deactivate_tx_ring(priv, &priv->tx_ring[tx_index]);
@@ -685,6 +683,8 @@ tx_err:
}
mlx4_en_release_rss_steer(priv);
+mac_err:
+ mlx4_unregister_mac(mdev->dev, priv->port, priv->base_qpn);
cq_err:
while (rx_index--)
mlx4_en_deactivate_cq(priv, &priv->rx_cq[rx_index]);
@@ -716,7 +716,7 @@ void mlx4_en_stop_port(struct net_device *dev)
mlx4_CLOSE_PORT(mdev->dev, priv->port);
/* Unregister Mac address for the port */
- mlx4_unregister_mac(mdev->dev, priv->port, priv->mac_index);
+ mlx4_unregister_mac(mdev->dev, priv->port, priv->base_qpn);
/* Free TX Rings */
for (i = 0; i < priv->tx_ring_num; i++) {
diff --git a/drivers/net/mlx4/en_port.c b/drivers/net/mlx4/en_port.c
index a29abe8..c099cb4 100644
--- a/drivers/net/mlx4/en_port.c
+++ b/drivers/net/mlx4/en_port.c
@@ -127,6 +127,7 @@ int mlx4_SET_PORT_qpn_calc(struct mlx4_dev *dev, u8 port, u32 base_qpn,
memset(context, 0, sizeof *context);
context->base_qpn = cpu_to_be32(base_qpn);
+ context->n_mac = 0x7;
context->promisc = cpu_to_be32(promisc << SET_PORT_PROMISC_SHIFT | base_qpn);
context->mcast = cpu_to_be32(1 << SET_PORT_PROMISC_SHIFT | base_qpn);
context->intra_no_vlan = 0;
diff --git a/drivers/net/mlx4/en_port.h b/drivers/net/mlx4/en_port.h
index 3892896..62c9135 100644
--- a/drivers/net/mlx4/en_port.h
+++ b/drivers/net/mlx4/en_port.h
@@ -54,14 +54,18 @@ struct mlx4_set_port_general_context {
struct mlx4_set_port_rqp_calc_context {
__be32 base_qpn;
+ u8 rererved;
+ u8 n_mac;
+ u8 n_vlan;
+ u8 n_prio;
__be32 flags;
- u8 reserved[3];
+ u8 reserved2[3];
u8 mac_miss;
u8 intra_no_vlan;
u8 no_vlan;
u8 intra_vlan_miss;
u8 vlan_miss;
- u8 reserved2[3];
+ u8 reserved3[3];
u8 no_vlan_prio;
__be32 promisc;
__be32 mcast;
diff --git a/drivers/net/mlx4/en_rx.c b/drivers/net/mlx4/en_rx.c
index 03b781a..d3884ce 100644
--- a/drivers/net/mlx4/en_rx.c
+++ b/drivers/net/mlx4/en_rx.c
@@ -865,16 +865,10 @@ int mlx4_en_config_rss_steer(struct mlx4_en_priv *priv)
}
/* Configure RSS indirection qp */
- err = mlx4_qp_reserve_range(mdev->dev, 1, 1, &priv->base_qpn);
- if (err) {
- en_err(priv, "Failed to reserve range for RSS "
- "indirection qp\n");
- goto rss_err;
- }
err = mlx4_qp_alloc(mdev->dev, priv->base_qpn, &rss_map->indir_qp);
if (err) {
en_err(priv, "Failed to allocate RSS indirection QP\n");
- goto reserve_err;
+ goto rss_err;
}
rss_map->indir_qp.event = mlx4_en_sqp_event;
mlx4_en_fill_qp_context(priv, 0, 0, 0, 1, priv->base_qpn,
@@ -900,8 +894,6 @@ indir_err:
MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->indir_qp);
mlx4_qp_remove(mdev->dev, &rss_map->indir_qp);
mlx4_qp_free(mdev->dev, &rss_map->indir_qp);
-reserve_err:
- mlx4_qp_release_range(mdev->dev, priv->base_qpn, 1);
rss_err:
for (i = 0; i < good_qps; i++) {
mlx4_qp_modify(mdev->dev, NULL, rss_map->state[i],
@@ -923,7 +915,6 @@ void mlx4_en_release_rss_steer(struct mlx4_en_priv *priv)
MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->indir_qp);
mlx4_qp_remove(mdev->dev, &rss_map->indir_qp);
mlx4_qp_free(mdev->dev, &rss_map->indir_qp);
- mlx4_qp_release_range(mdev->dev, priv->base_qpn, 1);
for (i = 0; i < priv->rx_ring_num; i++) {
mlx4_qp_modify(mdev->dev, NULL, rss_map->state[i],
diff --git a/drivers/net/mlx4/main.c b/drivers/net/mlx4/main.c
index 787bcde..45ce765 100644
--- a/drivers/net/mlx4/main.c
+++ b/drivers/net/mlx4/main.c
@@ -104,7 +104,7 @@ static struct mlx4_profile default_profile = {
.num_mtt = 1 << 20,
};
-static int log_num_mac = 2;
+static int log_num_mac = 7;
module_param_named(log_num_mac, log_num_mac, int, 0444);
MODULE_PARM_DESC(log_num_mac, "Log2 max number of MACs per ETH port (1-7)");
@@ -1155,9 +1155,12 @@ static int mlx4_init_port_info(struct mlx4_dev *dev, int port)
info->dev = dev;
info->port = port;
- mlx4_init_mac_table(dev, &info->mac_table);
- mlx4_init_vlan_table(dev, &info->vlan_table);
-
+ if (!mlx4_is_slave(dev)) {
+ mlx4_init_mac_table(dev, &info->mac_table);
+ mlx4_init_vlan_table(dev, &info->vlan_table);
+ info->base_qpn = dev->caps.reserved_qps_base[MLX4_QP_REGION_ETH_ADDR] +
+ (port - 1) * (1 << log_num_mac);
+ }
sprintf(info->dev_name, "mlx4_port%d", port);
info->port_attr.attr.name = info->dev_name;
info->port_attr.attr.mode = S_IRUGO | S_IWUSR;
diff --git a/drivers/net/mlx4/mlx4.h b/drivers/net/mlx4/mlx4.h
index b94c590..19c87e7 100644
--- a/drivers/net/mlx4/mlx4.h
+++ b/drivers/net/mlx4/mlx4.h
@@ -101,7 +101,10 @@ enum mlx4_resource {
RES_CQ,
RES_SRQ,
RES_MPT,
- RES_MTT
+ RES_MTT,
+ RES_MAC,
+ RES_VLAN,
+ RES_MCAST
};
enum mlx4_alloc_mode {
@@ -344,7 +347,6 @@ struct mlx4_catas_err {
struct mlx4_mac_table {
__be64 entries[MLX4_MAX_MAC_NUM];
- int refs[MLX4_MAX_MAC_NUM];
struct mutex mutex;
int total;
int max;
@@ -369,6 +371,7 @@ struct mlx4_port_info {
enum mlx4_port_type tmp_type;
struct mlx4_mac_table mac_table;
struct mlx4_vlan_table vlan_table;
+ int base_qpn;
};
struct mlx4_sense {
diff --git a/drivers/net/mlx4/port.c b/drivers/net/mlx4/port.c
index 67f0751..6e729dd 100644
--- a/drivers/net/mlx4/port.c
+++ b/drivers/net/mlx4/port.c
@@ -48,10 +48,8 @@ void mlx4_init_mac_table(struct mlx4_dev *dev, struct mlx4_mac_table *table)
int i;
mutex_init(&table->mutex);
- for (i = 0; i < MLX4_MAX_MAC_NUM; i++) {
+ for (i = 0; i < MLX4_MAX_MAC_NUM; i++)
table->entries[i] = 0;
- table->refs[i] = 0;
- }
table->max = 1 << dev->caps.log_num_macs;
table->total = 0;
}
@@ -90,24 +88,33 @@ static int mlx4_set_port_mac_table(struct mlx4_dev *dev, u8 port,
return err;
}
-int mlx4_register_mac(struct mlx4_dev *dev, u8 port, u64 mac, int *index)
+int mlx4_register_mac(struct mlx4_dev *dev, u8 port, u64 mac, int *qpn)
{
- struct mlx4_mac_table *table = &mlx4_priv(dev)->port[port].mac_table;
+ struct mlx4_port_info *info = &mlx4_priv(dev)->port[port];
+ struct mlx4_mac_table *table = &info->mac_table;
+ u64 out_param;
int i, err = 0;
int free = -1;
+ if (mlx4_is_slave(dev)) {
+ err = mlx4_cmd_imm(dev, mac, &out_param, RES_MAC, port,
+ MLX4_CMD_ALLOC_RES, MLX4_CMD_TIME_CLASS_A);
+ if (!err)
+ *qpn = out_param;
+ return err;
+ }
+
mlx4_dbg(dev, "Registering MAC: 0x%llx\n", (unsigned long long) mac);
mutex_lock(&table->mutex);
for (i = 0; i < MLX4_MAX_MAC_NUM - 1; i++) {
- if (free < 0 && !table->refs[i]) {
+ if (free < 0 && !table->entries[i]) {
free = i;
continue;
}
if (mac == (MLX4_MAC_MASK & be64_to_cpu(table->entries[i]))) {
- /* MAC already registered, increase refernce count */
- *index = i;
- ++table->refs[i];
+ /* MAC already registered, Must not have duplicates */
+ err = -EEXIST;
goto out;
}
}
@@ -120,18 +127,16 @@ int mlx4_register_mac(struct mlx4_dev *dev, u8 port, u64 mac, int *index)
}
/* Register new MAC */
- table->refs[free] = 1;
table->entries[free] = cpu_to_be64(mac | MLX4_MAC_VALID);
err = mlx4_set_port_mac_table(dev, port, table->entries);
if (unlikely(err)) {
mlx4_err(dev, "Failed adding MAC: 0x%llx\n", (unsigned long long) mac);
- table->refs[free] = 0;
table->entries[free] = 0;
goto out;
}
- *index = free;
+ *qpn = info->base_qpn + free;
++table->total;
out:
mutex_unlock(&table->mutex);
@@ -139,20 +144,35 @@ out:
}
EXPORT_SYMBOL_GPL(mlx4_register_mac);
-void mlx4_unregister_mac(struct mlx4_dev *dev, u8 port, int index)
+static int validate_index(struct mlx4_dev *dev,
+ struct mlx4_mac_table *table, int index)
{
- struct mlx4_mac_table *table = &mlx4_priv(dev)->port[port].mac_table;
+ int err = 0;
- mutex_lock(&table->mutex);
- if (!table->refs[index]) {
- mlx4_warn(dev, "No MAC entry for index %d\n", index);
- goto out;
+ if (index < 0 || index >= table->max || !table->entries[index]) {
+ mlx4_warn(dev, "No valid Mac entry for the given index\n");
+ err = -EINVAL;
}
- if (--table->refs[index]) {
- mlx4_warn(dev, "Have more references for index %d,"
- "no need to modify MAC table\n", index);
- goto out;
+ return err;
+}
+
+void mlx4_unregister_mac(struct mlx4_dev *dev, u8 port, int qpn)
+{
+ struct mlx4_port_info *info = &mlx4_priv(dev)->port[port];
+ struct mlx4_mac_table *table = &info->mac_table;
+ int index = qpn - info->base_qpn;
+
+ if (mlx4_is_slave(dev)) {
+ mlx4_cmd(dev, qpn, RES_MAC, port,
+ MLX4_CMD_FREE_RES, MLX4_CMD_TIME_CLASS_A);
+ return;
}
+
+ mutex_lock(&table->mutex);
+
+ if (validate_index(dev, table, index))
+ goto out;
+
table->entries[index] = 0;
mlx4_set_port_mac_table(dev, port, table->entries);
--table->total;
@@ -161,6 +181,38 @@ out:
}
EXPORT_SYMBOL_GPL(mlx4_unregister_mac);
+int mlx4_replace_mac(struct mlx4_dev *dev, u8 port, int qpn, u64 new_mac)
+{
+ struct mlx4_port_info *info = &mlx4_priv(dev)->port[port];
+ struct mlx4_mac_table *table = &info->mac_table;
+ int index = qpn - info->base_qpn;
+ int err;
+
+ if (mlx4_is_slave(dev)) {
+ err = mlx4_cmd_imm(dev, new_mac, (u64 *) &qpn, RES_MAC, port,
+ MLX4_CMD_REPLACE_RES, MLX4_CMD_TIME_CLASS_A);
+ return err;
+ }
+
+ mutex_lock(&table->mutex);
+
+ err = validate_index(dev, table, index);
+ if (err)
+ goto out;
+
+ table->entries[index] = cpu_to_be64(new_mac | MLX4_MAC_VALID);
+
+ err = mlx4_set_port_mac_table(dev, port, table->entries);
+ if (unlikely(err)) {
+ mlx4_err(dev, "Failed adding MAC: 0x%llx\n", (unsigned long long) new_mac);
+ table->entries[index] = 0;
+ }
+out:
+ mutex_unlock(&table->mutex);
+ return err;
+}
+EXPORT_SYMBOL_GPL(mlx4_replace_mac);
+
static int mlx4_set_port_vlan_table(struct mlx4_dev *dev, u8 port,
__be32 *entries)
{
diff --git a/include/linux/mlx4/cmd.h b/include/linux/mlx4/cmd.h
index 99145d4..c163d5e 100644
--- a/include/linux/mlx4/cmd.h
+++ b/include/linux/mlx4/cmd.h
@@ -127,10 +127,11 @@ enum {
/* virtual commands */
MLX4_CMD_ALLOC_RES = 0x50,
MLX4_CMD_FREE_RES = 0x51,
- MLX4_CMD_GET_EVENT = 0x52,
- MLX4_CMD_QUERY_SLAVE_CAP = 0x53,
- MLX4_CMD_MCAST_ATTACH = 0x54,
- MLX4_CMD_GET_SLAVE_SQP = 0x55,
+ MLX4_CMD_REPLACE_RES = 0x52,
+ MLX4_CMD_GET_EVENT = 0x53,
+ MLX4_CMD_QUERY_SLAVE_CAP = 0x54,
+ MLX4_CMD_MCAST_ATTACH = 0x55,
+ MLX4_CMD_GET_SLAVE_SQP = 0x56,
/* debug commands */
MLX4_CMD_QUERY_DEBUG_MSG = 0x2a,
diff --git a/include/linux/mlx4/device.h b/include/linux/mlx4/device.h
index bbd398b..246b7bc 100644
--- a/include/linux/mlx4/device.h
+++ b/include/linux/mlx4/device.h
@@ -491,8 +491,9 @@ int mlx4_multicast_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16],
int block_mcast_loopback);
int mlx4_multicast_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16]);
-int mlx4_register_mac(struct mlx4_dev *dev, u8 port, u64 mac, int *index);
-void mlx4_unregister_mac(struct mlx4_dev *dev, u8 port, int index);
+int mlx4_register_mac(struct mlx4_dev *dev, u8 port, u64 mac, int *qpn);
+void mlx4_unregister_mac(struct mlx4_dev *dev, u8 port, int qpn);
+int mlx4_replace_mac(struct mlx4_dev *dev, u8 port, int qpn, u64 new_mac);
int mlx4_register_vlan(struct mlx4_dev *dev, u8 port, u16 vlan, int *index);
void mlx4_unregister_vlan(struct mlx4_dev *dev, u8 port, int index);
--
1.5.3.7
From: Liran Liss <liranl-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 19/25 v2] mlx4: Randomizing mac addresses for slaves
From: Yevgeny Petrilin @ 2009-11-06 3:10 UTC (permalink / raw)
To: rdreier-FYB4Gu1CFyUAvxtiuMwx3w
Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA, netdev-u79uwXL29TY76Z2rM5mHXA,
liranl-VPRAkNaXOzVS1MOuV/RT9w, tziporet-VPRAkNaXOzVS1MOuV/RT9w,
yevgenyp-VPRAkNaXOzVS1MOuV/RT9w
Signed-off-by: Yevgeny Petrilin <yevgenyp-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
---
drivers/net/mlx4/fw.c | 8 +++++++-
1 files changed, 7 insertions(+), 1 deletions(-)
diff --git a/drivers/net/mlx4/fw.c b/drivers/net/mlx4/fw.c
index 60889d3..7343c70 100644
--- a/drivers/net/mlx4/fw.c
+++ b/drivers/net/mlx4/fw.c
@@ -32,6 +32,7 @@
* SOFTWARE.
*/
+#include <linux/etherdevice.h>
#include <linux/mlx4/cmd.h>
#include <linux/cache.h>
@@ -148,7 +149,8 @@ int mlx4_QUERY_SLAVE_CAP_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vh
struct mlx4_cmd_mailbox *outbox)
{
struct mlx4_caps *caps = outbox->buf;
- int i;
+ u8 rand_mac[6];
+ int i, j;
memcpy(caps, &dev->caps, sizeof *caps);
@@ -165,6 +167,10 @@ int mlx4_QUERY_SLAVE_CAP_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vh
for (i = 1; i <= dev->caps.num_ports; ++i) {
caps->gid_table_len[i] = 1;
caps->pkey_table_len[i] = 1;
+ random_ether_addr(rand_mac);
+ caps->def_mac[i] = 0;
+ for (j = 0; j < ETH_ALEN; j++)
+ caps->def_mac[i] |= ((u64)(rand_mac[1]) << 8 * j);
}
} else {
caps->sqp_demux = dev->num_slaves;
--
1.5.3.7
From: Liran Liss <liranl-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 21/25 v2] mlx4: Adding protocol fields to multicast group
From: Yevgeny Petrilin @ 2009-11-06 3:10 UTC (permalink / raw)
To: rdreier-FYB4Gu1CFyUAvxtiuMwx3w
Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA, netdev-u79uwXL29TY76Z2rM5mHXA,
liranl-VPRAkNaXOzVS1MOuV/RT9w, tziporet-VPRAkNaXOzVS1MOuV/RT9w,
yevgenyp-VPRAkNaXOzVS1MOuV/RT9w
The multicast attachment mechanism will be used both by
IB and Ethernet, so we need to specify for each multicast
address (whether it is gid or mac) its protocol.
Signed-off-by: Yevgeny Petrilin <yevgenyp-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
---
drivers/infiniband/hw/mlx4/main.c | 6 +++-
drivers/net/mlx4/mcg.c | 41 +++++++++++++++++++++---------------
include/linux/mlx4/device.h | 12 +++++++++-
3 files changed, 38 insertions(+), 21 deletions(-)
diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c
index 3cb3f47..5b67a31 100644
--- a/drivers/infiniband/hw/mlx4/main.c
+++ b/drivers/infiniband/hw/mlx4/main.c
@@ -450,13 +450,15 @@ static int mlx4_ib_mcg_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
return mlx4_multicast_attach(to_mdev(ibqp->device)->dev,
&to_mqp(ibqp)->mqp, gid->raw,
!!(to_mqp(ibqp)->flags &
- MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK));
+ MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK),
+ MLX4_PROT_IB_IPV6);
}
static int mlx4_ib_mcg_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
{
return mlx4_multicast_detach(to_mdev(ibqp->device)->dev,
- &to_mqp(ibqp)->mqp, gid->raw);
+ &to_mqp(ibqp)->mqp, gid->raw,
+ MLX4_PROT_IB_IPV6);
}
static int init_node_data(struct mlx4_ib_dev *dev)
diff --git a/drivers/net/mlx4/mcg.c b/drivers/net/mlx4/mcg.c
index daa08f1..c4f928a 100644
--- a/drivers/net/mlx4/mcg.c
+++ b/drivers/net/mlx4/mcg.c
@@ -96,7 +96,8 @@ static int mlx4_MGID_HASH(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox
* entry in hash chain and *mgm holds end of hash chain.
*/
static int find_mgm(struct mlx4_dev *dev,
- u8 *gid, struct mlx4_cmd_mailbox *mgm_mailbox,
+ u8 *gid, enum mlx4_protocol prot,
+ struct mlx4_cmd_mailbox *mgm_mailbox,
u16 *hash, int *prev, int *index)
{
struct mlx4_cmd_mailbox *mailbox;
@@ -135,8 +136,9 @@ static int find_mgm(struct mlx4_dev *dev,
return err;
}
- if (!memcmp(mgm->gid, gid, 16))
- return err;
+ if (!memcmp(mgm->gid, gid, 16) &&
+ (prot == be32_to_cpu(mgm->members_count) >> 30))
+ return err;
*prev = *index;
*index = be32_to_cpu(mgm->next_gid_index) >> 6;
@@ -154,14 +156,17 @@ int mlx4_MCAST_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr,
qp.qpn = vhcr->in_modifier & 0xffffff;
if (vhcr->op_modifier)
- return mlx4_multicast_attach(dev, &qp, inbox->buf, vhcr->in_modifier >> 31);
+ return mlx4_multicast_attach(dev, &qp, inbox->buf,
+ vhcr->in_modifier >> 31,
+ (vhcr->in_modifier >> 28) & 0x7);
else
- return mlx4_multicast_detach(dev, &qp, inbox->buf);
+ return mlx4_multicast_detach(dev, &qp, inbox->buf,
+ (vhcr->in_modifier >> 28) & 0x7);
}
static int mlx4_MCAST(struct mlx4_dev *dev, struct mlx4_qp *qp,
- u8 gid[16], u8 attach,
- u8 block_loopback)
+ u8 gid[16], u8 attach, u8 block_loopback,
+ enum mlx4_protocol prot)
{
struct mlx4_cmd_mailbox *mailbox;
int err;
@@ -176,6 +181,7 @@ static int mlx4_MCAST(struct mlx4_dev *dev, struct mlx4_qp *qp,
memcpy(mailbox->buf, gid, 16);
qpn = qp->qpn;
+ qpn |= (prot << 28);
if (attach && block_loopback)
qpn |= (1 << 31);
@@ -186,7 +192,7 @@ static int mlx4_MCAST(struct mlx4_dev *dev, struct mlx4_qp *qp,
}
int mlx4_multicast_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16],
- int block_mcast_loopback)
+ int block_mcast_loopback, enum mlx4_protocol prot)
{
struct mlx4_priv *priv = mlx4_priv(dev);
struct mlx4_cmd_mailbox *mailbox;
@@ -199,7 +205,7 @@ int mlx4_multicast_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16],
int err;
if (mlx4_is_slave(dev))
- return mlx4_MCAST(dev, qp, gid, 1, block_mcast_loopback);
+ return mlx4_MCAST(dev, qp, gid, 1, block_mcast_loopback, prot);
mailbox = mlx4_alloc_cmd_mailbox(dev);
if (IS_ERR(mailbox))
@@ -208,7 +214,7 @@ int mlx4_multicast_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16],
mutex_lock(&priv->mcg_table.mutex);
- err = find_mgm(dev, gid, mailbox, &hash, &prev, &index);
+ err = find_mgm(dev, gid, prot, mailbox, &hash, &prev, &index);
if (err)
goto out;
@@ -230,7 +236,7 @@ int mlx4_multicast_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16],
memcpy(mgm->gid, gid, 16);
}
- members_count = be32_to_cpu(mgm->members_count);
+ members_count = be32_to_cpu(mgm->members_count) & 0xffffff;
if (members_count == MLX4_QP_PER_MGM) {
mlx4_err(dev, "MGM at index %x is full.\n", index);
err = -ENOMEM;
@@ -250,7 +256,7 @@ int mlx4_multicast_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16],
else
mgm->qp[members_count++] = cpu_to_be32(qp->qpn & MGM_QPN_MASK);
- mgm->members_count = cpu_to_be32(members_count);
+ mgm->members_count = cpu_to_be32(members_count | ((u32) prot << 30));
err = mlx4_WRITE_MCG(dev, index, mailbox);
if (err)
@@ -285,7 +291,8 @@ out:
}
EXPORT_SYMBOL_GPL(mlx4_multicast_attach);
-int mlx4_multicast_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16])
+int mlx4_multicast_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16],
+ enum mlx4_protocol prot)
{
struct mlx4_priv *priv = mlx4_priv(dev);
struct mlx4_cmd_mailbox *mailbox;
@@ -297,7 +304,7 @@ int mlx4_multicast_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16])
int err;
if (mlx4_is_slave(dev))
- return mlx4_MCAST(dev, qp, gid, 0, 0);
+ return mlx4_MCAST(dev, qp, gid, 0, 0, prot);
mailbox = mlx4_alloc_cmd_mailbox(dev);
if (IS_ERR(mailbox))
@@ -306,7 +313,7 @@ int mlx4_multicast_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16])
mutex_lock(&priv->mcg_table.mutex);
- err = find_mgm(dev, gid, mailbox, &hash, &prev, &index);
+ err = find_mgm(dev, gid, prot, mailbox, &hash, &prev, &index);
if (err)
goto out;
@@ -316,7 +323,7 @@ int mlx4_multicast_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16])
goto out;
}
- members_count = be32_to_cpu(mgm->members_count);
+ members_count = be32_to_cpu(mgm->members_count) & 0xffffff;
for (loc = -1, i = 0; i < members_count; ++i)
if ((be32_to_cpu(mgm->qp[i]) & MGM_QPN_MASK) == qp->qpn)
loc = i;
@@ -328,7 +335,7 @@ int mlx4_multicast_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16])
}
- mgm->members_count = cpu_to_be32(--members_count);
+ mgm->members_count = cpu_to_be32(--members_count | ((u32) prot << 30));
mgm->qp[loc] = mgm->qp[i - 1];
mgm->qp[i - 1] = 0;
diff --git a/include/linux/mlx4/device.h b/include/linux/mlx4/device.h
index 246b7bc..3d74198 100644
--- a/include/linux/mlx4/device.h
+++ b/include/linux/mlx4/device.h
@@ -171,6 +171,13 @@ enum mlx4_special_vlan_idx {
MLX4_VLAN_REGULAR
};
+enum mlx4_protocol {
+ MLX4_PROT_IB_IPV6 = 0,
+ MLX4_PROT_ETH,
+ MLX4_PROT_IB_IPV4,
+ MLX4_PROT_FCOE
+};
+
enum {
MLX4_NUM_FEXCH = 64 * 1024,
};
@@ -488,8 +495,9 @@ int mlx4_INIT_PORT(struct mlx4_dev *dev, int port);
int mlx4_CLOSE_PORT(struct mlx4_dev *dev, int port);
int mlx4_multicast_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16],
- int block_mcast_loopback);
-int mlx4_multicast_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16]);
+ int block_mcast_loopback, enum mlx4_protocol prot);
+int mlx4_multicast_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16],
+ enum mlx4_protocol prot);
int mlx4_register_mac(struct mlx4_dev *dev, u8 port, u64 mac, int *qpn);
void mlx4_unregister_mac(struct mlx4_dev *dev, u8 port, int qpn);
--
1.5.3.7
From: Liran Liss <liranl-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* [PATCH 23/25 v2] mlx4: Configuring multicast filter
From: Yevgeny Petrilin @ 2009-11-06 3:10 UTC (permalink / raw)
To: rdreier-FYB4Gu1CFyUAvxtiuMwx3w
Cc: linux-rdma-u79uwXL29TY76Z2rM5mHXA, netdev-u79uwXL29TY76Z2rM5mHXA,
liranl-VPRAkNaXOzVS1MOuV/RT9w, tziporet-VPRAkNaXOzVS1MOuV/RT9w,
yevgenyp-VPRAkNaXOzVS1MOuV/RT9w
The Multicast filter configuration is done by the master,
that manages the filter which is common for all the functions.
The master holds a list of multicast addresses for all the
slaves, and adds them to the filter.
In case some slave wishes to flush the filter, only his addresses
are removed.
Signed-off-by: Yevgeny Petrilin <yevgenyp-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
---
drivers/net/mlx4/cmd.c | 6 ++-
drivers/net/mlx4/en_port.c | 7 ---
drivers/net/mlx4/mlx4.h | 10 ++++
drivers/net/mlx4/mlx4_en.h | 1 -
drivers/net/mlx4/port.c | 125 +++++++++++++++++++++++++++++++++++++++++++
include/linux/mlx4/device.h | 1 +
6 files changed, 140 insertions(+), 10 deletions(-)
diff --git a/drivers/net/mlx4/cmd.c b/drivers/net/mlx4/cmd.c
index 71d822a..f3d3a7b 100644
--- a/drivers/net/mlx4/cmd.c
+++ b/drivers/net/mlx4/cmd.c
@@ -947,7 +947,7 @@ static struct mlx4_cmd_info {
.has_outbox = false,
.out_is_imm = false,
.verify = NULL,
- .wrapper = NULL /* need wrapper*/
+ .wrapper = mlx4_SET_MCAST_FLTR_wrapper
},
{
.opcode = MLX4_CMD_DUMP_ETH_STATS,
@@ -1178,7 +1178,7 @@ static void mlx4_master_poll_comm(struct work_struct *work)
int mlx4_multi_func_init(struct mlx4_dev *dev)
{
struct mlx4_priv *priv = mlx4_priv(dev);
- int i;
+ int i, port;
priv->mfunc.vhcr = dma_alloc_coherent(&(dev->pdev->dev), PAGE_SIZE,
&priv->mfunc.vhcr_dma,
@@ -1211,6 +1211,8 @@ int mlx4_multi_func_init(struct mlx4_dev *dev)
for (i = 0; i < dev->num_slaves; ++i) {
priv->mfunc.master.slave_state[i].last_cmd = MLX4_COMM_CMD_RESET;
+ for (port = 1; port <= MLX4_MAX_PORTS; port++)
+ INIT_LIST_HEAD(&priv->mfunc.master.slave_state[i].mcast_filters[port]);
spin_lock_init(&priv->mfunc.master.slave_state[i].lock);
}
diff --git a/drivers/net/mlx4/en_port.c b/drivers/net/mlx4/en_port.c
index c7aa86e..6dc07c9 100644
--- a/drivers/net/mlx4/en_port.c
+++ b/drivers/net/mlx4/en_port.c
@@ -41,13 +41,6 @@
#include "mlx4_en.h"
-int mlx4_SET_MCAST_FLTR(struct mlx4_dev *dev, u8 port,
- u64 mac, u64 clear, u8 mode)
-{
- return mlx4_cmd(dev, (mac | (clear << 63)), port, mode,
- MLX4_CMD_SET_MCAST_FLTR, MLX4_CMD_TIME_CLASS_B);
-}
-
int mlx4_SET_VLAN_FLTR(struct mlx4_dev *dev, u8 port, struct vlan_group *grp)
{
struct mlx4_cmd_mailbox *mailbox;
diff --git a/drivers/net/mlx4/mlx4.h b/drivers/net/mlx4/mlx4.h
index 2c87cc5..2476fb5 100644
--- a/drivers/net/mlx4/mlx4.h
+++ b/drivers/net/mlx4/mlx4.h
@@ -220,6 +220,11 @@ struct mlx4_slave_eqe {
u32 param;
};
+struct mlx4_mcast_entry {
+ struct list_head list;
+ u64 addr;
+};
+
struct mlx4_slave_state {
u8 comm_toggle;
u8 last_cmd;
@@ -228,6 +233,7 @@ struct mlx4_slave_state {
u16 mtu[MLX4_MAX_PORTS + 1];
__be32 ib_cap_mask[MLX4_MAX_PORTS + 1];
struct mlx4_slave_eqe eq[MLX4_MFUNC_MAX_EQES];
+ struct list_head mcast_filters[MLX4_MAX_PORTS + 1];
u16 eq_pi;
u16 eq_ci;
int sqp_start;
@@ -238,6 +244,7 @@ struct mlx4_mfunc_master_ctx {
struct mlx4_slave_state *slave_state;
int init_port_ref[MLX4_MAX_PORTS + 1];
u16 max_mtu[MLX4_MAX_PORTS + 1];
+ int disable_mcast_ref[MLX4_MAX_PORTS + 1];
};
struct mlx4_vhcr {
@@ -554,5 +561,8 @@ int mlx4_GET_SLAVE_SQP_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vhcr
int mlx4_MCAST_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr,
struct mlx4_cmd_mailbox *inbox,
struct mlx4_cmd_mailbox *outbox);
+int mlx4_SET_MCAST_FLTR_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr,
+ struct mlx4_cmd_mailbox *inbox,
+ struct mlx4_cmd_mailbox *outbox);
#endif /* MLX4_H */
diff --git a/drivers/net/mlx4/mlx4_en.h b/drivers/net/mlx4/mlx4_en.h
index 6e0c9ed..b4731d5 100644
--- a/drivers/net/mlx4/mlx4_en.h
+++ b/drivers/net/mlx4/mlx4_en.h
@@ -556,7 +556,6 @@ void mlx4_en_release_rss_steer(struct mlx4_en_priv *priv);
int mlx4_en_free_tx_buf(struct net_device *dev, struct mlx4_en_tx_ring *ring);
void mlx4_en_rx_irq(struct mlx4_cq *mcq);
-int mlx4_SET_MCAST_FLTR(struct mlx4_dev *dev, u8 port, u64 mac, u64 clear, u8 mode);
int mlx4_SET_VLAN_FLTR(struct mlx4_dev *dev, u8 port, struct vlan_group *grp);
int mlx4_SET_PORT_general(struct mlx4_dev *dev, u8 port, int mtu,
u8 pptx, u8 pfctx, u8 pprx, u8 pfcrx);
diff --git a/drivers/net/mlx4/port.c b/drivers/net/mlx4/port.c
index 7317d0f..1bc527c 100644
--- a/drivers/net/mlx4/port.c
+++ b/drivers/net/mlx4/port.c
@@ -491,3 +491,128 @@ int mlx4_SET_PORT(struct mlx4_dev *dev, u8 port)
mlx4_free_cmd_mailbox(dev, mailbox);
return err;
}
+
+int mlx4_SET_MCAST_FLTR_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr,
+ struct mlx4_cmd_mailbox *inbox,
+ struct mlx4_cmd_mailbox *outbox)
+{
+ struct mlx4_priv *priv = mlx4_priv(dev);
+ int port = vhcr->in_modifier;
+ int err = 0;
+ u64 addr = vhcr->in_param & 0xffffffffffffULL;
+ u64 clear = vhcr->in_param >> 63;
+ struct mlx4_mcast_entry *entry, *tmp;
+ struct mlx4_slave_state *s_state = &priv->mfunc.master.slave_state[slave];
+ int i;
+
+ switch (vhcr->op_modifier) {
+ case MLX4_MCAST_DISABLE:
+ /* The multicast filter is disabled only once,
+ * If some other function already done it, operation
+ * is ignored */
+ if (!(priv->mfunc.master.disable_mcast_ref[port]++))
+ err = mlx4_cmd(dev, 0, port, MLX4_MCAST_DISABLE,
+ MLX4_CMD_SET_MCAST_FLTR,
+ MLX4_CMD_TIME_CLASS_B);
+ break;
+ case MLX4_MCAST_ENABLE:
+ /* We enable the muticast filter only if all functions
+ * have the filter enabled */
+ if (!(--priv->mfunc.master.disable_mcast_ref[port]))
+ err = mlx4_cmd(dev, 0, port, MLX4_MCAST_ENABLE,
+ MLX4_CMD_SET_MCAST_FLTR,
+ MLX4_CMD_TIME_CLASS_B);
+ break;
+ case MLX4_MCAST_CONFIG:
+ if (clear) {
+ /* Disable the muticast filter while updating it */
+ if (!priv->mfunc.master.disable_mcast_ref[port]) {
+ err = mlx4_cmd(dev, 0, port, MLX4_MCAST_DISABLE,
+ MLX4_CMD_SET_MCAST_FLTR,
+ MLX4_CMD_TIME_CLASS_B);
+ if (err) {
+ mlx4_warn(dev, "Failed to disable multicast "
+ "filter\n");
+ goto out;
+ }
+ }
+ /* Clear the multicast filter */
+ err = mlx4_cmd(dev, clear << 63, port,
+ MLX4_MCAST_CONFIG,
+ MLX4_CMD_SET_MCAST_FLTR,
+ MLX4_CMD_TIME_CLASS_B);
+ if (err) {
+ mlx4_warn(dev, "Failed clearing the multicast filter\n");
+ goto out;
+ }
+
+ /* Clear the multicast addresses for the given slave */
+ list_for_each_entry_safe(entry, tmp,
+ &s_state->mcast_filters[port],
+ list) {
+ list_del(&entry->list);
+ kfree(entry);
+ }
+
+ /* Assign all the multicast addresses that still exist */
+ for (i = 0; i < dev->num_slaves; i++) {
+ list_for_each_entry(entry,
+ &priv->mfunc.master.slave_state[slave].mcast_filters[port],
+ list) {
+ if (mlx4_cmd(dev, entry->addr, port,
+ MLX4_MCAST_CONFIG,
+ MLX4_CMD_SET_MCAST_FLTR,
+ MLX4_CMD_TIME_CLASS_B))
+ mlx4_warn(dev, "Failed to reconfigure "
+ "multicast address: 0x%llx\n",
+ entry->addr);
+ }
+ }
+ /* Enable the filter */
+ if (!priv->mfunc.master.disable_mcast_ref[port]) {
+ err = mlx4_cmd(dev, 0, port, MLX4_MCAST_ENABLE,
+ MLX4_CMD_SET_MCAST_FLTR,
+ MLX4_CMD_TIME_CLASS_B);
+ if (err) {
+ mlx4_warn(dev, "Failed to enable multicast "
+ "filter\n");
+ goto out;
+ }
+ }
+ }
+ /* Add the new address if exists */
+ if (addr) {
+ entry = kzalloc(sizeof (struct mlx4_mcast_entry),
+ GFP_KERNEL);
+ if (!entry) {
+ mlx4_warn(dev, "Failed to allocate entry for "
+ "muticast address\n");
+ err = -ENOMEM;
+ goto out;
+ }
+ INIT_LIST_HEAD(&entry->list);
+ entry->addr = addr;
+ list_add_tail(&entry->list, &s_state->mcast_filters[port]);
+ err = mlx4_cmd(dev, addr, port, MLX4_MCAST_CONFIG,
+ MLX4_CMD_SET_MCAST_FLTR,
+ MLX4_CMD_TIME_CLASS_B);
+ if (err)
+ mlx4_warn(dev, "Failed to add the new address:"
+ "0x%llx\n", addr);
+ }
+ break;
+ default:
+ mlx4_warn(dev, "SET_MCAST_FILTER called with illegal modifier\n");
+ err = -EINVAL;
+ }
+out:
+ return err;
+}
+
+int mlx4_SET_MCAST_FLTR(struct mlx4_dev *dev, u8 port,
+ u64 mac, u64 clear, u8 mode)
+{
+ return mlx4_cmd(dev, (mac | (clear << 63)), port, mode,
+ MLX4_CMD_SET_MCAST_FLTR, MLX4_CMD_TIME_CLASS_B);
+}
+EXPORT_SYMBOL(mlx4_SET_MCAST_FLTR);
diff --git a/include/linux/mlx4/device.h b/include/linux/mlx4/device.h
index 3d74198..10a5fb8 100644
--- a/include/linux/mlx4/device.h
+++ b/include/linux/mlx4/device.h
@@ -498,6 +498,7 @@ int mlx4_multicast_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16],
int block_mcast_loopback, enum mlx4_protocol prot);
int mlx4_multicast_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16],
enum mlx4_protocol prot);
+int mlx4_SET_MCAST_FLTR(struct mlx4_dev *dev, u8 port, u64 mac, u64 clear, u8 mode);
int mlx4_register_mac(struct mlx4_dev *dev, u8 port, u64 mac, int *qpn);
void mlx4_unregister_mac(struct mlx4_dev *dev, u8 port, int qpn);
--
1.5.3.7
From: Liran Liss <liranl-VPRAkNaXOzVS1MOuV/RT9w@public.gmane.org>
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply related
* Re: [RFC] move dma_head/dma_maps out of skb_shared_info and into sk_buff
From: Alexander Duyck @ 2009-11-06 3:24 UTC (permalink / raw)
To: Eric Dumazet; +Cc: Alexander Duyck, netdev@vger.kernel.org, davem@davemloft.net
In-Reply-To: <4AF3868B.7070409@gmail.com>
On Fri, 2009-11-06 at 03:14 +0100, Eric Dumazet wrote:
> Alexander Duyck a écrit :
> > During testing we found issues with the use of skb_dma_map/unmap on
> > systems that had iommu enabled and were configured to use a bridge. The
> > issue is that if two ports are members of the same bridge, and a
> > broadcast packet is sent out on the bridge skb_clone will be used to
> > send a copy to all ports, but the clones run into issues because the
> > dma mappings for the cloned skbs all share the shared_info structure
> > where the dma mappings are stored.
> >
> > To resolve that this patch moves those dma mappings out of the
> > shared_info structure and into the sk_buff itself. This allows cloned
> > skbs to be mapped separately without causing dma unmapping errors.
> >
> > Signed-off-by: Alexander Duyck <alexander.h.duyck@intel.com>
>
> Hello Alexander
>
> You probably know such a change is a major one ;)
> 1) a diffstat -p1 -w70 for this kind of patch would be nice.
>
> 2) Your patch is garbled (tabulations were replaced by spaces
> by your mailer)
I kind of figured that might be the case. I didn't really intend the
patch to be applied to the tree and just meant it to get conversation
going. That is why I had tagged it as [RFC].
> 3) Are you sure we need to clear dma_maps[] array and dma_head
> in __alloc_skb() ? I guess not.
> MAX_SKB_FRAGS = 18 on x86 -> 152 bytes on x86_64.
> Previous implementation was not clearing them.
> Thats would be a major slow down.
>
> 4) 152 bytes more in skb -> 304 bytes more in skbuff_fclone_cache
> Do we really want two copies of dma_maps[] when skb are allocated
> from fclone cache ?
The main problem that this was meant to address is the fact that
skb_dma_map is called dma_maps and dma_head needed to be maintained
until skb_dma_unmap was called. This wasn't happening with them being
stored in the skb_shared_info structure due to the fact that if two
clones of the skb were mapped on 2 different devices the 2nd mapping
would overwrite the first, and then the skb_dma_unmap call was being
called on the 2nd dma mapping twice which would trigger a dma_unmapping
error followed by an error of mappings still being held for the first
device on driver unload.
> 5) It seems to me this stuff is needed for xmit only and few drivers,
> could we find a way to not have it for RX path and drivers that dont
> need it ? Maybe drivers themselves should allocate storage for this
> stuff so we can remove it both from shared_info *and* skb
I'm thinking the best solution may be to drop the skb_dma_map/unmap
calls entirely and move things back to the old approach in which devices
maintained their list of mappings. At least until something better can
be figured out.
Thanks,
Alex
^ permalink raw reply
* Re: [rfc 4/4] igb: expose 82576 bandiwidth allocation
From: Simon Horman @ 2009-11-06 3:57 UTC (permalink / raw)
To: Alexander Duyck
Cc: e1000-devel@lists.sourceforge.net, netdev@vger.kernel.org,
Kirsher, Jeffrey T, Arnd Bergmann
In-Reply-To: <4AF362E4.3040507@intel.com>
On Thu, Nov 05, 2009 at 03:42:28PM -0800, Alexander Duyck wrote:
> Simon Horman wrote:
> >On Thu, Nov 05, 2009 at 03:00:02PM -0800, Alexander Duyck wrote:
> >>Simon Horman wrote:
> >>>The 82576 has support for bandwidth allocation to VFs.
> >>>
> >>>Contrary to the documentation in the 82576 datasheet v2.41 this
> >>>appears to work as follows:
> >>>
> >>>* The ratio supplied is always proportional to 1Gbit/s,
> >>> regardless of if the link speed.
> >>>* The ratio supplied is an upper-bound on bandwidth available
> >>> to the VF, not a minimun guarantee
> >>>
> >>>This patch exposes bandwidth control to userspace through a simple
> >>>per-device (PF) sysfs file, bandwidth_allocation.
> >>>
> >>>* The file contains a whitespace delimited list of values, one per VF.
> >>>* The first value corresponds to the first VF and so on.
> >>>* Valid values are integers from 0 to 1000
> >>>* A value of 0 indicates that bandwidth_allocation is disabled.
> >>>* Other values indicate the allocated bandwidth, in 1/1000ths of a gigabit/s
> >>>
> >>>e.g. The following for a PF with 4 VFs allocates ~20Mbits/ to VF 1,
> >>> ~100Mbit/s to VF 2, and leave the other 2 VFs with no allocation.
> >>>
> >>> echo "20 100 0 0" > /sys/class/net/eth3/device/bandwidth_allocation
> >>>
> >>>This interface is intended to allow testing of the hardware feature.
> >>>There are ongoing discussions about how to expose this feature
> >>>to user-space in a more generic way.
> >>>
> >>>Signed-off-by: Simon Horman <horms@verge.net.au>
> >>>
> >>Of the patches it looks like the only one that really has any issues
> >>is this one and it is mostly due to the sysfs implementation. The
> >>others I would say can be applied and pushed up into the
> >>net-next-2.6 tree.
> >
> >Thanks, I suspected as much.
> >
> >>We're currently working on an iproute2 based solution for
> >>configuring VFs and can incorporate this functionality into it at
> >>some point in the future.
> >
> >Do you have any pointers to discussions relating to that interface.
> >Do you think it would be worth putting in the sysfs interface in the
> >mean-time, or would you rather wait?
>
> I'm not the one working on the interface so I don't know much about
> it other than the fact it is being worked on. Hopefully we should
> see something in regards to that soon though.
Great, I look forward to hearing something soon.
> If anything it might be of some use to split this up into 2 patches.
> One that contains the sysfs bits, and another for enabling the
> bandwidth control registers. We won't be able to get the sysfs
> interface accepted upstream so there isn't much point in us keeping
> it around for any other purpose than testing to verify the registers
> work as you have described.
Understood with regards to the sysfs interface being umergable.
Do you think the non-sysfs portion would be acceptable even
though there would be no in-tree callers?
^ permalink raw reply
* Re: [PATCH 19/25] mlx4: Randomizing mac addresses for slaves
From: Simon Horman @ 2009-11-06 4:06 UTC (permalink / raw)
To: Roland Dreier
Cc: Liran Liss, Or Gerlitz, Yevgeny Petrilin, linux-rdma, netdev,
Tziporet Koren
In-Reply-To: <adabpjhc6az.fsf@roland-alpha.cisco.com>
On Thu, Nov 05, 2009 at 07:58:12AM -0800, Roland Dreier wrote:
>
> > This approach seems to be common practice now (e.g., drivers/net/igb/igb_main.c:1332).
> > In any case, the user can change the randomized mac.
>
> igb uses the full output of random_ether_addr(). I'd be fine with
> that. However setting the OUI means you only get 24 bits of randomness
> which makes a collision a lot more likely.
IIRC that was precisely why the OUI isn't used for the igb driver.
Perhaps some infrastructure (by which I mean a random_mac() function)
is warranted so at least this discussion can be concentrated around that
rather than repeating it for each driver that needs random mac addresses.
^ permalink raw reply
* Re: [PATCHv8 3/3] vhost_net: a kernel-level virtio server
From: Rusty Russell @ 2009-11-06 4:59 UTC (permalink / raw)
To: Michael S. Tsirkin
Cc: netdev, virtualization, kvm, linux-kernel, mingo, linux-mm, akpm,
hpa, gregory.haskins, s.hetze, Daniel Walker, Eric Dumazet,
Paul E. McKenney
In-Reply-To: <20091104155724.GD32673@redhat.com>
On Thu, 5 Nov 2009 02:27:24 am Michael S. Tsirkin wrote:
> What it is: vhost net is a character device that can be used to reduce
> the number of system calls involved in virtio networking.
Hi Michael,
Now everyone else has finally kicked all the tires and it seems to pass,
I've done a fairly complete review. Generally, it's really nice; just one
bug and a few minor suggestions for polishing.
> +/* Caller must have TX VQ lock */
> +static void tx_poll_stop(struct vhost_net *net)
> +{
> + if (likely(net->tx_poll_state != VHOST_NET_POLL_STARTED))
> + return;
likely? Really?
> + for (;;) {
> + head = vhost_get_vq_desc(&net->dev, vq, vq->iov, &out, &in,
> + NULL, NULL);
Danger! You need an arg to vhost_get_vq_desc to tell it the max desc size
you can handle. Otherwise, it's only limited by ring size, and a malicious
guest can overflow you here, and below:
> + /* Skip header. TODO: support TSO. */
> + s = move_iovec_hdr(vq->iov, vq->hdr, hdr_size, out);
...
> +
> + use_mm(net->dev.mm);
> + mutex_lock(&vq->mutex);
> + vhost_no_notify(vq);
I prefer a name like "vhost_disable_notify()".
> + /* OK, now we need to know about added descriptors. */
> + if (head == vq->num && vhost_notify(vq))
> + /* They could have slipped one in as we were doing that:
> + * check again. */
> + continue;
> + /* Nothing new? Wait for eventfd to tell us they refilled. */
> + if (head == vq->num)
> + break;
> + /* We don't need to be notified again. */
> + vhost_no_notify(vq);
Similarly, vhost_enable_notify. This one is particularly misleading since
it doesn't actually notify anything!
In particular, this code would be neater as:
if (head == vq->num) {
if (vhost_enable_notify(vq)) {
/* Try again, they could have slipped one in. */
continue;
}
/* Nothing more to do. */
break;
}
vhost_disable_notify(vq);
Now, AFAICT vhost_notify()/enable_notify() would be better rewritten to
return true only when there's more pending. Saves a loop around here most
of the time. Also, the vhost_no_notify/vhost_disable_notify() can be moved
out of the loop entirely. (It could be under an if (unlikely(enabled)), not
sure if it's worth it).
> + len = err;
> + err = memcpy_toiovec(vq->hdr, (unsigned char *)&hdr, hdr_size);
That unsigned char * arg to memcpy_toiovec is annoying. A patch might be
nice, separate from this effort.
> +static int vhost_net_open(struct inode *inode, struct file *f)
> +{
> + struct vhost_net *n = kzalloc(sizeof *n, GFP_KERNEL);
> + int r;
> + if (!n)
> + return -ENOMEM;
> + f->private_data = n;
> + n->vqs[VHOST_NET_VQ_TX].handle_kick = handle_tx_kick;
> + n->vqs[VHOST_NET_VQ_RX].handle_kick = handle_rx_kick;
I have a personal dislike of calloc for structures. In userspace, it's
because valgrind can't spot uninitialized fields. These days a similar
argument applies in the kernel, because we have KMEMCHECK now. If someone
adds a field to the struct and forgets to initialize it, we can spot it.
> +static void vhost_net_enable_vq(struct vhost_net *n, int index)
> +{
> + struct socket *sock = n->vqs[index].private_data;
OK, I can't help but this that presenting the vqs as an array doesn't buy
us very much. Esp. if you change vhost_dev_init to take a NULL-terminated
varargs. I think readability would improve. It means passing a vq around
rather than an index.
Not completely sure it'll be a win tho.
> +static long vhost_net_set_backend(struct vhost_net *n, unsigned index, int fd)
> +{
> + struct socket *sock, *oldsock = NULL;
...
> + sock = get_socket(fd);
> + if (IS_ERR(sock)) {
> + r = PTR_ERR(sock);
> + goto done;
> + }
> +
> + /* start polling new socket */
> + oldsock = vq->private_data;
...
> +done:
> + mutex_unlock(&n->dev.mutex);
> + if (oldsock) {
> + vhost_net_flush_vq(n, index);
> + fput(oldsock->file);
I dislike this style; I prefer multiple different goto points, one for when
oldsock is set, and one for when it's not.
That way, gcc warns us about uninitialized variables if we get it wrong.
> +static long vhost_net_reset_owner(struct vhost_net *n)
> +{
> + struct socket *tx_sock = NULL;
> + struct socket *rx_sock = NULL;
> + long r;
This should be called "err", since that's what it is.
> +static void vhost_net_set_features(struct vhost_net *n, u64 features)
> +{
> + size_t hdr_size = features & (1 << VHOST_NET_F_VIRTIO_NET_HDR) ?
> + sizeof(struct virtio_net_hdr) : 0;
> + int i;
> + mutex_lock(&n->dev.mutex);
> + n->dev.acked_features = features;
Why is this called "acked_features"? Not just "features"? I expected
to see code which exposed these back to userspace, and didn't.
> + case VHOST_GET_FEATURES:
> + features = VHOST_FEATURES;
> + return put_user(features, featurep);
> + case VHOST_ACK_FEATURES:
> + r = get_user(features, featurep);
> + /* No features for now */
> + if (r < 0)
> + return r;
> + if (features & ~VHOST_FEATURES)
> + return -EOPNOTSUPP;
> + vhost_net_set_features(n, features);
OK, from the userspace POV it's "get features" then "ack features". But
I think "VHOST_SET_FEATURES" is more consistent, despite this usage.
> + switch (ioctl) {
> + case VHOST_SET_VRING_NUM:
I haven't looked at your userspace implementation, but does a generic
VHOST_SET_VRING_STATE & VHOST_GET_VRING_STATE with a struct make more
sense? It'd be simpler here, but not sure if it'd be simpler to use?
(Not the fd-setting ioctls of course)
> + case VHOST_SET_VRING_LOG:
> + r = copy_from_user(&a, argp, sizeof a);
> + if (r < 0)
> + break;
> + if (a.padding) {
> + r = -EOPNOTSUPP;
> + break;
> + }
> + if (a.user_addr == VHOST_VRING_LOG_DISABLE) {
> + vq->log_used = false;
> + break;
> + }
> + if (a.user_addr & (sizeof *vq->used->ring - 1)) {
> + r = -EINVAL;
> + break;
> + }
> + vq->log_used = true;
> + vq->log_addr = a.user_addr;
> + break;
For future reference, this is *exactly* the kind of thing which would have
been nice as a followup patch. Easy to separate, easy to review, not critical
to the core.
> +/* TODO: This is really inefficient. We need something like get_user()
> + * (instruction directly accesses the data, with an exception table entry
> + * returning -EFAULT). See Documentation/x86/exception-tables.txt.
> + */
> +static int set_bit_to_user(int nr, void __user *addr)
> +{
I guess we won't be dealing with many contiguous pages, otherwise we could
get a cheap speedup making this set_bits_to_user(int nr, int num_bits...).
> +/* Each buffer in the virtqueues is actually a chain of descriptors. This
> + * function returns the next descriptor in the chain,
> + * or -1 if we're at the end. */
> +static unsigned next_desc(struct vring_desc *desc)
> +{
> + unsigned int next;
> +
> + /* If this descriptor says it doesn't chain, we're done. */
> + if (!(desc->flags & VRING_DESC_F_NEXT))
> + return -1;
Hmm, prefer s/-1/-1U/ in comment, here, and below. Clarifies a bit.
> +/* After we've used one of their buffers, we tell them about it. We'll then
> + * want to send them an interrupt, using vq->call. */
This comment has too much cut & paste:
... want to notify the guest, using the eventfd */
> +/* This actually sends the interrupt for this virtqueue */
> +void vhost_trigger_irq(struct vhost_dev *dev, struct vhost_virtqueue *vq)
> +{
Rename vhost_notify_eventfd() or something, and fix comments?
> +enum {
> + VHOST_NET_MAX_SG = MAX_SKB_FRAGS + 2,
+2? Believable, but is it correct?
> +/* Poll a file (eventfd or socket) */
> +/* Note: there's nothing vhost specific about this structure. */
> +struct vhost_poll {
This comment really helped while reading the code. Kudos!
Thanks!
Rusty.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply
* Re: [PATCHv7 3/3] vhost_net: a kernel-level virtio server
From: Rusty Russell @ 2009-11-06 5:01 UTC (permalink / raw)
To: paulmck
Cc: Michael S. Tsirkin, Gregory Haskins, Eric Dumazet, netdev,
virtualization, kvm, linux-kernel, mingo, linux-mm, akpm, hpa,
s.hetze
In-Reply-To: <20091104172542.GC6736@linux.vnet.ibm.com>
On Thu, 5 Nov 2009 03:55:42 am Paul E. McKenney wrote:
> On Wed, Nov 04, 2009 at 01:57:29PM +0200, Michael S. Tsirkin wrote:
> > Can you ack this usage please?
>
> I thought I had done so in my paragraph above, but if you would like
> something a bit more formal...
<snip verbose super-ack with qualifications>
That's great guys. And yes, this is a kind of read-copy-update. And no,
there's nothing wrong with it.
But it's still nasty to use half an API. If it were a few places I would
have open-coded it with a comment, or wrapped it. As it is, I don't think
that would be a win.
Cheers,
Rusty.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply
* Re: [PATCH net-next-2.6] bridge: remove dev_put() in add_del_if()
From: David Miller @ 2009-11-06 5:07 UTC (permalink / raw)
To: shemminger; +Cc: eric.dumazet, netdev
In-Reply-To: <20091104114020.5a2aef2c@nehalam>
From: Stephen Hemminger <shemminger@vyatta.com>
Date: Wed, 4 Nov 2009 11:40:20 -0800
> On Wed, 04 Nov 2009 20:09:07 +0100
> Eric Dumazet <eric.dumazet@gmail.com> wrote:
>
>> add_del_if() is called with RTNL, we can use __dev_get_by_index()
>> instead of [dev_get_by_index() + dev_put()]
>>
>> Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
...
>
> As part of the general ref count purge sure.
> Acked-by: Stephen Hemminger <shemminger@vyatta.com>
Applied, thanks everyone.
^ permalink raw reply
* Re: [PATCH 1/3] net: drop capability from protocol definitions
From: David Miller @ 2009-11-06 5:07 UTC (permalink / raw)
To: acme; +Cc: eparis, netdev, nhorman, dwalsh, linux-security-module
In-Reply-To: <20091104173104.GF2603@ghostprotocols.net>
From: Arnaldo Carvalho de Melo <acme@infradead.org>
Date: Wed, 4 Nov 2009 15:31:04 -0200
> Em Wed, Nov 04, 2009 at 11:32:11AM -0500, Eric Paris escreveu:
>> struct can_proto had a capability field which wasn't ever used. It is dropped
>> entirely.
>>
>> struct inet_protosw had a capability field which can be more clearly expressed
>> in the code by just checking if sock->type = SOCK_RAW.
>>
>> Signed-off-by: Eric Paris <eparis@redhat.com>
>
> Acked-by: Arnaldo Carvalho de Melo <acme@redhat.com>
Applied to net-next-2.6
^ permalink raw reply
* Re: [PATCH 2/3] net: pass kern to net_proto_family create function
From: David Miller @ 2009-11-06 5:08 UTC (permalink / raw)
To: acme; +Cc: eparis, netdev, nhorman, dwalsh, linux-security-module
In-Reply-To: <20091104173147.GG2603@ghostprotocols.net>
From: Arnaldo Carvalho de Melo <acme@infradead.org>
Date: Wed, 4 Nov 2009 15:31:47 -0200
> Em Wed, Nov 04, 2009 at 11:32:17AM -0500, Eric Paris escreveu:
>> The generic __sock_create function has a kern argument which allows the
>> security system to make decisions based on if a socket is being created by
>> the kernel or by userspace. This patch passes that flag to the
>> net_proto_family specific create function, so it can do the same thing.
>>
>> Signed-off-by: Eric Paris <eparis@redhat.com>
>
> Acked-by: Arnaldo Carvalho de Melo <acme@redhat.com>
Applied to net-next-2.6
^ permalink raw reply
* Re: [RFC] netlink: add socket destruction notification
From: David Miller @ 2009-11-06 5:08 UTC (permalink / raw)
To: johannes; +Cc: netdev, j, tgraf
In-Reply-To: <1254473048.3959.76.camel@johannes.local>
From: Johannes Berg <johannes@sipsolutions.net>
Date: Fri, 02 Oct 2009 10:44:08 +0200
> When we want to keep track of resources associated with applications, we
> need to know when an app is going away. Add a notification function to
> netlink that tells us that, and also hook it up to generic netlink so
> generic netlink can notify the families. Due to the way generic netlink
> works though, we need to notify all families and they have to sort out
> whatever resources some commands associated with the socket themselves.
>
> Signed-off-by: Johannes Berg <johannes@sipsolutions.net>
No fundamental objections.
However, as a followup, netlink_kernel_create() is becomming
function_that_takes_too_many_arguments().
At this point it's better to just pass two arguments, the network
namespace pointer, and a pointer to a "const struct netlink_kern_info"
that holds the rest of the parameters.
Could you make that change too?
Thanks!
^ permalink raw reply
* Re: [PATCH] bridge: prevent bridging wrong device
From: David Miller @ 2009-11-06 5:08 UTC (permalink / raw)
To: shemminger; +Cc: nneul, netdev
In-Reply-To: <20091104094713.1983a0b6@nehalam>
From: Stephen Hemminger <shemminger@vyatta.com>
Date: Wed, 4 Nov 2009 09:47:13 -0800
> The bridge code assumes ethernet addressing, so be more strict in
> the what is allowed. This showed up when GRE had a bug and was not
> using correct address format.
>
> Add some more comments for increased clarity.
>
> Signed-off-by: Stephen Hemminger <shemminger@vyatta.com>
Applied to net-2.6, thanks.
^ permalink raw reply
* Re: [PATCH net-next-2.6] netlabel: remove dev_put() calls
From: David Miller @ 2009-11-06 5:08 UTC (permalink / raw)
To: eric.dumazet; +Cc: netdev
In-Reply-To: <4AF1DE29.7010408@gmail.com>
From: Eric Dumazet <eric.dumazet@gmail.com>
Date: Wed, 04 Nov 2009 21:03:53 +0100
> Use dev_get_by_name_rcu() to avoid dev_put() calls,
> in sections already inside a rcu_read_lock()
>
> Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Applied to net-next-2.6
^ permalink raw reply
* Re: [PATCH] rose: device refcount leak
From: David Miller @ 2009-11-06 5:08 UTC (permalink / raw)
To: eric.dumazet; +Cc: netdev
In-Reply-To: <4AF1E4A9.7050404@gmail.com>
From: Eric Dumazet <eric.dumazet@gmail.com>
Date: Wed, 04 Nov 2009 21:31:37 +0100
> While hunting dev_put() for net-next-2.6, I found a device refcount
> leak in ROSE, ioctl(SIOCADDRT) error path.
>
> Fix is to not touch device refcount, as we hold RTNL
>
> Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Applied to net-2.6, thanks.
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox