[PATCH v1 19/23] net/mlx5: add support for GENEVE and option item in HWS

[Michael Baum <michaelba@nvidia.com>]

Add HW steering support for both "RTE_FLOW_ITEM_TYPE_GENEVE" and
"RTE_FLOW_ITEM_TYPE_GENEVE_OPT".

Signed-off-by: Michael Baum <michaelba@nvidia.com>
---
 doc/guides/nics/mlx5.rst               |  15 ++-
 doc/guides/rel_notes/release_24_03.rst |   5 +
 drivers/net/mlx5/mlx5_flow.h           |  21 +++++
 drivers/net/mlx5/mlx5_flow_geneve.c    | 121 ++++++++++++++++++++++++-
 drivers/net/mlx5/mlx5_flow_hw.c        |  44 ++++++++-
 5 files changed, 199 insertions(+), 7 deletions(-)

diff --git a/doc/guides/nics/mlx5.rst b/doc/guides/nics/mlx5.rst
index b0f2cdcd62..645b566d80 100644
--- a/doc/guides/nics/mlx5.rst
+++ b/doc/guides/nics/mlx5.rst
@@ -329,12 +329,25 @@ Limitations
      - Length
      - Data
 
-  Only one Class/Type/Length Geneve TLV option is supported per shared device.
   Class/Type/Length fields must be specified as well as masks.
   Class/Type/Length specified masks must be full.
   Matching Geneve TLV option without specifying data is not supported.
   Matching Geneve TLV option with ``data & mask == 0`` is not supported.
 
+  In SW steering (``dv_flow_en`` = 1):
+
+     - Only one Class/Type/Length Geneve TLV option is supported per shared
+       device.
+     - Supported only when ``FLEX_PARSER_PROFILE_ENABLE`` = 0.
+
+  In HW steering (``dv_flow_en`` = 2):
+
+     - Multiple Class/Type/Length Geneve TLV option are supported per physical
+       device. See :ref:`geneve_parser_api` for more information.
+     - Multiple of same Geneve TLV option isn't supported at the same pattern
+       template.
+     - Supported only when ``FLEX_PARSER_PROFILE_ENABLE`` = 8.
+
 - VF: flow rules created on VF devices can only match traffic targeted at the
   configured MAC addresses (see ``rte_eth_dev_mac_addr_add()``).
 
diff --git a/doc/guides/rel_notes/release_24_03.rst b/doc/guides/rel_notes/release_24_03.rst
index e9c9717706..bedef2a4c0 100644
--- a/doc/guides/rel_notes/release_24_03.rst
+++ b/doc/guides/rel_notes/release_24_03.rst
@@ -55,6 +55,11 @@ New Features
      Also, make sure to start the actual text at the margin.
      =======================================================
 
+* **Updated NVIDIA mlx5 net driver.**
+
+  * Added HW steering support for ``RTE_FLOW_ITEM_TYPE_GENEVE`` flow item.
+  * Added HW steering support for ``RTE_FLOW_ITEM_TYPE_GENEVE_OPT`` flow item.
+
 
 Removed Items
 -------------
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index dca3cacb65..04a2eb0b0c 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -1332,6 +1332,15 @@ struct mlx5_action_construct_data {
 
 #define MAX_GENEVE_OPTIONS_RESOURCES 7
 
+/* GENEVE TLV options manager structure. */
+struct mlx5_geneve_tlv_options_mng {
+	uint8_t nb_options; /* Number of options inside the template. */
+	struct {
+		uint8_t opt_type;
+		uint16_t opt_class;
+	} options[MAX_GENEVE_OPTIONS_RESOURCES];
+};
+
 /* Flow item template struct. */
 struct rte_flow_pattern_template {
 	LIST_ENTRY(rte_flow_pattern_template) next;
@@ -1351,6 +1360,8 @@ struct rte_flow_pattern_template {
 	 * tag pattern item for representor matching.
 	 */
 	bool implicit_tag;
+	/* Manages all GENEVE TLV options used by this pattern template. */
+	struct mlx5_geneve_tlv_options_mng geneve_opt_mng;
 	uint8_t flex_item; /* flex item index. */
 };
 
@@ -1799,6 +1810,16 @@ mlx5_geneve_tlv_parser_create(uint16_t port_id,
 			      const struct rte_pmd_mlx5_geneve_tlv tlv_list[],
 			      uint8_t nb_options);
 int mlx5_geneve_tlv_parser_destroy(void *handle);
+int mlx5_flow_geneve_tlv_option_validate(struct mlx5_priv *priv,
+					 const struct rte_flow_item *geneve_opt,
+					 struct rte_flow_error *error);
+
+struct mlx5_geneve_tlv_options_mng;
+int mlx5_geneve_tlv_option_register(struct mlx5_priv *priv,
+				    const struct rte_flow_item_geneve_opt *spec,
+				    struct mlx5_geneve_tlv_options_mng *mng);
+void mlx5_geneve_tlv_options_unregister(struct mlx5_priv *priv,
+					struct mlx5_geneve_tlv_options_mng *mng);
 
 void flow_hw_set_port_info(struct rte_eth_dev *dev);
 void flow_hw_clear_port_info(struct rte_eth_dev *dev);
diff --git a/drivers/net/mlx5/mlx5_flow_geneve.c b/drivers/net/mlx5/mlx5_flow_geneve.c
index 2d593b70ba..2c8dc39e74 100644
--- a/drivers/net/mlx5/mlx5_flow_geneve.c
+++ b/drivers/net/mlx5/mlx5_flow_geneve.c
@@ -152,6 +152,106 @@ mlx5_get_geneve_hl_data(const void *dr_ctx, uint8_t type, uint16_t class,
 	return -EINVAL;
 }
 
+/**
+ * Calculate total data size.
+ *
+ * @param[in] priv
+ *   Pointer to port's private data.
+ * @param[in] geneve_opt
+ *   Pointer to GENEVE option item structure.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_flow_geneve_tlv_option_validate(struct mlx5_priv *priv,
+				     const struct rte_flow_item *geneve_opt,
+				     struct rte_flow_error *error)
+{
+	const struct rte_flow_item_geneve_opt *spec = geneve_opt->spec;
+	const struct rte_flow_item_geneve_opt *mask = geneve_opt->mask;
+	struct mlx5_geneve_tlv_option *option;
+
+	option = mlx5_geneve_tlv_option_get(priv, spec->option_type, spec->option_class);
+	if (option == NULL)
+		return rte_flow_error_set(error, rte_errno,
+					  RTE_FLOW_ERROR_TYPE_ITEM, NULL,
+					  "Unregistered GENEVE option");
+	if (mask->option_type != UINT8_MAX)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM, NULL,
+					  "GENEVE option type must be fully masked");
+	if (option->class_mode == 1 && mask->option_class != UINT16_MAX)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM, NULL,
+					  "GENEVE option class must be fully masked");
+	return 0;
+}
+
+/**
+ * Register single GENEVE TLV option as used by pattern template.
+ *
+ * @param[in] priv
+ *   Pointer to port's private data.
+ * @param[in] spec
+ *   Pointer to GENEVE option item structure.
+ * @param[out] mng
+ *   Pointer to GENEVE option manager.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_geneve_tlv_option_register(struct mlx5_priv *priv,
+				const struct rte_flow_item_geneve_opt *spec,
+				struct mlx5_geneve_tlv_options_mng *mng)
+{
+	struct mlx5_geneve_tlv_option *option;
+
+	option = mlx5_geneve_tlv_option_get(priv, spec->option_type, spec->option_class);
+	if (option == NULL)
+		return -rte_errno;
+	/* Increase the option reference counter. */
+	rte_atomic_fetch_add_explicit(&option->refcnt, 1,
+				      rte_memory_order_relaxed);
+	/* Update the manager with option information. */
+	mng->options[mng->nb_options].opt_type = spec->option_type;
+	mng->options[mng->nb_options].opt_class = spec->option_class;
+	mng->nb_options++;
+	return 0;
+}
+
+/**
+ * Unregister all GENEVE TLV options used by pattern template.
+ *
+ * @param[in] priv
+ *   Pointer to port's private data.
+ * @param[in] mng
+ *   Pointer to GENEVE option manager.
+ */
+void
+mlx5_geneve_tlv_options_unregister(struct mlx5_priv *priv,
+				   struct mlx5_geneve_tlv_options_mng *mng)
+{
+	struct mlx5_geneve_tlv_option *option;
+	uint8_t i;
+
+	for (i = 0; i < mng->nb_options; ++i) {
+		option = mlx5_geneve_tlv_option_get(priv,
+						    mng->options[i].opt_type,
+						    mng->options[i].opt_class);
+		MLX5_ASSERT(option != NULL);
+		/* Decrease the option reference counter. */
+		rte_atomic_fetch_sub_explicit(&option->refcnt, 1,
+					      rte_memory_order_relaxed);
+		mng->options[i].opt_type = 0;
+		mng->options[i].opt_class = 0;
+	}
+	mng->nb_options = 0;
+}
+
 /**
  * Create single GENEVE TLV option sample.
  *
@@ -208,6 +308,24 @@ mlx5_geneve_tlv_option_destroy_sample(struct mlx5_geneve_tlv_resource *resource)
 	resource->obj = NULL;
 }
 
+/*
+ * Sample for DW0 are created when one of two conditions is met:
+ * 1. Header is matchable.
+ * 2. This option doesn't configure any data DW.
+ */
+static bool
+should_configure_sample_for_dw0(const struct rte_pmd_mlx5_geneve_tlv *spec)
+{
+	uint8_t i;
+
+	if (spec->match_on_class_mode == 2)
+		return true;
+	for (i = 0; i < spec->sample_len; ++i)
+		if (spec->match_data_mask[i] != 0)
+			return false;
+	return true;
+}
+
 /**
  * Create single GENEVE TLV option.
  *
@@ -237,8 +355,7 @@ mlx5_geneve_tlv_option_create(void *ctx, const struct rte_pmd_mlx5_geneve_tlv *s
 	uint8_t i, resource_id = 0;
 	int ret;
 
-	if (spec->match_on_class_mode == 2) {
-		/* Header is matchable, create sample for DW0. */
+	if (should_configure_sample_for_dw0(spec)) {
 		attr.sample_offset = 0;
 		resource = &option->resources[resource_id];
 		ret = mlx5_geneve_tlv_option_create_sample(ctx, &attr,
diff --git a/drivers/net/mlx5/mlx5_flow_hw.c b/drivers/net/mlx5/mlx5_flow_hw.c
index da873ae2e2..7c786c432f 100644
--- a/drivers/net/mlx5/mlx5_flow_hw.c
+++ b/drivers/net/mlx5/mlx5_flow_hw.c
@@ -6781,6 +6781,17 @@ flow_hw_pattern_validate(struct rte_eth_dev *dev,
 							  " attribute");
 			break;
 		}
+		case RTE_FLOW_ITEM_TYPE_GENEVE_OPT:
+		{
+			int ret;
+
+			ret = mlx5_flow_geneve_tlv_option_validate(priv,
+								   &items[i],
+								   error);
+			if (ret < 0)
+				return ret;
+			break;
+		}
 		case RTE_FLOW_ITEM_TYPE_VOID:
 		case RTE_FLOW_ITEM_TYPE_ETH:
 		case RTE_FLOW_ITEM_TYPE_VLAN:
@@ -6792,6 +6803,7 @@ flow_hw_pattern_validate(struct rte_eth_dev *dev,
 		case RTE_FLOW_ITEM_TYPE_GTP_PSC:
 		case RTE_FLOW_ITEM_TYPE_VXLAN:
 		case RTE_FLOW_ITEM_TYPE_MPLS:
+		case RTE_FLOW_ITEM_TYPE_GENEVE:
 		case MLX5_RTE_FLOW_ITEM_TYPE_SQ:
 		case RTE_FLOW_ITEM_TYPE_GRE:
 		case RTE_FLOW_ITEM_TYPE_GRE_KEY:
@@ -6959,24 +6971,45 @@ flow_hw_pattern_template_create(struct rte_eth_dev *dev,
 		}
 	}
 	for (i = 0; items[i].type != RTE_FLOW_ITEM_TYPE_END; ++i) {
-		if (items[i].type == RTE_FLOW_ITEM_TYPE_FLEX) {
+		switch (items[i].type) {
+		case RTE_FLOW_ITEM_TYPE_FLEX: {
 			const struct rte_flow_item_flex *spec =
 				(const struct rte_flow_item_flex *)items[i].spec;
 			struct rte_flow_item_flex_handle *handle = spec->handle;
 
 			if (flow_hw_flex_item_acquire(dev, handle, &it->flex_item)) {
-				claim_zero(mlx5dr_match_template_destroy(it->mt));
-				mlx5_free(it);
 				rte_flow_error_set(error, rte_errno,
 						   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 						   "Failed to acquire flex item");
-				return NULL;
+				goto error;
 			}
+			break;
+		}
+		case RTE_FLOW_ITEM_TYPE_GENEVE_OPT: {
+			const struct rte_flow_item_geneve_opt *spec = items[i].spec;
+
+			if (mlx5_geneve_tlv_option_register(priv, spec,
+							    &it->geneve_opt_mng)) {
+				rte_flow_error_set(error, rte_errno,
+						   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+						   "Failed to register GENEVE TLV option");
+				goto error;
+			}
+			break;
+		}
+		default:
+			break;
 		}
 	}
 	__atomic_fetch_add(&it->refcnt, 1, __ATOMIC_RELAXED);
 	LIST_INSERT_HEAD(&priv->flow_hw_itt, it, next);
 	return it;
+error:
+	flow_hw_flex_item_release(dev, &it->flex_item);
+	mlx5_geneve_tlv_options_unregister(priv, &it->geneve_opt_mng);
+	claim_zero(mlx5dr_match_template_destroy(it->mt));
+	mlx5_free(it);
+	return NULL;
 }
 
 /**
@@ -6997,6 +7030,8 @@ flow_hw_pattern_template_destroy(struct rte_eth_dev *dev,
 			      struct rte_flow_pattern_template *template,
 			      struct rte_flow_error *error __rte_unused)
 {
+	struct mlx5_priv *priv = dev->data->dev_private;
+
 	if (__atomic_load_n(&template->refcnt, __ATOMIC_RELAXED) > 1) {
 		DRV_LOG(WARNING, "Item template %p is still in use.",
 			(void *)template);
@@ -7010,6 +7045,7 @@ flow_hw_pattern_template_destroy(struct rte_eth_dev *dev,
 		mlx5_free_srh_flex_parser(dev);
 	LIST_REMOVE(template, next);
 	flow_hw_flex_item_release(dev, &template->flex_item);
+	mlx5_geneve_tlv_options_unregister(priv, &template->geneve_opt_mng);
 	claim_zero(mlx5dr_match_template_destroy(template->mt));
 	mlx5_free(template);
 	return 0;
-- 
2.25.1