[v5 18/18] net/mlx5/hws: Enable HWS

[Alex Vesker <valex@nvidia.com>]

Replace stub implenation of HWS with mlx5dr code.

Signed-off-by: Alex Vesker <valex@nvidia.com>
---
 doc/guides/nics/mlx5.rst                     |   5 +-
 doc/guides/rel_notes/release_22_11.rst       |   4 +
 drivers/common/mlx5/linux/meson.build        |   3 +
 drivers/net/mlx5/hws/meson.build             |  18 +
 drivers/net/mlx5/{mlx5_dr.h => hws/mlx5dr.h} | 209 ++++++++--
 drivers/net/mlx5/hws/mlx5dr_internal.h       |  93 +++++
 drivers/net/mlx5/meson.build                 |   7 +-
 drivers/net/mlx5/mlx5.c                      |   6 +-
 drivers/net/mlx5/mlx5.h                      |   7 +-
 drivers/net/mlx5/mlx5_devx.c                 |   2 +-
 drivers/net/mlx5/mlx5_dr.c                   | 383 -------------------
 drivers/net/mlx5/mlx5_flow.c                 |   2 +
 drivers/net/mlx5/mlx5_flow.h                 |  11 +-
 drivers/net/mlx5/mlx5_flow_hw.c              |  10 +-
 14 files changed, 328 insertions(+), 432 deletions(-)
 create mode 100644 drivers/net/mlx5/hws/meson.build
 rename drivers/net/mlx5/{mlx5_dr.h => hws/mlx5dr.h} (66%)
 create mode 100644 drivers/net/mlx5/hws/mlx5dr_internal.h
 delete mode 100644 drivers/net/mlx5/mlx5_dr.c

diff --git a/doc/guides/nics/mlx5.rst b/doc/guides/nics/mlx5.rst
index bb436892a0..303eb17714 100644
--- a/doc/guides/nics/mlx5.rst
+++ b/doc/guides/nics/mlx5.rst
@@ -539,7 +539,10 @@ Limitations
 
     - WQE based high scaling and safer flow insertion/destruction.
     - Set ``dv_flow_en`` to 2 in order to enable HW steering.
-    - Async queue-based ``rte_flow_q`` APIs supported only.
+    - Async queue-based ``rte_flow_async`` APIs supported only.
+    - NIC ConnectX-5 and before are not supported.
+    - Partial match with item template is not supported.
+    - IPv6 5-tuple matching is not supported.
 
 - Match on GRE header supports the following fields:
 
diff --git a/doc/guides/rel_notes/release_22_11.rst b/doc/guides/rel_notes/release_22_11.rst
index a3700bbb34..eed7acc838 100644
--- a/doc/guides/rel_notes/release_22_11.rst
+++ b/doc/guides/rel_notes/release_22_11.rst
@@ -237,6 +237,10 @@ New Features
   sysfs entries to adjust the minimum and maximum uncore frequency values,
   which works on Linux with Intel hardware only.
 
+* **Updated Nvidia mlx5 driver.**
+
+  * Added fully support for queue based async HW steering to the PMD.
+
 * **Rewritten pmdinfo script.**
 
   The ``dpdk-pmdinfo.py`` script was rewritten to produce valid JSON only.
diff --git a/drivers/common/mlx5/linux/meson.build b/drivers/common/mlx5/linux/meson.build
index e8b9a07db5..2c69c5e546 100644
--- a/drivers/common/mlx5/linux/meson.build
+++ b/drivers/common/mlx5/linux/meson.build
@@ -8,6 +8,7 @@ dlopen_ibverbs = (get_option('ibverbs_link') == 'dlopen')
 LIB_GLUE_BASE = 'librte_common_mlx5_glue.so'
 LIB_GLUE_VERSION = abi_version
 LIB_GLUE = LIB_GLUE_BASE + '.' + LIB_GLUE_VERSION
+MLX5_HAVE_IBV_FLOW_DV_SUPPORT = false
 if dlopen_ibverbs
     dpdk_conf.set('RTE_IBVERBS_LINK_DLOPEN', 1)
     cflags += [
@@ -231,6 +232,8 @@ foreach arg:has_member_args
 endforeach
 configure_file(output : 'mlx5_autoconf.h', configuration : config)
 
+MLX5_HAVE_IBV_FLOW_DV_SUPPORT=config.get('HAVE_IBV_FLOW_DV_SUPPORT')
+
 # Build Glue Library
 if dlopen_ibverbs
     dlopen_name = 'mlx5_glue'
diff --git a/drivers/net/mlx5/hws/meson.build b/drivers/net/mlx5/hws/meson.build
new file mode 100644
index 0000000000..f94798dd2d
--- /dev/null
+++ b/drivers/net/mlx5/hws/meson.build
@@ -0,0 +1,18 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2022 NVIDIA Corporation & Affiliates
+
+includes += include_directories('.')
+sources += files(
+	'mlx5dr_context.c',
+	'mlx5dr_table.c',
+	'mlx5dr_matcher.c',
+	'mlx5dr_rule.c',
+	'mlx5dr_action.c',
+	'mlx5dr_buddy.c',
+	'mlx5dr_pool.c',
+	'mlx5dr_cmd.c',
+	'mlx5dr_send.c',
+	'mlx5dr_definer.c',
+	'mlx5dr_debug.c',
+	'mlx5dr_pat_arg.c',
+)
diff --git a/drivers/net/mlx5/mlx5_dr.h b/drivers/net/mlx5/hws/mlx5dr.h
similarity index 66%
rename from drivers/net/mlx5/mlx5_dr.h
rename to drivers/net/mlx5/hws/mlx5dr.h
index d0b2c15652..664dadbcde 100644
--- a/drivers/net/mlx5/mlx5_dr.h
+++ b/drivers/net/mlx5/hws/mlx5dr.h
@@ -1,9 +1,9 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright (c) 2021 NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
  */
 
-#ifndef MLX5_DR_H_
-#define MLX5_DR_H_
+#ifndef MLX5DR_H_
+#define MLX5DR_H_
 
 #include <rte_flow.h>
 
@@ -11,6 +11,7 @@ struct mlx5dr_context;
 struct mlx5dr_table;
 struct mlx5dr_matcher;
 struct mlx5dr_rule;
+struct ibv_context;
 
 enum mlx5dr_table_type {
 	MLX5DR_TABLE_TYPE_NIC_RX,
@@ -26,6 +27,27 @@ enum mlx5dr_matcher_resource_mode {
 	MLX5DR_MATCHER_RESOURCE_MODE_HTABLE,
 };
 
+enum mlx5dr_action_type {
+	MLX5DR_ACTION_TYP_LAST,
+	MLX5DR_ACTION_TYP_TNL_L2_TO_L2,
+	MLX5DR_ACTION_TYP_L2_TO_TNL_L2,
+	MLX5DR_ACTION_TYP_TNL_L3_TO_L2,
+	MLX5DR_ACTION_TYP_L2_TO_TNL_L3,
+	MLX5DR_ACTION_TYP_DROP,
+	MLX5DR_ACTION_TYP_TIR,
+	MLX5DR_ACTION_TYP_FT,
+	MLX5DR_ACTION_TYP_CTR,
+	MLX5DR_ACTION_TYP_TAG,
+	MLX5DR_ACTION_TYP_MODIFY_HDR,
+	MLX5DR_ACTION_TYP_VPORT,
+	MLX5DR_ACTION_TYP_MISS,
+	MLX5DR_ACTION_TYP_POP_VLAN,
+	MLX5DR_ACTION_TYP_PUSH_VLAN,
+	MLX5DR_ACTION_TYP_ASO_METER,
+	MLX5DR_ACTION_TYP_ASO_CT,
+	MLX5DR_ACTION_TYP_MAX,
+};
+
 enum mlx5dr_action_flags {
 	MLX5DR_ACTION_FLAG_ROOT_RX = 1 << 0,
 	MLX5DR_ACTION_FLAG_ROOT_TX = 1 << 1,
@@ -33,7 +55,10 @@ enum mlx5dr_action_flags {
 	MLX5DR_ACTION_FLAG_HWS_RX = 1 << 3,
 	MLX5DR_ACTION_FLAG_HWS_TX = 1 << 4,
 	MLX5DR_ACTION_FLAG_HWS_FDB = 1 << 5,
-	MLX5DR_ACTION_FLAG_INLINE = 1 << 6,
+	/* Shared action can be used over a few threads, since data is written
+	 * only once at the creation of the action.
+	 */
+	MLX5DR_ACTION_FLAG_SHARED = 1 << 6,
 };
 
 enum mlx5dr_action_reformat_type {
@@ -43,6 +68,18 @@ enum mlx5dr_action_reformat_type {
 	MLX5DR_ACTION_REFORMAT_TYPE_L2_TO_TNL_L3,
 };
 
+enum mlx5dr_action_aso_meter_color {
+	MLX5DR_ACTION_ASO_METER_COLOR_RED = 0x0,
+	MLX5DR_ACTION_ASO_METER_COLOR_YELLOW = 0x1,
+	MLX5DR_ACTION_ASO_METER_COLOR_GREEN = 0x2,
+	MLX5DR_ACTION_ASO_METER_COLOR_UNDEFINED = 0x3,
+};
+
+enum mlx5dr_action_aso_ct_flags {
+	MLX5DR_ACTION_ASO_CT_DIRECTION_INITIATOR = 0 << 0,
+	MLX5DR_ACTION_ASO_CT_DIRECTION_RESPONDER = 1 << 0,
+};
+
 enum mlx5dr_match_template_flags {
 	/* Allow relaxed matching by skipping derived dependent match fields. */
 	MLX5DR_MATCH_TEMPLATE_FLAG_RELAXED_MATCH = 1,
@@ -56,7 +93,7 @@ enum mlx5dr_send_queue_actions {
 struct mlx5dr_context_attr {
 	uint16_t queues;
 	uint16_t queue_size;
-	size_t initial_log_ste_memory;
+	size_t initial_log_ste_memory; /* Currently not in use */
 	/* Optional PD used for allocating res ources */
 	struct ibv_pd *pd;
 };
@@ -66,9 +103,21 @@ struct mlx5dr_table_attr {
 	uint32_t level;
 };
 
+enum mlx5dr_matcher_flow_src {
+	MLX5DR_MATCHER_FLOW_SRC_ANY = 0x0,
+	MLX5DR_MATCHER_FLOW_SRC_WIRE = 0x1,
+	MLX5DR_MATCHER_FLOW_SRC_VPORT = 0x2,
+};
+
 struct mlx5dr_matcher_attr {
+	/* Processing priority inside table */
 	uint32_t priority;
+	/* Provide all rules with unique rule_idx in num_log range to reduce locking */
+	bool optimize_using_rule_idx;
+	/* Resource mode and corresponding size */
 	enum mlx5dr_matcher_resource_mode mode;
+	/* Optimize insertion in case packet origin is the same for all rules */
+	enum mlx5dr_matcher_flow_src optimize_flow_src;
 	union {
 		struct {
 			uint8_t sz_row_log;
@@ -84,6 +133,8 @@ struct mlx5dr_matcher_attr {
 struct mlx5dr_rule_attr {
 	uint16_t queue_id;
 	void *user_data;
+	/* Valid if matcher optimize_using_rule_idx is set */
+	uint32_t rule_idx;
 	uint32_t burst:1;
 };
 
@@ -92,6 +143,9 @@ struct mlx5dr_devx_obj {
 	uint32_t id;
 };
 
+/* In actions that take offset, the offset is unique, and the user should not
+ * reuse the same index because data changing is not atomic.
+ */
 struct mlx5dr_rule_action {
 	struct mlx5dr_action *action;
 	union {
@@ -116,31 +170,17 @@ struct mlx5dr_rule_action {
 		struct {
 			rte_be32_t vlan_hdr;
 		} push_vlan;
-	};
-};
-
-enum {
-	MLX5DR_MATCH_TAG_SZ = 32,
-	MLX5DR_JAMBO_TAG_SZ = 44,
-};
 
-enum mlx5dr_rule_status {
-	MLX5DR_RULE_STATUS_UNKNOWN,
-	MLX5DR_RULE_STATUS_CREATING,
-	MLX5DR_RULE_STATUS_CREATED,
-	MLX5DR_RULE_STATUS_DELETING,
-	MLX5DR_RULE_STATUS_DELETED,
-	MLX5DR_RULE_STATUS_FAILED,
-};
+		struct {
+			uint32_t offset;
+			enum mlx5dr_action_aso_meter_color init_color;
+		} aso_meter;
 
-struct mlx5dr_rule {
-	struct mlx5dr_matcher *matcher;
-	union {
-		uint8_t match_tag[MLX5DR_MATCH_TAG_SZ];
-		struct ibv_flow *flow;
+		struct {
+			uint32_t offset;
+			enum mlx5dr_action_aso_ct_flags direction;
+		} aso_ct;
 	};
-	enum mlx5dr_rule_status status;
-	uint32_t rtc_used; /* The RTC into which the STE was inserted */
 };
 
 /* Open a context used for direct rule insertion using hardware steering.
@@ -153,7 +193,7 @@ struct mlx5dr_rule {
  * @return pointer to mlx5dr_context on success NULL otherwise.
  */
 struct mlx5dr_context *
-mlx5dr_context_open(void *ibv_ctx,
+mlx5dr_context_open(struct ibv_context *ibv_ctx,
 		    struct mlx5dr_context_attr *attr);
 
 /* Close a context used for direct hardware steering.
@@ -205,6 +245,26 @@ mlx5dr_match_template_create(const struct rte_flow_item items[],
  */
 int mlx5dr_match_template_destroy(struct mlx5dr_match_template *mt);
 
+/* Create new action template based on action_type array, the action template
+ * will be used for matcher creation.
+ *
+ * @param[in] action_type
+ *	An array of actions based on the order of actions which will be provided
+ *	with rule_actions to mlx5dr_rule_create. The last action is marked
+ *	using MLX5DR_ACTION_TYP_LAST.
+ * @return pointer to mlx5dr_action_template on success NULL otherwise
+ */
+struct mlx5dr_action_template *
+mlx5dr_action_template_create(const enum mlx5dr_action_type action_type[]);
+
+/* Destroy action template.
+ *
+ * @param[in] at
+ *	Action template to destroy.
+ * @return zero on success non zero otherwise.
+ */
+int mlx5dr_action_template_destroy(struct mlx5dr_action_template *at);
+
 /* Create a new direct rule matcher. Each matcher can contain multiple rules.
  * Matchers on the table will be processed by priority. Matching fields and
  * mask are described by the match template. In some cases multiple match
@@ -216,6 +276,10 @@ int mlx5dr_match_template_destroy(struct mlx5dr_match_template *mt);
  *	Array of match templates to be used on matcher.
  * @param[in] num_of_mt
  *	Number of match templates in mt array.
+ * @param[in] at
+ *	Array of action templates to be used on matcher.
+ * @param[in] num_of_at
+ *	Number of action templates in mt array.
  * @param[in] attr
  *	Attributes used for matcher creation.
  * @return pointer to mlx5dr_matcher on success NULL otherwise.
@@ -224,6 +288,8 @@ struct mlx5dr_matcher *
 mlx5dr_matcher_create(struct mlx5dr_table *table,
 		      struct mlx5dr_match_template *mt[],
 		      uint8_t num_of_mt,
+		      struct mlx5dr_action_template *at[],
+		      uint8_t num_of_at,
 		      struct mlx5dr_matcher_attr *attr);
 
 /* Destroy direct rule matcher.
@@ -245,11 +311,13 @@ size_t mlx5dr_rule_get_handle_size(void);
  * @param[in] matcher
  *	The matcher in which the new rule will be created.
  * @param[in] mt_idx
- *	Match template index to create the rule with.
+ *	Match template index to create the match with.
  * @param[in] items
  *	The items used for the value matching.
  * @param[in] rule_actions
  *	Rule action to be executed on match.
+ * @param[in] at_idx
+ *	Action template index to apply the actions with.
  * @param[in] num_of_actions
  *	Number of rule actions.
  * @param[in] attr
@@ -261,8 +329,8 @@ size_t mlx5dr_rule_get_handle_size(void);
 int mlx5dr_rule_create(struct mlx5dr_matcher *matcher,
 		       uint8_t mt_idx,
 		       const struct rte_flow_item items[],
+		       uint8_t at_idx,
 		       struct mlx5dr_rule_action rule_actions[],
-		       uint8_t num_of_actions,
 		       struct mlx5dr_rule_attr *attr,
 		       struct mlx5dr_rule *rule_handle);
 
@@ -317,6 +385,21 @@ mlx5dr_action_create_dest_table(struct mlx5dr_context *ctx,
 				struct mlx5dr_table *tbl,
 				uint32_t flags);
 
+/* Create direct rule goto vport action.
+ *
+ * @param[in] ctx
+ *	The context in which the new action will be created.
+ * @param[in] ib_port_num
+ *	Destination ib_port number.
+ * @param[in] flags
+ *	Action creation flags. (enum mlx5dr_action_flags)
+ * @return pointer to mlx5dr_action on success NULL otherwise.
+ */
+struct mlx5dr_action *
+mlx5dr_action_create_dest_vport(struct mlx5dr_context *ctx,
+				uint32_t ib_port_num,
+				uint32_t flags);
+
 /*  Create direct rule goto TIR action.
  *
  * @param[in] ctx
@@ -400,10 +483,66 @@ mlx5dr_action_create_reformat(struct mlx5dr_context *ctx,
 struct mlx5dr_action *
 mlx5dr_action_create_modify_header(struct mlx5dr_context *ctx,
 				   size_t pattern_sz,
-				   rte_be64_t pattern[],
+				   __be64 pattern[],
 				   uint32_t log_bulk_size,
 				   uint32_t flags);
 
+/* Create direct rule ASO flow meter action.
+ *
+ * @param[in] ctx
+ *	The context in which the new action will be created.
+ * @param[in] devx_obj
+ *	The DEVX ASO object.
+ * @param[in] return_reg_c
+ *	Copy the ASO object value into this reg_c, after a packet hits a rule with this ASO object.
+ * @param[in] flags
+ *	Action creation flags. (enum mlx5dr_action_flags)
+ * @return pointer to mlx5dr_action on success NULL otherwise.
+ */
+struct mlx5dr_action *
+mlx5dr_action_create_aso_meter(struct mlx5dr_context *ctx,
+			       struct mlx5dr_devx_obj *devx_obj,
+			       uint8_t return_reg_c,
+			       uint32_t flags);
+
+/* Create direct rule ASO CT action.
+ *
+ * @param[in] ctx
+ *	The context in which the new action will be created.
+ * @param[in] devx_obj
+ *	The DEVX ASO object.
+ * @param[in] return_reg_id
+ * 	Copy the ASO object value into this reg_id, after a packet hits a rule with this ASO object.
+ * @param[in] flags
+ *	Action creation flags. (enum mlx5dr_action_flags)
+ * @return pointer to mlx5dr_action on success NULL otherwise.
+ */
+struct mlx5dr_action *
+mlx5dr_action_create_aso_ct(struct mlx5dr_context *ctx,
+			    struct mlx5dr_devx_obj *devx_obj,
+			    uint8_t return_reg_id,
+			    uint32_t flags);
+
+/* Create direct rule pop vlan action.
+ * @param[in] ctx
+ *	The context in which the new action will be created.
+ * @param[in] flags
+ *	Action creation flags. (enum mlx5dr_action_flags)
+ * @return pointer to mlx5dr_action on success NULL otherwise.
+ */
+struct mlx5dr_action *
+mlx5dr_action_create_pop_vlan(struct mlx5dr_context *ctx, uint32_t flags);
+
+/* Create direct rule push vlan action.
+ * @param[in] ctx
+ *	The context in which the new action will be created.
+ * @param[in] flags
+ *	Action creation flags. (enum mlx5dr_action_flags)
+ * @return pointer to mlx5dr_action on success NULL otherwise.
+ */
+struct mlx5dr_action *
+mlx5dr_action_create_push_vlan(struct mlx5dr_context *ctx, uint32_t flags);
+
 /* Destroy direct rule action.
  *
  * @param[in] action
@@ -432,11 +571,11 @@ int mlx5dr_send_queue_poll(struct mlx5dr_context *ctx,
 /* Perform an action on the queue
  *
  * @param[in] ctx
- *	The context to which the queue belong to.
+ * 	The context to which the queue belong to.
  * @param[in] queue_id
- *	The id of the queue to perform the action on.
+ * 	The id of the queue to perform the action on.
  * @param[in] actions
- *	Actions to perform on the queue. (enum mlx5dr_send_queue_actions)
+ * 	Actions to perform on the queue. (enum mlx5dr_send_queue_actions)
  * @return zero on success non zero otherwise.
  */
 int mlx5dr_send_queue_action(struct mlx5dr_context *ctx,
@@ -448,7 +587,7 @@ int mlx5dr_send_queue_action(struct mlx5dr_context *ctx,
  * @param[in] ctx
  *	The context which to dump the info from.
  * @param[in] f
- *	The file to write the dump to.
+ * 	The file to write the dump to.
  * @return zero on success non zero otherwise.
  */
 int mlx5dr_debug_dump(struct mlx5dr_context *ctx, FILE *f);
diff --git a/drivers/net/mlx5/hws/mlx5dr_internal.h b/drivers/net/mlx5/hws/mlx5dr_internal.h
new file mode 100644
index 0000000000..dbd77b9c66
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_internal.h
@@ -0,0 +1,93 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
+ */
+
+#ifndef MLX5DR_INTERNAL_H_
+#define MLX5DR_INTERNAL_H_
+
+#include <stdint.h>
+#include <sys/queue.h>
+/* Verbs headers do not support -pedantic. */
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+#include <infiniband/verbs.h>
+#include <infiniband/mlx5dv.h>
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+#include <rte_flow.h>
+#include <rte_gtp.h>
+
+#include "mlx5_prm.h"
+#include "mlx5_glue.h"
+#include "mlx5_flow.h"
+#include "mlx5_utils.h"
+#include "mlx5_malloc.h"
+
+#include "mlx5dr.h"
+#include "mlx5dr_pool.h"
+#include "mlx5dr_context.h"
+#include "mlx5dr_table.h"
+#include "mlx5dr_matcher.h"
+#include "mlx5dr_send.h"
+#include "mlx5dr_rule.h"
+#include "mlx5dr_cmd.h"
+#include "mlx5dr_action.h"
+#include "mlx5dr_definer.h"
+#include "mlx5dr_debug.h"
+#include "mlx5dr_pat_arg.h"
+
+#define DW_SIZE		4
+#define BITS_IN_BYTE	8
+#define BITS_IN_DW	(BITS_IN_BYTE * DW_SIZE)
+
+#define BIT(_bit) (1ULL << (_bit))
+#define IS_BIT_SET(_value, _bit) (_value & (1ULL << (_bit)))
+
+#ifndef ARRAY_SIZE
+#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
+#endif
+
+#ifdef RTE_LIBRTE_MLX5_DEBUG
+/* Prevent double function name print when debug is set */
+#define DR_LOG DRV_LOG
+#else
+/* Print function name as part of the log */
+#define DR_LOG(level, ...) \
+	DRV_LOG(level, RTE_FMT("[%s]: " RTE_FMT_HEAD(__VA_ARGS__,), __func__, RTE_FMT_TAIL(__VA_ARGS__,)))
+#endif
+
+static inline void *simple_malloc(size_t size)
+{
+	return mlx5_malloc(MLX5_MEM_SYS,
+			   size,
+			   MLX5_MALLOC_ALIGNMENT,
+			   SOCKET_ID_ANY);
+}
+
+static inline void *simple_calloc(size_t nmemb, size_t size)
+{
+	return mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO,
+			   nmemb * size,
+			   MLX5_MALLOC_ALIGNMENT,
+			   SOCKET_ID_ANY);
+}
+
+static inline void simple_free(void *addr)
+{
+	mlx5_free(addr);
+}
+
+static inline bool is_mem_zero(const uint8_t *mem, size_t size)
+{
+	assert(size);
+	return (*mem == 0) && memcmp(mem, mem + 1, size - 1) == 0;
+}
+
+static inline uint64_t roundup_pow_of_two(uint64_t n)
+{
+	return n == 1 ? 1 : 1ULL << log2above(n);
+}
+
+#endif /* MLX5DR_INTERNAL_H_ */
diff --git a/drivers/net/mlx5/meson.build b/drivers/net/mlx5/meson.build
index 6a84d96380..c3b8fa16d3 100644
--- a/drivers/net/mlx5/meson.build
+++ b/drivers/net/mlx5/meson.build
@@ -14,10 +14,8 @@ sources = files(
         'mlx5.c',
         'mlx5_ethdev.c',
         'mlx5_flow.c',
-        'mlx5_dr.c',
         'mlx5_flow_meter.c',
         'mlx5_flow_dv.c',
-        'mlx5_flow_hw.c',
         'mlx5_flow_aso.c',
         'mlx5_flow_flex.c',
         'mlx5_mac.c',
@@ -42,6 +40,7 @@ sources = files(
 
 if is_linux
     sources += files(
+            'mlx5_flow_hw.c',
             'mlx5_flow_verbs.c',
     )
     if (dpdk_conf.has('RTE_ARCH_X86_64')
@@ -72,3 +71,7 @@ endif
 testpmd_sources += files('mlx5_testpmd.c')
 
 subdir(exec_env)
+
+if (is_linux and MLX5_HAVE_IBV_FLOW_DV_SUPPORT)
+	subdir('hws')
+endif
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index b39ef1ecbe..a34fbcf74d 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1700,7 +1700,7 @@ mlx5_free_table_hash_list(struct mlx5_priv *priv)
 	*tbls = NULL;
 }
 
-#if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
+#ifdef HAVE_MLX5_HWS_SUPPORT
 /**
  * Allocate HW steering group hash list.
  *
@@ -1749,7 +1749,7 @@ mlx5_alloc_table_hash_list(struct mlx5_priv *priv __rte_unused)
 	int err = 0;
 
 	/* Tables are only used in DV and DR modes. */
-#if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
+#ifdef HAVE_MLX5_HWS_SUPPORT
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
 	char s[MLX5_NAME_SIZE];
 
@@ -1942,7 +1942,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	/* Free the eCPRI flex parser resource. */
 	mlx5_flex_parser_ecpri_release(dev);
 	mlx5_flex_item_port_cleanup(dev);
-#if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
+#ifdef HAVE_MLX5_HWS_SUPPORT
 	flow_hw_resource_release(dev);
 	flow_hw_clear_port_info(dev);
 	if (priv->sh->config.dv_flow_en == 2)
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 741be2df98..1d3c1ad93d 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -34,7 +34,12 @@
 #include "mlx5_os.h"
 #include "mlx5_autoconf.h"
 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
-#include "mlx5_dr.h"
+#ifndef RTE_EXEC_ENV_WINDOWS
+#define HAVE_MLX5_HWS_SUPPORT 1
+#else
+#define __be64 uint64_t
+#endif
+#include "hws/mlx5dr.h"
 #endif
 
 #define MLX5_SH(dev) (((struct mlx5_priv *)(dev)->data->dev_private)->sh)
diff --git a/drivers/net/mlx5/mlx5_devx.c b/drivers/net/mlx5/mlx5_devx.c
index fe303a73bb..137e7dd4ac 100644
--- a/drivers/net/mlx5/mlx5_devx.c
+++ b/drivers/net/mlx5/mlx5_devx.c
@@ -907,7 +907,7 @@ mlx5_devx_hrxq_new(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq,
 		rte_errno = errno;
 		goto error;
 	}
-#if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
+#ifdef HAVE_MLX5_HWS_SUPPORT
 	if (hrxq->hws_flags) {
 		hrxq->action = mlx5dr_action_create_dest_tir
 			(priv->dr_ctx,
diff --git a/drivers/net/mlx5/mlx5_dr.c b/drivers/net/mlx5/mlx5_dr.c
deleted file mode 100644
index 7218708986..0000000000
--- a/drivers/net/mlx5/mlx5_dr.c
+++ /dev/null
@@ -1,383 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright (c) 2021 NVIDIA CORPORATION. All rights reserved.
- */
-#include <rte_flow.h>
-
-#include "mlx5_defs.h"
-#if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
-#include "mlx5_dr.h"
-
-/*
- * The following null stubs are prepared in order not to break the linkage
- * before the HW steering low-level implementation is added.
- */
-
-/* Open a context used for direct rule insertion using hardware steering.
- * Each context can contain multiple tables of different types.
- *
- * @param[in] ibv_ctx
- *	The ibv context to used for HWS.
- * @param[in] attr
- *	Attributes used for context open.
- * @return pointer to mlx5dr_context on success NULL otherwise.
- */
-__rte_weak struct mlx5dr_context *
-mlx5dr_context_open(void *ibv_ctx,
-		    struct mlx5dr_context_attr *attr)
-{
-	(void)ibv_ctx;
-	(void)attr;
-	return NULL;
-}
-
-/* Close a context used for direct hardware steering.
- *
- * @param[in] ctx
- *	mlx5dr context to close.
- * @return zero on success non zero otherwise.
- */
-__rte_weak int
-mlx5dr_context_close(struct mlx5dr_context *ctx)
-{
-	(void)ctx;
-	return 0;
-}
-
-/* Create a new direct rule table. Each table can contain multiple matchers.
- *
- * @param[in] ctx
- *	The context in which the new table will be opened.
- * @param[in] attr
- *	Attributes used for table creation.
- * @return pointer to mlx5dr_table on success NULL otherwise.
- */
-__rte_weak struct mlx5dr_table *
-mlx5dr_table_create(struct mlx5dr_context *ctx,
-		    struct mlx5dr_table_attr *attr)
-{
-	(void)ctx;
-	(void)attr;
-	return NULL;
-}
-
-/* Destroy direct rule table.
- *
- * @param[in] tbl
- *	mlx5dr table to destroy.
- * @return zero on success non zero otherwise.
- */
-__rte_weak int mlx5dr_table_destroy(struct mlx5dr_table *tbl)
-{
-	(void)tbl;
-	return 0;
-}
-
-/* Create new match template based on items mask, the match template
- * will be used for matcher creation.
- *
- * @param[in] items
- *	Describe the mask for template creation
- * @param[in] flags
- *	Template creation flags
- * @return pointer to mlx5dr_match_template on success NULL otherwise
- */
-__rte_weak struct mlx5dr_match_template *
-mlx5dr_match_template_create(const struct rte_flow_item items[],
-			     enum mlx5dr_match_template_flags flags)
-{
-	(void)items;
-	(void)flags;
-	return NULL;
-}
-
-/* Destroy match template.
- *
- * @param[in] mt
- *	Match template to destroy.
- * @return zero on success non zero otherwise.
- */
-__rte_weak int
-mlx5dr_match_template_destroy(struct mlx5dr_match_template *mt)
-{
-	(void)mt;
-	return 0;
-}
-
-/* Create a new direct rule matcher. Each matcher can contain multiple rules.
- * Matchers on the table will be processed by priority. Matching fields and
- * mask are described by the match template. In some cases multiple match
- * templates can be used on the same matcher.
- *
- * @param[in] table
- *	The table in which the new matcher will be opened.
- * @param[in] mt
- *	Array of match templates to be used on matcher.
- * @param[in] num_of_mt
- *	Number of match templates in mt array.
- * @param[in] attr
- *	Attributes used for matcher creation.
- * @return pointer to mlx5dr_matcher on success NULL otherwise.
- */
-__rte_weak struct mlx5dr_matcher *
-mlx5dr_matcher_create(struct mlx5dr_table *table __rte_unused,
-		      struct mlx5dr_match_template *mt[] __rte_unused,
-		      uint8_t num_of_mt __rte_unused,
-		      struct mlx5dr_matcher_attr *attr __rte_unused)
-{
-	return NULL;
-}
-
-/* Destroy direct rule matcher.
- *
- * @param[in] matcher
- *	Matcher to destroy.
- * @return zero on success non zero otherwise.
- */
-__rte_weak int
-mlx5dr_matcher_destroy(struct mlx5dr_matcher *matcher __rte_unused)
-{
-	return 0;
-}
-
-/* Enqueue create rule operation.
- *
- * @param[in] matcher
- *	The matcher in which the new rule will be created.
- * @param[in] mt_idx
- *	Match template index to create the rule with.
- * @param[in] items
- *	The items used for the value matching.
- * @param[in] rule_actions
- *	Rule action to be executed on match.
- * @param[in] num_of_actions
- *	Number of rule actions.
- * @param[in] attr
- *	Rule creation attributes.
- * @param[in, out] rule_handle
- *	A valid rule handle. The handle doesn't require any initialization.
- * @return zero on successful enqueue non zero otherwise.
- */
-__rte_weak int
-mlx5dr_rule_create(struct mlx5dr_matcher *matcher __rte_unused,
-		   uint8_t mt_idx __rte_unused,
-		   const struct rte_flow_item items[] __rte_unused,
-		   struct mlx5dr_rule_action rule_actions[] __rte_unused,
-		   uint8_t num_of_actions __rte_unused,
-		   struct mlx5dr_rule_attr *attr __rte_unused,
-		   struct mlx5dr_rule *rule_handle __rte_unused)
-{
-	return 0;
-}
-
-/* Enqueue destroy rule operation.
- *
- * @param[in] rule
- *	The rule destruction to enqueue.
- * @param[in] attr
- *	Rule destruction attributes.
- * @return zero on successful enqueue non zero otherwise.
- */
-__rte_weak int
-mlx5dr_rule_destroy(struct mlx5dr_rule *rule __rte_unused,
-		    struct mlx5dr_rule_attr *attr __rte_unused)
-{
-	return 0;
-}
-
-/* Create direct rule drop action.
- *
- * @param[in] ctx
- *	The context in which the new action will be created.
- * @param[in] flags
- *	Action creation flags. (enum mlx5dr_action_flags)
- * @return pointer to mlx5dr_action on success NULL otherwise.
- */
-__rte_weak struct mlx5dr_action *
-mlx5dr_action_create_dest_drop(struct mlx5dr_context *ctx __rte_unused,
-			       uint32_t flags __rte_unused)
-{
-	return NULL;
-}
-
-/* Create direct rule default miss action.
- * Defaults are RX: Drop TX: Wire.
- *
- * @param[in] ctx
- *	The context in which the new action will be created.
- * @param[in] flags
- *	Action creation flags. (enum mlx5dr_action_flags)
- * @return pointer to mlx5dr_action on success NULL otherwise.
- */
-__rte_weak struct mlx5dr_action *
-mlx5dr_action_create_default_miss(struct mlx5dr_context *ctx __rte_unused,
-				  uint32_t flags __rte_unused)
-{
-	return NULL;
-}
-
-/* Create direct rule goto table action.
- *
- * @param[in] ctx
- *	The context in which the new action will be created.
- * @param[in] tbl
- *	Destination table.
- * @param[in] flags
- *	Action creation flags. (enum mlx5dr_action_flags)
- * @return pointer to mlx5dr_action on success NULL otherwise.
- */
-__rte_weak struct mlx5dr_action *
-mlx5dr_action_create_dest_table(struct mlx5dr_context *ctx __rte_unused,
-				struct mlx5dr_table *tbl __rte_unused,
-				uint32_t flags __rte_unused)
-{
-	return NULL;
-}
-
-/*  Create direct rule goto TIR action.
- *
- * @param[in] ctx
- *	The context in which the new action will be created.
- * @param[in] obj
- *	Direct rule TIR devx object.
- * @param[in] flags
- *	Action creation flags. (enum mlx5dr_action_flags)
- * @return pointer to mlx5dr_action on success NULL otherwise.
- */
-__rte_weak struct mlx5dr_action *
-mlx5dr_action_create_dest_tir(struct mlx5dr_context *ctx __rte_unused,
-			      struct mlx5dr_devx_obj *obj __rte_unused,
-			      uint32_t flags __rte_unused)
-{
-	return NULL;
-}
-
-/* Create direct rule TAG action.
- *
- * @param[in] ctx
- *	The context in which the new action will be created.
- * @param[in] flags
- *	Action creation flags. (enum mlx5dr_action_flags)
- * @return pointer to mlx5dr_action on success NULL otherwise.
- */
-__rte_weak struct mlx5dr_action *
-mlx5dr_action_create_tag(struct mlx5dr_context *ctx __rte_unused,
-			 uint32_t flags __rte_unused)
-{
-	return NULL;
-}
-
-/* Create direct rule counter action.
- *
- * @param[in] ctx
- *	The context in which the new action will be created.
- * @param[in] obj
- *	Direct rule counter devx object.
- * @param[in] flags
- *	Action creation flags. (enum mlx5dr_action_flags)
- * @return pointer to mlx5dr_action on success NULL otherwise.
- */
-__rte_weak struct mlx5dr_action *
-mlx5dr_action_create_counter(struct mlx5dr_context *ctx,
-			     struct mlx5dr_devx_obj *obj,
-			     uint32_t flags);
-
-/* Create direct rule reformat action.
- *
- * @param[in] ctx
- *	The context in which the new action will be created.
- * @param[in] reformat_type
- *	Type of reformat.
- * @param[in] data_sz
- *	Size in bytes of data.
- * @param[in] inline_data
- *	Header data array in case of inline action.
- * @param[in] log_bulk_size
- *	Number of unique values used with this pattern.
- * @param[in] flags
- *	Action creation flags. (enum mlx5dr_action_flags)
- * @return pointer to mlx5dr_action on success NULL otherwise.
- */
-__rte_weak struct mlx5dr_action *
-mlx5dr_action_create_reformat(struct mlx5dr_context *ctx __rte_unused,
-	      enum mlx5dr_action_reformat_type reformat_type __rte_unused,
-			      size_t data_sz __rte_unused,
-			      void *inline_data __rte_unused,
-			      uint32_t log_bulk_size __rte_unused,
-			      uint32_t flags __rte_unused)
-{
-	return NULL;
-}
-
-/* Create direct rule modify header action.
- *
- * @param[in] ctx
- *	The context in which the new action will be created.
- * @param[in] pattern_sz
- *	Byte size of the pattern array.
- * @param[in] pattern
- *	PRM format modify pattern action array.
- * @param[in] log_bulk_size
- *	Number of unique values used with this pattern.
- * @param[in] flags
- *	Action creation flags. (enum mlx5dr_action_flags)
- * @return pointer to mlx5dr_action on success NULL otherwise.
- */
-__rte_weak struct mlx5dr_action *
-mlx5dr_action_create_modify_header(struct mlx5dr_context *ctx,
-				   size_t pattern_sz,
-				   rte_be64_t pattern[],
-				   uint32_t log_bulk_size,
-				   uint32_t flags);
-
-/* Destroy direct rule action.
- *
- * @param[in] action
- *	The action to destroy.
- * @return zero on success non zero otherwise.
- */
-__rte_weak int
-mlx5dr_action_destroy(struct mlx5dr_action *action __rte_unused)
-{
-	return 0;
-}
-
-/* Poll queue for rule creation and deletions completions.
- *
- * @param[in] ctx
- *	The context to which the queue belong to.
- * @param[in] queue_id
- *	The id of the queue to poll.
- * @param[in, out] res
- *	Completion array.
- * @param[in] res_nb
- *	Maximum number of results to return.
- * @return negative number on failure, the number of completions otherwise.
- */
-__rte_weak int
-mlx5dr_send_queue_poll(struct mlx5dr_context *ctx __rte_unused,
-		       uint16_t queue_id __rte_unused,
-		       struct rte_flow_op_result res[] __rte_unused,
-		       uint32_t res_nb __rte_unused)
-{
-	return 0;
-}
-
-/* Perform an action on the queue
- *
- * @param[in] ctx
- *	The context to which the queue belong to.
- * @param[in] queue_id
- *	The id of the queue to perform the action on.
- * @param[in] actions
- *	Actions to perform on the queue. (enum mlx5dr_send_queue_actions)
- * @return zero on success non zero otherwise.
- */
-__rte_weak int
-mlx5dr_send_queue_action(struct mlx5dr_context *ctx __rte_unused,
-			 uint16_t queue_id __rte_unused,
-			 uint32_t actions __rte_unused)
-{
-	return 0;
-}
-
-#endif
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index dd3d2bb1a4..2c6acd551c 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -93,6 +93,8 @@ const struct mlx5_flow_driver_ops *flow_drv_ops[] = {
 	[MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops,
 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
 	[MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops,
+#endif
+#ifdef HAVE_MLX5_HWS_SUPPORT
 	[MLX5_FLOW_TYPE_HW] = &mlx5_flow_hw_drv_ops,
 #endif
 	[MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops,
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 2002f6ef4b..cde602d3a1 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -17,6 +17,7 @@
 #include <mlx5_prm.h>
 
 #include "mlx5.h"
+#include "hws/mlx5dr.h"
 
 /* E-Switch Manager port, used for rte_flow_item_port_id. */
 #define MLX5_PORT_ESW_MGR UINT32_MAX
@@ -1043,6 +1044,10 @@ struct rte_flow {
 
 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
 
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+
 /* HWS flow struct. */
 struct rte_flow_hw {
 	uint32_t idx; /* Flow index from indexed pool. */
@@ -1053,9 +1058,13 @@ struct rte_flow_hw {
 		struct mlx5_hrxq *hrxq; /* TIR action. */
 	};
 	struct rte_flow_template_table *table; /* The table flow allcated from. */
-	struct mlx5dr_rule rule; /* HWS layer data struct. */
+	uint8_t rule[0]; /* HWS layer data struct. */
 } __rte_packed;
 
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
 /* rte flow action translate to DR action struct. */
 struct mlx5_action_construct_data {
 	LIST_ENTRY(mlx5_action_construct_data) next;
diff --git a/drivers/net/mlx5/mlx5_flow_hw.c b/drivers/net/mlx5/mlx5_flow_hw.c
index 78c741bb91..fecf28c1ca 100644
--- a/drivers/net/mlx5/mlx5_flow_hw.c
+++ b/drivers/net/mlx5/mlx5_flow_hw.c
@@ -1107,8 +1107,8 @@ flow_hw_async_flow_create(struct rte_eth_dev *dev,
 				  actions, rule_acts, &acts_num);
 	ret = mlx5dr_rule_create(table->matcher,
 				 pattern_template_index, items,
-				 rule_acts, acts_num,
-				 &rule_attr, &flow->rule);
+				 action_template_index, rule_acts,
+				 &rule_attr, (struct mlx5dr_rule *)flow->rule);
 	if (likely(!ret))
 		return (struct rte_flow *)flow;
 	/* Flow created fail, return the descriptor and flow memory. */
@@ -1171,7 +1171,7 @@ flow_hw_async_flow_destroy(struct rte_eth_dev *dev,
 	job->user_data = user_data;
 	job->flow = fh;
 	rule_attr.user_data = job;
-	ret = mlx5dr_rule_destroy(&fh->rule, &rule_attr);
+	ret = mlx5dr_rule_destroy((struct mlx5dr_rule *)fh->rule, &rule_attr);
 	if (likely(!ret))
 		return 0;
 	priv->hw_q[queue].job_idx++;
@@ -1437,7 +1437,7 @@ flow_hw_table_create(struct rte_eth_dev *dev,
 		.data = &flow_attr,
 	};
 	struct mlx5_indexed_pool_config cfg = {
-		.size = sizeof(struct rte_flow_hw),
+		.size = sizeof(struct rte_flow_hw) + mlx5dr_rule_get_handle_size(),
 		.trunk_size = 1 << 12,
 		.per_core_cache = 1 << 13,
 		.need_lock = 1,
@@ -1498,7 +1498,7 @@ flow_hw_table_create(struct rte_eth_dev *dev,
 		tbl->its[i] = item_templates[i];
 	}
 	tbl->matcher = mlx5dr_matcher_create
-		(tbl->grp->tbl, mt, nb_item_templates, &matcher_attr);
+		(tbl->grp->tbl, mt, nb_item_templates, NULL, 0, &matcher_attr);
 	if (!tbl->matcher)
 		goto it_error;
 	tbl->nb_item_templates = nb_item_templates;
-- 
2.18.1