* [PATCH v12 03/10] netdev: cavium: octeon: Add Octeon III BGX Ethernet Nexus
From: Steven J. Hill @ 2018-06-27 21:25 UTC (permalink / raw)
To: netdev; +Cc: Carlos Munoz, Chandrakala Chavva, Steven J. Hill
In-Reply-To: <1530134719-19407-1-git-send-email-steven.hill@cavium.com>
From: Carlos Munoz <cmunoz@cavium.com>
Add the BGX nexus architeture for Octeon III BGX Ethernet.
Signed-off-by: Carlos Munoz <cmunoz@cavium.com>
Signed-off-by: Steven J. Hill <Steven.Hill@cavium.com>
---
.../net/ethernet/cavium/octeon/octeon3-bgx-nexus.c | 670 +++++++++++++++++++++
drivers/net/ethernet/cavium/octeon/octeon3-bgx.h | 281 +++++----
2 files changed, 831 insertions(+), 120 deletions(-)
create mode 100644 drivers/net/ethernet/cavium/octeon/octeon3-bgx-nexus.c
diff --git a/drivers/net/ethernet/cavium/octeon/octeon3-bgx-nexus.c b/drivers/net/ethernet/cavium/octeon/octeon3-bgx-nexus.c
new file mode 100644
index 0000000..fced298
--- /dev/null
+++ b/drivers/net/ethernet/cavium/octeon/octeon3-bgx-nexus.c
@@ -0,0 +1,670 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Octeon III BGX Nexus Ethernet driver
+ *
+ * Copyright (C) 2018 Cavium, Inc.
+ */
+
+#include <linux/ctype.h>
+#include <linux/of_address.h>
+#include <linux/of_platform.h>
+
+#include "octeon3.h"
+
+static atomic_t request_mgmt_once;
+static atomic_t load_driver_once;
+static atomic_t pki_id;
+
+static char *mix_port;
+module_param(mix_port, charp, 0444);
+MODULE_PARM_DESC(mix_port, "Specifies which ports connect to MIX interfaces.");
+
+static char *pki_port;
+module_param(pki_port, charp, 0444);
+MODULE_PARM_DESC(pki_port, "Specifies which ports connect to the PKI.");
+
+#define MAX_MIX_PER_NODE 2
+#define MAX_MIX (MAX_NODES * MAX_MIX_PER_NODE)
+
+/* struct mix_port_lmac - Describes a lmac that connects to a mix port. The lmac
+ * must be on the same node as the mix.
+ * @node: Node of the lmac.
+ * @bgx: Bgx of the lmac.
+ * @lmac: Lmac index.
+ */
+struct mix_port_lmac {
+ int node;
+ int bgx;
+ int lmac;
+};
+
+/* mix_ports_lmacs contains all the lmacs connected to mix ports */
+static struct mix_port_lmac mix_port_lmacs[MAX_MIX];
+
+/* pki_ports keeps track of the lmacs connected to the pki */
+static bool pki_ports[MAX_NODES][MAX_BGX_PER_NODE][MAX_LMAC_PER_BGX];
+
+/* Created platform devices get added to this list */
+static struct list_head pdev_list;
+static struct mutex pdev_list_lock;
+
+/* Created platform device use this structure to add themselves to the list */
+struct pdev_list_item {
+ struct list_head list;
+ struct platform_device *pdev;
+};
+
+/* is_lmac_to_mix - Search the list of lmacs connected to mix'es for a match.
+ * @node: Numa node of lmac to search for.
+ * @bgx: Bgx of lmac to search for.
+ * @lmac: Lmac index to search for.
+ *
+ * Returns true if the lmac is connected to a mix.
+ * Returns false if the lmac is not connected to a mix.
+ */
+static bool is_lmac_to_mix(int node, int bgx, int lmac)
+{
+ int i;
+
+ for (i = 0; i < MAX_MIX; i++) {
+ if (mix_port_lmacs[i].node == node &&
+ mix_port_lmacs[i].bgx == bgx &&
+ mix_port_lmacs[i].lmac == lmac)
+ return true;
+ }
+
+ return false;
+}
+
+/* is_lmac_to_pki - Search the list of lmacs connected to the pki for a match.
+ * @node: Numa node of lmac to search for.
+ * @bgx: Bgx of lmac to search for.
+ * @lmac: Lmac index to search for.
+ *
+ * Returns true if the lmac is connected to the pki.
+ * Returns false if the lmac is not connected to the pki.
+ */
+static bool is_lmac_to_pki(int node, int bgx, int lmac)
+{
+ return pki_ports[node][bgx][lmac];
+}
+
+/* is_lmac_to_xcv - Check if this lmac is connected to the xcv block (rgmii).
+ * @of_node: Device node to check.
+ *
+ * Returns true if the lmac is connected to the xcv port.
+ * Returns false if the lmac is not connected to the xcv port.
+ */
+static bool is_lmac_to_xcv(struct device_node *of_node)
+{
+ return of_device_is_compatible(of_node, "cavium,octeon-7360-xcv");
+}
+
+static int bgx_probe(struct platform_device *pdev)
+{
+ struct platform_device *new_dev, *pki_dev;
+ struct mac_platform_data platform_data;
+ int i, interface, numa_node, r = 0;
+ struct device_node *child;
+ const __be32 *reg;
+ u64 addr, data;
+ char id[64];
+ u32 port;
+
+ reg = of_get_property(pdev->dev.of_node, "reg", NULL);
+ addr = of_translate_address(pdev->dev.of_node, reg);
+ interface = (addr >> 24) & 0xf;
+ numa_node = (addr >> 36) & 0x7;
+
+ /* Assign 8 CAM entries per LMAC */
+ for (i = 0; i < 32; i++) {
+ data = i >> 3;
+ oct_csr_write(data,
+ BGX_CMR_RX_ADRX_CAM(numa_node, interface, i));
+ }
+
+ for_each_available_child_of_node(pdev->dev.of_node, child) {
+ struct pdev_list_item *pdev_item;
+ bool is_mix = false;
+ bool is_pki = false;
+ bool is_xcv = false;
+
+ if (!of_device_is_compatible(child, "cavium,octeon-7890-bgx-port") &&
+ !of_device_is_compatible(child, "cavium,octeon-7360-xcv"))
+ continue;
+ r = of_property_read_u32(child, "reg", &port);
+ if (r)
+ return -ENODEV;
+
+ is_mix = is_lmac_to_mix(numa_node, interface, port);
+ is_pki = is_lmac_to_pki(numa_node, interface, port);
+ is_xcv = is_lmac_to_xcv(child);
+
+ /* Check if this port should be configured */
+ if (!is_mix && !is_pki)
+ continue;
+
+ /* Connect to PKI/PKO */
+ data = oct_csr_read(BGX_CMR_CONFIG(numa_node, interface, port));
+ if (is_mix)
+ data |= BGX_CMR_CONFIG_MIX_EN;
+ else
+ data &= ~BGX_CMR_CONFIG_MIX_EN;
+ oct_csr_write(data, BGX_CMR_CONFIG(numa_node, interface, port));
+
+ /* Unreset the mix bgx interface or it will interfare with the
+ * other ports.
+ */
+ if (is_mix) {
+ data = oct_csr_read(BGX_CMR_GLOBAL_CONFIG(numa_node,
+ interface));
+ if (!port)
+ data &= ~BGX_CMR_GLOBAL_CONFIG_CMR_MIX0_RST;
+ else if (port == 1)
+ data &= ~BGX_CMR_GLOBAL_CONFIG_CMR_MIX1_RST;
+ oct_csr_write(data, BGX_CMR_GLOBAL_CONFIG(numa_node,
+ interface));
+ }
+
+ snprintf(id, sizeof(id), "%llx.%u.ethernet-mac",
+ (unsigned long long)addr, port);
+ new_dev = of_platform_device_create(child, id, &pdev->dev);
+ if (!new_dev) {
+ dev_err(&pdev->dev, "Error creating %s\n", id);
+ continue;
+ }
+ platform_data.mac_type = BGX_MAC;
+ platform_data.numa_node = numa_node;
+ platform_data.interface = interface;
+ platform_data.port = port;
+ if (is_xcv)
+ platform_data.src_type = XCV;
+ else
+ platform_data.src_type = QLM;
+
+ /* Add device to the list of created devices so we can remove it
+ * on exit.
+ */
+ pdev_item = kmalloc(sizeof(*pdev_item), GFP_KERNEL);
+ pdev_item->pdev = new_dev;
+ mutex_lock(&pdev_list_lock);
+ list_add(&pdev_item->list, &pdev_list);
+ mutex_unlock(&pdev_list_lock);
+
+ atomic_inc(&pki_id);
+ pki_dev = platform_device_register_data(&new_dev->dev,
+ is_mix ? "octeon_mgmt" : "ethernet-mac-pki",
+ (int)atomic_read(&pki_id),
+ &platform_data, sizeof(platform_data));
+ dev_info(&pdev->dev, "Created %s %u: %p\n",
+ is_mix ? "MIX" : "PKI", pki_dev->id, pki_dev);
+
+ /* Add device to the list of created devices so we can remove it
+ * on exit.
+ */
+ pdev_item = kmalloc(sizeof(*pdev_item), GFP_KERNEL);
+ pdev_item->pdev = pki_dev;
+ mutex_lock(&pdev_list_lock);
+ list_add(&pdev_item->list, &pdev_list);
+ mutex_unlock(&pdev_list_lock);
+
+#ifdef CONFIG_NUMA
+ new_dev->dev.numa_node = pdev->dev.numa_node;
+ pki_dev->dev.numa_node = pdev->dev.numa_node;
+#endif
+ /* One time request driver module */
+ if (is_mix) {
+ if (atomic_cmpxchg(&request_mgmt_once, 0, 1) == 0)
+ request_module_nowait("octeon_mgmt");
+ }
+ if (is_pki) {
+ if (atomic_cmpxchg(&load_driver_once, 0, 1) == 0)
+ request_module_nowait("octeon3-ethernet");
+ }
+ }
+
+ dev_info(&pdev->dev, "Probed\n");
+ return 0;
+}
+
+/* bgx_mix_init_from_fdt - Initialize the list of lmacs that connect to mix
+ * ports from information in the device tree.
+ *
+ * Returns 0 if successful.
+ * Returns <0 for error codes.
+ */
+static int bgx_mix_init_from_fdt(void)
+{
+ struct device_node *node, *parent = NULL;
+ int mix = 0;
+
+ for_each_compatible_node(node, NULL, "cavium,octeon-7890-mix") {
+ struct device_node *lmac_fdt_node;
+ const __be32 *reg;
+ u64 addr;
+
+ /* Get the fdt node of the lmac connected to this mix */
+ lmac_fdt_node = of_parse_phandle(node, "cavium,mac-handle", 0);
+ if (!lmac_fdt_node)
+ goto err;
+
+ /* Get the numa node and bgx of the lmac */
+ parent = of_get_parent(lmac_fdt_node);
+ if (!parent)
+ goto err;
+ reg = of_get_property(parent, "reg", NULL);
+ if (!reg)
+ goto err;
+ addr = of_translate_address(parent, reg);
+ of_node_put(parent);
+ parent = NULL;
+
+ mix_port_lmacs[mix].node = (addr >> 36) & 0x7;
+ mix_port_lmacs[mix].bgx = (addr >> 24) & 0xf;
+
+ /* Get the lmac index */
+ reg = of_get_property(lmac_fdt_node, "reg", NULL);
+ if (!reg)
+ goto err;
+
+ mix_port_lmacs[mix].lmac = *reg;
+
+ mix++;
+ if (mix >= MAX_MIX)
+ break;
+ }
+
+ return 0;
+err:
+ pr_warn("Invalid device tree mix port information\n");
+ for (mix = 0; mix < MAX_MIX; mix++) {
+ mix_port_lmacs[mix].node = -1;
+ mix_port_lmacs[mix].bgx = -1;
+ mix_port_lmacs[mix].lmac = -1;
+ }
+ if (parent)
+ of_node_put(parent);
+
+ return -EINVAL;
+}
+
+/* bgx_mix_init_from_param - Initialize the list of lmacs that connect to mix
+ * ports from information in the "mix_port" parameter.
+ * The mix_port parameter format is as follows:
+ * mix_port=nbl
+ * where:
+ * n = node
+ * b = bgx
+ * l = lmac
+ * There can be up to 4 lmacs defined separated by
+ * commas. For example to select node0, bgx0, lmac0
+ * and node0, bgx4, lamc0, the mix_port parameter
+ * would be: mix_port=000,040
+ *
+ * Returns 0 if successful.
+ * Returns <0 for error codes.
+ */
+static int bgx_mix_init_from_param(void)
+{
+ char *p = mix_port;
+ int i, mix = 0;
+
+ while (*p) {
+ int node = -1;
+ int bgx = -1;
+ int lmac = -1;
+
+ if (strlen(p) < 3)
+ goto err;
+
+ /* Get the numa node */
+ if (!isdigit(*p))
+ goto err;
+ node = *p - '0';
+ if (node >= MAX_NODES)
+ goto err;
+
+ /* Get the bgx */
+ p++;
+ if (!isdigit(*p))
+ goto err;
+ bgx = *p - '0';
+ if (bgx >= MAX_BGX_PER_NODE)
+ goto err;
+
+ /* Get the lmac index */
+ p++;
+ if (!isdigit(*p))
+ goto err;
+ lmac = *p - '0';
+ if (lmac >= 2)
+ goto err;
+
+ /* Only one lmac0 and one lmac1 per node is supported */
+ for (i = 0; i < MAX_MIX; i++) {
+ if (mix_port_lmacs[i].node == node &&
+ mix_port_lmacs[i].lmac == lmac)
+ goto err;
+ }
+
+ mix_port_lmacs[mix].node = node;
+ mix_port_lmacs[mix].bgx = bgx;
+ mix_port_lmacs[mix].lmac = lmac;
+
+ p++;
+ if (*p == ',')
+ p++;
+
+ mix++;
+ if (mix >= MAX_MIX)
+ break;
+ }
+
+ return 0;
+err:
+ pr_warn("Invalid parameter mix_port=%s\n", mix_port);
+ for (mix = 0; mix < MAX_MIX; mix++) {
+ mix_port_lmacs[mix].node = -1;
+ mix_port_lmacs[mix].bgx = -1;
+ mix_port_lmacs[mix].lmac = -1;
+ }
+ return -EINVAL;
+}
+
+/* bgx_mix_port_lmacs_init - Initialize the mix_port_lmacs variable with the
+ * lmacs that connect to mic ports.
+ *
+ * Returns 0 if successful.
+ * Returns <0 for error codes.
+ */
+static int bgx_mix_port_lmacs_init(void)
+{
+ int mix;
+
+ /* Start with no mix ports configured */
+ for (mix = 0; mix < MAX_MIX; mix++) {
+ mix_port_lmacs[mix].node = -1;
+ mix_port_lmacs[mix].bgx = -1;
+ mix_port_lmacs[mix].lmac = -1;
+ }
+
+ /* Check if no mix port should be configured */
+ if (mix_port && !strcmp(mix_port, "none"))
+ return 0;
+
+ /* Configure the mix ports using information from the device tree if no
+ * parameter was passed. Otherwise, use the information in the module
+ * parameter.
+ */
+ if (!mix_port)
+ bgx_mix_init_from_fdt();
+ else
+ bgx_mix_init_from_param();
+
+ return 0;
+}
+
+/* bgx_parse_pki_elem - Parse a single element (node, bgx, or lmac) out a pki
+ * lmac string and set its bitmap accordingly.
+ * @str: Pki lmac string to parse.
+ * @bitmap: Updated with the bits selected by str.
+ * @size: Maximum size of the bitmap.
+ *
+ * Returns number of characters processed from str.
+ * Returns <0 for error codes.
+ */
+static int bgx_parse_pki_elem(const char *str, unsigned long *bitmap, int size)
+{
+ const char *p = str;
+ int bit, len = -1;
+
+ if (*p == 0) {
+ /* If identifier is missing, the whole subset is allowed */
+ bitmap_set(bitmap, 0, size);
+ len = 0;
+ } else if (*p == '*') {
+ /* If identifier is an asterisk, the whole subset is allowed */
+ bitmap_set(bitmap, 0, size);
+ len = 1;
+ } else if (isdigit(*p)) {
+ /* If identifier is a digit, only the bit corresponding to the
+ * digit is set.
+ */
+ bit = *p - '0';
+ if (bit < size) {
+ bitmap_set(bitmap, bit, 1);
+ len = 1;
+ }
+ } else if (*p == '[') {
+ /* If identifier is a bracket, all the bits corresponding to
+ * the digits inside the bracket are set.
+ */
+ p++;
+ len = 1;
+ do {
+ if (isdigit(*p)) {
+ bit = *p - '0';
+ if (bit < size)
+ bitmap_set(bitmap, bit, 1);
+ else
+ return -1;
+ } else {
+ return -1;
+ }
+ p++;
+ len++;
+ } while (*p != ']');
+ len++;
+ } else {
+ len = -1;
+ }
+
+ return len;
+}
+
+/* bgx_pki_bitmap_set - Set the bitmap bits for all elements (node, bgx, and
+ * lmac) selected by a pki lmac string.
+ * @str: Pki lmac string to process.
+ * @node: Updated with the nodes specified in the pki lmac string.
+ * @bgx: Updated with the bgx's specified in the pki lmac string.
+ * @lmac: Updated with the lmacs specified in the pki lmac string.
+ *
+ * Returns 0 if successful.
+ * Returns <0 for error codes.
+ */
+static unsigned long bgx_pki_bitmap_set(const char *str, unsigned long *node,
+ unsigned long *bgx, unsigned long *lmac)
+{
+ const char *p = str;
+ int len;
+
+ /* Parse the node */
+ len = bgx_parse_pki_elem(p, node, MAX_NODES);
+ if (len < 0)
+ goto err;
+
+ /* Parse the bgx */
+ p += len;
+ len = bgx_parse_pki_elem(p, bgx, MAX_BGX_PER_NODE);
+ if (len < 0)
+ goto err;
+
+ /* Parse the lmac */
+ p += len;
+ len = bgx_parse_pki_elem(p, lmac, MAX_LMAC_PER_BGX);
+ if (len < 0)
+ goto err;
+
+ return 0;
+err:
+ bitmap_zero(node, MAX_NODES);
+ bitmap_zero(bgx, MAX_BGX_PER_NODE);
+ bitmap_zero(lmac, MAX_LMAC_PER_BGX);
+ return len;
+}
+
+/* bgx_pki_init_from_param - Initialize the list of lmacs that connect to the
+ * pki from information in the "pki_port" parameter.
+ *
+ * The pki_port parameter format is as follows:
+ * pki_port=nbl
+ * where:
+ * n = node
+ * b = bgx
+ * l = lmac
+ *
+ * Commas must be used to separate multiple lmacs:
+ * pki_port=000,100,110
+ *
+ * Asterisks (*) specify all possible characters in
+ * the subset:
+ * pki_port=00* (all lmacs of node0 bgx0).
+ *
+ * Missing lmacs identifiers default to all
+ * possible characters in the subset:
+ * pki_port=00 (all lmacs on node0 bgx0)
+ *
+ * Brackets ('[' and ']') specify the valid
+ * characters in the subset:
+ * pki_port=00[01] (lmac0 and lmac1 of node0 bgx0).
+ *
+ * Returns 0 if successful.
+ * Returns <0 for error codes.
+ */
+static int bgx_pki_init_from_param(void)
+{
+ DECLARE_BITMAP(lmac_bitmap, MAX_LMAC_PER_BGX);
+ DECLARE_BITMAP(bgx_bitmap, MAX_BGX_PER_NODE);
+ DECLARE_BITMAP(node_bitmap, MAX_NODES);
+ char *cur, *next;
+
+ /* Parse each comma separated lmac specifier */
+ cur = pki_port;
+ while (cur) {
+ unsigned long bgx, lmac, node;
+
+ bitmap_zero(node_bitmap, BITS_PER_LONG);
+ bitmap_zero(bgx_bitmap, BITS_PER_LONG);
+ bitmap_zero(lmac_bitmap, BITS_PER_LONG);
+
+ next = strchr(cur, ',');
+ if (next)
+ *next++ = '\0';
+
+ /* Convert the specifier into a bitmap */
+ bgx_pki_bitmap_set(cur, node_bitmap, bgx_bitmap, lmac_bitmap);
+
+ /* Mark the lmacs to be connected to the pki */
+ for_each_set_bit(node, node_bitmap, MAX_NODES) {
+ for_each_set_bit(bgx, bgx_bitmap, MAX_BGX_PER_NODE) {
+ for_each_set_bit(lmac, lmac_bitmap,
+ MAX_LMAC_PER_BGX)
+ pki_ports[node][bgx][lmac] = true;
+ }
+ }
+
+ cur = next;
+ }
+
+ return 0;
+}
+
+/* bgx_pki_ports_init - Initialize the pki_ports variable with the lmacs that
+ * connect to the pki.
+ *
+ * Returns 0 if successful.
+ * Returns <0 for error codes.
+ */
+static int bgx_pki_ports_init(void)
+{
+ bool def_val;
+ int i, j, k;
+
+ /* Whether all ports default to connect to the pki or not depend on the
+ * passed module parameter (if any).
+ */
+ if (pki_port)
+ def_val = false;
+ else
+ def_val = true;
+
+ for (i = 0; i < MAX_NODES; i++) {
+ for (j = 0; j < MAX_BGX_PER_NODE; j++) {
+ for (k = 0; k < MAX_LMAC_PER_BGX; k++)
+ pki_ports[i][j][k] = def_val;
+ }
+ }
+
+ /* Check if ports have to be individually configured */
+ if (pki_port && strcmp(pki_port, "none"))
+ bgx_pki_init_from_param();
+
+ return 0;
+}
+
+static int bgx_remove(struct platform_device *pdev)
+{
+ return 0;
+}
+
+static void bgx_shutdown(struct platform_device *pdev)
+{
+}
+
+static const struct of_device_id bgx_match[] = {
+ {
+ .compatible = "cavium,octeon-7890-bgx",
+ },
+ {},
+};
+MODULE_DEVICE_TABLE(of, bgx_match);
+
+static struct platform_driver bgx_driver = {
+ .probe = bgx_probe,
+ .remove = bgx_remove,
+ .shutdown = bgx_shutdown,
+ .driver = {
+ .owner = THIS_MODULE,
+ .name = KBUILD_MODNAME,
+ .of_match_table = bgx_match,
+ },
+};
+
+/* Allow bgx_port driver to force this driver to load */
+void bgx_nexus_load(void)
+{
+}
+EXPORT_SYMBOL(bgx_nexus_load);
+
+static int __init bgx_driver_init(void)
+{
+ INIT_LIST_HEAD(&pdev_list);
+ mutex_init(&pdev_list_lock);
+ bgx_mix_port_lmacs_init();
+ bgx_pki_ports_init();
+
+ return platform_driver_register(&bgx_driver);
+}
+
+static void __exit bgx_driver_exit(void)
+{
+ struct pdev_list_item *pdev_item;
+
+ mutex_lock(&pdev_list_lock);
+ while (!list_empty(&pdev_list)) {
+ pdev_item = list_first_entry(&pdev_list,
+ struct pdev_list_item, list);
+ list_del(&pdev_item->list);
+ platform_device_unregister(pdev_item->pdev);
+ kfree(pdev_item);
+ }
+ mutex_unlock(&pdev_list_lock);
+
+ platform_driver_unregister(&bgx_driver);
+}
+
+module_init(bgx_driver_init);
+module_exit(bgx_driver_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Cavium, Inc. <support@caviumnetworks.com>");
+MODULE_DESCRIPTION("Cavium, Inc. BGX MAC Nexus driver.");
diff --git a/drivers/net/ethernet/cavium/octeon/octeon3-bgx.h b/drivers/net/ethernet/cavium/octeon/octeon3-bgx.h
index df794f5..58ed870 100644
--- a/drivers/net/ethernet/cavium/octeon/octeon3-bgx.h
+++ b/drivers/net/ethernet/cavium/octeon/octeon3-bgx.h
@@ -8,14 +8,83 @@
#include <linux/bitops.h>
-#define BGX_CMR_CHAN_MSK_MASK GENMASK_ULL(15, 0)
-#define BGX_CMR_CHAN_MSK_SHIFT 16
-#define BGX_CMR_CONFIG_DATA_PKT_RX_EN BIT(14)
-#define BGX_CMR_CONFIG_DATA_PKT_TX_EN BIT(13)
-#define BGX_CMR_CONFIG_ENABLE BIT(15)
-#define BGX_CMR_CONFIG_LMAC_TYPE_MASK GENMASK_ULL(10, 8)
-#define BGX_CMR_CONFIG_LMAC_TYPE_SHIFT 8
-#define BGX_CMR_CONFIG_MIX_EN BIT(11)
+#define BGX_RX_FIFO_SIZE (64 * 1024)
+#define BGX_TX_FIFO_SIZE (32 * 1024)
+
+/* Bgx register definitions */
+#define BGX_BASE 0x11800e0000000ull
+#define BGX_ADDR(node, bgx, index) (BGX_BASE + SET_XKPHYS + \
+ NODE_OFFSET(node) + ((bgx) << 24) + \
+ ((index) << 20))
+#define BGX_ADDR_CAM(n, b, mac) (BGX_ADDR(n, b, 0) + ((mac) << 3))
+
+#define BGX_CMR_CONFIG(n, b, i) (BGX_ADDR(n, b, i) + 0x00000)
+#define BGX_CMR_GLOBAL_CONFIG(n, b) (BGX_ADDR(n, b, 0) + 0x00008)
+#define BGX_CMR_RX_ID_MAP(n, b, i) (BGX_ADDR(n, b, i) + 0x00028)
+#define BGX_CMR_RX_BP_ON(n, b, i) (BGX_ADDR(n, b, i) + 0x00088)
+#define BGX_CMR_RX_ADR_CTL(n, b, i) (BGX_ADDR(n, b, i) + 0x000a0)
+#define BGX_CMR_RX_FIFO_LEN(n, b, i) (BGX_ADDR(n, b, i) + 0x000c0)
+#define BGX_CMR_RX_ADRX_CAM(n, b, m) (BGX_ADDR_CAM(n, b, m) + 0x00100)
+#define BGX_CMR_CHAN_MSK_AND(n, b) (BGX_ADDR(n, b, 0) + 0x00200)
+#define BGX_CMR_CHAN_MSK_OR(n, b) (BGX_ADDR(n, b, 0) + 0x00208)
+#define BGX_CMR_TX_FIFO_LEN(n, b, i) (BGX_ADDR(n, b, i) + 0x00418)
+#define BGX_CMR_TX_LMACS(n, b) (BGX_ADDR(n, b, 0) + 0x01000)
+
+#define BGX_SPU_CONTROL1(n, b, i) (BGX_ADDR(n, b, i) + 0x10000)
+#define BGX_SPU_STATUS1(n, b, i) (BGX_ADDR(n, b, i) + 0x10008)
+#define BGX_SPU_STATUS2(n, b, i) (BGX_ADDR(n, b, i) + 0x10020)
+#define BGX_SPU_BX_STATUS(n, b, i) (BGX_ADDR(n, b, i) + 0x10028)
+#define BGX_SPU_BR_STATUS1(n, b, i) (BGX_ADDR(n, b, i) + 0x10030)
+#define BGX_SPU_BR_STATUS2(n, b, i) (BGX_ADDR(n, b, i) + 0x10038)
+#define BGX_SPU_BR_BIP_ERR_CNT(n, b, i) (BGX_ADDR(n, b, i) + 0x10058)
+#define BGX_SPU_BR_PMD_CONTROL(n, b, i) (BGX_ADDR(n, b, i) + 0x10068)
+#define BGX_SPU_BR_PMD_LP_CUP(n, b, i) (BGX_ADDR(n, b, i) + 0x10078)
+#define BGX_SPU_BR_PMD_LD_CUP(n, b, i) (BGX_ADDR(n, b, i) + 0x10088)
+#define BGX_SPU_BR_PMD_LD_REP(n, b, i) (BGX_ADDR(n, b, i) + 0x10090)
+#define BGX_SPU_FEC_CONTROL(n, b, i) (BGX_ADDR(n, b, i) + 0x100a0)
+#define BGX_SPU_AN_CONTROL(n, b, i) (BGX_ADDR(n, b, i) + 0x100c8)
+#define BGX_SPU_AN_STATUS(n, b, i) (BGX_ADDR(n, b, i) + 0x100d0)
+#define BGX_SPU_AN_ADV(n, b, i) (BGX_ADDR(n, b, i) + 0x100d8)
+#define BGX_SPU_MISC_CONTROL(n, b, i) (BGX_ADDR(n, b, i) + 0x10218)
+#define BGX_SPU_INT(n, b, i) (BGX_ADDR(n, b, i) + 0x10220)
+#define BGX_SPU_DBG_CONTROL(n, b) (BGX_ADDR(n, b, 0) + 0x10300)
+
+#define BGX_SMU_RX_INT(n, b, i) (BGX_ADDR(n, b, i) + 0x20000)
+#define BGX_SMU_RX_FRM_CTL(n, b, i) (BGX_ADDR(n, b, i) + 0x20008)
+#define BGX_SMU_RX_JABBER(n, b, i) (BGX_ADDR(n, b, i) + 0x20018)
+#define BGX_SMU_RX_CTL(n, b, i) (BGX_ADDR(n, b, i) + 0x20030)
+#define BGX_SMU_TX_APPEND(n, b, i) (BGX_ADDR(n, b, i) + 0x20100)
+#define BGX_SMU_TX_MIN_PKT(n, b, i) (BGX_ADDR(n, b, i) + 0x20118)
+#define BGX_SMU_TX_INT(n, b, i) (BGX_ADDR(n, b, i) + 0x20140)
+#define BGX_SMU_TX_CTL(n, b, i) (BGX_ADDR(n, b, i) + 0x20160)
+#define BGX_SMU_TX_THRESH(n, b, i) (BGX_ADDR(n, b, i) + 0x20168)
+#define BGX_SMU_CTRL(n, b, i) (BGX_ADDR(n, b, i) + 0x20200)
+
+#define BGX_GMP_PCS_MR_CONTROL(n, b, i) (BGX_ADDR(n, b, i) + 0x30000)
+#define BGX_GMP_PCS_MR_STATUS(n, b, i) (BGX_ADDR(n, b, i) + 0x30008)
+#define BGX_GMP_PCS_AN_ADV(n, b, i) (BGX_ADDR(n, b, i) + 0x30010)
+#define BGX_GMP_PCS_LINK_TIMER(n, b, i) (BGX_ADDR(n, b, i) + 0x30040)
+#define BGX_GMP_PCS_SGM_AN_ADV(n, b, i) (BGX_ADDR(n, b, i) + 0x30068)
+#define BGX_GMP_PCS_MISC_CTL(n, b, i) (BGX_ADDR(n, b, i) + 0x30078)
+#define BGX_GMP_GMI_PRT_CFG(n, b, i) (BGX_ADDR(n, b, i) + 0x38010)
+#define BGX_GMP_GMI_RX_FRM_CTL(n, b, i) (BGX_ADDR(n, b, i) + 0x38018)
+#define BGX_GMP_GMI_RX_JABBER(n, b, i) (BGX_ADDR(n, b, i) + 0x38038)
+#define BGX_GMP_GMI_TX_THRESH(n, b, i) (BGX_ADDR(n, b, i) + 0x38210)
+#define BGX_GMP_GMI_TX_APPEND(n, b, i) (BGX_ADDR(n, b, i) + 0x38218)
+#define BGX_GMP_GMI_TX_SLOT(n, b, i) (BGX_ADDR(n, b, i) + 0x38220)
+#define BGX_GMP_GMI_TX_BURST(n, b, i) (BGX_ADDR(n, b, i) + 0x38228)
+#define BGX_GMP_GMI_TX_MIN_PKT(n, b, i) (BGX_ADDR(n, b, i) + 0x38240)
+#define BGX_GMP_GMI_TX_SGMII_CTL(n, b, i) (BGX_ADDR(n, b, i) + 0x38300)
+
+/* Bgx register bitfields */
+#define BGX_CMR_CHAN_MSK_MASK GENMASK_ULL(15, 0)
+#define BGX_CMR_CHAN_MSK_SHIFT 16
+#define BGX_CMR_CONFIG_DATA_PKT_RX_EN BIT(14)
+#define BGX_CMR_CONFIG_DATA_PKT_TX_EN BIT(13)
+#define BGX_CMR_CONFIG_ENABLE BIT(15)
+#define BGX_CMR_CONFIG_LMAC_TYPE_MASK GENMASK_ULL(10, 8)
+#define BGX_CMR_CONFIG_LMAC_TYPE_SHIFT 8
+#define BGX_CMR_CONFIG_MIX_EN BIT(11)
#define BGX_CMR_GLOBAL_CONFIG_CMR_MIX0_RST BIT(3)
#define BGX_CMR_GLOBAL_CONFIG_CMR_MIX1_RST BIT(4)
#define BGX_CMR_RX_ADR_CTL_ACCEPT_ALL_MCST BIT(1)
@@ -23,128 +92,100 @@
#define BGX_CMR_RX_ADR_CTL_CAM_ACCEPT BIT(3)
#define BGX_CMR_RX_ADR_CTL_MCST_MODE_MASK GENMASK_ULL(2, 1)
#define BGX_CMR_RX_ADR_CTL_USE_CAM_FILTER BIT(2)
-#define BGX_CMR_RX_ADRX_CAM_EN BIT(48)
-#define BGX_CMR_RX_ADRX_CAM_ID_SHIFT 52
-#define BGX_CMR_RX_FIFO_LEN_MASK GENMASK_ULL(12, 0)
-#define BGX_CMR_RX_ID_MAP_PKND_MASK GENMASK_ULL(7, 0)
-#define BGX_CMR_RX_ID_MAP_RID_MASK GENMASK_ULL(14, 8)
-#define BGX_CMR_RX_ID_MAP_RID_SHIFT 8
-#define BGX_CMR_TX_FIFO_LEN_LMAC_IDLE BIT(13)
-#define BGX_CMR_TX_LMACS_MASK GENMASK_ULL(2, 0)
-#define BGX_GMP_GMI_PRT_CFG_DUPLEX BIT(2)
-#define BGX_GMP_GMI_PRT_CFG_RX_IDLE BIT(12)
-#define BGX_GMP_GMI_PRT_CFG_SLOTTIME BIT(3)
-#define BGX_GMP_GMI_PRT_CFG_SPEED BIT(1)
-#define BGX_GMP_GMI_PRT_CFG_SPEED_MSB BIT(8)
-#define BGX_GMP_GMI_PRT_CFG_TX_IDLE BIT(13)
+#define BGX_CMR_RX_ADRX_CAM_EN BIT(48)
+#define BGX_CMR_RX_ADRX_CAM_ID_SHIFT 52
+#define BGX_CMR_RX_FIFO_LEN_MASK GENMASK_ULL(12, 0)
+#define BGX_CMR_RX_ID_MAP_PKND_MASK GENMASK_ULL(7, 0)
+#define BGX_CMR_RX_ID_MAP_RID_MASK GENMASK_ULL(14, 8)
+#define BGX_CMR_RX_ID_MAP_RID_SHIFT 8
+#define BGX_CMR_TX_FIFO_LEN_LMAC_IDLE BIT(13)
+#define BGX_CMR_TX_LMACS_MASK GENMASK_ULL(2, 0)
+
+#define BGX_SPU_AN_ADV_FEC_ABLE BIT(46)
+#define BGX_SPU_AN_ADV_FEC_REQ BIT(47)
+#define BGX_SPU_AN_ADV_A100G_CR10 BIT(26)
+#define BGX_SPU_AN_ADV_A40G_CR4 BIT(25)
+#define BGX_SPU_AN_ADV_A40G_KR4 BIT(24)
+#define BGX_SPU_AN_ADV_A10G_KR BIT(23)
+#define BGX_SPU_AN_ADV_A10G_KX4 BIT(22)
+#define BGX_SPU_AN_ADV_A1G_KX BIT(21)
+#define BGX_SPU_AN_ADV_RF BIT(13)
+#define BGX_SPU_AN_ADV_XNP_ABLE BIT(12)
+#define BGX_SPU_AN_CONTROL_XNP_EN BIT(13)
+#define BGX_SPU_AN_CONTROL_AN_EN BIT(12)
+#define BGX_SPU_AN_CONTROL_AN_RESTART BIT(9)
+#define BGX_SPU_AN_STATUS_AN_COMPLETE BIT(5)
+#define BGX_SPU_BR_PMD_CONTROL_TRAIN_EN BIT(1)
+#define BGX_SPU_BR_PMD_CONTROL_TRAIN_RESTART BIT(0)
+#define BGX_SPU_BR_STATUS1_BLK_LOCK BIT(0)
+#define BGX_SPU_BR_STATUS2_LATCHED_LOCK BIT(15)
+#define BGX_SPU_BR_STATUS2_LATCHED_BER BIT(14)
+#define BGX_SPU_BX_STATUS_ALIGND BIT(12)
+#define BGX_SPU_CONTROL1_RESET BIT(15)
+#define BGX_SPU_CONTROL1_LOOPBACK BIT(14)
+#define BGX_SPU_CONTROL1_LO_PWR BIT(11)
+#define BGX_SPU_DBG_CONTROL_US_CLK_PERIOD_MASK GENMASK_ULL(43, 32)
+#define BGX_SPU_DBG_CONTROL_US_CLK_PERIOD_SHIFT 32
+#define BGX_SPU_DBG_CONTROL_AN_NONCE_MATCH_DIS BIT(29)
+#define BGX_SPU_DBG_CONTROL_AN_ARB_LINK_CHK_EN BIT(18)
+#define BGX_SPU_FEC_CONTROL_FEC_EN BIT(0)
+#define BGX_SPU_INT_AN_COMPLETE BIT(12)
+#define BGX_SPU_INT_AN_LINK_GOOD BIT(11)
+#define BGX_SPU_INT_AN_PAGE_RX BIT(10)
+#define BGX_SPU_INT_TRAINING_FAILURE BIT(14)
+#define BGX_SPU_INT_TRAINING_DONE BIT(13)
+#define BGX_SPU_MISC_CONTROL_RX_PACKET_DIS BIT(12)
+#define BGX_SPU_MISC_CONTROL_INTLV_RDISP BIT(10)
+#define BGX_SPU_STATUS1_RCV_LINK BIT(2)
+#define BGX_SPU_STATUS2_RCVFLT BIT(10)
+
+#define BGX_SMU_CTRL_TX_IDLE BIT(1)
+#define BGX_SMU_CTRL_RX_IDLE BIT(0)
+#define BGX_SMU_RX_CTL_STATUS_MASK GENMASK_ULL(1, 0)
+#define BGX_SMU_TX_APPEND_FCS_C BIT(3)
+#define BGX_SMU_TX_APPEND_FCS_D BIT(2)
+#define BGX_SMU_TX_APPEND_PAD BIT(1)
+#define BGX_SMU_TX_CTL_LS_MASK GENMASK_ULL(5, 4)
+#define BGX_SMU_TX_CTL_UNI_EN BIT(1)
+#define BGX_SMU_TX_CTL_DIC_EN BIT(0)
+
+#define BGX_GMP_GMI_PRT_CFG_TX_IDLE BIT(13)
+#define BGX_GMP_GMI_PRT_CFG_RX_IDLE BIT(12)
+#define BGX_GMP_GMI_PRT_CFG_SPEED_MSB BIT(8)
+#define BGX_GMP_GMI_PRT_CFG_SLOTTIME BIT(3)
+#define BGX_GMP_GMI_PRT_CFG_DUPLEX BIT(2)
+#define BGX_GMP_GMI_PRT_CFG_SPEED BIT(1)
#define BGX_GMP_GMI_TX_APPEND_FCS BIT(2)
#define BGX_GMP_GMI_TX_APPEND_PAD BIT(1)
#define BGX_GMP_GMI_TX_APPEND_PREAMBLE BIT(0)
#define BGX_GMP_GMI_TX_THRESH_DEFAULT 0x20
-#define BGX_GMP_PCS_AN_ADV_FULL_DUPLEX BIT(5)
-#define BGX_GMP_PCS_AN_ADV_HALF_DUPLEX BIT(6)
-#define BGX_GMP_PCS_AN_ADV_PAUSE_ASYMMETRIC 2
-#define BGX_GMP_PCS_AN_ADV_PAUSE_BOTH 3
-#define BGX_GMP_PCS_AN_ADV_PAUSE_MASK GENMASK_ULL(8, 7)
+#define BGX_GMP_PCS_AN_ADV_REM_FLT_MASK GENMASK_ULL(13, 12)
+#define BGX_GMP_PCS_AN_ADV_PAUSE_MASK GENMASK_ULL(8, 7)
+#define BGX_GMP_PCS_AN_ADV_PAUSE_SHIFT 7
#define BGX_GMP_PCS_AN_ADV_PAUSE_NONE 0
-#define BGX_GMP_PCS_AN_ADV_PAUSE_SHIFT 7
#define BGX_GMP_PCS_AN_ADV_PAUSE_SYMMETRIC 1
-#define BGX_GMP_PCS_AN_ADV_REM_FLT_MASK GENMASK_ULL(13, 12)
+#define BGX_GMP_PCS_AN_ADV_PAUSE_ASYMMETRIC 2
+#define BGX_GMP_PCS_AN_ADV_PAUSE_BOTH 3
+#define BGX_GMP_PCS_AN_ADV_HALF_DUPLEX BIT(6)
+#define BGX_GMP_PCS_AN_ADV_FULL_DUPLEX BIT(5)
#define BGX_GMP_PCS_LINK_TIMER_COUNT_SHIFT 10
-#define BGX_GMP_PCS_MISC_CTL_GMXENO BIT(11)
-#define BGX_GMP_PCS_MISC_CTL_MAC_PHY BIT(9)
-#define BGX_GMP_PCS_MISC_CTL_MODE BIT(8)
+#define BGX_GMP_PCS_MISC_CTL_GMXENO BIT(11)
+#define BGX_GMP_PCS_MISC_CTL_MAC_PHY BIT(9)
+#define BGX_GMP_PCS_MISC_CTL_MODE BIT(8)
#define BGX_GMP_PCS_MISC_CTL_SAMP_PT_MASK GENMASK_ULL(6, 0)
-#define BGX_GMP_PCS_MR_CONTROL_AN_EN BIT(12)
-#define BGX_GMP_PCS_MR_CONTROL_PWR_DN BIT(11)
-#define BGX_GMP_PCS_MR_CONTROL_RESET BIT(15)
-#define BGX_GMP_PCS_MR_CONTROL_RST_AN BIT(9)
-#define BGX_GMP_PCS_MR_CONTROL_SPDLSB BIT(13)
-#define BGX_GMP_PCS_MR_CONTROL_SPDMSB BIT(6)
-#define BGX_GMP_PCS_MR_STATUS_AN_CPT BIT(5)
+#define BGX_GMP_PCS_MR_CONTROL_RESET BIT(15)
+#define BGX_GMP_PCS_MR_CONTROL_SPDLSB BIT(13)
+#define BGX_GMP_PCS_MR_CONTROL_AN_EN BIT(12)
+#define BGX_GMP_PCS_MR_CONTROL_PWR_DN BIT(11)
+#define BGX_GMP_PCS_MR_CONTROL_RST_AN BIT(9)
+#define BGX_GMP_PCS_MR_CONTROL_SPDMSB BIT(6)
+#define BGX_GMP_PCS_MR_STATUS_AN_CPT BIT(5)
#define BGX_GMP_PCS_SGM_AN_ADV_DUPLEX_FULL BIT(12)
-#define BGX_GMP_PCS_SGM_AN_ADV_SPEED_10 0
-#define BGX_GMP_PCS_SGM_AN_ADV_SPEED_10000 3
-#define BGX_GMP_PCS_SGM_AN_ADV_SPEED_1000 2
-#define BGX_GMP_PCS_SGM_AN_ADV_SPEED_100 1
#define BGX_GMP_PCS_SGM_AN_ADV_SPEED_MASK GENMASK_ULL(11, 10)
#define BGX_GMP_PCS_SGM_AN_ADV_SPEED_SHIFT 10
-#define BGX_SMU_CTRL_RX_IDLE BIT(0)
-#define BGX_SMU_CTRL_TX_IDLE BIT(1)
-#define BGX_SMU_RX_CTL_STATUS_MASK GENMASK_ULL(1, 0)
-#define BGX_SMU_TX_APPEND_FCS_C BIT(3)
-#define BGX_SMU_TX_APPEND_FCS_D BIT(2)
-#define BGX_SMU_TX_APPEND_PAD BIT(1)
-#define BGX_SMU_TX_CTL_DIC_EN BIT(0)
-#define BGX_SMU_TX_CTL_LS_MASK GENMASK_ULL(5, 4)
-#define BGX_SMU_TX_CTL_UNI_EN BIT(1)
-#define BGX_SPU_AN_ADV_A100G_CR10 BIT(26)
-#define BGX_SPU_AN_ADV_A10G_KR BIT(23)
-#define BGX_SPU_AN_ADV_A10G_KX4 BIT(22)
-#define BGX_SPU_AN_ADV_A1G_KX BIT(21)
-#define BGX_SPU_AN_ADV_A40G_CR4 BIT(25)
-#define BGX_SPU_AN_ADV_A40G_KR4 BIT(24)
-#define BGX_SPU_AN_ADV_FEC_ABLE BIT(46)
-#define BGX_SPU_AN_ADV_FEC_REQ BIT(47)
-#define BGX_SPU_AN_ADV_RF BIT(13)
-#define BGX_SPU_AN_ADV_XNP_ABLE BIT(12)
-#define BGX_SPU_AN_CONTROL_AN_EN BIT(12)
-#define BGX_SPU_AN_CONTROL_AN_RESTART BIT(9)
-#define BGX_SPU_AN_CONTROL_XNP_EN BIT(13)
-#define BGX_SPU_AN_STATUS_AN_COMPLETE BIT(5)
-#define BGX_SPU_BR_PMD_CONTROL_TRAIN_EN BIT(1)
-#define BGX_SPU_BR_PMD_CONTROL_TRAIN_RESTART BIT(0)
-#define BGX_SPU_BR_STATUS1_BLK_LOCK BIT(0)
-#define BGX_SPU_BR_STATUS2_LATCHED_BER BIT(14)
-#define BGX_SPU_BR_STATUS2_LATCHED_LOCK BIT(15)
-#define BGX_SPU_BX_STATUS_ALIGND BIT(12)
-#define BGX_SPU_CONTROL1_LOOPBACK BIT(14)
-#define BGX_SPU_CONTROL1_LO_PWR BIT(11)
-#define BGX_SPU_CONTROL1_RESET BIT(15)
-#define BGX_SPU_DBG_CONTROL_AN_ARB_LINK_CHK_EN BIT(18)
-#define BGX_SPU_DBG_CONTROL_AN_NONCE_MATCH_DIS BIT(29)
-#define BGX_SPU_DBG_CONTROL_US_CLK_PERIOD_MASK GENMASK_ULL(43, 32)
-#define BGX_SPU_DBG_CONTROL_US_CLK_PERIOD_SHIFT 32
-#define BGX_SPU_FEC_CONTROL_FEC_EN BIT(0)
-#define BGX_SPU_INT_AN_COMPLETE BIT(12)
-#define BGX_SPU_INT_AN_LINK_GOOD BIT(11)
-#define BGX_SPU_INT_AN_PAGE_RX BIT(10)
-#define BGX_SPU_INT_TRAINING_DONE BIT(13)
-#define BGX_SPU_INT_TRAINING_FAILURE BIT(14)
-#define BGX_SPU_MISC_CONTROL_INTLV_RDISP BIT(10)
-#define BGX_SPU_MISC_CONTROL_RX_PACKET_DIS BIT(12)
-#define BGX_SPU_STATUS1_RCV_LINK BIT(2)
-#define BGX_SPU_STATUS2_RCVFLT BIT(10)
-#define GSER_BR_RX_CTL_RXT_EER BIT(15)
-#define GSER_BR_RX_CTL_RXT_ESV BIT(14)
-#define GSER_BR_RX_CTL_RXT_SWM BIT(2)
-#define GSER_BR_RX_EER_RXT_EER BIT(15)
-#define GSER_BR_RX_EER_RXT_ESV BIT(14)
-#define GSER_LANE_LBERT_CFG_LBERT_PM_EN BIT(6)
-#define GSER_LANE_MODE_LMODE_MASK GENMASK_ULL(3, 0)
-#define GSER_LANE_MODE_MASK GENMASK_ULL(3, 0)
-#define GSER_LANE_PCS_CTLIFC_0_CFG_TX_COEFF_REQ_OVRRD_VAL BIT(12)
-#define GSER_LANE_PCS_CTLIFC_2_CFG_TX_COEFF_REQ_OVRRD_EN BIT(7)
-#define GSER_LANE_PCS_CTLIFC_2_CTLIFC_OVRRD_REQ BIT(15)
-#define GSER_LANE_P_MODE_1_VMA_MM BIT(14)
-#define GSER_PHY_CTL_PHY_PD BIT(0)
-#define GSER_PHY_CTL_PHY_RESET BIT(1)
-#define GSER_RX_EIE_DETSTS_CDRLOCK_SHIFT 8
-#define XCV_BATCH_CRD_RET_CRD_RET BIT(0)
-#define XCV_COMP_CTL_DRV_BYP BIT(63)
-#define XCV_CTL_LPBK_INT BIT(2)
-#define XCV_CTL_SPEED_MASK GENMASK_ULL(1, 0)
-#define XCV_DLL_CTL_CLKRX_BYP BIT(23)
-#define XCV_DLL_CTL_CLKRX_SET_MASK GENMASK_ULL(22, 16)
-#define XCV_DLL_CTL_CLKTX_BYP BIT(15)
-#define XCV_DLL_CTL_REFCLK_SEL_MASK GENMASK_ULL(1, 0)
-#define XCV_RESET_CLKRST BIT(15)
-#define XCV_RESET_COMP BIT(7)
-#define XCV_RESET_DLLRST BIT(11)
-#define XCV_RESET_ENABLE BIT(63)
-#define XCV_RESET_RX_DAT_RST_N BIT(0)
-#define XCV_RESET_RX_PKT_RST_N BIT(1)
-#define XCV_RESET_TX_DAT_RST_N BIT(2)
-#define XCV_RESET_TX_PKT_RST_N BIT(3)
+#define BGX_GMP_PCS_SGM_AN_ADV_SPEED_10 0
+#define BGX_GMP_PCS_SGM_AN_ADV_SPEED_100 1
+#define BGX_GMP_PCS_SGM_AN_ADV_SPEED_1000 2
+#define BGX_GMP_PCS_SGM_AN_ADV_SPEED_10000 3
#endif /* _OCTEON3_BGX_H_ */
--
2.1.4
^ permalink raw reply related
* [PATCH v12 04/10] netdev: cavium: octeon: Add Octeon III BGX Ports
From: Steven J. Hill @ 2018-06-27 21:25 UTC (permalink / raw)
To: netdev; +Cc: Carlos Munoz, Chandrakala Chavva, Steven J. Hill
In-Reply-To: <1530134719-19407-1-git-send-email-steven.hill@cavium.com>
From: Carlos Munoz <cmunoz@cavium.com>
Add individual BGX nexus port support for Octeon III BGX Ethernet.
Signed-off-by: Carlos Munoz <cmunoz@cavium.com>
Signed-off-by: Steven J. Hill <Steven.Hill@cavium.com>
---
.../net/ethernet/cavium/octeon/octeon3-bgx-port.c | 2192 ++++++++++++++++++++
1 file changed, 2192 insertions(+)
create mode 100644 drivers/net/ethernet/cavium/octeon/octeon3-bgx-port.c
diff --git a/drivers/net/ethernet/cavium/octeon/octeon3-bgx-port.c b/drivers/net/ethernet/cavium/octeon/octeon3-bgx-port.c
new file mode 100644
index 0000000..eb5921b
--- /dev/null
+++ b/drivers/net/ethernet/cavium/octeon/octeon3-bgx-port.c
@@ -0,0 +1,2192 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Octeon III BGX Nexus Ethernet driver
+ *
+ * Copyright (C) 2018 Cavium, Inc.
+ */
+
+#include <linux/etherdevice.h>
+#include <linux/of_address.h>
+#include <linux/of_mdio.h>
+#include <linux/of_net.h>
+
+#include "octeon3.h"
+
+struct bgx_port_priv {
+ int node;
+ int bgx;
+ int index; /* Port index on BGX block*/
+ enum port_mode mode;
+ int pknd;
+ int qlm;
+ const u8 *mac_addr;
+ struct phy_device *phydev;
+ struct device_node *phy_np;
+ bool mode_1000basex;
+ bool bgx_as_phy;
+ struct net_device *netdev;
+ struct mutex lock; /* Serializes delayed work */
+ struct port_status (*get_link)(struct bgx_port_priv *priv);
+ int (*set_link)(struct bgx_port_priv *priv, struct port_status status);
+ struct port_status last_status;
+ struct delayed_work dwork;
+ bool work_queued;
+};
+
+/* lmac_pknd keeps track of the port kinds assigned to the lmacs */
+static int lmac_pknd[MAX_NODES][MAX_BGX_PER_NODE][MAX_LMAC_PER_BGX];
+
+static struct workqueue_struct *check_state_wq;
+static DEFINE_MUTEX(check_state_wq_mutex);
+
+int bgx_port_get_qlm(int node, int bgx, int index)
+{
+ int qlm = -1;
+ u64 data;
+
+ if (OCTEON_IS_MODEL(OCTEON_CN78XX)) {
+ if (bgx < 2) {
+ data = oct_csr_read(BGX_CMR_GLOBAL_CONFIG(node, bgx));
+ if (data & 1)
+ qlm = bgx + 2;
+ else
+ qlm = bgx;
+ } else {
+ qlm = bgx + 2;
+ }
+ } else if (OCTEON_IS_MODEL(OCTEON_CN73XX)) {
+ if (bgx < 2) {
+ qlm = bgx + 2;
+ } else {
+ /* Ports on bgx2 can be connected to qlm5 or qlm6 */
+ if (index < 2)
+ qlm = 5;
+ else
+ qlm = 6;
+ }
+ } else if (OCTEON_IS_MODEL(OCTEON_CNF75XX)) {
+ /* Ports on bgx0 can be connected to qlm4 or qlm5 */
+ if (index < 2)
+ qlm = 4;
+ else
+ qlm = 5;
+ }
+
+ return qlm;
+}
+EXPORT_SYMBOL(bgx_port_get_qlm);
+
+/* Returns the mode of the bgx port */
+enum port_mode bgx_port_get_mode(int node, int bgx, int index)
+{
+ enum port_mode mode;
+ u64 data;
+
+ data = oct_csr_read(BGX_CMR_CONFIG(node, bgx, index)) &
+ BGX_CMR_CONFIG_LMAC_TYPE_MASK;
+
+ switch (data >> BGX_CMR_CONFIG_LMAC_TYPE_SHIFT) {
+ case 0:
+ mode = PORT_MODE_SGMII;
+ break;
+ case 1:
+ mode = PORT_MODE_XAUI;
+ break;
+ case 2:
+ mode = PORT_MODE_RXAUI;
+ break;
+ case 3:
+ data = oct_csr_read(BGX_SPU_BR_PMD_CONTROL(node, bgx, index));
+ /* The use of training differentiates 10G_KR from xfi */
+ if (data & BGX_SPU_BR_PMD_CONTROL_TRAIN_EN)
+ mode = PORT_MODE_10G_KR;
+ else
+ mode = PORT_MODE_XFI;
+ break;
+ case 4:
+ data = oct_csr_read(BGX_SPU_BR_PMD_CONTROL(node, bgx, index));
+ /* The use of training differentiates 40G_KR4 from xlaui */
+ if (data & BGX_SPU_BR_PMD_CONTROL_TRAIN_EN)
+ mode = PORT_MODE_40G_KR4;
+ else
+ mode = PORT_MODE_XLAUI;
+ break;
+ default:
+ mode = PORT_MODE_DISABLED;
+ break;
+ }
+
+ return mode;
+}
+EXPORT_SYMBOL(bgx_port_get_mode);
+
+int bgx_port_allocate_pknd(int node)
+{
+ struct global_resource_tag tag;
+ char buf[16];
+ int pknd;
+
+ strncpy((char *)&tag.lo, "cvm_pknd", 8);
+ snprintf(buf, 16, "_%d......", node);
+ memcpy(&tag.hi, buf, 8);
+
+ res_mgr_create_resource(tag, 64);
+ pknd = res_mgr_alloc(tag, -1, false);
+ if (pknd < 0) {
+ pr_err("bgx-port: Failed to allocate pknd\n");
+ return -ENODEV;
+ }
+
+ return pknd;
+}
+EXPORT_SYMBOL(bgx_port_allocate_pknd);
+
+int bgx_port_get_pknd(int node, int bgx, int index)
+{
+ return lmac_pknd[node][bgx][index];
+}
+EXPORT_SYMBOL(bgx_port_get_pknd);
+
+/* GSER-20075 */
+static void bgx_port_gser_20075(struct bgx_port_priv *priv, int qlm, int lane)
+{
+ u64 addr, data;
+
+ if (OCTEON_IS_MODEL(OCTEON_CN78XX_PASS1_X) &&
+ (lane == -1 || lane == 3)) {
+ /* Enable software control */
+ addr = GSER_BR_RX_CTL(priv->node, qlm, 3);
+ data = oct_csr_read(addr);
+ data |= GSER_BR_RX_CTL_RXT_SWM;
+ oct_csr_write(data, addr);
+
+ /* Clear the completion flag */
+ addr = GSER_BR_RX_EER(priv->node, qlm, 3);
+ data = oct_csr_read(addr);
+ data &= ~GSER_BR_RX_CTL_RXT_ESV;
+ oct_csr_write(data, addr);
+
+ /* Initiate a new request on lane 2 */
+ if (lane == 3) {
+ addr = GSER_BR_RX_EER(priv->node, qlm, 2);
+ data = oct_csr_read(addr);
+ data |= GSER_BR_RX_CTL_RXT_EER;
+ oct_csr_write(data, addr);
+ }
+ }
+}
+
+static void bgx_common_init_pknd(struct bgx_port_priv *priv)
+{
+ int num_ports;
+ u64 data;
+
+ /* Setup pkind */
+ priv->pknd = bgx_port_allocate_pknd(priv->node);
+ lmac_pknd[priv->node][priv->bgx][priv->index] = priv->pknd;
+ data = oct_csr_read(BGX_CMR_RX_ID_MAP(priv->node, priv->bgx,
+ priv->index));
+ data &= ~BGX_CMR_RX_ID_MAP_PKND_MASK;
+ data |= priv->pknd;
+ if (OCTEON_IS_MODEL(OCTEON_CN73XX)) {
+ /* Change the default reassembly id (max allowed is 14) */
+ data &= ~BGX_CMR_RX_ID_MAP_RID_MASK;
+ data |= ((4 * priv->bgx) + 2 + priv->index) <<
+ BGX_CMR_RX_ID_MAP_RID_SHIFT;
+ }
+ oct_csr_write(data, BGX_CMR_RX_ID_MAP(priv->node, priv->bgx,
+ priv->index));
+
+ /* Set backpressure channel mask AND/OR registers */
+ data = oct_csr_read(BGX_CMR_CHAN_MSK_AND(priv->node, priv->bgx));
+ data |= BGX_CMR_CHAN_MSK_MASK << (BGX_CMR_CHAN_MSK_SHIFT * priv->index);
+ oct_csr_write(data, BGX_CMR_CHAN_MSK_AND(priv->node, priv->bgx));
+
+ data = oct_csr_read(BGX_CMR_CHAN_MSK_OR(priv->node, priv->bgx));
+ data |= BGX_CMR_CHAN_MSK_MASK << (BGX_CMR_CHAN_MSK_SHIFT * priv->index);
+ oct_csr_write(data, BGX_CMR_CHAN_MSK_OR(priv->node, priv->bgx));
+
+ /* Rx back pressure watermark:
+ * Set to 1/4 of the available lmacs buffer (in multiple of 16 bytes)
+ */
+ data = oct_csr_read(BGX_CMR_TX_LMACS(priv->node, priv->bgx));
+ num_ports = data & BGX_CMR_TX_LMACS_MASK;
+ data = BGX_RX_FIFO_SIZE / (num_ports * 4 * 16);
+ oct_csr_write(data, BGX_CMR_RX_BP_ON(priv->node, priv->bgx,
+ priv->index));
+}
+
+static int bgx_xgmii_hardware_init(struct bgx_port_priv *priv)
+{
+ u64 ctl, clock_mhz, data;
+
+ /* Set TX Threshold */
+ data = BGX_GMP_GMI_TX_THRESH_DEFAULT;
+ oct_csr_write(data, BGX_GMP_GMI_TX_THRESH(priv->node, priv->bgx,
+ priv->index));
+
+ data = oct_csr_read(BGX_GMP_PCS_MISC_CTL(priv->node, priv->bgx,
+ priv->index));
+ data &= ~(BGX_GMP_PCS_MISC_CTL_MAC_PHY | BGX_GMP_PCS_MISC_CTL_MODE);
+ if (priv->mode_1000basex)
+ data |= BGX_GMP_PCS_MISC_CTL_MODE;
+ if (priv->bgx_as_phy)
+ data |= BGX_GMP_PCS_MISC_CTL_MAC_PHY;
+ oct_csr_write(data, BGX_GMP_PCS_MISC_CTL(priv->node, priv->bgx,
+ priv->index));
+
+ data = oct_csr_read(BGX_GMP_PCS_LINK_TIMER(priv->node, priv->bgx,
+ priv->index));
+ clock_mhz = octeon_get_io_clock_rate() / 1000000;
+ if (priv->mode_1000basex)
+ data = (10000ull * clock_mhz) >>
+ BGX_GMP_PCS_LINK_TIMER_COUNT_SHIFT;
+ else
+ data = (1600ull * clock_mhz) >>
+ BGX_GMP_PCS_LINK_TIMER_COUNT_SHIFT;
+ oct_csr_write(data, BGX_GMP_PCS_LINK_TIMER(priv->node, priv->bgx,
+ priv->index));
+
+ if (priv->mode_1000basex) {
+ data = oct_csr_read(BGX_GMP_PCS_AN_ADV(priv->node, priv->bgx,
+ priv->index));
+ data &= ~(BGX_GMP_PCS_AN_ADV_REM_FLT_MASK |
+ BGX_GMP_PCS_AN_ADV_PAUSE_MASK);
+ data |= BGX_GMP_PCS_AN_ADV_PAUSE_BOTH <<
+ BGX_GMP_PCS_AN_ADV_PAUSE_SHIFT;
+ data |= BGX_GMP_PCS_AN_ADV_HALF_DUPLEX |
+ BGX_GMP_PCS_AN_ADV_FULL_DUPLEX;
+ oct_csr_write(data, BGX_GMP_PCS_AN_ADV(priv->node, priv->bgx,
+ priv->index));
+ } else if (priv->bgx_as_phy) {
+ data = oct_csr_read(BGX_GMP_PCS_SGM_AN_ADV(priv->node,
+ priv->bgx,
+ priv->index));
+ data |= BGX_GMP_PCS_SGM_AN_ADV_DUPLEX_FULL;
+ data &= ~BGX_GMP_PCS_SGM_AN_ADV_SPEED_MASK;
+ data |= BGX_GMP_PCS_SGM_AN_ADV_SPEED_1000 <<
+ BGX_GMP_PCS_SGM_AN_ADV_SPEED_SHIFT;
+ oct_csr_write(data, BGX_GMP_PCS_SGM_AN_ADV(priv->node,
+ priv->bgx,
+ priv->index));
+ }
+
+ data = oct_csr_read(BGX_GMP_GMI_TX_APPEND(priv->node, priv->bgx,
+ priv->index));
+ ctl = oct_csr_read(BGX_GMP_GMI_TX_SGMII_CTL(priv->node, priv->bgx,
+ priv->index));
+ ctl &= ~BGX_GMP_GMI_TX_APPEND_PREAMBLE;
+ ctl |= (data & BGX_GMP_GMI_TX_APPEND_PREAMBLE) ? 0 : 1;
+ oct_csr_write(ctl, BGX_GMP_GMI_TX_SGMII_CTL(priv->node, priv->bgx,
+ priv->index));
+
+ if (priv->mode == PORT_MODE_RGMII) {
+ /* Disable XCV interface when initialized */
+ data = oct_csr_read(XCV_RESET(priv->node));
+ data &= ~(XCV_RESET_ENABLE | XCV_RESET_TX_PKT_RST_N |
+ XCV_RESET_RX_PKT_RST_N);
+ oct_csr_write(data, XCV_RESET(priv->node));
+ }
+
+ return 0;
+}
+
+int bgx_get_tx_fifo_size(struct bgx_port_priv *priv)
+{
+ int num_ports;
+ u64 data;
+
+ data = oct_csr_read(BGX_CMR_TX_LMACS(priv->node, priv->bgx));
+ num_ports = (data & BGX_CMR_TX_LMACS_MASK);
+
+ switch (num_ports) {
+ case 1:
+ return BGX_TX_FIFO_SIZE;
+ case 2:
+ return BGX_TX_FIFO_SIZE / 2;
+ case 3:
+ case 4:
+ return BGX_TX_FIFO_SIZE / 4;
+ default:
+ return 0;
+ }
+}
+
+static int bgx_xaui_hardware_init(struct bgx_port_priv *priv)
+{
+ u64 clock_mhz, data, tx_fifo_size;
+
+ if (octeon_is_simulation()) {
+ /* Enable the port */
+ data = oct_csr_read(BGX_CMR_CONFIG(priv->node, priv->bgx,
+ priv->index));
+ data |= BGX_CMR_CONFIG_ENABLE;
+ oct_csr_write(data, BGX_CMR_CONFIG(priv->node, priv->bgx,
+ priv->index));
+ } else {
+ /* Reset the port */
+ data = oct_csr_read(BGX_SPU_CONTROL1(priv->node, priv->bgx,
+ priv->index));
+ data |= BGX_SPU_CONTROL1_RESET;
+ oct_csr_write(data, BGX_SPU_CONTROL1(priv->node, priv->bgx,
+ priv->index));
+
+ /* Wait for reset to complete */
+ udelay(1);
+ data = oct_csr_read(BGX_SPU_CONTROL1(priv->node, priv->bgx,
+ priv->index));
+ if (data & BGX_SPU_CONTROL1_RESET) {
+ netdev_err(priv->netdev,
+ "BGX%d:%d: SPU stuck in reset\n", priv->bgx,
+ priv->node);
+ return -1;
+ }
+
+ /* Reset the SerDes lanes */
+ data = oct_csr_read(BGX_SPU_CONTROL1(priv->node, priv->bgx,
+ priv->index));
+ data |= BGX_SPU_CONTROL1_LO_PWR;
+ oct_csr_write(data, BGX_SPU_CONTROL1(priv->node, priv->bgx,
+ priv->index));
+
+ /* Disable packet reception */
+ data = oct_csr_read(BGX_SPU_MISC_CONTROL(priv->node, priv->bgx,
+ priv->index));
+ data |= BGX_SPU_MISC_CONTROL_RX_PACKET_DIS;
+ oct_csr_write(data, BGX_SPU_MISC_CONTROL(priv->node, priv->bgx,
+ priv->index));
+
+ /* Clear/disable interrupts */
+ data = oct_csr_read(BGX_SMU_RX_INT(priv->node, priv->bgx,
+ priv->index));
+ oct_csr_write(data, BGX_SMU_RX_INT(priv->node, priv->bgx,
+ priv->index));
+ data = oct_csr_read(BGX_SMU_TX_INT(priv->node, priv->bgx,
+ priv->index));
+ oct_csr_write(data, BGX_SMU_TX_INT(priv->node, priv->bgx,
+ priv->index));
+ data = oct_csr_read(BGX_SPU_INT(priv->node, priv->bgx,
+ priv->index));
+ oct_csr_write(data, BGX_SPU_INT(priv->node, priv->bgx,
+ priv->index));
+
+ if ((priv->mode == PORT_MODE_10G_KR ||
+ priv->mode == PORT_MODE_40G_KR4) &&
+ !OCTEON_IS_MODEL(OCTEON_CN78XX_PASS1_X)) {
+ oct_csr_write(0, BGX_SPU_BR_PMD_LP_CUP(priv->node,
+ priv->bgx,
+ priv->index));
+ oct_csr_write(0, BGX_SPU_BR_PMD_LD_CUP(priv->node,
+ priv->bgx,
+ priv->index));
+ oct_csr_write(0, BGX_SPU_BR_PMD_LD_REP(priv->node,
+ priv->bgx,
+ priv->index));
+ data = oct_csr_read(BGX_SPU_BR_PMD_CONTROL(priv->node,
+ priv->bgx,
+ priv->index));
+ data |= BGX_SPU_BR_PMD_CONTROL_TRAIN_EN;
+ oct_csr_write(data, BGX_SPU_BR_PMD_CONTROL(priv->node,
+ priv->bgx,
+ priv->index));
+ }
+ }
+
+ data = oct_csr_read(BGX_SMU_TX_APPEND(priv->node, priv->bgx,
+ priv->index));
+ data |= BGX_SMU_TX_APPEND_FCS_C;
+ oct_csr_write(data, BGX_SMU_TX_APPEND(priv->node, priv->bgx,
+ priv->index));
+
+ if (!octeon_is_simulation()) {
+ /* Disable fec */
+ data = oct_csr_read(BGX_SPU_FEC_CONTROL(priv->node, priv->bgx,
+ priv->index));
+ data &= ~BGX_SPU_FEC_CONTROL_FEC_EN;
+ oct_csr_write(data, BGX_SPU_FEC_CONTROL(priv->node, priv->bgx,
+ priv->index));
+
+ /* Disable/configure auto negotiation */
+ data = oct_csr_read(BGX_SPU_AN_CONTROL(priv->node, priv->bgx,
+ priv->index));
+ data &= ~(BGX_SPU_AN_CONTROL_XNP_EN | BGX_SPU_AN_CONTROL_AN_EN);
+ oct_csr_write(data, BGX_SPU_AN_CONTROL(priv->node, priv->bgx,
+ priv->index));
+
+ data = oct_csr_read(BGX_SPU_AN_ADV(priv->node, priv->bgx,
+ priv->index));
+ data &= ~(BGX_SPU_AN_ADV_FEC_REQ | BGX_SPU_AN_ADV_A100G_CR10 |
+ BGX_SPU_AN_ADV_A40G_CR4 | BGX_SPU_AN_ADV_A10G_KX4 |
+ BGX_SPU_AN_ADV_A1G_KX | BGX_SPU_AN_ADV_RF |
+ BGX_SPU_AN_ADV_XNP_ABLE);
+ data |= BGX_SPU_AN_ADV_FEC_REQ;
+ if (priv->mode == PORT_MODE_40G_KR4)
+ data |= BGX_SPU_AN_ADV_A40G_KR4;
+ else
+ data &= ~BGX_SPU_AN_ADV_A40G_KR4;
+ if (priv->mode == PORT_MODE_10G_KR)
+ data |= BGX_SPU_AN_ADV_A10G_KR;
+ else
+ data &= ~BGX_SPU_AN_ADV_A10G_KR;
+ oct_csr_write(data, BGX_SPU_AN_ADV(priv->node, priv->bgx,
+ priv->index));
+
+ data = oct_csr_read(BGX_SPU_DBG_CONTROL(priv->node, priv->bgx));
+ data |= BGX_SPU_DBG_CONTROL_AN_NONCE_MATCH_DIS;
+ if (priv->mode == PORT_MODE_10G_KR ||
+ priv->mode == PORT_MODE_40G_KR4)
+ data |= BGX_SPU_DBG_CONTROL_AN_ARB_LINK_CHK_EN;
+ else
+ data &= ~BGX_SPU_DBG_CONTROL_AN_ARB_LINK_CHK_EN;
+ oct_csr_write(data, BGX_SPU_DBG_CONTROL(priv->node, priv->bgx));
+
+ /* Enable the port */
+ data = oct_csr_read(BGX_CMR_CONFIG(priv->node, priv->bgx,
+ priv->index));
+ data |= BGX_CMR_CONFIG_ENABLE;
+ oct_csr_write(data, BGX_CMR_CONFIG(priv->node, priv->bgx,
+ priv->index));
+
+ if (OCTEON_IS_MODEL(OCTEON_CN78XX_PASS1_X) && priv->index) {
+ /* BGX-22429 */
+ data = oct_csr_read(BGX_CMR_CONFIG(priv->node,
+ priv->bgx, 0));
+ data |= BGX_CMR_CONFIG_ENABLE;
+ oct_csr_write(data, BGX_CMR_CONFIG(priv->node,
+ priv->bgx, 0));
+ }
+ }
+
+ data = oct_csr_read(BGX_SPU_CONTROL1(priv->node, priv->bgx,
+ priv->index));
+ data &= ~BGX_SPU_CONTROL1_LO_PWR;
+ oct_csr_write(data, BGX_SPU_CONTROL1(priv->node, priv->bgx,
+ priv->index));
+
+ data = oct_csr_read(BGX_SMU_TX_CTL(priv->node, priv->bgx, priv->index));
+ data |= BGX_SMU_TX_CTL_DIC_EN;
+ data &= ~BGX_SMU_TX_CTL_UNI_EN;
+ oct_csr_write(data, BGX_SMU_TX_CTL(priv->node, priv->bgx, priv->index));
+
+ clock_mhz = octeon_get_io_clock_rate() / 1000000;
+ data = oct_csr_read(BGX_SPU_DBG_CONTROL(priv->node, priv->bgx));
+ data &= ~BGX_SPU_DBG_CONTROL_US_CLK_PERIOD_MASK;
+ data |= (clock_mhz - 1) << BGX_SPU_DBG_CONTROL_US_CLK_PERIOD_SHIFT;
+ oct_csr_write(data, BGX_SPU_DBG_CONTROL(priv->node, priv->bgx));
+
+ /* Fifo in 16-byte words */
+ tx_fifo_size = bgx_get_tx_fifo_size(priv);
+ tx_fifo_size >>= 4;
+ oct_csr_write(tx_fifo_size - 10, BGX_SMU_TX_THRESH(priv->node,
+ priv->bgx,
+ priv->index));
+
+ if (priv->mode == PORT_MODE_RXAUI && priv->phy_np) {
+ data = oct_csr_read(BGX_SPU_MISC_CONTROL(priv->node, priv->bgx,
+ priv->index));
+ data |= BGX_SPU_MISC_CONTROL_INTLV_RDISP;
+ oct_csr_write(data, BGX_SPU_MISC_CONTROL(priv->node, priv->bgx,
+ priv->index));
+ }
+
+ /* Some PHYs take up to 250ms to stabilize */
+ if (!octeon_is_simulation())
+ usleep_range(250000, 300000);
+
+ return 0;
+}
+
+/* Configure/initialize a bgx port.
+ */
+static int bgx_port_init(struct bgx_port_priv *priv)
+{
+ int rc = 0;
+ u64 data;
+
+ /* GSER-20956 */
+ if (OCTEON_IS_MODEL(OCTEON_CN78XX_PASS1_X) &&
+ (priv->mode == PORT_MODE_10G_KR ||
+ priv->mode == PORT_MODE_XFI ||
+ priv->mode == PORT_MODE_40G_KR4 ||
+ priv->mode == PORT_MODE_XLAUI)) {
+ /* Disable link training */
+ data = oct_csr_read(BGX_SPU_BR_PMD_CONTROL(priv->node,
+ priv->bgx,
+ priv->index));
+
+ data &= ~BGX_SPU_BR_PMD_CONTROL_TRAIN_EN;
+ oct_csr_write(data, BGX_SPU_BR_PMD_CONTROL(priv->node,
+ priv->bgx,
+ priv->index));
+ }
+
+ bgx_common_init_pknd(priv);
+
+ if (priv->mode == PORT_MODE_SGMII ||
+ priv->mode == PORT_MODE_RGMII)
+ rc = bgx_xgmii_hardware_init(priv);
+ else
+ rc = bgx_xaui_hardware_init(priv);
+
+ return rc;
+}
+
+static int bgx_port_get_qlm_speed(struct bgx_port_priv *priv, int qlm)
+{
+ enum lane_mode lmode;
+ u64 data;
+
+ data = oct_csr_read(GSER_LANE_MODE(priv->node, qlm));
+ lmode = data & GSER_LANE_MODE_LMODE_MASK;
+
+ switch (lmode) {
+ case R_125G_REFCLK15625_KX:
+ case R_125G_REFCLK15625_SGMII:
+ return 1250;
+ case R_25G_REFCLK100:
+ case R_25G_REFCLK125:
+ return 2500;
+ case R_3125G_REFCLK15625_XAUI:
+ return 3125;
+ case R_5G_REFCLK100:
+ case R_5G_REFCLK15625_QSGMII:
+ case R_5G_REFCLK125:
+ return 5000;
+ case R_8G_REFCLK100:
+ case R_8G_REFCLK125:
+ return 8000;
+ case R_625G_REFCLK15625_RXAUI:
+ return 6250;
+ case R_103125G_REFCLK15625_KR:
+ return 10312;
+ default:
+ return 0;
+ }
+}
+
+static struct port_status bgx_port_get_sgmii_link(struct bgx_port_priv *priv)
+{
+ struct port_status status;
+
+ status.link = 1;
+ status.duplex = DUPLEX_FULL;
+ status.speed = 1000;
+
+ return status;
+}
+
+static int bgx_port_xgmii_set_link_up(struct bgx_port_priv *priv)
+{
+ int timeout;
+ u64 data;
+
+ if (!octeon_is_simulation()) {
+ /* PCS reset sequence */
+ data = oct_csr_read(BGX_GMP_PCS_MR_CONTROL(priv->node,
+ priv->bgx,
+ priv->index));
+ data |= BGX_GMP_PCS_MR_CONTROL_RESET;
+ oct_csr_write(data, BGX_GMP_PCS_MR_CONTROL(priv->node,
+ priv->bgx,
+ priv->index));
+
+ /* Wait for reset to complete */
+ udelay(1);
+ data = oct_csr_read(BGX_GMP_PCS_MR_CONTROL(priv->node,
+ priv->bgx,
+ priv->index));
+ if (data & BGX_GMP_PCS_MR_CONTROL_RESET) {
+ netdev_err(priv->netdev,
+ "BGX%d:%d: PCS stuck in reset\n", priv->bgx,
+ priv->node);
+ return -1;
+ }
+ }
+
+ /* Autonegotiation */
+ if (priv->phy_np) {
+ data = oct_csr_read(BGX_GMP_PCS_MR_CONTROL(priv->node,
+ priv->bgx,
+ priv->index));
+ data |= BGX_GMP_PCS_MR_CONTROL_RST_AN;
+ if (priv->mode != PORT_MODE_RGMII)
+ data |= BGX_GMP_PCS_MR_CONTROL_AN_EN;
+ else
+ data &= ~BGX_GMP_PCS_MR_CONTROL_AN_EN;
+ data &= ~BGX_GMP_PCS_MR_CONTROL_PWR_DN;
+ oct_csr_write(data, BGX_GMP_PCS_MR_CONTROL(priv->node,
+ priv->bgx,
+ priv->index));
+ } else {
+ data = oct_csr_read(BGX_GMP_PCS_MR_CONTROL(priv->node,
+ priv->bgx,
+ priv->index));
+ data |= BGX_GMP_PCS_MR_CONTROL_SPDMSB;
+ data &= ~(BGX_GMP_PCS_MR_CONTROL_SPDLSB |
+ BGX_GMP_PCS_MR_CONTROL_AN_EN |
+ BGX_GMP_PCS_MR_CONTROL_PWR_DN);
+ oct_csr_write(data, BGX_GMP_PCS_MR_CONTROL(priv->node,
+ priv->bgx,
+ priv->index));
+ }
+
+ data = oct_csr_read(BGX_GMP_PCS_MISC_CTL(priv->node, priv->bgx,
+ priv->index));
+ data &= ~(BGX_GMP_PCS_MISC_CTL_MAC_PHY | BGX_GMP_PCS_MISC_CTL_MODE);
+ if (priv->mode_1000basex)
+ data |= BGX_GMP_PCS_MISC_CTL_MODE;
+ if (priv->bgx_as_phy)
+ data |= BGX_GMP_PCS_MISC_CTL_MAC_PHY;
+ oct_csr_write(data, BGX_GMP_PCS_MISC_CTL(priv->node, priv->bgx,
+ priv->index));
+
+ /* Wait for autonegotiation to complete */
+ if (!octeon_is_simulation() && !priv->bgx_as_phy &&
+ priv->mode != PORT_MODE_RGMII) {
+ timeout = 10000;
+ do {
+ data = oct_csr_read(BGX_GMP_PCS_MR_STATUS(priv->node,
+ priv->bgx,
+ priv->index));
+ if (data & BGX_GMP_PCS_MR_STATUS_AN_CPT)
+ break;
+ timeout--;
+ udelay(1);
+ } while (timeout);
+ if (!timeout) {
+ netdev_err(priv->netdev, "BGX%d:%d: AN timeout\n",
+ priv->bgx, priv->node);
+ return -1;
+ }
+ }
+
+ return 0;
+}
+
+static void bgx_port_rgmii_set_link_down(struct bgx_port_priv *priv)
+{
+ int rx_fifo_len;
+ u64 data;
+
+ data = oct_csr_read(XCV_RESET(priv->node));
+ data &= ~XCV_RESET_RX_PKT_RST_N;
+ oct_csr_write(data, XCV_RESET(priv->node));
+ /* Is this read really needed? TODO */
+ data = oct_csr_read(XCV_RESET(priv->node));
+
+ /* Wait for 2 MTUs */
+ mdelay(10);
+
+ data = oct_csr_read(BGX_CMR_CONFIG(priv->node, priv->bgx, priv->index));
+ data &= ~BGX_CMR_CONFIG_DATA_PKT_RX_EN;
+ oct_csr_write(data, BGX_CMR_CONFIG(priv->node, priv->bgx, priv->index));
+
+ /* Wait for the rx and tx fifos to drain */
+ do {
+ data = oct_csr_read(BGX_CMR_RX_FIFO_LEN(priv->node, priv->bgx,
+ priv->index));
+ rx_fifo_len = data & BGX_CMR_RX_FIFO_LEN_MASK;
+ data = oct_csr_read(BGX_CMR_TX_FIFO_LEN(priv->node, priv->bgx,
+ priv->index));
+ } while (rx_fifo_len > 0 || !(data & BGX_CMR_TX_FIFO_LEN_LMAC_IDLE));
+
+ data = oct_csr_read(BGX_CMR_CONFIG(priv->node, priv->bgx, priv->index));
+ data &= ~BGX_CMR_CONFIG_DATA_PKT_TX_EN;
+ oct_csr_write(data, BGX_CMR_CONFIG(priv->node, priv->bgx, priv->index));
+
+ data = oct_csr_read(XCV_RESET(priv->node));
+ data &= ~XCV_RESET_TX_PKT_RST_N;
+ oct_csr_write(data, XCV_RESET(priv->node));
+
+ data = oct_csr_read(BGX_GMP_PCS_MR_CONTROL(priv->node, priv->bgx,
+ priv->index));
+ data |= BGX_GMP_PCS_MR_CONTROL_PWR_DN;
+ oct_csr_write(data, BGX_GMP_PCS_MR_CONTROL(priv->node, priv->bgx,
+ priv->index));
+}
+
+static void bgx_port_sgmii_set_link_down(struct bgx_port_priv *priv)
+{
+ u64 data;
+
+ data = oct_csr_read(BGX_CMR_CONFIG(priv->node, priv->bgx, priv->index));
+ data &= ~(BGX_CMR_CONFIG_DATA_PKT_RX_EN |
+ BGX_CMR_CONFIG_DATA_PKT_TX_EN);
+ oct_csr_write(data, BGX_CMR_CONFIG(priv->node, priv->bgx, priv->index));
+
+ data = oct_csr_read(BGX_GMP_PCS_MR_CONTROL(priv->node, priv->bgx,
+ priv->index));
+ data &= ~BGX_GMP_PCS_MR_CONTROL_AN_EN;
+ oct_csr_write(data, BGX_GMP_PCS_MR_CONTROL(priv->node, priv->bgx,
+ priv->index));
+
+ data = oct_csr_read(BGX_GMP_PCS_MISC_CTL(priv->node, priv->bgx,
+ priv->index));
+ data |= BGX_GMP_PCS_MISC_CTL_GMXENO;
+ oct_csr_write(data, BGX_GMP_PCS_MISC_CTL(priv->node, priv->bgx,
+ priv->index));
+ data = oct_csr_read(BGX_GMP_PCS_MISC_CTL(priv->node, priv->bgx,
+ priv->index));
+}
+
+static int bgx_port_sgmii_set_link_speed(struct bgx_port_priv *priv,
+ struct port_status status)
+{
+ u64 data, miscx, prtx;
+ int timeout;
+
+ data = oct_csr_read(BGX_CMR_CONFIG(priv->node, priv->bgx, priv->index));
+ data &= ~(BGX_CMR_CONFIG_DATA_PKT_RX_EN |
+ BGX_CMR_CONFIG_DATA_PKT_TX_EN);
+ oct_csr_write(data, BGX_CMR_CONFIG(priv->node, priv->bgx, priv->index));
+
+ timeout = 10000;
+ do {
+ prtx = oct_csr_read(BGX_GMP_GMI_PRT_CFG(priv->node, priv->bgx,
+ priv->index));
+ if ((prtx & BGX_GMP_GMI_PRT_CFG_TX_IDLE) &&
+ (prtx & BGX_GMP_GMI_PRT_CFG_RX_IDLE))
+ break;
+ timeout--;
+ udelay(1);
+ } while (timeout);
+ if (!timeout) {
+ netdev_err(priv->netdev, "BGX%d:%d: GMP idle timeout\n",
+ priv->bgx, priv->node);
+ return -1;
+ }
+
+ prtx = oct_csr_read(BGX_GMP_GMI_PRT_CFG(priv->node, priv->bgx,
+ priv->index));
+ miscx = oct_csr_read(BGX_GMP_PCS_MISC_CTL(priv->node, priv->bgx,
+ priv->index));
+ if (status.link) {
+ miscx &= ~BGX_GMP_PCS_MISC_CTL_GMXENO;
+ if (status.duplex == DUPLEX_FULL)
+ prtx |= BGX_GMP_GMI_PRT_CFG_DUPLEX;
+ else
+ prtx &= ~BGX_GMP_GMI_PRT_CFG_DUPLEX;
+ } else {
+ miscx |= BGX_GMP_PCS_MISC_CTL_GMXENO;
+ }
+
+ switch (status.speed) {
+ case 10:
+ prtx &= ~(BGX_GMP_GMI_PRT_CFG_SLOTTIME |
+ BGX_GMP_GMI_PRT_CFG_SPEED);
+ prtx |= BGX_GMP_GMI_PRT_CFG_SPEED_MSB;
+ miscx &= ~BGX_GMP_PCS_MISC_CTL_SAMP_PT_MASK;
+ miscx |= 25;
+ oct_csr_write(64, BGX_GMP_GMI_TX_SLOT(priv->node, priv->bgx,
+ priv->index));
+ oct_csr_write(0, BGX_GMP_GMI_TX_BURST(priv->node, priv->bgx,
+ priv->index));
+ break;
+ case 100:
+ prtx &= ~(BGX_GMP_GMI_PRT_CFG_SPEED_MSB |
+ BGX_GMP_GMI_PRT_CFG_SLOTTIME |
+ BGX_GMP_GMI_PRT_CFG_SPEED);
+ miscx &= ~BGX_GMP_PCS_MISC_CTL_SAMP_PT_MASK;
+ miscx |= 5;
+ oct_csr_write(64, BGX_GMP_GMI_TX_SLOT(priv->node, priv->bgx,
+ priv->index));
+ oct_csr_write(0, BGX_GMP_GMI_TX_BURST(priv->node, priv->bgx,
+ priv->index));
+ break;
+ case 1000:
+ prtx |= (BGX_GMP_GMI_PRT_CFG_SLOTTIME |
+ BGX_GMP_GMI_PRT_CFG_SPEED);
+ prtx &= ~BGX_GMP_GMI_PRT_CFG_SPEED_MSB;
+ miscx &= ~BGX_GMP_PCS_MISC_CTL_SAMP_PT_MASK;
+ miscx |= 1;
+ oct_csr_write(512, BGX_GMP_GMI_TX_SLOT(priv->node, priv->bgx,
+ priv->index));
+ if (status.duplex == DUPLEX_FULL)
+ oct_csr_write(0, BGX_GMP_GMI_TX_BURST(priv->node,
+ priv->bgx,
+ priv->index));
+ else
+ oct_csr_write(8192, BGX_GMP_GMI_TX_BURST(priv->node,
+ priv->bgx,
+ priv->index));
+ break;
+ default:
+ break;
+ }
+
+ oct_csr_write(miscx, BGX_GMP_PCS_MISC_CTL(priv->node, priv->bgx,
+ priv->index));
+ oct_csr_write(prtx, BGX_GMP_GMI_PRT_CFG(priv->node, priv->bgx,
+ priv->index));
+ /* This read verifies the write completed */
+ prtx = oct_csr_read(BGX_GMP_GMI_PRT_CFG(priv->node, priv->bgx,
+ priv->index));
+
+ data = oct_csr_read(BGX_CMR_CONFIG(priv->node, priv->bgx, priv->index));
+ data |= BGX_CMR_CONFIG_DATA_PKT_RX_EN | BGX_CMR_CONFIG_DATA_PKT_TX_EN;
+ oct_csr_write(data, BGX_CMR_CONFIG(priv->node, priv->bgx, priv->index));
+
+ return 0;
+}
+
+static int bgx_port_rgmii_set_link_speed(struct bgx_port_priv *priv,
+ struct port_status status)
+{
+ bool do_credits, int_lpbk = false, speed_changed = false;
+ int speed;
+ u64 data;
+
+ switch (status.speed) {
+ case 10:
+ speed = 0;
+ break;
+ case 100:
+ speed = 1;
+ break;
+ case 1000:
+ default:
+ speed = 2;
+ break;
+ }
+
+ /* Do credits if link came up */
+ data = oct_csr_read(XCV_RESET(priv->node));
+ do_credits = status.link && !(data & XCV_RESET_ENABLE);
+
+ /* Was there a speed change */
+ data = oct_csr_read(XCV_CTL(priv->node));
+ if ((data & XCV_CTL_SPEED_MASK) != speed)
+ speed_changed = true;
+
+ /* Clear clkrst when in internal loopback */
+ if (data & XCV_CTL_LPBK_INT) {
+ int_lpbk = true;
+ data = oct_csr_read(XCV_RESET(priv->node));
+ data &= ~XCV_RESET_CLKRST;
+ oct_csr_write(data, XCV_RESET(priv->node));
+ }
+
+ /* Link came up or there was a speed change */
+ data = oct_csr_read(XCV_RESET(priv->node));
+ if (status.link && (!(data & XCV_RESET_ENABLE) || speed_changed)) {
+ data |= XCV_RESET_ENABLE;
+ oct_csr_write(data, XCV_RESET(priv->node));
+
+ data = oct_csr_read(XCV_CTL(priv->node));
+ data &= ~XCV_CTL_SPEED_MASK;
+ data |= speed;
+ oct_csr_write(data, XCV_CTL(priv->node));
+
+ data = oct_csr_read(XCV_DLL_CTL(priv->node));
+ data |= XCV_DLL_CTL_CLKRX_BYP;
+ data &= ~XCV_DLL_CTL_CLKRX_SET_MASK;
+ data &= ~XCV_DLL_CTL_CLKTX_BYP;
+ oct_csr_write(data, XCV_DLL_CTL(priv->node));
+
+ data = oct_csr_read(XCV_DLL_CTL(priv->node));
+ data &= ~XCV_DLL_CTL_REFCLK_SEL_MASK;
+ oct_csr_write(data, XCV_DLL_CTL(priv->node));
+
+ data = oct_csr_read(XCV_RESET(priv->node));
+ data &= ~XCV_RESET_DLLRST;
+ oct_csr_write(data, XCV_RESET(priv->node));
+
+ usleep_range(10, 100);
+
+ data = oct_csr_read(XCV_COMP_CTL(priv->node));
+ data &= ~XCV_COMP_CTL_DRV_BYP;
+ oct_csr_write(data, XCV_COMP_CTL(priv->node));
+
+ data = oct_csr_read(XCV_RESET(priv->node));
+ data |= XCV_RESET_COMP;
+ oct_csr_write(data, XCV_RESET(priv->node));
+
+ data = oct_csr_read(XCV_RESET(priv->node));
+ if (int_lpbk)
+ data &= ~XCV_RESET_CLKRST;
+ else
+ data |= XCV_RESET_CLKRST;
+ oct_csr_write(data, XCV_RESET(priv->node));
+
+ data = oct_csr_read(XCV_RESET(priv->node));
+ data |= XCV_RESET_TX_DAT_RST_N | XCV_RESET_RX_DAT_RST_N;
+ oct_csr_write(data, XCV_RESET(priv->node));
+ }
+
+ data = oct_csr_read(XCV_RESET(priv->node));
+ if (status.link)
+ data |= XCV_RESET_TX_PKT_RST_N | XCV_RESET_RX_PKT_RST_N;
+ else
+ data &= ~(XCV_RESET_TX_PKT_RST_N | XCV_RESET_RX_PKT_RST_N);
+ oct_csr_write(data, XCV_RESET(priv->node));
+
+ if (!status.link) {
+ mdelay(10);
+ oct_csr_write(0, XCV_RESET(priv->node));
+ }
+
+ /* Grant pko tx credits */
+ if (do_credits) {
+ data = oct_csr_read(XCV_BATCH_CRD_RET(priv->node));
+ data |= XCV_BATCH_CRD_RET_CRD_RET;
+ oct_csr_write(data, XCV_BATCH_CRD_RET(priv->node));
+ }
+
+ return 0;
+}
+
+static int bgx_port_set_xgmii_link(struct bgx_port_priv *priv,
+ struct port_status status)
+{
+ int rc = 0;
+ u64 data;
+
+ if (status.link) {
+ /* Link up */
+ data = oct_csr_read(BGX_CMR_CONFIG(priv->node, priv->bgx,
+ priv->index));
+ data |= BGX_CMR_CONFIG_ENABLE;
+ oct_csr_write(data, BGX_CMR_CONFIG(priv->node, priv->bgx,
+ priv->index));
+
+ /* BGX-22429 */
+ if (OCTEON_IS_MODEL(OCTEON_CN78XX_PASS1_X) && priv->index) {
+ data = oct_csr_read(BGX_CMR_CONFIG(priv->node,
+ priv->bgx, 0));
+ data |= BGX_CMR_CONFIG_ENABLE;
+ oct_csr_write(data, BGX_CMR_CONFIG(priv->node,
+ priv->bgx, 0));
+ }
+
+ rc = bgx_port_xgmii_set_link_up(priv);
+ if (rc)
+ return rc;
+ rc = bgx_port_sgmii_set_link_speed(priv, status);
+ if (rc)
+ return rc;
+ if (priv->mode == PORT_MODE_RGMII)
+ rc = bgx_port_rgmii_set_link_speed(priv, status);
+ } else {
+ /* Link down */
+ if (priv->mode == PORT_MODE_RGMII) {
+ bgx_port_rgmii_set_link_down(priv);
+ rc = bgx_port_sgmii_set_link_speed(priv, status);
+ if (rc)
+ return rc;
+ rc = bgx_port_rgmii_set_link_speed(priv, status);
+ } else {
+ bgx_port_sgmii_set_link_down(priv);
+ }
+ }
+
+ return rc;
+}
+
+static struct port_status bgx_port_get_xaui_link(struct bgx_port_priv *priv)
+{
+ struct port_status status;
+ int lanes, speed;
+ u64 data;
+
+ status.link = 0;
+ status.duplex = DUPLEX_HALF;
+ status.speed = 0;
+
+ /* Get the link state */
+ data = oct_csr_read(BGX_SMU_TX_CTL(priv->node, priv->bgx, priv->index));
+ data &= BGX_SMU_TX_CTL_LS_MASK;
+ if (!data) {
+ data = oct_csr_read(BGX_SMU_RX_CTL(priv->node, priv->bgx,
+ priv->index));
+ data &= BGX_SMU_RX_CTL_STATUS_MASK;
+ if (!data) {
+ data = oct_csr_read(BGX_SPU_STATUS1(priv->node,
+ priv->bgx,
+ priv->index));
+ if (data & BGX_SPU_STATUS1_RCV_LINK)
+ status.link = 1;
+ }
+ }
+
+ if (status.link) {
+ /* Always full duplex */
+ status.duplex = DUPLEX_FULL;
+
+ /* Speed */
+ speed = bgx_port_get_qlm_speed(priv, priv->qlm);
+ data = oct_csr_read(BGX_CMR_CONFIG(priv->node, priv->bgx,
+ priv->index));
+ switch ((data & BGX_CMR_CONFIG_LMAC_TYPE_MASK) >>
+ BGX_CMR_CONFIG_LMAC_TYPE_SHIFT) {
+ default:
+ case 1:
+ speed = (speed * 8 + 5) / 10;
+ lanes = 4;
+ break;
+ case 2:
+ speed = (speed * 8 + 5) / 10;
+ lanes = 2;
+ break;
+ case 3:
+ speed = (speed * 64 + 33) / 66;
+ lanes = 1;
+ break;
+ case 4:
+ if (speed == 6250)
+ speed = 6445;
+ speed = (speed * 64 + 33) / 66;
+ lanes = 4;
+ break;
+ }
+
+ speed *= lanes;
+ status.speed = speed;
+ }
+
+ return status;
+}
+
+static int bgx_port_init_xaui_an(struct bgx_port_priv *priv)
+{
+ u64 data;
+
+ if (OCTEON_IS_MODEL(OCTEON_CN78XX_PASS1_X)) {
+ data = oct_csr_read(BGX_SPU_INT(priv->node, priv->bgx,
+ priv->index));
+ /* If autonegotiation is no good */
+ if (!(data & BGX_SPU_INT_AN_LINK_GOOD)) {
+ data = BGX_SPU_INT_AN_COMPLETE |
+ BGX_SPU_INT_AN_LINK_GOOD |
+ BGX_SPU_INT_AN_PAGE_RX;
+ oct_csr_write(data, BGX_SPU_INT(priv->node, priv->bgx,
+ priv->index));
+
+ data = oct_csr_read(BGX_SPU_AN_CONTROL(priv->node,
+ priv->bgx,
+ priv->index));
+ data |= BGX_SPU_AN_CONTROL_AN_RESTART;
+ oct_csr_write(data, BGX_SPU_AN_CONTROL(priv->node,
+ priv->bgx,
+ priv->index));
+ return -1;
+ }
+ } else {
+ data = oct_csr_read(BGX_SPU_AN_STATUS(priv->node, priv->bgx,
+ priv->index));
+ /* If autonegotiation hasn't completed */
+ if (!(data & BGX_SPU_AN_STATUS_AN_COMPLETE)) {
+ data = oct_csr_read(BGX_SPU_AN_CONTROL(priv->node,
+ priv->bgx,
+ priv->index));
+ data |= BGX_SPU_AN_CONTROL_AN_RESTART;
+ oct_csr_write(data, BGX_SPU_AN_CONTROL(priv->node,
+ priv->bgx,
+ priv->index));
+ return -1;
+ }
+ }
+
+ return 0;
+}
+
+static void bgx_port_xaui_start_training(struct bgx_port_priv *priv)
+{
+ u64 data;
+
+ data = BGX_SPU_INT_TRAINING_FAILURE | BGX_SPU_INT_TRAINING_DONE;
+ oct_csr_write(data, BGX_SPU_INT(priv->node, priv->bgx, priv->index));
+
+ /* BGX-20968 */
+ oct_csr_write(0, BGX_SPU_BR_PMD_LP_CUP(priv->node, priv->bgx,
+ priv->index));
+ oct_csr_write(0, BGX_SPU_BR_PMD_LD_CUP(priv->node, priv->bgx,
+ priv->index));
+ oct_csr_write(0, BGX_SPU_BR_PMD_LD_REP(priv->node, priv->bgx,
+ priv->index));
+ data = oct_csr_read(BGX_SPU_AN_CONTROL(priv->node, priv->bgx,
+ priv->index));
+ data &= ~BGX_SPU_AN_CONTROL_AN_EN;
+ oct_csr_write(data, BGX_SPU_AN_CONTROL(priv->node, priv->bgx,
+ priv->index));
+ udelay(1);
+
+ data = oct_csr_read(BGX_SPU_BR_PMD_CONTROL(priv->node, priv->bgx,
+ priv->index));
+ data |= BGX_SPU_BR_PMD_CONTROL_TRAIN_EN;
+ oct_csr_write(data, BGX_SPU_BR_PMD_CONTROL(priv->node, priv->bgx,
+ priv->index));
+ udelay(1);
+
+ data = oct_csr_read(BGX_SPU_BR_PMD_CONTROL(priv->node, priv->bgx,
+ priv->index));
+ data |= BGX_SPU_BR_PMD_CONTROL_TRAIN_RESTART;
+ oct_csr_write(data, BGX_SPU_BR_PMD_CONTROL(priv->node, priv->bgx,
+ priv->index));
+}
+
+static int bgx_port_gser_27882(struct bgx_port_priv *priv)
+{
+ u64 addr, data;
+ int timeout;
+
+ timeout = 200;
+ do {
+ data = oct_csr_read(GSER_RX_EIE_DETSTS(priv->node, priv->qlm));
+ if (data &
+ (1 << (priv->index + GSER_RX_EIE_DETSTS_CDRLCK_SHIFT)))
+ break;
+ timeout--;
+ udelay(1);
+ } while (timeout);
+ if (!timeout)
+ return -1;
+
+ addr = GSER_LANE_PCS_CTLIFC_0(priv->node, priv->qlm, priv->index);
+ data = oct_csr_read(addr);
+ data |= GSER_LANE_PCS_CTLIFC_0_CFG_TX_COEFF_REQ_OVRRD_VAL;
+ oct_csr_write(data, addr);
+
+ addr = GSER_LANE_PCS_CTLIFC_2(priv->node, priv->qlm, priv->index);
+ data = oct_csr_read(addr);
+ data |= GSER_LANE_PCS_CTLIFC_2_CFG_TX_COEFF_REQ_OVRRD_EN;
+ oct_csr_write(data, addr);
+
+ data = oct_csr_read(addr);
+ data |= GSER_LANE_PCS_CTLIFC_2_CTLIFC_OVRRD_REQ;
+ oct_csr_write(data, addr);
+
+ data = oct_csr_read(addr);
+ data &= ~GSER_LANE_PCS_CTLIFC_2_CFG_TX_COEFF_REQ_OVRRD_EN;
+ oct_csr_write(data, addr);
+
+ data = oct_csr_read(addr);
+ data |= GSER_LANE_PCS_CTLIFC_2_CTLIFC_OVRRD_REQ;
+ oct_csr_write(data, addr);
+
+ return 0;
+}
+
+static void bgx_port_xaui_restart_training(struct bgx_port_priv *priv)
+{
+ u64 data;
+
+ data = BGX_SPU_INT_TRAINING_FAILURE | BGX_SPU_INT_TRAINING_DONE;
+ oct_csr_write(data, BGX_SPU_INT(priv->node, priv->bgx, priv->index));
+ usleep_range(1700, 2000);
+
+ /* BGX-20968 */
+ oct_csr_write(0, BGX_SPU_BR_PMD_LP_CUP(priv->node, priv->bgx,
+ priv->index));
+ oct_csr_write(0, BGX_SPU_BR_PMD_LD_CUP(priv->node, priv->bgx,
+ priv->index));
+ oct_csr_write(0, BGX_SPU_BR_PMD_LD_REP(priv->node, priv->bgx,
+ priv->index));
+
+ /* Restart training */
+ data = oct_csr_read(BGX_SPU_BR_PMD_CONTROL(priv->node, priv->bgx,
+ priv->index));
+ data |= BGX_SPU_BR_PMD_CONTROL_TRAIN_RESTART;
+ oct_csr_write(data, BGX_SPU_BR_PMD_CONTROL(priv->node, priv->bgx,
+ priv->index));
+}
+
+static int bgx_port_get_max_qlm_lanes(int qlm)
+{
+ if (OCTEON_IS_MODEL(OCTEON_CN73XX))
+ return (qlm < 4) ? 4 : 2;
+ else if (OCTEON_IS_MODEL(OCTEON_CNF75XX))
+ return 2;
+ return 4;
+}
+
+static int bgx_port_qlm_rx_equalization(struct bgx_port_priv *priv, int qlm,
+ int lane)
+{
+ int i, lane_mask, max_lanes, timeout, rc = 0;
+ u64 addr, data, lmode;
+
+ max_lanes = bgx_port_get_max_qlm_lanes(qlm);
+ lane_mask = lane == -1 ? ((1 << max_lanes) - 1) : (1 << lane);
+
+ /* Nothing to do for qlms in reset */
+ data = oct_csr_read(GSER_PHY_CTL(priv->node, qlm));
+ if (data & (GSER_PHY_CTL_PHY_RESET | GSER_PHY_CTL_PHY_PD))
+ return -1;
+
+ for (i = 0; i < max_lanes; i++) {
+ if (!(i & lane_mask))
+ continue;
+
+ addr = GSER_LANE_LBERT_CFG(priv->node, qlm, i);
+ data = oct_csr_read(addr);
+ /* Rx equalization can't be completed while pattern matcher is
+ * enabled because it causes errors.
+ */
+ if (data & GSER_LANE_LBERT_CFG_LBERT_PM_EN)
+ return -1;
+ }
+
+ lmode = oct_csr_read(GSER_LANE_MODE(priv->node, qlm));
+ lmode &= GSER_LANE_MODE_LMODE_MASK;
+ addr = GSER_LANE_P_MODE_1(priv->node, qlm, lmode);
+ data = oct_csr_read(addr);
+ /* Don't complete rx equalization if in VMA manual mode */
+ if (data & GSER_LANE_P_MODE_1_VMA_MM)
+ return 0;
+
+ /* Apply rx equalization for speed > 6250 */
+ if (bgx_port_get_qlm_speed(priv, qlm) < 6250)
+ return 0;
+
+ /* Wait until rx data is valid (CDRLOCK) */
+ timeout = 500;
+ addr = GSER_RX_EIE_DETSTS(priv->node, qlm);
+ do {
+ data = oct_csr_read(addr);
+ data >>= GSER_RX_EIE_DETSTS_CDRLCK_SHIFT;
+ data &= lane_mask;
+ if (data == lane_mask)
+ break;
+ timeout--;
+ udelay(1);
+ } while (timeout);
+ if (!timeout) {
+ pr_debug("QLM%d:%d: CDRLOCK timeout\n", qlm, priv->node);
+ return -1;
+ }
+
+ bgx_port_gser_20075(priv, qlm, lane);
+
+ for (i = 0; i < max_lanes; i++) {
+ if (!(i & lane_mask))
+ continue;
+ /* Skip lane 3 on 78p1.x due to gser-20075. Handled above */
+ if (OCTEON_IS_MODEL(OCTEON_CN78XX_PASS1_X) && i == 3)
+ continue;
+
+ /* Enable software control */
+ addr = GSER_BR_RX_CTL(priv->node, qlm, i);
+ data = oct_csr_read(addr);
+ data |= GSER_BR_RX_CTL_RXT_SWM;
+ oct_csr_write(data, addr);
+
+ /* Clear the completion flag */
+ addr = GSER_BR_RX_EER(priv->node, qlm, i);
+ data = oct_csr_read(addr);
+ data &= ~GSER_BR_RX_EER_RXT_ESV;
+ data |= GSER_BR_RX_EER_RXT_EER;
+ oct_csr_write(data, addr);
+ }
+
+ /* Wait for rx equalization to complete */
+ for (i = 0; i < max_lanes; i++) {
+ if (!(i & lane_mask))
+ continue;
+
+ timeout = 250000;
+ addr = GSER_BR_RX_EER(priv->node, qlm, i);
+ do {
+ data = oct_csr_read(addr);
+ if (data & GSER_BR_RX_EER_RXT_ESV)
+ break;
+ timeout--;
+ udelay(1);
+ } while (timeout);
+ if (!timeout) {
+ pr_debug("QLM%d:%d: RXT_ESV timeout\n",
+ qlm, priv->node);
+ rc = -1;
+ }
+
+ /* Switch back to hardware control */
+ addr = GSER_BR_RX_CTL(priv->node, qlm, i);
+ data = oct_csr_read(addr);
+ data &= ~GSER_BR_RX_CTL_RXT_SWM;
+ oct_csr_write(data, addr);
+ }
+
+ return rc;
+}
+
+static int bgx_port_xaui_equalization(struct bgx_port_priv *priv)
+{
+ u64 data;
+ int lane;
+
+ /* Nothing to do for loopback mode */
+ data = oct_csr_read(BGX_SPU_CONTROL1(priv->node, priv->bgx,
+ priv->index));
+ if (data & BGX_SPU_CONTROL1_LOOPBACK)
+ return 0;
+
+ if (priv->mode == PORT_MODE_XAUI || priv->mode == PORT_MODE_XLAUI) {
+ if (bgx_port_qlm_rx_equalization(priv, priv->qlm, -1))
+ return -1;
+
+ /* BGX2 of 73xx uses 2 dlms */
+ if (OCTEON_IS_MODEL(OCTEON_CN73XX) && priv->bgx == 2) {
+ if (bgx_port_qlm_rx_equalization(priv, priv->qlm + 1,
+ -1))
+ return -1;
+ }
+ } else if (priv->mode == PORT_MODE_RXAUI) {
+ /* Rxaui always uses 2 lanes */
+ if (bgx_port_qlm_rx_equalization(priv, priv->qlm, -1))
+ return -1;
+ } else if (priv->mode == PORT_MODE_XFI) {
+ lane = priv->index;
+ if ((OCTEON_IS_MODEL(OCTEON_CN73XX) && priv->qlm == 6) ||
+ (OCTEON_IS_MODEL(OCTEON_CNF75XX) && priv->qlm == 5))
+ lane -= 2;
+
+ if (bgx_port_qlm_rx_equalization(priv, priv->qlm, lane))
+ return -1;
+ }
+
+ return 0;
+}
+
+static int bgx_port_init_xaui_link(struct bgx_port_priv *priv)
+{
+ int use_ber = 0, use_training = 0;
+ int rc = 0, timeout;
+ u64 data;
+
+ if (priv->mode == PORT_MODE_10G_KR || priv->mode == PORT_MODE_40G_KR4)
+ use_training = 1;
+
+ if (!octeon_is_simulation() &&
+ (priv->mode == PORT_MODE_XFI || priv->mode == PORT_MODE_XLAUI ||
+ priv->mode == PORT_MODE_10G_KR || priv->mode == PORT_MODE_40G_KR4))
+ use_ber = 1;
+
+ data = oct_csr_read(BGX_CMR_CONFIG(priv->node, priv->bgx, priv->index));
+ data &= ~(BGX_CMR_CONFIG_DATA_PKT_RX_EN |
+ BGX_CMR_CONFIG_DATA_PKT_TX_EN);
+ oct_csr_write(data, BGX_CMR_CONFIG(priv->node, priv->bgx, priv->index));
+
+ data = oct_csr_read(BGX_SPU_MISC_CONTROL(priv->node, priv->bgx,
+ priv->index));
+ data |= BGX_SPU_MISC_CONTROL_RX_PACKET_DIS;
+ oct_csr_write(data, BGX_SPU_MISC_CONTROL(priv->node, priv->bgx,
+ priv->index));
+
+ if (!octeon_is_simulation()) {
+ data = oct_csr_read(BGX_SPU_AN_CONTROL(priv->node, priv->bgx,
+ priv->index));
+ /* Restart autonegotiation */
+ if (data & BGX_SPU_AN_CONTROL_AN_EN) {
+ rc = bgx_port_init_xaui_an(priv);
+ if (rc)
+ return rc;
+ }
+
+ if (use_training) {
+ data = oct_csr_read(BGX_SPU_BR_PMD_CONTROL(priv->node,
+ priv->bgx,
+ priv->index));
+ /* Check if training is enabled */
+ if (OCTEON_IS_MODEL(OCTEON_CN78XX_PASS1_X) &&
+ !(data & BGX_SPU_BR_PMD_CONTROL_TRAIN_EN)) {
+ bgx_port_xaui_start_training(priv);
+ return -1;
+ }
+
+ if (OCTEON_IS_MODEL(OCTEON_CN73XX) ||
+ OCTEON_IS_MODEL(OCTEON_CNF75XX) ||
+ OCTEON_IS_MODEL(OCTEON_CN78XX))
+ bgx_port_gser_27882(priv);
+
+ data = oct_csr_read(BGX_SPU_INT(priv->node, priv->bgx,
+ priv->index));
+
+ /* Restart training if it failed */
+ if ((data & BGX_SPU_INT_TRAINING_FAILURE) &&
+ !OCTEON_IS_MODEL(OCTEON_CN78XX_PASS1_X)) {
+ bgx_port_xaui_restart_training(priv);
+ return -1;
+ }
+
+ if (!(data & BGX_SPU_INT_TRAINING_DONE)) {
+ pr_debug("Waiting for link training\n");
+ return -1;
+ }
+ } else {
+ bgx_port_xaui_equalization(priv);
+ }
+
+ /* Wait until the reset is complete */
+ timeout = 10000;
+ do {
+ data = oct_csr_read(BGX_SPU_CONTROL1(priv->node,
+ priv->bgx,
+ priv->index));
+ if (!(data & BGX_SPU_CONTROL1_RESET))
+ break;
+ timeout--;
+ udelay(1);
+ } while (timeout);
+ if (!timeout) {
+ pr_debug("BGX%d:%d:%d: Reset timeout\n", priv->bgx,
+ priv->index, priv->node);
+ return -1;
+ }
+
+ if (use_ber) {
+ timeout = 10000;
+ do {
+ data =
+ oct_csr_read(BGX_SPU_BR_STATUS1(priv->node,
+ priv->bgx,
+ priv->index));
+ if (data & BGX_SPU_BR_STATUS1_BLK_LOCK)
+ break;
+ timeout--;
+ udelay(1);
+ } while (timeout);
+ if (!timeout) {
+ pr_debug("BGX%d:%d:%d: BLK_LOCK timeout\n",
+ priv->bgx, priv->index, priv->node);
+ return -1;
+ }
+ } else {
+ timeout = 10000;
+ do {
+ data =
+ oct_csr_read(BGX_SPU_BX_STATUS(priv->node,
+ priv->bgx,
+ priv->index));
+ if (data & BGX_SPU_BX_STATUS_ALIGND)
+ break;
+ timeout--;
+ udelay(1);
+ } while (timeout);
+ if (!timeout) {
+ pr_debug("BGX%d:%d:%d: Lanes align timeout\n",
+ priv->bgx, priv->index, priv->node);
+ return -1;
+ }
+ }
+
+ if (use_ber) {
+ data = oct_csr_read(BGX_SPU_BR_STATUS2(priv->node,
+ priv->bgx,
+ priv->index));
+ data |= BGX_SPU_BR_STATUS2_LATCHED_LOCK;
+ oct_csr_write(data, BGX_SPU_BR_STATUS2(priv->node,
+ priv->bgx,
+ priv->index));
+ }
+
+ data = oct_csr_read(BGX_SPU_STATUS2(priv->node, priv->bgx,
+ priv->index));
+ data |= BGX_SPU_STATUS2_RCVFLT;
+ oct_csr_write(data, BGX_SPU_STATUS2(priv->node, priv->bgx,
+ priv->index));
+
+ data = oct_csr_read(BGX_SPU_STATUS2(priv->node, priv->bgx,
+ priv->index));
+ if (data & BGX_SPU_STATUS2_RCVFLT) {
+ if (OCTEON_IS_MODEL(OCTEON_CN78XX_PASS1_X) &&
+ use_training)
+ bgx_port_xaui_restart_training(priv);
+ return -1;
+ }
+
+ /* Wait for mac rx to be ready */
+ timeout = 10000;
+ do {
+ data = oct_csr_read(BGX_SMU_RX_CTL(priv->node,
+ priv->bgx,
+ priv->index));
+ data &= BGX_SMU_RX_CTL_STATUS_MASK;
+ if (!data)
+ break;
+ timeout--;
+ udelay(1);
+ } while (timeout);
+ if (!timeout) {
+ pr_debug("BGX%d:%d:%d: mac ready timeout\n", priv->bgx,
+ priv->index, priv->node);
+ return -1;
+ }
+
+ /* Wait for bgx rx to be idle */
+ timeout = 10000;
+ do {
+ data = oct_csr_read(BGX_SMU_CTRL(priv->node, priv->bgx,
+ priv->index));
+ if (data & BGX_SMU_CTRL_RX_IDLE)
+ break;
+ timeout--;
+ udelay(1);
+ } while (timeout);
+ if (!timeout) {
+ pr_debug("BGX%d:%d:%d: rx idle timeout\n", priv->bgx,
+ priv->index, priv->node);
+ return -1;
+ }
+
+ /* Wait for gmx tx to be idle */
+ timeout = 10000;
+ do {
+ data = oct_csr_read(BGX_SMU_CTRL(priv->node, priv->bgx,
+ priv->index));
+ if (data & BGX_SMU_CTRL_TX_IDLE)
+ break;
+ timeout--;
+ udelay(1);
+ } while (timeout);
+ if (!timeout) {
+ pr_debug("BGX%d:%d:%d: tx idle timeout\n", priv->bgx,
+ priv->index, priv->node);
+ return -1;
+ }
+
+ /* Check rcvflt is still be 0 */
+ data = oct_csr_read(BGX_SPU_STATUS2(priv->node, priv->bgx,
+ priv->index));
+ if (data & BGX_SPU_STATUS2_RCVFLT) {
+ pr_debug("BGX%d:%d:%d: receive fault\n", priv->bgx,
+ priv->index, priv->node);
+ return -1;
+ }
+
+ /* Receive link is latching low. Force it high and verify it */
+ data = oct_csr_read(BGX_SPU_STATUS1(priv->node, priv->bgx,
+ priv->index));
+ data |= BGX_SPU_STATUS1_RCV_LINK;
+ oct_csr_write(data, BGX_SPU_STATUS1(priv->node, priv->bgx,
+ priv->index));
+ timeout = 10000;
+ do {
+ data = oct_csr_read(BGX_SPU_STATUS1(priv->node,
+ priv->bgx,
+ priv->index));
+ if (data & BGX_SPU_STATUS1_RCV_LINK)
+ break;
+ timeout--;
+ udelay(1);
+ } while (timeout);
+ if (!timeout) {
+ pr_debug("BGX%d:%d:%d: rx link down\n", priv->bgx,
+ priv->index, priv->node);
+ return -1;
+ }
+ }
+
+ if (use_ber) {
+ /* Read error counters to clear */
+ data = oct_csr_read(BGX_SPU_BR_BIP_ERR_CNT(priv->node,
+ priv->bgx,
+ priv->index));
+ data = oct_csr_read(BGX_SPU_BR_STATUS2(priv->node, priv->bgx,
+ priv->index));
+
+ /* Verify latch lock is set */
+ if (!(data & BGX_SPU_BR_STATUS2_LATCHED_LOCK)) {
+ pr_debug("BGX%d:%d:%d: latch lock lost\n",
+ priv->bgx, priv->index, priv->node);
+ return -1;
+ }
+
+ /* LATCHED_BER is cleared by writing 1 to it */
+ if (data & BGX_SPU_BR_STATUS2_LATCHED_BER)
+ oct_csr_write(data, BGX_SPU_BR_STATUS2(priv->node,
+ priv->bgx,
+ priv->index));
+
+ usleep_range(1500, 2000);
+ data = oct_csr_read(BGX_SPU_BR_STATUS2(priv->node, priv->bgx,
+ priv->index));
+ if (data & BGX_SPU_BR_STATUS2_LATCHED_BER) {
+ pr_debug("BGX%d:%d:%d: BER test failed\n",
+ priv->bgx, priv->index, priv->node);
+ return -1;
+ }
+ }
+
+ /* Enable packet transmit and receive */
+ data = oct_csr_read(BGX_SPU_MISC_CONTROL(priv->node, priv->bgx,
+ priv->index));
+ data &= ~BGX_SPU_MISC_CONTROL_RX_PACKET_DIS;
+ oct_csr_write(data, BGX_SPU_MISC_CONTROL(priv->node, priv->bgx,
+ priv->index));
+ data = oct_csr_read(BGX_CMR_CONFIG(priv->node, priv->bgx, priv->index));
+ data |= BGX_CMR_CONFIG_DATA_PKT_RX_EN | BGX_CMR_CONFIG_DATA_PKT_TX_EN;
+ oct_csr_write(data, BGX_CMR_CONFIG(priv->node, priv->bgx, priv->index));
+
+ return 0;
+}
+
+static int bgx_port_set_xaui_link(struct bgx_port_priv *priv,
+ struct port_status status)
+{
+ bool smu_rx_ok = false, smu_tx_ok = false, spu_link_ok = false;
+ int rc = 0;
+ u64 data;
+
+ /* Initialize hardware if link is up but hardware is not happy */
+ if (status.link) {
+ data = oct_csr_read(BGX_SMU_TX_CTL(priv->node, priv->bgx,
+ priv->index));
+ data &= BGX_SMU_TX_CTL_LS_MASK;
+ smu_tx_ok = data == 0;
+
+ data = oct_csr_read(BGX_SMU_RX_CTL(priv->node, priv->bgx,
+ priv->index));
+ data &= BGX_SMU_RX_CTL_STATUS_MASK;
+ smu_rx_ok = data == 0;
+
+ data = oct_csr_read(BGX_SPU_STATUS1(priv->node, priv->bgx,
+ priv->index));
+ data &= BGX_SPU_STATUS1_RCV_LINK;
+ spu_link_ok = data == BGX_SPU_STATUS1_RCV_LINK;
+
+ if (!smu_tx_ok || !smu_rx_ok || !spu_link_ok)
+ rc = bgx_port_init_xaui_link(priv);
+ }
+
+ return rc;
+}
+
+static struct bgx_port_priv *bgx_port_netdev2priv(struct net_device *netdev)
+{
+ struct bgx_port_netdev_priv *nd_priv = netdev_priv(netdev);
+
+ return nd_priv->bgx_priv;
+}
+
+void bgx_port_set_netdev(struct device *dev, struct net_device *netdev)
+{
+ struct bgx_port_priv *priv = dev_get_drvdata(dev);
+
+ if (netdev) {
+ struct bgx_port_netdev_priv *nd_priv = netdev_priv(netdev);
+
+ nd_priv->bgx_priv = priv;
+ }
+
+ priv->netdev = netdev;
+}
+EXPORT_SYMBOL(bgx_port_set_netdev);
+
+int bgx_port_ethtool_get_link_ksettings(struct net_device *netdev,
+ struct ethtool_link_ksettings *cmd)
+{
+ struct bgx_port_priv *priv = bgx_port_netdev2priv(netdev);
+
+ if (priv->phydev) {
+ phy_ethtool_ksettings_get(priv->phydev, cmd);
+ return 0;
+ } else {
+ return -EINVAL;
+ }
+}
+EXPORT_SYMBOL(bgx_port_ethtool_get_link_ksettings);
+
+int bgx_port_ethtool_set_settings(struct net_device *netdev,
+ struct ethtool_cmd *cmd)
+{
+ struct bgx_port_priv *p = bgx_port_netdev2priv(netdev);
+
+ if (!capable(CAP_NET_ADMIN))
+ return -EPERM;
+
+ if (p->phydev)
+ return phy_ethtool_sset(p->phydev, cmd);
+
+ return -EOPNOTSUPP;
+}
+EXPORT_SYMBOL(bgx_port_ethtool_set_settings);
+
+int bgx_port_ethtool_nway_reset(struct net_device *netdev)
+{
+ struct bgx_port_priv *p = bgx_port_netdev2priv(netdev);
+
+ if (!capable(CAP_NET_ADMIN))
+ return -EPERM;
+
+ if (p->phydev)
+ return phy_start_aneg(p->phydev);
+
+ return -EOPNOTSUPP;
+}
+EXPORT_SYMBOL(bgx_port_ethtool_nway_reset);
+
+const u8 *bgx_port_get_mac(struct net_device *netdev)
+{
+ struct bgx_port_priv *priv = bgx_port_netdev2priv(netdev);
+
+ return priv->mac_addr;
+}
+EXPORT_SYMBOL(bgx_port_get_mac);
+
+int bgx_port_do_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
+{
+ struct bgx_port_priv *p = bgx_port_netdev2priv(netdev);
+
+ if (p->phydev)
+ return phy_mii_ioctl(p->phydev, ifr, cmd);
+ return -EOPNOTSUPP;
+}
+EXPORT_SYMBOL(bgx_port_do_ioctl);
+
+static void bgx_port_write_cam(struct bgx_port_priv *priv, int cam,
+ const u8 *mac)
+{
+ u64 m = 0;
+ int i;
+
+ if (mac) {
+ for (i = 0; i < 6; i++)
+ m |= (((u64)mac[i]) << ((5 - i) * 8));
+ m |= BGX_CMR_RX_ADRX_CAM_EN;
+ }
+
+ m |= (u64)priv->index << BGX_CMR_RX_ADRX_CAM_ID_SHIFT;
+ oct_csr_write(m, BGX_CMR_RX_ADRX_CAM(priv->node, priv->bgx,
+ priv->index * 8 + cam));
+}
+
+/* Set MAC address for the net_device that is attached. */
+void bgx_port_set_rx_filtering(struct net_device *netdev)
+{
+ struct bgx_port_priv *priv = bgx_port_netdev2priv(netdev);
+ int available_cam_entries, current_cam_entry;
+ struct netdev_hw_addr *ha;
+ u64 data;
+
+ available_cam_entries = 8;
+ data = BGX_CMR_RX_ADR_CTL_BCST_ACCEPT;
+
+ if ((netdev->flags & IFF_PROMISC) || netdev->uc.count > 7) {
+ data &= ~BGX_CMR_RX_ADR_CTL_CAM_ACCEPT;
+ available_cam_entries = 0;
+ } else {
+ /* One CAM entry for the primary address, leaves seven
+ * for the secondary addresses.
+ */
+ data |= BGX_CMR_RX_ADR_CTL_CAM_ACCEPT;
+ available_cam_entries = 7 - netdev->uc.count;
+ }
+
+ if (netdev->flags & IFF_PROMISC) {
+ data |= BGX_CMR_RX_ADR_CTL_ACCEPT_ALL_MCST;
+ } else {
+ if (netdev->flags & IFF_MULTICAST) {
+ if ((netdev->flags & IFF_ALLMULTI) ||
+ netdev_mc_count(netdev) > available_cam_entries)
+ data |= BGX_CMR_RX_ADR_CTL_ACCEPT_ALL_MCST;
+ else
+ data |= BGX_CMR_RX_ADR_CTL_USE_CAM_FILTER;
+ }
+ }
+ current_cam_entry = 0;
+ if (data & BGX_CMR_RX_ADR_CTL_CAM_ACCEPT) {
+ bgx_port_write_cam(priv, current_cam_entry, netdev->dev_addr);
+ current_cam_entry++;
+ netdev_for_each_uc_addr(ha, netdev) {
+ bgx_port_write_cam(priv, current_cam_entry, ha->addr);
+ current_cam_entry++;
+ }
+ }
+ if ((data & BGX_CMR_RX_ADR_CTL_MCST_MODE_MASK) ==
+ BGX_CMR_RX_ADR_CTL_USE_CAM_FILTER) {
+ /* Accept all Multicast via CAM */
+ netdev_for_each_mc_addr(ha, netdev) {
+ bgx_port_write_cam(priv, current_cam_entry, ha->addr);
+ current_cam_entry++;
+ }
+ }
+ while (current_cam_entry < 8) {
+ bgx_port_write_cam(priv, current_cam_entry, NULL);
+ current_cam_entry++;
+ }
+ oct_csr_write(data, BGX_CMR_RX_ADR_CTL(priv->node, priv->bgx,
+ priv->index));
+}
+EXPORT_SYMBOL(bgx_port_set_rx_filtering);
+
+static void bgx_port_adjust_link(struct net_device *netdev)
+{
+ struct bgx_port_priv *priv = bgx_port_netdev2priv(netdev);
+ unsigned int duplex, link, speed;
+ int link_changed = 0;
+
+ mutex_lock(&priv->lock);
+
+ if (!priv->phydev->link && priv->last_status.link)
+ link_changed = -1;
+
+ if (priv->phydev->link &&
+ (priv->last_status.link != priv->phydev->link ||
+ priv->last_status.duplex != priv->phydev->duplex ||
+ priv->last_status.speed != priv->phydev->speed))
+ link_changed = 1;
+
+ link = priv->phydev->link;
+ priv->last_status.link = priv->phydev->link;
+
+ speed = priv->phydev->speed;
+ priv->last_status.speed = priv->phydev->speed;
+
+ duplex = priv->phydev->duplex;
+ priv->last_status.duplex = priv->phydev->duplex;
+
+ mutex_unlock(&priv->lock);
+
+ if (link_changed != 0) {
+ struct port_status status;
+
+ if (link_changed > 0) {
+ netdev_info(netdev, "Link is up - %d/%s\n",
+ priv->phydev->speed,
+ priv->phydev->duplex == DUPLEX_FULL ?
+ "Full" : "Half");
+ } else {
+ netdev_info(netdev, "Link is down\n");
+ }
+ status.link = link ? 1 : 0;
+ status.duplex = duplex;
+ status.speed = speed;
+ priv->set_link(priv, status);
+ }
+}
+
+static void bgx_port_check_state(struct work_struct *work)
+{
+ struct bgx_port_priv *priv;
+ struct port_status status;
+
+ priv = container_of(work, struct bgx_port_priv, dwork.work);
+
+ status = priv->get_link(priv);
+
+ if (!status.link &&
+ priv->mode != PORT_MODE_SGMII && priv->mode != PORT_MODE_RGMII)
+ bgx_port_init_xaui_link(priv);
+
+ if (priv->last_status.link != status.link) {
+ priv->last_status.link = status.link;
+ if (status.link)
+ netdev_info(priv->netdev, "Link is up - %d/%s\n",
+ status.speed,
+ status.duplex == DUPLEX_FULL ?
+ "Full" : "Half");
+ else
+ netdev_info(priv->netdev, "Link is down\n");
+ }
+
+ mutex_lock(&priv->lock);
+ if (priv->work_queued)
+ queue_delayed_work(check_state_wq, &priv->dwork, HZ);
+ mutex_unlock(&priv->lock);
+}
+
+int bgx_port_enable(struct net_device *netdev)
+{
+ struct bgx_port_priv *priv = bgx_port_netdev2priv(netdev);
+ struct port_status status;
+ bool dont_use_phy;
+ u64 data;
+
+ if (priv->mode == PORT_MODE_SGMII || priv->mode == PORT_MODE_RGMII) {
+ /* 1G */
+ data = oct_csr_read(BGX_GMP_GMI_TX_APPEND(priv->node,
+ priv->bgx,
+ priv->index));
+ data |= BGX_GMP_GMI_TX_APPEND_FCS | BGX_GMP_GMI_TX_APPEND_PAD;
+ oct_csr_write(data, BGX_GMP_GMI_TX_APPEND(priv->node,
+ priv->bgx,
+ priv->index));
+
+ /* Packets are padded (without FCS) to MIN_SIZE + 1 in SGMII */
+ data = 60 - 1;
+ oct_csr_write(data, BGX_GMP_GMI_TX_MIN_PKT(priv->node,
+ priv->bgx,
+ priv->index));
+ } else {
+ /* 10G or higher */
+ data = oct_csr_read(BGX_SMU_TX_APPEND(priv->node,
+ priv->bgx,
+ priv->index));
+ data |= BGX_SMU_TX_APPEND_FCS_D | BGX_SMU_TX_APPEND_PAD;
+ oct_csr_write(data, BGX_SMU_TX_APPEND(priv->node,
+ priv->bgx,
+ priv->index));
+
+ /* Packets are padded(with FCS) to MIN_SIZE in non-SGMII */
+ data = 60 + 4;
+ oct_csr_write(data, BGX_SMU_TX_MIN_PKT(priv->node,
+ priv->bgx,
+ priv->index));
+ }
+
+ switch (priv->mode) {
+ case PORT_MODE_XLAUI:
+ case PORT_MODE_XFI:
+ case PORT_MODE_10G_KR:
+ case PORT_MODE_40G_KR4:
+ dont_use_phy = true;
+ break;
+ default:
+ dont_use_phy = false;
+ break;
+ }
+
+ if (!priv->phy_np || dont_use_phy) {
+ status = priv->get_link(priv);
+ priv->set_link(priv, status);
+
+ mutex_lock(&check_state_wq_mutex);
+ if (!check_state_wq) {
+ check_state_wq =
+ alloc_workqueue("check_state_wq",
+ WQ_UNBOUND | WQ_MEM_RECLAIM, 1);
+ }
+ mutex_unlock(&check_state_wq_mutex);
+ if (!check_state_wq)
+ return -ENOMEM;
+
+ mutex_lock(&priv->lock);
+ INIT_DELAYED_WORK(&priv->dwork, bgx_port_check_state);
+ queue_delayed_work(check_state_wq, &priv->dwork, 0);
+ priv->work_queued = true;
+ mutex_unlock(&priv->lock);
+
+ netdev_info(priv->netdev, "Link is not ready\n");
+
+ } else {
+ priv->phydev = of_phy_connect(netdev, priv->phy_np,
+ bgx_port_adjust_link, 0,
+ of_get_phy_mode(priv->phy_np));
+ if (!priv->phydev)
+ return -ENODEV;
+
+ if (priv->phydev)
+ phy_start_aneg(priv->phydev);
+ }
+
+ return 0;
+}
+EXPORT_SYMBOL(bgx_port_enable);
+
+int bgx_port_disable(struct net_device *netdev)
+{
+ struct bgx_port_priv *priv = bgx_port_netdev2priv(netdev);
+ struct port_status status;
+
+ if (priv->phydev)
+ phy_disconnect(priv->phydev);
+ priv->phydev = NULL;
+
+ memset(&status, 0, sizeof(status));
+ priv->last_status.link = 0;
+ priv->set_link(priv, status);
+
+ mutex_lock(&priv->lock);
+ if (priv->work_queued) {
+ cancel_delayed_work_sync(&priv->dwork);
+ priv->work_queued = false;
+ }
+ mutex_unlock(&priv->lock);
+
+ return 0;
+}
+EXPORT_SYMBOL(bgx_port_disable);
+
+int bgx_port_change_mtu(struct net_device *netdev, int new_mtu)
+{
+ struct bgx_port_priv *priv = bgx_port_netdev2priv(netdev);
+ int max_frame;
+
+ if (new_mtu < 60 || new_mtu > 65392) {
+ netdev_warn(netdev, "Maximum MTU supported is 65392\n");
+ return -EINVAL;
+ }
+
+ netdev->mtu = new_mtu;
+
+ max_frame = round_up(new_mtu + ETH_HLEN + ETH_FCS_LEN, 8);
+
+ if (priv->mode == PORT_MODE_SGMII || priv->mode == PORT_MODE_RGMII) {
+ /* 1G */
+ oct_csr_write(max_frame, BGX_GMP_GMI_RX_JABBER(priv->node,
+ priv->bgx,
+ priv->index));
+ } else {
+ /* 10G or higher */
+ oct_csr_write(max_frame, BGX_SMU_RX_JABBER(priv->node,
+ priv->bgx,
+ priv->index));
+ }
+
+ return 0;
+}
+EXPORT_SYMBOL(bgx_port_change_mtu);
+
+void bgx_port_mix_assert_reset(struct net_device *netdev, int mix, bool v)
+{
+ struct bgx_port_priv *priv = bgx_port_netdev2priv(netdev);
+ u64 data, mask = 1ull << (3 + (mix & 1));
+
+ if (OCTEON_IS_MODEL(OCTEON_CN78XX_PASS1_X) && v) {
+ /* Need to disable the mix before resetting the bgx-mix
+ * interface as not doing so confuses the other already up
+ * lmacs.
+ */
+ data = oct_csr_read(BGX_CMR_CONFIG(priv->node, priv->bgx,
+ priv->index));
+ data &= ~BGX_CMR_CONFIG_MIX_EN;
+ oct_csr_write(data, BGX_CMR_CONFIG(priv->node, priv->bgx,
+ priv->index));
+ }
+
+ data = oct_csr_read(BGX_CMR_GLOBAL_CONFIG(priv->node, priv->bgx));
+ if (v)
+ data |= mask;
+ else
+ data &= ~mask;
+ oct_csr_write(data, BGX_CMR_GLOBAL_CONFIG(priv->node, priv->bgx));
+
+ if (OCTEON_IS_MODEL(OCTEON_CN78XX_PASS1_X) && !v) {
+ data = oct_csr_read(BGX_CMR_CONFIG(priv->node, priv->bgx,
+ priv->index));
+ data |= BGX_CMR_CONFIG_MIX_EN;
+ oct_csr_write(data, BGX_CMR_CONFIG(priv->node, priv->bgx,
+ priv->index));
+ }
+}
+EXPORT_SYMBOL(bgx_port_mix_assert_reset);
+
+static int bgx_port_probe(struct platform_device *pdev)
+{
+ struct bgx_port_priv *priv;
+ const __be32 *reg;
+ int numa_node, rc;
+ const u8 *mac;
+ u32 index;
+ u64 addr;
+
+ reg = of_get_property(pdev->dev.parent->of_node, "reg", NULL);
+ addr = of_translate_address(pdev->dev.parent->of_node, reg);
+ mac = of_get_mac_address(pdev->dev.of_node);
+
+ numa_node = (addr >> 36) & 0x7;
+
+ rc = of_property_read_u32(pdev->dev.of_node, "reg", &index);
+ if (rc)
+ return -ENODEV;
+ priv = kzalloc_node(sizeof(*priv), GFP_KERNEL, numa_node);
+ if (!priv)
+ return -ENOMEM;
+ priv->phy_np = of_parse_phandle(pdev->dev.of_node, "phy-handle", 0);
+ if (of_get_property(pdev->dev.of_node, "cavium,sgmii-mac-1000x-mode",
+ NULL))
+ priv->mode_1000basex = true;
+ if (of_get_property(pdev->dev.of_node, "cavium,sgmii-mac-phy-mode",
+ NULL))
+ priv->bgx_as_phy = true;
+
+ mutex_init(&priv->lock);
+ priv->node = numa_node;
+ priv->bgx = (addr >> 24) & 0xf;
+ priv->index = index;
+ if (mac)
+ priv->mac_addr = mac;
+
+ priv->qlm = bgx_port_get_qlm(priv->node, priv->bgx, priv->index);
+ priv->mode = bgx_port_get_mode(priv->node, priv->bgx, priv->index);
+
+ switch (priv->mode) {
+ case PORT_MODE_SGMII:
+ case PORT_MODE_RGMII:
+ priv->get_link = bgx_port_get_sgmii_link;
+ priv->set_link = bgx_port_set_xgmii_link;
+ break;
+ case PORT_MODE_XAUI:
+ case PORT_MODE_RXAUI:
+ case PORT_MODE_XLAUI:
+ case PORT_MODE_XFI:
+ case PORT_MODE_10G_KR:
+ case PORT_MODE_40G_KR4:
+ priv->get_link = bgx_port_get_xaui_link;
+ priv->set_link = bgx_port_set_xaui_link;
+ break;
+ default:
+ goto err;
+ }
+
+ dev_set_drvdata(&pdev->dev, priv);
+
+ bgx_port_init(priv);
+
+ dev_info(&pdev->dev, "Probed\n");
+ return 0;
+ err:
+ kfree(priv);
+ return rc;
+}
+
+static int bgx_port_remove(struct platform_device *pdev)
+{
+ struct bgx_port_priv *priv = dev_get_drvdata(&pdev->dev);
+
+ kfree(priv);
+ return 0;
+}
+
+static void bgx_port_shutdown(struct platform_device *pdev)
+{
+}
+
+static const struct of_device_id bgx_port_match[] = {
+ {
+ .compatible = "cavium,octeon-7890-bgx-port",
+ },
+ {
+ .compatible = "cavium,octeon-7360-xcv",
+ },
+ {},
+};
+MODULE_DEVICE_TABLE(of, bgx_port_match);
+
+static struct platform_driver bgx_port_driver = {
+ .probe = bgx_port_probe,
+ .remove = bgx_port_remove,
+ .shutdown = bgx_port_shutdown,
+ .driver = {
+ .owner = THIS_MODULE,
+ .name = KBUILD_MODNAME,
+ .of_match_table = bgx_port_match,
+ },
+};
+
+static int __init bgx_port_driver_init(void)
+{
+ int i, j, k, r;
+
+ for (i = 0; i < MAX_NODES; i++) {
+ for (j = 0; j < MAX_BGX_PER_NODE; j++) {
+ for (k = 0; k < MAX_LMAC_PER_BGX; k++)
+ lmac_pknd[i][j][k] = -1;
+ }
+ }
+
+ bgx_nexus_load();
+ r = platform_driver_register(&bgx_port_driver);
+ return r;
+}
+module_init(bgx_port_driver_init);
+
+static void __exit bgx_port_driver_exit(void)
+{
+ platform_driver_unregister(&bgx_port_driver);
+ if (check_state_wq)
+ destroy_workqueue(check_state_wq);
+}
+module_exit(bgx_port_driver_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Cavium, Inc. <support@caviumnetworks.com>");
+MODULE_DESCRIPTION("BGX Nexus Ethernet MAC driver.");
--
2.1.4
^ permalink raw reply related
* [PATCH v12 05/10] netdev: cavium: octeon: Add Octeon III PKI Support
From: Steven J. Hill @ 2018-06-27 21:25 UTC (permalink / raw)
To: netdev; +Cc: Carlos Munoz, Chandrakala Chavva, Steven J. Hill
In-Reply-To: <1530134719-19407-1-git-send-email-steven.hill@cavium.com>
From: Carlos Munoz <cmunoz@cavium.com>
Add support for Octeon III PKI logic block for BGX Ethernet.
Signed-off-by: Carlos Munoz <cmunoz@cavium.com>
Signed-off-by: Steven J. Hill <Steven.Hill@cavium.com>
---
drivers/net/ethernet/cavium/octeon/octeon3-pki.c | 789 +++++++++++++++++++++++
drivers/net/ethernet/cavium/octeon/octeon3-pki.h | 113 ++++
2 files changed, 902 insertions(+)
create mode 100644 drivers/net/ethernet/cavium/octeon/octeon3-pki.c
create mode 100644 drivers/net/ethernet/cavium/octeon/octeon3-pki.h
diff --git a/drivers/net/ethernet/cavium/octeon/octeon3-pki.c b/drivers/net/ethernet/cavium/octeon/octeon3-pki.c
new file mode 100644
index 0000000..9782ba1
--- /dev/null
+++ b/drivers/net/ethernet/cavium/octeon/octeon3-pki.c
@@ -0,0 +1,789 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Octeon III Packet Input Unit (PKI)
+ *
+ * Copyright (C) 2018 Cavium, Inc.
+ */
+
+#include <linux/firmware.h>
+
+#include "octeon3.h"
+#include "octeon3-pki.h"
+
+#define PKI_CLUSTER_FIRMWARE "cavium/pki-cluster.bin"
+#define VERSION_LEN 8
+
+#define PKI_MAX_PCAM_BANKS 2
+#define PKI_MAX_PCAM_BANK_ENTRIES 192
+#define PKI_MAX_PKNDS 64
+#define PKI_NUM_QPG_ENTRIES 2048
+#define PKI_NUM_STYLES 256
+#define PKI_NUM_FINAL_STYLES 64
+
+#define PKI_BASE 0x1180044000000ull
+#define PKI_ADDR(n) (PKI_BASE + SET_XKPHYS + NODE_OFFSET(n))
+#define PKI_CL_ADDR(n, c) (PKI_ADDR(n) + ((c) << 16))
+#define PKI_CL_PCAM(n, c, b, e) (PKI_CL_ADDR(n, c) + ((b) << 12) + \
+ ((e) << 3))
+#define PKI_CL_PKIND(n, c, p) (PKI_CL_ADDR(n, c) + ((p) << 8))
+#define PKI_CL_STYLE(n, c, s) (PKI_CL_ADDR(n, c) + ((s) << 3))
+
+#define PKI_AURA_CFG(n, a) (PKI_ADDR(n) + 0x900000 + ((a) << 3))
+#define PKI_BUF_CTL(n) (PKI_ADDR(n) + 0x000100)
+#define PKI_ICG_CFG(n) (PKI_ADDR(n) + 0x00a000)
+#define PKI_IMEM(n, i) (PKI_ADDR(n) + 0x100000 + ((i) << 3))
+#define PKI_LTYPE_MAP(n, l) (PKI_ADDR(n) + 0x005000 + ((l) << 3))
+#define PKI_QPG_TBL(n, i) (PKI_ADDR(n) + 0x800000 + ((i) << 3))
+#define PKI_SFT_RST(n) (PKI_ADDR(n) + 0x000010)
+#define PKI_STAT_CTL(n) (PKI_ADDR(n) + 0x000110)
+#define PKI_STAT_STAT0(n, p) (PKI_ADDR(n) + 0xe00038 + ((p) << 8))
+#define PKI_STAT_STAT1(n, p) (PKI_ADDR(n) + 0xe00040 + ((p) << 8))
+#define PKI_STAT_STAT3(n, p) (PKI_ADDR(n) + 0xe00050 + ((p) << 8))
+#define PKI_STYLE_BUF(n, s) (PKI_ADDR(n) + 0x024000 + ((s) << 3))
+
+#define PKI_CL_ECC_CTL(n, c) (PKI_CL_ADDR(n, c) + 0x00c020)
+#define PKI_CL_PCAM_TERM(n, c, b, e) (PKI_CL_PCAM(n, c, b, e) + 0x700000)
+#define PKI_CL_PCAM_MATCH(n, c, b, e) (PKI_CL_PCAM(n, c, b, e) + 0x704000)
+#define PKI_CL_PCAM_ACTION(n, c, b, e) (PKI_CL_PCAM(n, c, b, e) + 0x708000)
+#define PKI_CL_PKIND_CFG(n, c, p) (PKI_CL_PKIND(n, c, p) + 0x300040)
+#define PKI_CL_PKIND_STYLE(n, c, p) (PKI_CL_PKIND(n, c, p) + 0x300048)
+#define PKI_CL_PKIND_SKIP(n, c, p) (PKI_CL_PKIND(n, c, p) + 0x300050)
+#define PKI_CL_PKIND_L2_CUSTOM(n, c, p) (PKI_CL_PKIND(n, c, p) + 0x300058)
+#define PKI_CL_PKIND_LG_CUSTOM(n, c, p) (PKI_CL_PKIND(n, c, p) + 0x300060)
+#define PKI_CL_STYLE_CFG(n, c, s) (PKI_CL_STYLE(n, c, s) + 0x500000)
+#define PKI_CL_STYLE_CFG2(n, c, s) (PKI_CL_STYLE(n, c, s) + 0x500800)
+#define PKI_CL_STYLE_ALG(n, c, s) (PKI_CL_STYLE(n, c, s) + 0x501000)
+
+enum pcam_term {
+ NONE,
+ L2_CUSTOM = 0x2,
+ HIGIGD = 0x4,
+ HIGIG = 0x5,
+ SMACH = 0x8,
+ SMACL = 0x9,
+ DMACH = 0xa,
+ DMACL = 0xb,
+ GLORT = 0x12,
+ DSA = 0x13,
+ ETHTYPE0 = 0x18,
+ ETHTYPE1 = 0x19,
+ ETHTYPE2 = 0x1a,
+ ETHTYPE3 = 0x1b,
+ MPLS0 = 0x1e,
+ L3_SIPHH = 0x1f,
+ L3_SIPMH = 0x20,
+ L3_SIPML = 0x21,
+ L3_SIPLL = 0x22,
+ L3_FLAGS = 0x23,
+ L3_DIPHH = 0x24,
+ L3_DIPMH = 0x25,
+ L3_DIPML = 0x26,
+ L3_DIPLL = 0x27,
+ LD_VNI = 0x28,
+ IL3_FLAGS = 0x2b,
+ LF_SPI = 0x2e,
+ L4_SPORT = 0x2f,
+ L4_PORT = 0x30,
+ LG_CUSTOM = 0x39
+};
+
+enum pki_ltype {
+ LTYPE_NONE,
+ LTYPE_ENET,
+ LTYPE_VLAN,
+ LTYPE_SNAP_PAYLD = 0x05,
+ LTYPE_ARP = 0x06,
+ LTYPE_RARP = 0x07,
+ LTYPE_IP4 = 0x08,
+ LTYPE_IP4_OPT = 0x09,
+ LTYPE_IP6 = 0x0a,
+ LTYPE_IP6_OPT = 0x0b,
+ LTYPE_IPSEC_ESP = 0x0c,
+ LTYPE_IPFRAG = 0x0d,
+ LTYPE_IPCOMP = 0x0e,
+ LTYPE_TCP = 0x10,
+ LTYPE_UDP = 0x11,
+ LTYPE_SCTP = 0x12,
+ LTYPE_UDP_VXLAN = 0x13,
+ LTYPE_GRE = 0x14,
+ LTYPE_NVGRE = 0x15,
+ LTYPE_GTP = 0x16,
+ LTYPE_UDP_GENEVE = 0x17,
+ LTYPE_SW28 = 0x1c,
+ LTYPE_SW29 = 0x1d,
+ LTYPE_SW30 = 0x1e,
+ LTYPE_SW31 = 0x1f
+};
+
+enum pki_beltype {
+ BELTYPE_NONE,
+ BELTYPE_MISC,
+ BELTYPE_IP4,
+ BELTYPE_IP6,
+ BELTYPE_TCP,
+ BELTYPE_UDP,
+ BELTYPE_SCTP,
+ BELTYPE_SNAP
+};
+
+struct ltype_beltype {
+ enum pki_ltype ltype;
+ enum pki_beltype beltype;
+};
+
+/* struct pcam_term_info - Describes a term to configure in the PCAM.
+ * @term: Identifies the term to configure.
+ * @term_mask: Specifies don't cares in the term.
+ * @style: Style to compare.
+ * @style_mask: Specifies don't cares in the style.
+ * @data: Data to compare.
+ * @data_mask: Specifies don't cares in the data.
+ */
+struct pcam_term_info {
+ u8 term;
+ u8 term_mask;
+ u8 style;
+ u8 style_mask;
+ u32 data;
+ u32 data_mask;
+};
+
+/* struct fw_hdr - Describes the firmware.
+ * @version: Firmware version.
+ * @size: Size of the data in bytes.
+ * @data: Actual firmware data.
+ */
+struct fw_hdr {
+ char version[VERSION_LEN];
+ u64 size;
+ u64 data[];
+};
+
+static struct ltype_beltype dflt_ltype_config[] = {
+ { LTYPE_NONE, BELTYPE_NONE },
+ { LTYPE_ENET, BELTYPE_MISC },
+ { LTYPE_VLAN, BELTYPE_MISC },
+ { LTYPE_SNAP_PAYLD, BELTYPE_MISC },
+ { LTYPE_ARP, BELTYPE_MISC },
+ { LTYPE_RARP, BELTYPE_MISC },
+ { LTYPE_IP4, BELTYPE_IP4 },
+ { LTYPE_IP4_OPT, BELTYPE_IP4 },
+ { LTYPE_IP6, BELTYPE_IP6 },
+ { LTYPE_IP6_OPT, BELTYPE_IP6 },
+ { LTYPE_IPSEC_ESP, BELTYPE_MISC },
+ { LTYPE_IPFRAG, BELTYPE_MISC },
+ { LTYPE_IPCOMP, BELTYPE_MISC },
+ { LTYPE_TCP, BELTYPE_TCP },
+ { LTYPE_UDP, BELTYPE_UDP },
+ { LTYPE_SCTP, BELTYPE_SCTP },
+ { LTYPE_UDP_VXLAN, BELTYPE_UDP },
+ { LTYPE_GRE, BELTYPE_MISC },
+ { LTYPE_NVGRE, BELTYPE_MISC },
+ { LTYPE_GTP, BELTYPE_MISC },
+ { LTYPE_UDP_GENEVE, BELTYPE_UDP },
+ { LTYPE_SW28, BELTYPE_MISC },
+ { LTYPE_SW29, BELTYPE_MISC },
+ { LTYPE_SW30, BELTYPE_MISC },
+ { LTYPE_SW31, BELTYPE_MISC }
+};
+
+static int get_num_clusters(void)
+{
+ if (OCTEON_IS_MODEL(OCTEON_CN73XX) || OCTEON_IS_MODEL(OCTEON_CNF75XX))
+ return 2;
+ return 4;
+}
+
+static int octeon3_pki_pcam_alloc_entry(int node, int entry, int bank)
+{
+ struct global_resource_tag tag;
+ int i, rc, num_clusters;
+ char buf[16];
+
+ /* Allocate a PCAM entry for cluster0. */
+ strncpy((char *)&tag.lo, "cvm_pcam", 8);
+ snprintf(buf, 16, "_%d%d%d....", node, 0, bank);
+ memcpy(&tag.hi, buf, 8);
+
+ res_mgr_create_resource(tag, PKI_MAX_PCAM_BANK_ENTRIES);
+ rc = res_mgr_alloc(tag, entry, false);
+ if (rc < 0)
+ return rc;
+
+ entry = rc;
+
+ /* Allocate entries for all clusters. */
+ num_clusters = get_num_clusters();
+ for (i = 1; i < num_clusters; i++) {
+ strncpy((char *)&tag.lo, "cvm_pcam", 8);
+ snprintf(buf, 16, "_%d%d%d....", node, i, bank);
+ memcpy(&tag.hi, buf, 8);
+
+ res_mgr_create_resource(tag, PKI_MAX_PCAM_BANK_ENTRIES);
+ rc = res_mgr_alloc(tag, entry, false);
+ if (rc < 0) {
+ int j;
+
+ pr_err("%s: Failed to allocate PCAM entry\n", __FILE__);
+ /* Undo whatever we've did */
+ for (j = 0; i < i; j++) {
+ strncpy((char *)&tag.lo, "cvm_pcam", 8);
+ snprintf(buf, 16, "_%d%d%d....", node, j, bank);
+ memcpy(&tag.hi, buf, 8);
+ res_mgr_free(tag, entry);
+ }
+
+ return -1;
+ }
+ }
+
+ return entry;
+}
+
+static int octeon3_pki_pcam_write_entry(int node,
+ struct pcam_term_info *term_info)
+{
+ int bank, entry, i, num_clusters;
+ u64 action, match, term;
+
+ /* Bit 0 of the PCAM term determines the bank to use. */
+ bank = term_info->term & 1;
+
+ /* Allocate a PCAM entry. */
+ entry = octeon3_pki_pcam_alloc_entry(node, -1, bank);
+ if (entry < 0)
+ return entry;
+
+ term = PKI_CL_PCAM_TERM_VALID;
+ term |= (u64)(term_info->term & term_info->term_mask)
+ << PKI_CL_PCAM_TERM_TERM1_SHIFT;
+ term |= (~term_info->term & term_info->term_mask)
+ << PKI_CL_PCAM_TERM_TERM0_SHIFT;
+ term |= (u64)(term_info->style & term_info->style_mask)
+ << PKI_CL_PCAM_TERM_STYLE1_SHIFT;
+ term |= ~term_info->style & term_info->style_mask;
+
+ match = (u64)(term_info->data & term_info->data_mask)
+ << PKI_CL_PCAM_MATCH_DATA1_SHIFT;
+ match |= ~term_info->data & term_info->data_mask;
+
+ action = 0;
+ if (term_info->term >= ETHTYPE0 && term_info->term <= ETHTYPE3) {
+ action |= (PKI_CL_PCAM_ACTION_L2_CUSTOM <<
+ PKI_CL_PCAM_ACTION_SETTY_SHIFT);
+ action |= PKI_CL_PCAM_ACTION_ADVANCE_4B;
+ }
+
+ /* Must write the term to all clusters. */
+ num_clusters = get_num_clusters();
+ for (i = 0; i < num_clusters; i++) {
+ oct_csr_write(0, PKI_CL_PCAM_TERM(node, i, bank, entry));
+ oct_csr_write(match, PKI_CL_PCAM_MATCH(node, i, bank, entry));
+ oct_csr_write(action, PKI_CL_PCAM_ACTION(node, i, bank, entry));
+ oct_csr_write(term, PKI_CL_PCAM_TERM(node, i, bank, entry));
+ }
+
+ return 0;
+}
+
+static int octeon3_pki_alloc_qpg_entry(int node)
+{
+ struct global_resource_tag tag;
+ char buf[16];
+ int entry;
+
+ /* Allocate a Qpg entry. */
+ strncpy((char *)&tag.lo, "cvm_qpge", 8);
+ snprintf(buf, 16, "t_%d.....", node);
+ memcpy(&tag.hi, buf, 8);
+
+ res_mgr_create_resource(tag, PKI_NUM_QPG_ENTRIES);
+ entry = res_mgr_alloc(tag, -1, false);
+ if (entry < 0)
+ pr_err("%s: Failed to allocate qpg entry", __FILE__);
+
+ return entry;
+}
+
+static int octeon3_pki_alloc_style(int node)
+{
+ struct global_resource_tag tag;
+ char buf[16];
+ int entry;
+
+ /* Allocate a style entry. */
+ strncpy((char *)&tag.lo, "cvm_styl", 8);
+ snprintf(buf, 16, "e_%d.....", node);
+ memcpy(&tag.hi, buf, 8);
+
+ res_mgr_create_resource(tag, PKI_NUM_STYLES);
+ entry = res_mgr_alloc(tag, -1, false);
+ if (entry < 0)
+ pr_err("%s: Failed to allocate style", __FILE__);
+
+ return entry;
+}
+
+int octeon3_pki_set_ptp_skip(int node, int pknd, int skip)
+{
+ int i, num_clusters;
+ u64 data;
+
+ num_clusters = get_num_clusters();
+ for (i = 0; i < num_clusters; i++) {
+ data = oct_csr_read(PKI_CL_PKIND_SKIP(node, i, pknd));
+ data &= ~(PKI_CL_PKIND_SKIP_INST_SKIP_MASK |
+ PKI_CL_PKIND_SKIP_FCS_SKIP_MASK);
+ data |= (skip << PKI_CL_PKIND_SKIP_FCS_SKIP_SHIFT) | skip;
+ oct_csr_write(data, PKI_CL_PKIND_SKIP(node, i, pknd));
+
+ data = oct_csr_read(PKI_CL_PKIND_L2_CUSTOM(node, i, pknd));
+ data &= ~PKI_CL_PKIND_SKIP_INST_SKIP_MASK;
+ data |= skip;
+ oct_csr_write(data, PKI_CL_PKIND_L2_CUSTOM(node, i, pknd));
+ }
+
+ return 0;
+}
+EXPORT_SYMBOL(octeon3_pki_set_ptp_skip);
+
+/* octeon3_pki_get_stats - Get the statistics for a given pknd (port).
+ * @node: Node to get statistics for.
+ * @pknd: Pknd to get statistis for.
+ * @packets: Updated with the number of packets received.
+ * @octets: Updated with the number of octets received.
+ * @dropped: Updated with the number of dropped packets.
+ *
+ * Returns 0 if successful.
+ * Returns <0 for error codes.
+ */
+int octeon3_pki_get_stats(int node, int pknd, u64 *packets, u64 *octets,
+ u64 *dropped)
+{
+ /* PKI-20775, must read until not all ones. */
+ do {
+ *packets = oct_csr_read(PKI_STAT_STAT0(node, pknd));
+ } while (*packets == 0xffffffffffffffffull);
+
+ do {
+ *octets = oct_csr_read(PKI_STAT_STAT1(node, pknd));
+ } while (*octets == 0xffffffffffffffffull);
+
+ do {
+ *dropped = oct_csr_read(PKI_STAT_STAT3(node, pknd));
+ } while (*dropped == 0xffffffffffffffffull);
+
+ return 0;
+}
+EXPORT_SYMBOL(octeon3_pki_get_stats);
+
+/* octeon3_pki_port_init - Initialize a port.
+ * @node: Node port is using.
+ * @aura: Aura to use for packet buffers.
+ * @grp: SSO group packets will be queued up for.
+ * @skip: Extra bytes to skip before packet data.
+ * @mb_size: Size of packet buffers.
+ * @pknd: Port kind assigned to the port.
+ * @num_rx_cxt: Number of SSO groups used by the port.
+ *
+ * Returns 0 if successful.
+ * Returns <0 for error codes.
+ */
+int octeon3_pki_port_init(int node, int aura, int grp, int skip, int mb_size,
+ int pknd, int num_rx_cxt)
+{
+ int num_clusters, qpg_entry, style;
+ u64 data, i;
+
+ /* Allocate and configure a Qpg table entry for the port's group. */
+ i = 0;
+ while ((num_rx_cxt & (1 << i)) == 0)
+ i++;
+ qpg_entry = octeon3_pki_alloc_qpg_entry(node);
+ data = oct_csr_read(PKI_QPG_TBL(node, qpg_entry));
+ data &= ~(PKI_QPG_TBL_PADD_MASK | PKI_QPG_TBL_GRPTAG_OK_MASK |
+ PKI_QPG_TBL_GRP_OK_MASK | PKI_QPG_TBL_GRPTAG_BAD_MASK |
+ PKI_QPG_TBL_GRP_BAD_MASK | PKI_QPG_TBL_LAURA_MASK);
+ data |= i << PKI_QPG_TBL_GRPTAG_OK_SHIFT;
+ data |= ((u64)((node << PKI_QPG_TBL_GRP_NUM_BITS_SHIFT) | grp)
+ << PKI_QPG_TBL_GRP_OK_SHIFT);
+ data |= i << PKI_QPG_TBL_GRPTAG_BAD_SHIFT;
+ data |= (((node << PKI_QPG_TBL_GRP_NUM_BITS_SHIFT) | grp)
+ << PKI_QPG_TBL_GRP_BAD_SHIFT);
+ data |= aura;
+ oct_csr_write(data, PKI_QPG_TBL(node, qpg_entry));
+
+ /* Allocate a style for the port */
+ style = octeon3_pki_alloc_style(node);
+
+ /* Map the Qpg table entry to the style. */
+ num_clusters = get_num_clusters();
+ for (i = 0; i < num_clusters; i++) {
+ data = PKI_CL_STYLE_CFG_LENERR_EN | qpg_entry |
+ PKI_CL_STYLE_CFG_FCS_CHK;
+ oct_csr_write(data, PKI_CL_STYLE_CFG(node, i, style));
+
+ /* Specify the tag generation rules and checksum to use. */
+ oct_csr_write((PKI_CL_STYLE_CFG2_CSUM_ALL |
+ PKI_CL_STYLE_CFG2_LEN_TAG_ALL |
+ PKI_CL_STYLE_CFG2_LEN_LF |
+ PKI_CL_STYLE_CFG2_LEN_LC),
+ PKI_CL_STYLE_CFG2(node, i, style));
+
+ data = SSO_TAG_TYPE_UNTAGGED << PKI_CL_STYLE_ALG_TT_SHIFT;
+ oct_csr_write(data, PKI_CL_STYLE_ALG(node, i, style));
+ }
+
+ /* Set the style's buffer size and skips:
+ * Every buffer has 128 bytes reserved for Linux.
+ * The first buffer must also skip the WQE (40 bytes).
+ * SRIO also requires skipping its header (skip).
+ */
+ data = 1 << PKI_STYLE_BUF_WQE_SKIP_SHIFT;
+ data |= ((128 + 40 + skip) / 8) << PKI_STYLE_BUF_FIRST_SKIP_SHIFT;
+ data |= (128 / 8) << PKI_STYLE_BUF_LATER_SKIP_SHIFT;
+ data |= (mb_size & ~0xf) / 8;
+ oct_csr_write(data, PKI_STYLE_BUF(node, style));
+
+ /* Assign the initial style to the port via the pknd. */
+ for (i = 0; i < num_clusters; i++) {
+ data = oct_csr_read(PKI_CL_PKIND_STYLE(node, i, pknd));
+ data &= ~PKI_CL_PKIND_STYLE_STYLE_MASK;
+ data |= style;
+ oct_csr_write(data, PKI_CL_PKIND_STYLE(node, i, pknd));
+ }
+
+ /* Enable red. */
+ data = PKI_AURA_CFG_ENA_RED;
+ oct_csr_write(data, PKI_AURA_CFG(node, aura));
+
+ /* Clear statistic counters. */
+ oct_csr_write(0, PKI_STAT_STAT0(node, pknd));
+ oct_csr_write(0, PKI_STAT_STAT1(node, pknd));
+ oct_csr_write(0, PKI_STAT_STAT3(node, pknd));
+
+ return 0;
+}
+EXPORT_SYMBOL(octeon3_pki_port_init);
+
+/* octeon3_pki_port_shutdown - Release all the resources used by a port.
+ * @node: Node the port is on.
+ * @pknd: Pknd assigned to the port.
+ *
+ * Returns 0 if successful.
+ * Returns <0 for error codes.
+ */
+int octeon3_pki_port_shutdown(int node, int pknd)
+{
+ /* Nothing at the moment. */
+ return 0;
+}
+EXPORT_SYMBOL(octeon3_pki_port_shutdown);
+
+/* octeon3_pki_cluster_init - Loads cluster firmware into the PKI clusters.
+ * @node: Node to configure.
+ * @pdev: Device requesting the firmware.
+ *
+ * Returns 0 if successful.
+ * Returns <0 for error codes.
+ */
+int octeon3_pki_cluster_init(int node, struct platform_device *pdev)
+{
+ const struct firmware *pki_fw;
+ const struct fw_hdr *hdr;
+ const u64 *data;
+ int i, rc;
+
+ rc = request_firmware(&pki_fw, PKI_CLUSTER_FIRMWARE, &pdev->dev);
+ if (rc) {
+ dev_err(&pdev->dev, "%s: Failed to load %s error=%d\n",
+ __FILE__, PKI_CLUSTER_FIRMWARE, rc);
+ return rc;
+ }
+
+ /* Verify the firmware is valid. */
+ hdr = (const struct fw_hdr *)pki_fw->data;
+ if ((pki_fw->size - sizeof(const struct fw_hdr) != hdr->size) ||
+ hdr->size % 8) {
+ dev_err(&pdev->dev, "%s: Corrupted PKI firmware\n", __FILE__);
+ goto err;
+ }
+
+ dev_info(&pdev->dev, "%s: Loading PKI firmware %s\n", __FILE__,
+ hdr->version);
+ data = hdr->data;
+ for (i = 0; i < hdr->size / 8; i++) {
+ oct_csr_write(cpu_to_be64(*data), PKI_IMEM(node, i));
+ data++;
+ }
+err:
+ release_firmware(pki_fw);
+
+ return 0;
+}
+EXPORT_SYMBOL(octeon3_pki_cluster_init);
+
+/* octeon3_pki_vlan_init - Configure PCAM to recognize the VLAN ethtypes.
+ * @node: Node to configure.
+ *
+ * Returns 0 if successful.
+ * Returns <0 for error codes.
+ */
+int octeon3_pki_vlan_init(int node)
+{
+ int i, rc;
+ u64 data;
+
+ /* PKI-20858 */
+ if (OCTEON_IS_MODEL(OCTEON_CN78XX_PASS1_X)) {
+ for (i = 0; i < 4; i++) {
+ data = oct_csr_read(PKI_CL_ECC_CTL(node, i));
+ data &= ~PKI_CL_ECC_CTL_PCAM_EN;
+ data |= PKI_CL_ECC_CTL_PCAM1_CDIS |
+ PKI_CL_ECC_CTL_PCAM0_CDIS;
+ oct_csr_write(data, PKI_CL_ECC_CTL(node, i));
+ }
+ }
+
+ /* Configure the pcam ethtype0 and ethtype1 terms */
+ for (i = ETHTYPE0; i <= ETHTYPE1; i++) {
+ struct pcam_term_info term_info;
+
+ /* Term for 0x8100 ethtype */
+ term_info.term = i;
+ term_info.term_mask = 0xfd;
+ term_info.style = 0;
+ term_info.style_mask = 0;
+ term_info.data = 0x81000000;
+ term_info.data_mask = 0xffff0000;
+ rc = octeon3_pki_pcam_write_entry(node, &term_info);
+ if (rc)
+ return rc;
+
+ /* Term for 0x88a8 ethtype */
+ term_info.data = 0x88a80000;
+ rc = octeon3_pki_pcam_write_entry(node, &term_info);
+ if (rc)
+ return rc;
+
+ /* Term for 0x9200 ethtype */
+ term_info.data = 0x92000000;
+ rc = octeon3_pki_pcam_write_entry(node, &term_info);
+ if (rc)
+ return rc;
+
+ /* Term for 0x9100 ethtype */
+ term_info.data = 0x91000000;
+ rc = octeon3_pki_pcam_write_entry(node, &term_info);
+ if (rc)
+ return rc;
+ }
+
+ return 0;
+}
+EXPORT_SYMBOL(octeon3_pki_vlan_init);
+
+/* octeon3_pki_ltype_init - Configures the PKI layer types.
+ * @node: Node to configure.
+ *
+ * Returns 0 if successful.
+ * Returns <0 for error codes.
+ */
+int octeon3_pki_ltype_init(int node)
+{
+ enum pki_ltype ltype;
+ u64 data;
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(dflt_ltype_config); i++) {
+ ltype = dflt_ltype_config[i].ltype;
+ data = oct_csr_read(PKI_LTYPE_MAP(node, ltype));
+ data &= ~PKI_LTYPE_MAP_BELTYPE_MASK;
+ data |= dflt_ltype_config[i].beltype;
+ oct_csr_write(data, PKI_LTYPE_MAP(node, ltype));
+ }
+
+ return 0;
+}
+EXPORT_SYMBOL(octeon3_pki_ltype_init);
+
+int octeon3_pki_srio_init(int node, int pknd)
+{
+ int i, num_clusters, style;
+ u64 data;
+
+ num_clusters = get_num_clusters();
+ for (i = 0; i < num_clusters; i++) {
+ data = oct_csr_read(PKI_CL_PKIND_STYLE(node, i, pknd));
+ style = data & PKI_CL_PKIND_STYLE_STYLE_MASK;
+ data &= ~PKI_CL_PKIND_STYLE_PARSE_MODE_MASK;
+ oct_csr_write(data, PKI_CL_PKIND_STYLE(node, i, pknd));
+
+ /* Disable packet length errors and FCS. */
+ data = oct_csr_read(PKI_CL_STYLE_CFG(node, i, style));
+ data &= ~(PKI_CL_STYLE_CFG_LENERR_EN |
+ PKI_CL_STYLE_CFG_MAXERR_EN |
+ PKI_CL_STYLE_CFG_MINERR_EN |
+ PKI_CL_STYLE_CFG_FCS_STRIP |
+ PKI_CL_STYLE_CFG_FCS_CHK);
+ oct_csr_write(data, PKI_CL_STYLE_CFG(node, i, style));
+
+ /* Packets have no FCS. */
+ data = oct_csr_read(PKI_CL_PKIND_CFG(node, i, pknd));
+ data &= ~PKI_CL_PKIND_CFG_FCS_PRES;
+ oct_csr_write(data, PKI_CL_PKIND_CFG(node, i, pknd));
+
+ /* Skip the SRIO header and the INST_HDR_S data. */
+ data = oct_csr_read(PKI_CL_PKIND_SKIP(node, i, pknd));
+ data &= ~(PKI_CL_PKIND_SKIP_FCS_SKIP_MASK |
+ PKI_CL_PKIND_SKIP_INST_SKIP_MASK);
+ data |= (16 << PKI_CL_PKIND_SKIP_FCS_SKIP_SHIFT) | 16;
+ oct_csr_write(data, PKI_CL_PKIND_SKIP(node, i, pknd));
+
+ /* Exclude port number from Qpg. */
+ data = oct_csr_read(PKI_CL_STYLE_ALG(node, i, style));
+ data &= ~PKI_CL_STYLE_ALG_QPG_PORT_MSB_MASK;
+ oct_csr_write(data, PKI_CL_STYLE_ALG(node, i, style));
+ }
+
+ return 0;
+}
+EXPORT_SYMBOL(octeon3_pki_srio_init);
+
+int octeon3_pki_enable(int node)
+{
+ int timeout;
+ u64 data;
+
+ /* Enable backpressure. */
+ data = oct_csr_read(PKI_BUF_CTL(node));
+ data |= PKI_BUF_CTL_PBP_EN;
+ oct_csr_write(data, PKI_BUF_CTL(node));
+
+ /* Enable cluster parsing. */
+ data = oct_csr_read(PKI_ICG_CFG(node));
+ data |= PKI_ICG_CFG_PENA;
+ oct_csr_write(data, PKI_ICG_CFG(node));
+
+ /* Wait until the PKI is out of reset. */
+ timeout = 10000;
+ do {
+ data = oct_csr_read(PKI_SFT_RST(node));
+ if (!(data & PKI_SFT_RST_BUSY))
+ break;
+ timeout--;
+ udelay(1);
+ } while (timeout);
+ if (!timeout) {
+ pr_err("%s: timeout waiting for reset\n", __FILE__);
+ return -1;
+ }
+
+ /* Enable the PKI. */
+ data = oct_csr_read(PKI_BUF_CTL(node));
+ data |= PKI_BUF_CTL_PKI_EN;
+ oct_csr_write(data, PKI_BUF_CTL(node));
+
+ /* Statistics are kept per pknd. */
+ oct_csr_write(0, PKI_STAT_CTL(node));
+
+ return 0;
+}
+EXPORT_SYMBOL(octeon3_pki_enable);
+
+void octeon3_pki_shutdown(int node)
+{
+ struct global_resource_tag tag;
+ int i, j, k, timeout;
+ char buf[16];
+ u64 data;
+
+ /* Disable the PKI. */
+ data = oct_csr_read(PKI_BUF_CTL(node));
+ if (data & PKI_BUF_CTL_PKI_EN) {
+ data &= ~PKI_BUF_CTL_PKI_EN;
+ oct_csr_write(data, PKI_BUF_CTL(node));
+
+ /* Wait until the PKI has finished processing packets. */
+ timeout = 10000;
+ do {
+ data = oct_csr_read(PKI_SFT_RST(node));
+ if (data & PKI_SFT_RST_ACTIVE)
+ break;
+ timeout--;
+ udelay(1);
+ } while (timeout);
+ if (!timeout)
+ pr_warn("%s: disable timeout\n", __FILE__);
+ }
+
+ /* Give all prefetched buffers back to the FPA. */
+ data = oct_csr_read(PKI_BUF_CTL(node));
+ data |= PKI_BUF_CTL_FPA_WAIT | PKI_BUF_CTL_PKT_OFF;
+ oct_csr_write(data, PKI_BUF_CTL(node));
+
+ /* Dummy read to get the register write to take effect. */
+ data = oct_csr_read(PKI_BUF_CTL(node));
+
+ /* Now we can reset the PKI. */
+ data = oct_csr_read(PKI_SFT_RST(node));
+ data |= PKI_SFT_RST_RST;
+ oct_csr_write(data, PKI_SFT_RST(node));
+ timeout = 10000;
+ do {
+ data = oct_csr_read(PKI_SFT_RST(node));
+ if ((data & PKI_SFT_RST_BUSY) == 0)
+ break;
+ timeout--;
+ udelay(1);
+ } while (timeout);
+ if (!timeout)
+ pr_warn("%s: reset timeout\n", __FILE__);
+
+ /* Free all the allocated resources. */
+ for (i = 0; i < PKI_NUM_STYLES; i++) {
+ strncpy((char *)&tag.lo, "cvm_styl", 8);
+ snprintf(buf, 16, "e_%d.....", node);
+ memcpy(&tag.hi, buf, 8);
+ res_mgr_free(tag, i);
+ }
+ for (i = 0; i < PKI_NUM_QPG_ENTRIES; i++) {
+ strncpy((char *)&tag.lo, "cvm_qpge", 8);
+ snprintf(buf, 16, "t_%d.....", node);
+ memcpy(&tag.hi, buf, 8);
+ res_mgr_free(tag, i);
+ }
+ for (i = 0; i < get_num_clusters(); i++) {
+ for (j = 0; j < PKI_MAX_PCAM_BANKS; j++) {
+ strncpy((char *)&tag.lo, "cvm_pcam", 8);
+ snprintf(buf, 16, "_%d%d%d....", node, i, j);
+ memcpy(&tag.hi, buf, 8);
+ for (k = 0; k < PKI_MAX_PCAM_BANK_ENTRIES; k++)
+ res_mgr_free(tag, k);
+ }
+ }
+
+ /* Restore the registers back to their reset state. */
+ for (i = 0; i < get_num_clusters(); i++) {
+ for (j = 0; j < PKI_MAX_PKNDS; j++) {
+ oct_csr_write(0, PKI_CL_PKIND_CFG(node, i, j));
+ oct_csr_write(0, PKI_CL_PKIND_STYLE(node, i, j));
+ oct_csr_write(0, PKI_CL_PKIND_SKIP(node, i, j));
+ oct_csr_write(0, PKI_CL_PKIND_L2_CUSTOM(node, i, j));
+ oct_csr_write(0, PKI_CL_PKIND_LG_CUSTOM(node, i, j));
+ }
+ for (j = 0; j < PKI_NUM_FINAL_STYLES; j++) {
+ oct_csr_write(0, PKI_CL_STYLE_CFG(node, i, j));
+ oct_csr_write(0, PKI_CL_STYLE_CFG2(node, i, j));
+ oct_csr_write(0, PKI_CL_STYLE_ALG(node, i, j));
+ }
+ }
+ for (i = 0; i < PKI_NUM_FINAL_STYLES; i++)
+ oct_csr_write((5 << PKI_STYLE_BUF_FIRST_SKIP_SHIFT) | 32,
+ PKI_STYLE_BUF(node, i));
+}
+EXPORT_SYMBOL(octeon3_pki_shutdown);
+
+MODULE_LICENSE("GPL");
+MODULE_FIRMWARE(PKI_CLUSTER_FIRMWARE);
+MODULE_AUTHOR("Carlos Munoz <cmunoz@cavium.com>");
+MODULE_DESCRIPTION("Octeon III PKI management.");
diff --git a/drivers/net/ethernet/cavium/octeon/octeon3-pki.h b/drivers/net/ethernet/cavium/octeon/octeon3-pki.h
new file mode 100644
index 0000000..6c3899e
--- /dev/null
+++ b/drivers/net/ethernet/cavium/octeon/octeon3-pki.h
@@ -0,0 +1,113 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Octeon III Packet Input Unit (PKI)
+ *
+ * Copyright (C) 2018 Cavium, Inc.
+ */
+#ifndef _OCTEON3_PKI_H_
+#define _OCTEON3_PKI_H_
+
+#include <linux/bitops.h>
+
+#define PKI_AURA_CFG_ENA_RED BIT(18)
+
+#define PKI_BUF_CTL_FPA_WAIT BIT(10)
+#define PKI_BUF_CTL_PKT_OFF BIT(5)
+#define PKI_BUF_CTL_PBP_EN BIT(2)
+#define PKI_BUF_CTL_PKI_EN BIT(0)
+
+#define PKI_CL_ECC_CTL_PCAM_EN BIT(63)
+#define PKI_CL_ECC_CTL_PCAM1_CDIS BIT(4)
+#define PKI_CL_ECC_CTL_PCAM0_CDIS BIT(3)
+
+#define PKI_CL_PCAM_ACTION_ADVANCE_4B BIT(2)
+#define PKI_CL_PCAM_ACTION_L2_CUSTOM BIT(1)
+#define PKI_CL_PCAM_ACTION_SETTY_SHIFT 8
+#define PKI_CL_PCAM_MATCH_DATA1_SHIFT 32
+#define PKI_CL_PCAM_TERM_VALID BIT(63)
+#define PKI_CL_PCAM_TERM_TERM1_SHIFT 40
+#define PKI_CL_PCAM_TERM_STYLE1_SHIFT 32
+#define PKI_CL_PCAM_TERM_TERM0_SHIFT 8
+
+#define PKI_CL_PKIND_CFG_FCS_PRES BIT(7)
+
+#define PKI_CL_PKIND_SKIP_FCS_SKIP_MASK GENMASK_ULL(15, 8)
+#define PKI_CL_PKIND_SKIP_FCS_SKIP_SHIFT 8
+#define PKI_CL_PKIND_SKIP_INST_SKIP_MASK GENMASK_ULL(7, 0)
+
+#define PKI_CL_PKIND_STYLE_PARSE_MODE_MASK GENMASK_ULL(14, 8)
+#define PKI_CL_PKIND_STYLE_STYLE_MASK GENMASK_ULL(7, 0)
+
+#define PKI_CL_STYLE_ALG_QPG_PORT_MSB_MASK GENMASK_ULL(20, 17)
+#define PKI_CL_STYLE_ALG_TT_SHIFT 30
+
+#define PKI_CL_STYLE_CFG_LENERR_EN BIT(29)
+#define PKI_CL_STYLE_CFG_MAXERR_EN BIT(26)
+#define PKI_CL_STYLE_CFG_MINERR_EN BIT(25)
+#define PKI_CL_STYLE_CFG_FCS_STRIP BIT(23)
+#define PKI_CL_STYLE_CFG_FCS_CHK BIT(22)
+
+#define PKI_CL_STYLE_CFG2_LEN_TAG_SRC_LF BIT(22)
+#define PKI_CL_STYLE_CFG2_LEN_TAG_SRC_LE BIT(21)
+#define PKI_CL_STYLE_CFG2_LEN_TAG_SRC_LD BIT(20)
+#define PKI_CL_STYLE_CFG2_LEN_TAG_SRC_LC BIT(19)
+#define PKI_CL_STYLE_CFG2_LEN_TAG_DST_LF BIT(16)
+#define PKI_CL_STYLE_CFG2_LEN_TAG_DST_LE BIT(15)
+#define PKI_CL_STYLE_CFG2_LEN_TAG_DST_LD BIT(14)
+#define PKI_CL_STYLE_CFG2_LEN_TAG_DST_LC BIT(13)
+#define PKI_CL_STYLE_CFG2_LEN_LF BIT(10)
+#define PKI_CL_STYLE_CFG2_LEN_LC BIT(7)
+#define PKI_CL_STYLE_CFG2_CSUM_LF BIT(4)
+#define PKI_CL_STYLE_CFG2_CSUM_LE BIT(3)
+#define PKI_CL_STYLE_CFG2_CSUM_LD BIT(2)
+#define PKI_CL_STYLE_CFG2_CSUM_LC BIT(1)
+
+#define PKI_CL_STYLE_CFG2_CSUM_ALL (PKI_CL_STYLE_CFG2_CSUM_LF | \
+ PKI_CL_STYLE_CFG2_CSUM_LE | \
+ PKI_CL_STYLE_CFG2_CSUM_LD | \
+ PKI_CL_STYLE_CFG2_CSUM_LC)
+
+#define PKI_CL_STYLE_CFG2_LEN_TAG_ALL (PKI_CL_STYLE_CFG2_LEN_TAG_DST_LF | \
+ PKI_CL_STYLE_CFG2_LEN_TAG_DST_LE | \
+ PKI_CL_STYLE_CFG2_LEN_TAG_DST_LD | \
+ PKI_CL_STYLE_CFG2_LEN_TAG_DST_LC)
+
+#define PKI_ICG_CFG_PENA BIT(24)
+
+#define PKI_LTYPE_MAP_BELTYPE_MASK GENMASK_ULL(2, 0)
+
+#define PKI_QPG_TBL_PADD_MASK GENMASK_ULL(59, 48)
+#define PKI_QPG_TBL_GRPTAG_OK_MASK GENMASK_ULL(47, 45)
+#define PKI_QPG_TBL_GRPTAG_OK_SHIFT 45
+#define PKI_QPG_TBL_GRP_OK_MASK GENMASK_ULL(41, 32)
+#define PKI_QPG_TBL_GRP_OK_SHIFT 32
+#define PKI_QPG_TBL_GRPTAG_BAD_MASK GENMASK_ULL(31, 29)
+#define PKI_QPG_TBL_GRPTAG_BAD_SHIFT 29
+#define PKI_QPG_TBL_GRP_BAD_MASK GENMASK_ULL(25, 16)
+#define PKI_QPG_TBL_GRP_BAD_SHIFT 16
+#define PKI_QPG_TBL_LAURA_MASK GENMASK_ULL(9, 0)
+#define PKI_QPG_TBL_GRP_NUM_BITS_SHIFT 8
+
+#define PKI_SFT_RST_BUSY BIT(63)
+#define PKI_SFT_RST_ACTIVE BIT(32)
+#define PKI_SFT_RST_RST BIT(0)
+
+#define PKI_STYLE_BUF_WQE_SKIP_SHIFT 28
+#define PKI_STYLE_BUF_FIRST_SKIP_SHIFT 22
+#define PKI_STYLE_BUF_LATER_SKIP_SHIFT 16
+
+/* Values for wqe word2 [ERRLEV] */
+#define PKI_ERRLEV_LA 0x01
+
+/* Values for wqe word2 [OPCODE] */
+#define PKI_OPCODE_NONE 0x00
+#define PKI_OPCODE_JABBER 0x02
+#define PKI_OPCODE_FCS 0x07
+
+/* Values for layer type in wqe */
+#define PKI_LTYPE_IP4 0x08
+#define PKI_LTYPE_IP6 0x0a
+#define PKI_LTYPE_TCP 0x10
+#define PKI_LTYPE_UDP 0x11
+#define PKI_LTYPE_SCTP 0x12
+
+#endif /* _OCTEON3_PKI_H_ */
--
2.1.4
^ permalink raw reply related
* [PATCH v12 06/10] netdev: cavium: octeon: Add Octeon III PKO Support
From: Steven J. Hill @ 2018-06-27 21:25 UTC (permalink / raw)
To: netdev; +Cc: Carlos Munoz, Chandrakala Chavva, Steven J. Hill
In-Reply-To: <1530134719-19407-1-git-send-email-steven.hill@cavium.com>
From: Carlos Munoz <cmunoz@cavium.com>
Add support for Octeon III PKO logic block for BGX Ethernet.
Signed-off-by: Carlos Munoz <cmunoz@cavium.com>
Signed-off-by: Steven J. Hill <Steven.Hill@cavium.com>
---
drivers/net/ethernet/cavium/octeon/octeon3-pko.c | 1638 ++++++++++++++++++++++
drivers/net/ethernet/cavium/octeon/octeon3-pko.h | 159 +++
2 files changed, 1797 insertions(+)
create mode 100644 drivers/net/ethernet/cavium/octeon/octeon3-pko.c
create mode 100644 drivers/net/ethernet/cavium/octeon/octeon3-pko.h
diff --git a/drivers/net/ethernet/cavium/octeon/octeon3-pko.c b/drivers/net/ethernet/cavium/octeon/octeon3-pko.c
new file mode 100644
index 0000000..238bf51
--- /dev/null
+++ b/drivers/net/ethernet/cavium/octeon/octeon3-pko.c
@@ -0,0 +1,1638 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Octeon III Packet-Output Processing Unit (PKO)
+ *
+ * Copyright (C) 2018 Cavium, Inc.
+ */
+
+#include "octeon3.h"
+
+#define PKO_MAX_FIFO_GROUPS 8
+#define PKO_MAX_OUTPUT_MACS 28
+#define PKO_FIFO_SIZE 2560
+
+#define PKO_BASE 0x1540000000000ull
+#define PKO_ADDR(n) (PKO_BASE + SET_XKPHYS + NODE_OFFSET(n))
+#define PKO_Q_ADDR(n, q) (PKO_ADDR(n) + ((q) << 9))
+
+#define PKO_CHANNEL_LEVEL(n) (PKO_ADDR(n) + 0x0800f0)
+#define PKO_DPFI_ENA(n) (PKO_ADDR(n) + 0xc00018)
+#define PKO_DPFI_FLUSH(n) (PKO_ADDR(n) + 0xc00008)
+#define PKO_DPFI_FPA_AURA(n) (PKO_ADDR(n) + 0xc00010)
+#define PKO_DPFI_STATUS(n) (PKO_ADDR(n) + 0xc00000)
+#define PKO_DQ_SCHEDULE(n, q) (PKO_Q_ADDR(n, q) + 0x280008)
+#define PKO_DQ_SW_XOFF(n, q) (PKO_Q_ADDR(n, q) + 0x2800e0)
+#define PKO_DQ_TOPOLOGY(n, q) (PKO_Q_ADDR(n, q) + 0x300000)
+#define PKO_DQ_WM_CTL(n, q) (PKO_Q_ADDR(n, q) + 0x000040)
+#define PKO_ENABLE(n) (PKO_ADDR(n) + 0xd00008)
+#define PKO_L1_SQ_LINK(n, q) (PKO_Q_ADDR(n, q) + 0x000038)
+#define PKO_L1_SQ_SHAPE(n, q) (PKO_Q_ADDR(n, q) + 0x000010)
+#define PKO_L1_SQ_TOPOLOGY(n, q) (PKO_Q_ADDR(n, q) + 0x080000)
+#define PKO_L2_SQ_SCHEDULE(n, q) (PKO_Q_ADDR(n, q) + 0x080008)
+#define PKO_L2_SQ_TOPOLOGY(n, q) (PKO_Q_ADDR(n, q) + 0x100000)
+#define PKO_L3_L2_SQ_CHANNEL(n, q) (PKO_Q_ADDR(n, q) + 0x080038)
+#define PKO_L3_SQ_SCHEDULE(n, q) (PKO_Q_ADDR(n, q) + 0x100008)
+#define PKO_L3_SQ_TOPOLOGY(n, q) (PKO_Q_ADDR(n, q) + 0x180000)
+#define PKO_L4_SQ_SCHEDULE(n, q) (PKO_Q_ADDR(n, q) + 0x180008)
+#define PKO_L4_SQ_TOPOLOGY(n, q) (PKO_Q_ADDR(n, q) + 0x200000)
+#define PKO_L5_SQ_SCHEDULE(n, q) (PKO_Q_ADDR(n, q) + 0x200008)
+#define PKO_L5_SQ_TOPOLOGY(n, q) (PKO_Q_ADDR(n, q) + 0x280000)
+#define PKO_LUT(n, c) (PKO_ADDR(n) + ((c) << 3) + 0xb00000)
+#define PKO_MAC_CFG(n, m) (PKO_ADDR(n) + ((m) << 3) + 0x900000)
+#define PKO_MCI0_MAX_CRED(n, m) (PKO_ADDR(n) + ((m) << 3) + 0xa00000)
+#define PKO_MCI1_MAX_CRED(n, m) (PKO_ADDR(n) + ((m) << 3) + 0xa80000)
+#define PKO_PDM_CFG(n) (PKO_ADDR(n) + 0x800000)
+#define PKO_PDM_DQ_MINPAD(n, q) (PKO_ADDR(n) + ((q) << 3) + 0x8f0000)
+#define PKO_PTF_IOBP_CFG(n) (PKO_ADDR(n) + 0x900300)
+#define PKO_PTF_STATUS(n, f) (PKO_ADDR(n) + ((f) << 3) + 0x900100)
+#define PKO_PTGF_CFG(n, g) (PKO_ADDR(n) + ((g) << 3) + 0x900200)
+#define PKO_SHAPER_CFG(n) (PKO_ADDR(n) + 0x0800f8)
+#define PKO_STATUS(n) (PKO_ADDR(n) + 0xd00000)
+
+/* These levels mimic the PKO internal linked queue structure */
+enum queue_level {
+ PQ = 1,
+ L2_SQ = 2,
+ L3_SQ = 3,
+ L4_SQ = 4,
+ L5_SQ = 5,
+ DQ = 6
+};
+
+enum pko_dqop_e {
+ DQOP_SEND,
+ DQOP_OPEN,
+ DQOP_CLOSE,
+ DQOP_QUERY
+};
+
+enum pko_dqstatus_e {
+ PASS = 0,
+ BADSTATE = 0x8,
+ NOFPABUF = 0x9,
+ NOPKOBUF = 0xa,
+ FAILRTNPTR = 0xb,
+ ALREADY = 0xc,
+ NOTCREATED = 0xd,
+ NOTEMPTY = 0xe,
+ SENDPKTDROP = 0xf
+};
+
+struct mac_info {
+ int fifo_cnt;
+ int prio;
+ int speed;
+ int fifo;
+ int num_lmacs;
+};
+
+struct fifo_grp_info {
+ int speed;
+ int size;
+};
+
+static const int lut_index_78xx[] = {
+ 0x200,
+ 0x240,
+ 0x280,
+ 0x2c0,
+ 0x300,
+ 0x340
+};
+
+static const int lut_index_73xx[] = {
+ 0x000,
+ 0x040,
+ 0x080
+};
+
+static enum queue_level max_sq_level(void)
+{
+ /* 73xx and 75xx only have 3 scheduler queue levels */
+ if (OCTEON_IS_MODEL(OCTEON_CN73XX) || OCTEON_IS_MODEL(OCTEON_CNF75XX))
+ return L3_SQ;
+ return L5_SQ;
+}
+
+static int get_num_fifos(void)
+{
+ if (OCTEON_IS_MODEL(OCTEON_CN73XX) || OCTEON_IS_MODEL(OCTEON_CNF75XX))
+ return 16;
+ return 28;
+}
+
+static int get_num_fifo_groups(void)
+{
+ if (OCTEON_IS_MODEL(OCTEON_CN73XX) || OCTEON_IS_MODEL(OCTEON_CNF75XX))
+ return 5;
+ return 8;
+}
+
+static int get_num_output_macs(void)
+{
+ if (OCTEON_IS_MODEL(OCTEON_CN78XX))
+ return 28;
+ else if (OCTEON_IS_MODEL(OCTEON_CNF75XX))
+ return 10;
+ else if (OCTEON_IS_MODEL(OCTEON_CN73XX))
+ return 14;
+ return 0;
+}
+
+static int get_output_mac(int interface, int index,
+ enum octeon3_mac_type mac_type)
+{
+ int mac;
+
+ /* Output macs are hardcoded in the hardware. See PKO Output MACs
+ * section in the HRM.
+ */
+ if (OCTEON_IS_MODEL(OCTEON_CN73XX) || OCTEON_IS_MODEL(OCTEON_CNF75XX)) {
+ if (mac_type == SRIO_MAC)
+ mac = 4 + 2 * interface + index;
+ else
+ mac = 2 + 4 * interface + index;
+ } else {
+ mac = 4 + 4 * interface + index;
+ }
+ return mac;
+}
+
+static int get_num_port_queues(void)
+{
+ if (OCTEON_IS_MODEL(OCTEON_CN73XX) || OCTEON_IS_MODEL(OCTEON_CNF75XX))
+ return 16;
+ return 32;
+}
+
+static int allocate_queues(int node, enum queue_level level, int num_queues,
+ int *queues)
+{
+ struct global_resource_tag tag;
+ int rc, max_queues = 0;
+ char buf[16];
+
+ if (level == PQ) {
+ strncpy((char *)&tag.lo, "cvm_pkop", 8);
+ snprintf(buf, 16, "oq_%d....", node);
+ memcpy(&tag.hi, buf, 8);
+
+ if (OCTEON_IS_MODEL(OCTEON_CN78XX))
+ max_queues = 32;
+ else
+ max_queues = 16;
+ } else if (level == L2_SQ) {
+ strncpy((char *)&tag.lo, "cvm_pkol", 8);
+ snprintf(buf, 16, "2q_%d....", node);
+ memcpy(&tag.hi, buf, 8);
+
+ if (OCTEON_IS_MODEL(OCTEON_CN78XX))
+ max_queues = 512;
+ else
+ max_queues = 256;
+ } else if (level == L3_SQ) {
+ strncpy((char *)&tag.lo, "cvm_pkol", 8);
+ snprintf(buf, 16, "3q_%d....", node);
+ memcpy(&tag.hi, buf, 8);
+
+ if (OCTEON_IS_MODEL(OCTEON_CN78XX))
+ max_queues = 512;
+ else
+ max_queues = 256;
+ } else if (level == L4_SQ) {
+ strncpy((char *)&tag.lo, "cvm_pkol", 8);
+ snprintf(buf, 16, "4q_%d....", node);
+ memcpy(&tag.hi, buf, 8);
+
+ if (OCTEON_IS_MODEL(OCTEON_CN78XX))
+ max_queues = 1024;
+ else
+ max_queues = 0;
+ } else if (level == L5_SQ) {
+ strncpy((char *)&tag.lo, "cvm_pkol", 8);
+ snprintf(buf, 16, "5q_%d....", node);
+ memcpy(&tag.hi, buf, 8);
+
+ if (OCTEON_IS_MODEL(OCTEON_CN78XX))
+ max_queues = 1024;
+ else
+ max_queues = 0;
+ } else if (level == DQ) {
+ strncpy((char *)&tag.lo, "cvm_pkod", 8);
+ snprintf(buf, 16, "eq_%d....", node);
+ memcpy(&tag.hi, buf, 8);
+
+ if (OCTEON_IS_MODEL(OCTEON_CN78XX))
+ max_queues = 1024;
+ else
+ max_queues = 256;
+ }
+
+ res_mgr_create_resource(tag, max_queues);
+ rc = res_mgr_alloc_range(tag, -1, num_queues, false, queues);
+ if (rc < 0)
+ return rc;
+
+ return 0;
+}
+
+static void free_queues(int node, enum queue_level level, int num_queues,
+ const int *queues)
+{
+ struct global_resource_tag tag;
+ char buf[16];
+
+ if (level == PQ) {
+ strncpy((char *)&tag.lo, "cvm_pkop", 8);
+ snprintf(buf, 16, "oq_%d....", node);
+ memcpy(&tag.hi, buf, 8);
+ } else if (level == L2_SQ) {
+ strncpy((char *)&tag.lo, "cvm_pkol", 8);
+ snprintf(buf, 16, "2q_%d....", node);
+ memcpy(&tag.hi, buf, 8);
+ } else if (level == L3_SQ) {
+ strncpy((char *)&tag.lo, "cvm_pkol", 8);
+ snprintf(buf, 16, "3q_%d....", node);
+ memcpy(&tag.hi, buf, 8);
+ } else if (level == L4_SQ) {
+ strncpy((char *)&tag.lo, "cvm_pkol", 8);
+ snprintf(buf, 16, "4q_%d....", node);
+ memcpy(&tag.hi, buf, 8);
+ } else if (level == L5_SQ) {
+ strncpy((char *)&tag.lo, "cvm_pkol", 8);
+ snprintf(buf, 16, "5q_%d....", node);
+ memcpy(&tag.hi, buf, 8);
+ } else if (level == DQ) {
+ strncpy((char *)&tag.lo, "cvm_pkod", 8);
+ snprintf(buf, 16, "eq_%d....", node);
+ memcpy(&tag.hi, buf, 8);
+ }
+
+ res_mgr_free_range(tag, queues, num_queues);
+}
+
+static int port_queue_init(int node, int pq, int mac)
+{
+ u64 data;
+
+ data = mac << PKO_SQ_TOPOLOGY_LINK_SHIFT;
+ oct_csr_write(data, PKO_L1_SQ_TOPOLOGY(node, pq));
+
+ data = mac << PKO_L1_SQ_SHAPE_LINK_SHIFT;
+ oct_csr_write(data, PKO_L1_SQ_SHAPE(node, pq));
+
+ data = mac;
+ data <<= PKO_L1_SQ_LINK_LINK_SHIFT;
+ oct_csr_write(data, PKO_L1_SQ_LINK(node, pq));
+
+ return 0;
+}
+
+static int scheduler_queue_l2_init(int node, int queue, int parent_q)
+{
+ u64 data;
+
+ data = oct_csr_read(PKO_L1_SQ_TOPOLOGY(node, parent_q));
+ data &= ~(PKO_L12_SQ_TOPOLOGY_PRIO_ANCHOR_MASK |
+ PKO_SQ_TOPOLOGY_RR_PRIO_MASK);
+ data |= (u64)queue << PKO_SQ_TOPOLOGY_PRIO_ANCHOR_SHIFT;
+ data |= PKO_SQ_TOPOLOGY_RR_PRIO_SHAPER << PKO_SQ_TOPOLOGY_RR_PRIO_SHIFT;
+ oct_csr_write(data, PKO_L1_SQ_TOPOLOGY(node, parent_q));
+
+ oct_csr_write(0, PKO_L2_SQ_SCHEDULE(node, queue));
+
+ data = parent_q << PKO_SQ_TOPOLOGY_LINK_SHIFT;
+ oct_csr_write(data, PKO_L2_SQ_TOPOLOGY(node, queue));
+
+ return 0;
+}
+
+static int scheduler_queue_l3_init(int node, int queue, int parent_q)
+{
+ u64 data;
+
+ data = oct_csr_read(PKO_L2_SQ_TOPOLOGY(node, parent_q));
+ data &= ~(PKO_L345_SQ_TOPOLOGY_PRIO_ANCHOR_MASK |
+ PKO_SQ_TOPOLOGY_RR_PRIO_MASK);
+ data |= (u64)queue << PKO_SQ_TOPOLOGY_PRIO_ANCHOR_SHIFT;
+ data |= PKO_SQ_TOPOLOGY_RR_PRIO_SHAPER << PKO_SQ_TOPOLOGY_RR_PRIO_SHIFT;
+ oct_csr_write(data, PKO_L2_SQ_TOPOLOGY(node, parent_q));
+
+ oct_csr_write(0, PKO_L3_SQ_SCHEDULE(node, queue));
+
+ data = parent_q << PKO_SQ_TOPOLOGY_LINK_SHIFT;
+ oct_csr_write(data, PKO_L3_SQ_TOPOLOGY(node, queue));
+
+ return 0;
+}
+
+static int scheduler_queue_l4_init(int node, int queue, int parent_q)
+{
+ u64 data;
+
+ data = oct_csr_read(PKO_L3_SQ_TOPOLOGY(node, parent_q));
+ data &= ~(PKO_L345_SQ_TOPOLOGY_PRIO_ANCHOR_MASK |
+ PKO_SQ_TOPOLOGY_RR_PRIO_MASK);
+ data |= (u64)queue << PKO_SQ_TOPOLOGY_PRIO_ANCHOR_SHIFT;
+ data |= PKO_SQ_TOPOLOGY_RR_PRIO_SHAPER << PKO_SQ_TOPOLOGY_RR_PRIO_SHIFT;
+ oct_csr_write(data, PKO_L3_SQ_TOPOLOGY(node, parent_q));
+
+ oct_csr_write(0, PKO_L4_SQ_SCHEDULE(node, queue));
+
+ data = parent_q << PKO_SQ_TOPOLOGY_LINK_SHIFT;
+ oct_csr_write(data, PKO_L4_SQ_TOPOLOGY(node, queue));
+
+ return 0;
+}
+
+static int scheduler_queue_l5_init(int node, int queue, int parent_q)
+{
+ u64 data;
+
+ data = oct_csr_read(PKO_L4_SQ_TOPOLOGY(node, parent_q));
+ data &= ~(PKO_L345_SQ_TOPOLOGY_PRIO_ANCHOR_MASK |
+ PKO_SQ_TOPOLOGY_RR_PRIO_MASK);
+ data |= (u64)queue << PKO_SQ_TOPOLOGY_PRIO_ANCHOR_SHIFT;
+ data |= PKO_SQ_TOPOLOGY_RR_PRIO_SHAPER << PKO_SQ_TOPOLOGY_RR_PRIO_SHIFT;
+ oct_csr_write(data, PKO_L4_SQ_TOPOLOGY(node, parent_q));
+
+ oct_csr_write(0, PKO_L5_SQ_SCHEDULE(node, queue));
+
+ data = parent_q << PKO_SQ_TOPOLOGY_LINK_SHIFT;
+ oct_csr_write(data, PKO_L5_SQ_TOPOLOGY(node, queue));
+
+ return 0;
+}
+
+static int descriptor_queue_init(int node, const int *queue, int parent_q,
+ int num_dq)
+{
+ int i, prio, rr_prio, rr_quantum;
+ u64 addr, data;
+
+ /* Limit static priorities to the available prio field bits */
+ if (num_dq > 9) {
+ pr_err("%s: Invalid number of dqs\n", __FILE__);
+ return -1;
+ }
+
+ prio = 0;
+
+ if (num_dq == 1) {
+ /* Single dq */
+ rr_prio = PKO_SQ_TOPOLOGY_RR_PRIO_SHAPER;
+ rr_quantum = 0x10;
+ } else {
+ /* Multiple dqs */
+ rr_prio = num_dq;
+ rr_quantum = 0;
+ }
+
+ if (OCTEON_IS_MODEL(OCTEON_CN78XX))
+ addr = PKO_L5_SQ_TOPOLOGY(node, parent_q);
+ else
+ addr = PKO_L3_SQ_TOPOLOGY(node, parent_q);
+
+ data = oct_csr_read(addr);
+ data &= ~(PKO_L345_SQ_TOPOLOGY_PRIO_ANCHOR_MASK |
+ PKO_SQ_TOPOLOGY_RR_PRIO_MASK);
+ data |= (u64)queue[0] << PKO_SQ_TOPOLOGY_PRIO_ANCHOR_SHIFT;
+ data |= rr_prio << PKO_SQ_TOPOLOGY_RR_PRIO_SHIFT;
+ oct_csr_write(data, addr);
+
+ for (i = 0; i < num_dq; i++) {
+ data = (prio << PKO_DQ_SCHEDULE_PRIO_SHIFT) | rr_quantum;
+ oct_csr_write(data, PKO_DQ_SCHEDULE(node, queue[i]));
+
+ data = parent_q << PKO_SQ_TOPOLOGY_LINK_SHIFT;
+ oct_csr_write(data, PKO_DQ_TOPOLOGY(node, queue[i]));
+
+ data = PKO_DQ_WM_CTL_KIND;
+ oct_csr_write(data, PKO_DQ_WM_CTL(node, queue[i]));
+
+ if (prio << rr_prio)
+ prio++;
+ }
+
+ return 0;
+}
+
+static int map_channel(int node, int pq, int queue, int ipd_port)
+{
+ int table_index, lut_index = 0;
+ u64 data;
+
+ data = oct_csr_read(PKO_L3_L2_SQ_CHANNEL(node, queue));
+ data &= ~PKO_L3_L2_CHANNEL_CC_CHANNEL_MASK;
+ data |= (u64)ipd_port << PKO_L3_L2_CHANNEL_CC_CHANNEL_SHIFT;
+ oct_csr_write(data, PKO_L3_L2_SQ_CHANNEL(node, queue));
+
+ /* See PKO_LUT register description in the HRM for how to compose the
+ * lut_index.
+ */
+ if (OCTEON_IS_MODEL(OCTEON_CN78XX)) {
+ table_index = ((ipd_port & PKO_LUT_PORT_MASK) -
+ PKO_LUT_PORT_TO_INDEX) >> PKO_LUT_QUEUE_NUMBER_SHIFT;
+ lut_index = lut_index_78xx[table_index];
+ lut_index += ipd_port & PKO_LUT_QUEUE_NUMBER_MASK;
+ } else if (OCTEON_IS_MODEL(OCTEON_CN73XX)) {
+ table_index = ((ipd_port & PKO_LUT_PORT_MASK) -
+ PKO_LUT_PORT_TO_INDEX) >> PKO_LUT_QUEUE_NUMBER_SHIFT;
+ lut_index = lut_index_73xx[table_index];
+ lut_index += ipd_port & PKO_LUT_QUEUE_NUMBER_MASK;
+ } else if (OCTEON_IS_MODEL(OCTEON_CNF75XX)) {
+ if ((ipd_port & PKO_LUT_PORT_MASK) != PKO_LUT_PORT_TO_INDEX)
+ return -1;
+ lut_index = ipd_port & PKO_LUT_QUEUE_NUMBER_MASK;
+ }
+
+ data = PKO_LUT_VALID;
+ data |= pq << PKO_LUT_PQ_IDX_SHIFT;
+ data |= queue;
+ oct_csr_write(data, PKO_LUT(node, lut_index));
+
+ return 0;
+}
+
+static int open_dq(int node, int dq)
+{
+ u64 data, *iobdma_addr, *scratch_addr;
+ enum pko_dqstatus_e status;
+
+ /* Build the dq open query. See PKO_QUERY_DMA_S in the HRM for the
+ * query format.
+ */
+ data = (LMTDMA_SCR_OFFSET >> PKO_LMTDMA_SCRADDR_SHIFT)
+ << PKO_QUERY_DMA_SCRADDR_SHIFT;
+ data |= 1ull << PKO_QUERY_DMA_RTNLEN_SHIFT; /* Must always be 1 */
+ data |= 0x51ull << PKO_QUERY_DMA_DID_SHIFT; /* Must always be 51h */
+ data |= (u64)node << PKO_QUERY_DMA_NODE_SHIFT;
+ data |= (u64)DQOP_OPEN << PKO_QUERY_DMA_DQOP_SHIFT;
+ data |= dq << PKO_QUERY_DMA_DQ_SHIFT;
+
+ CVMX_SYNCWS;
+ preempt_disable();
+
+ /* Clear return location */
+ scratch_addr = (u64 *)(SCRATCH_BASE_ADDR + LMTDMA_SCR_OFFSET);
+ *scratch_addr = ~0ull;
+
+ /* Issue pko lmtdma command */
+ iobdma_addr = (u64 *)(IOBDMA_ORDERED_IO_ADDR);
+ *iobdma_addr = data;
+
+ /* Wait for lmtdma command to complete and get response*/
+ CVMX_SYNCIOBDMA;
+ data = *scratch_addr;
+
+ preempt_enable();
+
+ /* See PKO_QUERY_RTN_S in the HRM for response format */
+ status = (data & PKO_QUERY_RTN_DQSTATUS_MASK)
+ >> PKO_QUERY_RTN_DQSTATUS_SHIFT;
+ if (status != PASS && status != ALREADY) {
+ pr_err("%s: Failed to open dq\n", __FILE__);
+ return -1;
+ }
+
+ return 0;
+}
+
+static s64 query_dq(int node, int dq)
+{
+ u64 data, *iobdma_addr, *scratch_addr;
+ enum pko_dqstatus_e status;
+ s64 depth;
+
+ /* Build the dq open query. See PKO_QUERY_DMA_S in the HRM for the
+ * query format.
+ */
+ data = (LMTDMA_SCR_OFFSET >> PKO_LMTDMA_SCRADDR_SHIFT) <<
+ PKO_QUERY_DMA_SCRADDR_SHIFT;
+ data |= 1ull << PKO_QUERY_DMA_RTNLEN_SHIFT; /* Must always be 1 */
+ data |= 0x51ull << PKO_QUERY_DMA_DID_SHIFT; /* Must always be 51h */
+ data |= (u64)node << PKO_QUERY_DMA_NODE_SHIFT;
+ data |= (u64)DQOP_QUERY << PKO_QUERY_DMA_DQOP_SHIFT;
+ data |= dq << PKO_QUERY_DMA_DQ_SHIFT;
+
+ CVMX_SYNCWS;
+ preempt_disable();
+
+ /* Clear return location */
+ scratch_addr = (u64 *)(SCRATCH_BASE_ADDR + LMTDMA_SCR_OFFSET);
+ *scratch_addr = ~0ull;
+
+ /* Issue pko lmtdma command */
+ iobdma_addr = (u64 *)(IOBDMA_ORDERED_IO_ADDR);
+ *iobdma_addr = data;
+
+ /* Wait for lmtdma command to complete and get response*/
+ CVMX_SYNCIOBDMA;
+ data = *scratch_addr;
+
+ preempt_enable();
+
+ /* See PKO_QUERY_RTN_S in the HRM for response format */
+ status = (data & PKO_QUERY_RTN_DQSTATUS_MASK)
+ >> PKO_QUERY_RTN_DQSTATUS_SHIFT;
+ if (status != PASS) {
+ pr_err("%s: Failed to query dq=%d\n", __FILE__, dq);
+ return -1;
+ }
+
+ depth = data & PKO_QUERY_RTN_DEPTH_MASK;
+
+ return depth;
+}
+
+static u64 close_dq(int node, int dq)
+{
+ u64 data, *iobdma_addr, *scratch_addr;
+ enum pko_dqstatus_e status;
+
+ /* Build the dq open query. See PKO_QUERY_DMA_S in the HRM for the
+ * query format.
+ */
+ data = (LMTDMA_SCR_OFFSET >> PKO_LMTDMA_SCRADDR_SHIFT) <<
+ PKO_QUERY_DMA_SCRADDR_SHIFT;
+ data |= 1ull << PKO_QUERY_DMA_RTNLEN_SHIFT; /* Must always be 1 */
+ data |= 0x51ull << PKO_QUERY_DMA_DID_SHIFT; /* Must always be 51h */
+ data |= (u64)node << PKO_QUERY_DMA_NODE_SHIFT;
+ data |= (u64)DQOP_CLOSE << PKO_QUERY_DMA_DQOP_SHIFT;
+ data |= dq << PKO_QUERY_DMA_DQ_SHIFT;
+
+ CVMX_SYNCWS;
+ preempt_disable();
+
+ /* Clear return location */
+ scratch_addr = (u64 *)(SCRATCH_BASE_ADDR + LMTDMA_SCR_OFFSET);
+ *scratch_addr = ~0ull;
+
+ /* Issue pko lmtdma command */
+ iobdma_addr = (u64 *)(IOBDMA_ORDERED_IO_ADDR);
+ *iobdma_addr = data;
+
+ /* Wait for lmtdma command to complete and get response*/
+ CVMX_SYNCIOBDMA;
+ data = *scratch_addr;
+
+ preempt_enable();
+
+ /* See PKO_QUERY_RTN_S in the HRM for response format */
+ status = (data & PKO_QUERY_RTN_DQSTATUS_MASK)
+ >> PKO_QUERY_RTN_DQSTATUS_SHIFT;
+ if (status != PASS) {
+ pr_err("%s: Failed to close dq\n", __FILE__);
+ return -1;
+ }
+
+ return 0;
+}
+
+static int get_78xx_fifos_required(int node, struct mac_info *macs)
+{
+ int bgx, cnt, i, index, num_lmacs, prio, qlm, fifo_cnt = 0;
+ enum port_mode mode;
+ u64 data;
+
+ /* The loopback mac gets 1 fifo by default */
+ macs[0].fifo_cnt = 1;
+ macs[0].speed = 1;
+ fifo_cnt += 1;
+
+ /* The dpi mac gets 1 fifo by default */
+ macs[1].fifo_cnt = 1;
+ macs[1].speed = 50;
+ fifo_cnt += 1;
+
+ /* The ilk macs get default number of fifos (module param) */
+ macs[2].fifo_cnt = ilk0_lanes <= 4 ? ilk0_lanes : 4;
+ macs[2].speed = 40;
+ fifo_cnt += macs[2].fifo_cnt;
+ macs[3].fifo_cnt = ilk1_lanes <= 4 ? ilk1_lanes : 4;
+ macs[3].speed = 40;
+ fifo_cnt += macs[3].fifo_cnt;
+
+ /* Assign fifos to the active bgx macs */
+ for (i = 4; i < get_num_output_macs(); i += 4) {
+ bgx = (i - 4) / 4;
+ qlm = bgx_port_get_qlm(node, bgx, 0);
+
+ data = oct_csr_read(GSER_CFG(node, qlm));
+ if (data & GSER_CFG_BGX) {
+ data = oct_csr_read(BGX_CMR_TX_LMACS(node, bgx));
+ num_lmacs = data & 7;
+
+ for (index = 0; index < num_lmacs; index++) {
+ switch (num_lmacs) {
+ case 1:
+ macs[i + index].num_lmacs = 4;
+ break;
+ case 2:
+ macs[i + index].num_lmacs = 2;
+ break;
+ case 4:
+ default:
+ macs[i + index].num_lmacs = 1;
+ break;
+ }
+
+ mode = bgx_port_get_mode(node, bgx, 0);
+ switch (mode) {
+ case PORT_MODE_SGMII:
+ case PORT_MODE_RGMII:
+ macs[i + index].fifo_cnt = 1;
+ macs[i + index].prio = 1;
+ macs[i + index].speed = 1;
+ break;
+
+ case PORT_MODE_XAUI:
+ case PORT_MODE_RXAUI:
+ macs[i + index].fifo_cnt = 4;
+ macs[i + index].prio = 2;
+ macs[i + index].speed = 20;
+ break;
+
+ case PORT_MODE_10G_KR:
+ case PORT_MODE_XFI:
+ macs[i + index].fifo_cnt = 4;
+ macs[i + index].prio = 2;
+ macs[i + index].speed = 10;
+ break;
+
+ case PORT_MODE_40G_KR4:
+ case PORT_MODE_XLAUI:
+ macs[i + index].fifo_cnt = 4;
+ macs[i + index].prio = 3;
+ macs[i + index].speed = 40;
+ break;
+
+ default:
+ macs[i + index].fifo_cnt = 0;
+ macs[i + index].prio = 0;
+ macs[i + index].speed = 0;
+ macs[i + index].num_lmacs = 0;
+ break;
+ }
+
+ fifo_cnt += macs[i + index].fifo_cnt;
+ }
+ }
+ }
+
+ /* If more fifos than available were assigned, reduce the number of
+ * fifos until within limit. Start with the lowest priority macs with 4
+ * fifos.
+ */
+ prio = 1;
+ cnt = 4;
+ while (fifo_cnt > get_num_fifos()) {
+ for (i = 0; i < get_num_output_macs(); i++) {
+ if (macs[i].prio == prio && macs[i].fifo_cnt == cnt) {
+ macs[i].fifo_cnt >>= 1;
+ fifo_cnt -= macs[i].fifo_cnt;
+ }
+
+ if (fifo_cnt <= get_num_fifos())
+ break;
+ }
+
+ if (prio >= 3) {
+ prio = 1;
+ cnt >>= 1;
+ } else {
+ prio++;
+ }
+
+ if (cnt == 0)
+ break;
+ }
+
+ /* Assign left over fifos to dpi */
+ if (get_num_fifos() - fifo_cnt > 0) {
+ if (get_num_fifos() - fifo_cnt >= 3) {
+ macs[1].fifo_cnt += 3;
+ fifo_cnt -= 3;
+ } else {
+ macs[1].fifo_cnt += 1;
+ fifo_cnt -= 1;
+ }
+ }
+
+ return 0;
+}
+
+static int get_75xx_fifos_required(int node, struct mac_info *macs)
+{
+ int bgx, cnt, i, index, prio, qlm, fifo_cnt = 0;
+ enum port_mode mode;
+ u64 data;
+
+ /* The loopback mac gets 1 fifo by default */
+ macs[0].fifo_cnt = 1;
+ macs[0].speed = 1;
+ fifo_cnt += 1;
+
+ /* The dpi mac gets 1 fifo by default */
+ macs[1].fifo_cnt = 1;
+ macs[1].speed = 50;
+ fifo_cnt += 1;
+
+ /* Assign fifos to the active bgx macs */
+ bgx = 0;
+ for (i = 2; i < 6; i++) {
+ index = i - 2;
+ qlm = bgx_port_get_qlm(node, bgx, index);
+ data = oct_csr_read(GSER_CFG(node, qlm));
+ if (data & GSER_CFG_BGX) {
+ macs[i].num_lmacs = 1;
+
+ mode = bgx_port_get_mode(node, bgx, index);
+ switch (mode) {
+ case PORT_MODE_SGMII:
+ case PORT_MODE_RGMII:
+ macs[i].fifo_cnt = 1;
+ macs[i].prio = 1;
+ macs[i].speed = 1;
+ break;
+
+ case PORT_MODE_10G_KR:
+ case PORT_MODE_XFI:
+ macs[i].fifo_cnt = 4;
+ macs[i].prio = 2;
+ macs[i].speed = 10;
+ break;
+
+ default:
+ macs[i].fifo_cnt = 0;
+ macs[i].prio = 0;
+ macs[i].speed = 0;
+ macs[i].num_lmacs = 0;
+ break;
+ }
+
+ fifo_cnt += macs[i].fifo_cnt;
+ }
+ }
+
+ /* If more fifos than available were assigned, reduce the number of
+ * fifos until within limit. Start with the lowest priority macs with 4
+ * fifos.
+ */
+ prio = 1;
+ cnt = 4;
+ while (fifo_cnt > get_num_fifos()) {
+ for (i = 0; i < get_num_output_macs(); i++) {
+ if (macs[i].prio == prio && macs[i].fifo_cnt == cnt) {
+ macs[i].fifo_cnt >>= 1;
+ fifo_cnt -= macs[i].fifo_cnt;
+ }
+
+ if (fifo_cnt <= get_num_fifos())
+ break;
+ }
+
+ if (prio >= 3) {
+ prio = 1;
+ cnt >>= 1;
+ } else {
+ prio++;
+ }
+
+ if (cnt == 0)
+ break;
+ }
+
+ /* Assign left over fifos to dpi */
+ if (get_num_fifos() - fifo_cnt > 0) {
+ if (get_num_fifos() - fifo_cnt >= 3) {
+ macs[1].fifo_cnt += 3;
+ fifo_cnt -= 3;
+ } else {
+ macs[1].fifo_cnt += 1;
+ fifo_cnt -= 1;
+ }
+ }
+
+ return 0;
+}
+
+static int get_73xx_fifos_required(int node, struct mac_info *macs)
+{
+ int bgx, cnt, i, index, num_lmacs, prio, qlm, fifo_cnt = 0;
+ enum port_mode mode;
+ u64 data;
+
+ /* The loopback mac gets 1 fifo by default */
+ macs[0].fifo_cnt = 1;
+ macs[0].speed = 1;
+ fifo_cnt += 1;
+
+ /* The dpi mac gets 1 fifo by default */
+ macs[1].fifo_cnt = 1;
+ macs[1].speed = 50;
+ fifo_cnt += 1;
+
+ /* Assign fifos to the active bgx macs */
+ for (i = 2; i < get_num_output_macs(); i += 4) {
+ bgx = (i - 2) / 4;
+ qlm = bgx_port_get_qlm(node, bgx, 0);
+ data = oct_csr_read(GSER_CFG(node, qlm));
+
+ /* Bgx2 can be connected to dlm 5, 6, or both */
+ if (bgx == 2) {
+ if (!(data & GSER_CFG_BGX)) {
+ qlm = bgx_port_get_qlm(node, bgx, 2);
+ data = oct_csr_read(GSER_CFG(node, qlm));
+ }
+ }
+
+ if (data & GSER_CFG_BGX) {
+ data = oct_csr_read(BGX_CMR_TX_LMACS(node, bgx));
+ num_lmacs = data & 7;
+
+ for (index = 0; index < num_lmacs; index++) {
+ switch (num_lmacs) {
+ case 1:
+ macs[i + index].num_lmacs = 4;
+ break;
+ case 2:
+ macs[i + index].num_lmacs = 2;
+ break;
+ case 4:
+ default:
+ macs[i + index].num_lmacs = 1;
+ break;
+ }
+
+ mode = bgx_port_get_mode(node, bgx, index);
+ switch (mode) {
+ case PORT_MODE_SGMII:
+ case PORT_MODE_RGMII:
+ macs[i + index].fifo_cnt = 1;
+ macs[i + index].prio = 1;
+ macs[i + index].speed = 1;
+ break;
+
+ case PORT_MODE_XAUI:
+ case PORT_MODE_RXAUI:
+ macs[i + index].fifo_cnt = 4;
+ macs[i + index].prio = 2;
+ macs[i + index].speed = 20;
+ break;
+
+ case PORT_MODE_10G_KR:
+ case PORT_MODE_XFI:
+ macs[i + index].fifo_cnt = 4;
+ macs[i + index].prio = 2;
+ macs[i + index].speed = 10;
+ break;
+
+ case PORT_MODE_40G_KR4:
+ case PORT_MODE_XLAUI:
+ macs[i + index].fifo_cnt = 4;
+ macs[i + index].prio = 3;
+ macs[i + index].speed = 40;
+ break;
+
+ default:
+ macs[i + index].fifo_cnt = 0;
+ macs[i + index].prio = 0;
+ macs[i + index].speed = 0;
+ break;
+ }
+
+ fifo_cnt += macs[i + index].fifo_cnt;
+ }
+ }
+ }
+
+ /* If more fifos than available were assigned, reduce the number of
+ * fifos until within limit. Start with the lowest priority macs with 4
+ * fifos.
+ */
+ prio = 1;
+ cnt = 4;
+ while (fifo_cnt > get_num_fifos()) {
+ for (i = 0; i < get_num_output_macs(); i++) {
+ if (macs[i].prio == prio && macs[i].fifo_cnt == cnt) {
+ macs[i].fifo_cnt >>= 1;
+ fifo_cnt -= macs[i].fifo_cnt;
+ }
+
+ if (fifo_cnt <= get_num_fifos())
+ break;
+ }
+
+ if (prio >= 3) {
+ prio = 1;
+ cnt >>= 1;
+ } else {
+ prio++;
+ }
+
+ if (cnt == 0)
+ break;
+ }
+
+ /* Assign left over fifos to dpi */
+ if (get_num_fifos() - fifo_cnt > 0) {
+ if (get_num_fifos() - fifo_cnt >= 3) {
+ macs[1].fifo_cnt += 3;
+ fifo_cnt -= 3;
+ } else {
+ macs[1].fifo_cnt += 1;
+ fifo_cnt -= 1;
+ }
+ }
+
+ return 0;
+}
+
+static int setup_macs(int node)
+{
+ struct fifo_grp_info fifo_grp[PKO_MAX_FIFO_GROUPS];
+ struct mac_info macs[PKO_MAX_OUTPUT_MACS];
+ int cnt, fifo, grp, i, size;
+ u64 data;
+
+ memset(macs, 0, sizeof(macs));
+ memset(fifo_grp, 0, sizeof(fifo_grp));
+
+ /* Get the number of fifos required by each mac */
+ if (OCTEON_IS_MODEL(OCTEON_CN78XX)) {
+ get_78xx_fifos_required(node, macs);
+ } else if (OCTEON_IS_MODEL(OCTEON_CNF75XX)) {
+ get_75xx_fifos_required(node, macs);
+ } else if (OCTEON_IS_MODEL(OCTEON_CN73XX)) {
+ get_73xx_fifos_required(node, macs);
+ } else {
+ pr_err("%s: Unsupported board type\n", __FILE__);
+ return -1;
+ }
+
+ /* Assign fifos to each mac. Start with macs requiring 4 fifos */
+ fifo = 0;
+ for (cnt = 4; cnt > 0; cnt >>= 1) {
+ for (i = 0; i < get_num_output_macs(); i++) {
+ if (macs[i].fifo_cnt != cnt)
+ continue;
+
+ macs[i].fifo = fifo;
+ grp = fifo / 4;
+
+ fifo_grp[grp].speed += macs[i].speed;
+
+ if (cnt == 4) {
+ /* 10, 0, 0, 0 */
+ fifo_grp[grp].size = 4;
+ } else if (cnt == 2) {
+ /* 5, 0, 5, 0 */
+ fifo_grp[grp].size = 3;
+ } else if (cnt == 1) {
+ if ((fifo & 0x2) && fifo_grp[grp].size == 3) {
+ /* 5, 0, 2.5, 2.5 */
+ fifo_grp[grp].size = 1;
+ } else {
+ /* 2.5, 2.5, 2.5, 2.5 */
+ fifo_grp[grp].size = 0;
+ }
+ }
+
+ fifo += cnt;
+ }
+ }
+
+ /* Configure the fifo groups */
+ for (i = 0; i < get_num_fifo_groups(); i++) {
+ data = oct_csr_read(PKO_PTGF_CFG(node, i));
+ size = data & PKO_PTGF_CFG_SIZE_MASK;
+ if (size != fifo_grp[i].size)
+ data |= PKO_PTGF_CFG_RESET;
+ data &= ~PKO_PTGF_CFG_SIZE_MASK;
+ data |= fifo_grp[i].size;
+
+ data &= ~PKO_PTGF_CFG_RATE_MASK;
+ if (fifo_grp[i].speed >= 40) {
+ if (fifo_grp[i].size >= 3) {
+ /* 50 Gbps */
+ data |= 0x3 << PKO_PTGF_CFG_RATE_SHIFT;
+ } else {
+ /* 25 Gbps */
+ data |= 0x2 << PKO_PTGF_CFG_RATE_SHIFT;
+ }
+ } else if (fifo_grp[i].speed >= 20) {
+ /* 25 Gbps */
+ data |= 0x2 << PKO_PTGF_CFG_RATE_SHIFT;
+ } else if (fifo_grp[i].speed >= 10) {
+ /* 12.5 Gbps */
+ data |= 0x1 << PKO_PTGF_CFG_RATE_SHIFT;
+ }
+ oct_csr_write(data, PKO_PTGF_CFG(node, i));
+ data &= ~PKO_PTGF_CFG_RESET;
+ oct_csr_write(data, PKO_PTGF_CFG(node, i));
+ }
+
+ /* Configure the macs with their assigned fifo */
+ for (i = 0; i < get_num_output_macs(); i++) {
+ data = oct_csr_read(PKO_MAC_CFG(node, i));
+ data &= ~PKO_MAC_CFG_FIFO_NUM_MASK;
+ if (!macs[i].fifo_cnt)
+ data |= PKO_MAC_CFG_FIFO_UNDEFINED;
+ else
+ data |= macs[i].fifo;
+ oct_csr_write(data, PKO_MAC_CFG(node, i));
+ }
+
+ /* Setup mci0/mci1/skid credits */
+ for (i = 0; i < get_num_output_macs(); i++) {
+ int fifo_credit, mac_credit, skid_credit;
+
+ if (!macs[i].fifo_cnt)
+ continue;
+
+ if (i == 0) {
+ /* Loopback */
+ mac_credit = 4 * 1024;
+ skid_credit = 0;
+ } else if (i == 1) {
+ /* Dpi */
+ mac_credit = 2 * 1024;
+ skid_credit = 0;
+ } else if (OCTEON_IS_MODEL(OCTEON_CN78XX) &&
+ ((i == 2 || i == 3))) {
+ /* ILK */
+ mac_credit = 4 * 1024;
+ skid_credit = 0;
+ } else if (OCTEON_IS_MODEL(OCTEON_CNF75XX) &&
+ ((i >= 6 && i <= 9))) {
+ /* Srio */
+ mac_credit = 1024 / 2;
+ skid_credit = 0;
+ } else {
+ /* Bgx */
+ mac_credit = macs[i].num_lmacs * 8 * 1024;
+ skid_credit = macs[i].num_lmacs * 256;
+ }
+
+ if (OCTEON_IS_MODEL(OCTEON_CN78XX_PASS1_X)) {
+ fifo_credit = macs[i].fifo_cnt * PKO_FIFO_SIZE;
+ data = (fifo_credit + mac_credit) / 16;
+ oct_csr_write(data, PKO_MCI0_MAX_CRED(node, i));
+ }
+
+ data = mac_credit / 16;
+ oct_csr_write(data, PKO_MCI1_MAX_CRED(node, i));
+
+ data = oct_csr_read(PKO_MAC_CFG(node, i));
+ data &= ~PKO_MAC_CFG_SKID_MAX_CNT_MASK;
+ data |= ((skid_credit / 256) >> 1) << 5;
+ oct_csr_write(data, PKO_MAC_CFG(node, i));
+ }
+
+ return 0;
+}
+
+static int hw_init_global(int node, int aura)
+{
+ int timeout;
+ u64 data;
+
+ data = oct_csr_read(PKO_ENABLE(node));
+ if (data & PKO_ENABLE_ENABLE) {
+ pr_info("%s: Pko already enabled on node %d\n", __FILE__, node);
+ return 0;
+ }
+
+ /* Enable color awareness */
+ data = oct_csr_read(PKO_SHAPER_CFG(node));
+ data |= PKO_SHAPER_CFG_COLOR_AWARE;
+ oct_csr_write(data, PKO_SHAPER_CFG(node));
+
+ /* Clear flush command */
+ oct_csr_write(0, PKO_DPFI_FLUSH(node));
+
+ /* Set the aura number */
+ data = (node << PKO_DPFI_FPA_AURA_NODE_SHIFT) | aura;
+ oct_csr_write(data, PKO_DPFI_FPA_AURA(node));
+
+ data = PKO_DPFI_ENA_ENABLE;
+ oct_csr_write(data, PKO_DPFI_ENA(node));
+
+ /* Wait until all pointers have been returned */
+ timeout = 100000;
+ do {
+ data = oct_csr_read(PKO_STATUS(node));
+ if (data & PKO_STATUS_PKO_RDY)
+ break;
+ udelay(1);
+ timeout--;
+ } while (timeout);
+ if (!timeout) {
+ pr_err("%s: Pko dfpi failed on node %d\n", __FILE__, node);
+ return -1;
+ }
+
+ /* Set max outstanding requests in IOBP for any FIFO.*/
+ data = oct_csr_read(PKO_PTF_IOBP_CFG(node));
+ data &= ~PKO_PTF_IOBP_CFG_MAX_READ_SIZE_MASK;
+ if (OCTEON_IS_MODEL(OCTEON_CN78XX))
+ data |= 16;
+ else
+ data |= 3;
+ oct_csr_write(data, PKO_PTF_IOBP_CFG(node));
+
+ /* Set minimum packet size per Ethernet standard */
+ data = (PKO_PDM_PAD_MINLEN_TYPICAL << PKO_PDM_PAD_MINLEN_SHIFT);
+ oct_csr_write(data, PKO_PDM_CFG(node));
+
+ /* Initialize macs and fifos */
+ setup_macs(node);
+
+ /* Enable pko */
+ data = PKO_ENABLE_ENABLE;
+ oct_csr_write(data, PKO_ENABLE(node));
+
+ /* Verify pko is ready */
+ data = oct_csr_read(PKO_STATUS(node));
+ if (!(data & PKO_STATUS_PKO_RDY)) {
+ pr_err("%s: pko is not ready\n", __FILE__);
+ return -1;
+ }
+
+ return 0;
+}
+
+static int hw_exit_global(int node)
+{
+ int i, timeout;
+ u64 data;
+
+ /* Wait until there are no in-flight packets */
+ for (i = 0; i < get_num_fifos(); i++) {
+ data = oct_csr_read(PKO_PTF_STATUS(node, i));
+ if ((data & PKO_PTF_STATUS_MAC_NUM_MASK) ==
+ PKO_MAC_CFG_FIFO_UNDEFINED)
+ continue;
+
+ timeout = 10000;
+ do {
+ if (!(data & PKO_PTF_STATUS_IN_FLIGHT_CNT_MASK))
+ break;
+ udelay(1);
+ timeout--;
+ data = oct_csr_read(PKO_PTF_STATUS(node, i));
+ } while (timeout);
+ if (!timeout) {
+ pr_err("%s: Timeout in-flight fifo\n", __FILE__);
+ return -1;
+ }
+ }
+
+ /* Disable pko */
+ oct_csr_write(0, PKO_ENABLE(node));
+
+ /* Reset all port queues to the virtual mac */
+ for (i = 0; i < get_num_port_queues(); i++) {
+ data = get_num_output_macs() << PKO_L1_SQ_TOPOLOGY_LINK_SHIFT;
+ oct_csr_write(data, PKO_L1_SQ_TOPOLOGY(node, i));
+
+ data = get_num_output_macs() << PKO_L1_SQ_SHAPE_LINK_SHIFT;
+ oct_csr_write(data, PKO_L1_SQ_SHAPE(node, i));
+
+ data = (u64)get_num_output_macs() << PKO_L1_SQ_LINK_LINK_SHIFT;
+ oct_csr_write(data, PKO_L1_SQ_LINK(node, i));
+ }
+
+ /* Reset all output macs */
+ for (i = 0; i < get_num_output_macs(); i++) {
+ data = PKO_MAC_CFG_FIFO_NUM_UNDEFINED;
+ oct_csr_write(data, PKO_MAC_CFG(node, i));
+ }
+
+ /* Reset all fifo groups */
+ for (i = 0; i < get_num_fifo_groups(); i++) {
+ data = oct_csr_read(PKO_PTGF_CFG(node, i));
+ /* Simulator asserts if an unused group is reset */
+ if (data == 0)
+ continue;
+ data = PKO_PTGF_CFG_RESET;
+ oct_csr_write(data, PKO_PTGF_CFG(node, i));
+ }
+
+ /* Return cache pointers to fpa */
+ data = PKO_DPFI_FLUSH_FLUSH_EN;
+ oct_csr_write(data, PKO_DPFI_FLUSH(node));
+ timeout = 10000;
+ do {
+ data = oct_csr_read(PKO_DPFI_STATUS(node));
+ if (data & PKO_DPFI_STATUS_CACHE_FLUSHED)
+ break;
+ udelay(1);
+ timeout--;
+ } while (timeout);
+ if (!timeout) {
+ pr_err("%s: Timeout flushing cache\n", __FILE__);
+ return -1;
+ }
+ oct_csr_write(0, PKO_DPFI_ENA(node));
+ oct_csr_write(0, PKO_DPFI_FLUSH(node));
+
+ return 0;
+}
+
+static int virtual_mac_config(int node)
+{
+ int dq[8], i, num_dq, parent_q, pq, queue, rc, vmac;
+ enum queue_level level;
+
+ /* The virtual mac is after the last output mac. Note: for the 73xx it
+ * might be 2 after the last output mac (15).
+ */
+ vmac = get_num_output_macs();
+
+ /* Allocate a port queue */
+ rc = allocate_queues(node, PQ, 1, &pq);
+ if (rc < 0) {
+ pr_err("%s: Failed to allocate port queue\n", __FILE__);
+ return rc;
+ }
+
+ /* Connect the port queue to the output mac */
+ port_queue_init(node, pq, vmac);
+
+ parent_q = pq;
+ for (level = L2_SQ; level <= max_sq_level(); level++) {
+ rc = allocate_queues(node, level, 1, &queue);
+ if (rc < 0) {
+ pr_err("%s: Failed to allocate queue\n", __FILE__);
+ return rc;
+ }
+
+ switch (level) {
+ case L2_SQ:
+ scheduler_queue_l2_init(node, queue, parent_q);
+ break;
+ case L3_SQ:
+ scheduler_queue_l3_init(node, queue, parent_q);
+ break;
+ case L4_SQ:
+ scheduler_queue_l4_init(node, queue, parent_q);
+ break;
+ case L5_SQ:
+ scheduler_queue_l5_init(node, queue, parent_q);
+ break;
+ default:
+ break;
+ }
+
+ parent_q = queue;
+ }
+
+ if (OCTEON_IS_MODEL(OCTEON_CN78XX_PASS1_0))
+ num_dq = 8;
+ else
+ num_dq = 1;
+
+ rc = allocate_queues(node, DQ, num_dq, dq);
+ if (rc < 0) {
+ pr_err("%s: Failed to allocate description queues\n", __FILE__);
+ return rc;
+ }
+
+ /* By convention the dq must be zero */
+ if (dq[0] != 0) {
+ pr_err("%s: Failed to reserve description queues\n", __FILE__);
+ return -1;
+ }
+ descriptor_queue_init(node, dq, parent_q, num_dq);
+
+ /* Open the dqs */
+ for (i = 0; i < num_dq; i++)
+ open_dq(node, dq[i]);
+
+ return 0;
+}
+
+static int drain_dq(int node, int dq)
+{
+ int timeout;
+ u64 data;
+ s64 rc;
+
+ data = PKO_DQ_SW_XOFF_SC_RAM_FLIP | PKO_DQ_SW_XOFF_SC_RAM_CDIS;
+ oct_csr_write(data, PKO_DQ_SW_XOFF(node, dq));
+
+ usleep_range(1000, 2000);
+
+ data = 0;
+ oct_csr_write(data, PKO_DQ_SW_XOFF(node, dq));
+
+ /* Wait for the dq to drain */
+ timeout = 10000;
+ do {
+ rc = query_dq(node, dq);
+ if (!rc)
+ break;
+ else if (rc < 0)
+ return rc;
+ udelay(1);
+ timeout--;
+ } while (timeout);
+ if (!timeout) {
+ pr_err("%s: Timeout waiting for dq to drain\n", __FILE__);
+ return -1;
+ }
+
+ /* Close the queue anf free internal buffers */
+ close_dq(node, dq);
+
+ return 0;
+}
+
+int octeon3_pko_exit_global(int node)
+{
+ int dq[8], i, num_dq;
+
+ if (OCTEON_IS_MODEL(OCTEON_CN78XX_PASS1_0))
+ num_dq = 8;
+ else
+ num_dq = 1;
+
+ /* Shutdown the virtual/null interface */
+ for (i = 0; i < ARRAY_SIZE(dq); i++)
+ dq[i] = i;
+ octeon3_pko_interface_uninit(node, dq, num_dq);
+
+ /* Shutdown pko */
+ hw_exit_global(node);
+
+ return 0;
+}
+EXPORT_SYMBOL(octeon3_pko_exit_global);
+
+int octeon3_pko_init_global(int node, int aura)
+{
+ int rc;
+
+ rc = hw_init_global(node, aura);
+ if (rc)
+ return rc;
+
+ /* Channel credit level at level 2 */
+ oct_csr_write(0, PKO_CHANNEL_LEVEL(node));
+
+ /* Configure the null mac */
+ rc = virtual_mac_config(node);
+ if (rc)
+ return rc;
+
+ return 0;
+}
+EXPORT_SYMBOL(octeon3_pko_init_global);
+
+int octeon3_pko_set_mac_options(int node, int interface, int index,
+ enum octeon3_mac_type mac_type, bool fcs_en,
+ bool pad_en, int fcs_sop_off)
+{
+ int fifo_num, mac;
+ u64 data;
+
+ mac = get_output_mac(interface, index, mac_type);
+
+ data = oct_csr_read(PKO_MAC_CFG(node, mac));
+ fifo_num = data & PKO_MAC_CFG_FIFO_NUM_MASK;
+ if (fifo_num == PKO_MAC_CFG_FIFO_UNDEFINED) {
+ pr_err("%s: mac not configured %d:%d:%d\n", __FILE__, node,
+ interface, index);
+ return -ENODEV;
+ }
+
+ /* Some silicon requires fifo_num=0x1f to change padding, fcs */
+ data &= ~PKO_MAC_CFG_FIFO_NUM_MASK;
+ data |= PKO_MAC_CFG_FIFO_UNDEFINED;
+
+ data &= ~(PKO_MAC_CFG_MIN_PAD_ENA | PKO_MAC_CFG_FCS_ENA |
+ PKO_MAC_CFG_FCS_SOP_OFF_MASK);
+ if (pad_en)
+ data |= PKO_MAC_CFG_MIN_PAD_ENA;
+ if (fcs_en)
+ data |= PKO_MAC_CFG_FCS_ENA;
+ if (fcs_sop_off)
+ data |= fcs_sop_off << PKO_MAC_CFG_FCS_SOP_OFF_SHIFT;
+
+ oct_csr_write(data, PKO_MAC_CFG(node, mac));
+
+ data &= ~PKO_MAC_CFG_FIFO_NUM_MASK;
+ data |= fifo_num;
+ oct_csr_write(data, PKO_MAC_CFG(node, mac));
+
+ return 0;
+}
+EXPORT_SYMBOL(octeon3_pko_set_mac_options);
+
+int octeon3_pko_get_fifo_size(int node, int interface, int index,
+ enum octeon3_mac_type mac_type)
+{
+ int fifo_grp, fifo_off, mac, size;
+ u64 data;
+
+ /* Set fifo size to 2.4 KB */
+ size = PKO_FIFO_SIZE;
+
+ mac = get_output_mac(interface, index, mac_type);
+
+ data = oct_csr_read(PKO_MAC_CFG(node, mac));
+ if ((data & PKO_MAC_CFG_FIFO_NUM_MASK) == PKO_MAC_CFG_FIFO_UNDEFINED) {
+ pr_err("%s: mac not configured %d:%d:%d\n", __FILE__, node,
+ interface, index);
+ return -ENODEV;
+ }
+ fifo_grp = (data & PKO_MAC_CFG_FIFO_NUM_MASK)
+ >> PKO_MAC_CFG_FIFO_GRP_SHIFT;
+ fifo_off = data & PKO_MAC_CFG_FIFO_OFF_MASK;
+
+ data = oct_csr_read(PKO_PTGF_CFG(node, fifo_grp));
+ data &= PKO_PTGF_CFG_SIZE_MASK;
+ switch (data) {
+ case 0:
+ /* 2.5l, 2.5k, 2.5k, 2.5k */
+ break;
+ case 1:
+ /* 5.0k, 0.0k, 2.5k, 2.5k */
+ if (fifo_off == 0)
+ size *= 2;
+ if (fifo_off == 1)
+ size = 0;
+ break;
+ case 2:
+ /* 2.5k, 2.5k, 5.0k, 0.0k */
+ if (fifo_off == 2)
+ size *= 2;
+ if (fifo_off == 3)
+ size = 0;
+ break;
+ case 3:
+ /* 5k, 0, 5k, 0 */
+ if ((fifo_off & 1) != 0)
+ size = 0;
+ size *= 2;
+ break;
+ case 4:
+ /* 10k, 0, 0, 0 */
+ if (fifo_off != 0)
+ size = 0;
+ size *= 4;
+ break;
+ default:
+ size = -1;
+ }
+
+ return size;
+}
+EXPORT_SYMBOL(octeon3_pko_get_fifo_size);
+
+int octeon3_pko_activate_dq(int node, int dq, int cnt)
+{
+ int i, rc = 0;
+ u64 data;
+
+ for (i = 0; i < cnt; i++) {
+ rc = open_dq(node, dq + i);
+ if (rc)
+ break;
+
+ data = oct_csr_read(PKO_PDM_DQ_MINPAD(node, dq + i));
+ data &= ~PKO_PDM_DQ_MINPAD_MINPAD;
+ oct_csr_write(data, PKO_PDM_DQ_MINPAD(node, dq + i));
+ }
+
+ return rc;
+}
+EXPORT_SYMBOL(octeon3_pko_activate_dq);
+
+int octeon3_pko_interface_init(int node, int interface, int index,
+ enum octeon3_mac_type mac_type, int ipd_port)
+{
+ int mac, pq, parent_q, queue, rc;
+ enum queue_level level;
+
+ mac = get_output_mac(interface, index, mac_type);
+
+ /* Allocate a port queue for this interface */
+ rc = allocate_queues(node, PQ, 1, &pq);
+ if (rc < 0) {
+ pr_err("%s: Failed to allocate port queue\n", __FILE__);
+ return rc;
+ }
+
+ /* Connect the port queue to the output mac */
+ port_queue_init(node, pq, mac);
+
+ /* Link scheduler queues to the port queue */
+ parent_q = pq;
+ for (level = L2_SQ; level <= max_sq_level(); level++) {
+ rc = allocate_queues(node, level, 1, &queue);
+ if (rc < 0) {
+ pr_err("%s: Failed to allocate queue\n", __FILE__);
+ return rc;
+ }
+
+ switch (level) {
+ case L2_SQ:
+ scheduler_queue_l2_init(node, queue, parent_q);
+ map_channel(node, pq, queue, ipd_port);
+ break;
+ case L3_SQ:
+ scheduler_queue_l3_init(node, queue, parent_q);
+ break;
+ case L4_SQ:
+ scheduler_queue_l4_init(node, queue, parent_q);
+ break;
+ case L5_SQ:
+ scheduler_queue_l5_init(node, queue, parent_q);
+ break;
+ default:
+ break;
+ }
+
+ parent_q = queue;
+ }
+
+ /* Link the descriptor queue */
+ rc = allocate_queues(node, DQ, 1, &queue);
+ if (rc < 0) {
+ pr_err("%s: Failed to allocate descriptor queue\n", __FILE__);
+ return rc;
+ }
+ descriptor_queue_init(node, &queue, parent_q, 1);
+
+ return queue;
+}
+EXPORT_SYMBOL(octeon3_pko_interface_init);
+
+int octeon3_pko_interface_uninit(int node, const int *dq, int num_dq)
+{
+ int i, parent_q, queue, rc;
+ enum queue_level level;
+ u64 addr, data, mask, shift = PKO_DQ_TOPOLOGY_PARENT_SHIFT;
+
+ /* Drain all dqs */
+ for (i = 0; i < num_dq; i++) {
+ rc = drain_dq(node, dq[i]);
+ if (rc)
+ return rc;
+
+ /* Free the dq */
+ data = oct_csr_read(PKO_DQ_TOPOLOGY(node, dq[i]));
+
+ parent_q = (data & PKO_DQ_TOPOLOGY_PARENT_MASK) >> shift;
+ free_queues(node, DQ, 1, &dq[i]);
+
+ /* Free all the scheduler queues */
+ queue = parent_q;
+ for (level = max_sq_level(); (signed int)level >= PQ; level--) {
+ switch (level) {
+ case L5_SQ:
+ addr = PKO_L5_SQ_TOPOLOGY(node, queue);
+ data = oct_csr_read(addr);
+ mask = PKO_DQ_TOPOLOGY_PARENT_MASK;
+ parent_q = (data & mask) >> shift;
+ break;
+
+ case L4_SQ:
+ addr = PKO_L4_SQ_TOPOLOGY(node, queue);
+ data = oct_csr_read(addr);
+ mask = PKO_L34_DQ_TOPOLOGY_PARENT_MASK;
+ parent_q = (data & mask) >> shift;
+ break;
+
+ case L3_SQ:
+ addr = PKO_L3_SQ_TOPOLOGY(node, queue);
+ data = oct_csr_read(addr);
+ mask = PKO_L34_DQ_TOPOLOGY_PARENT_MASK;
+ parent_q = (data & mask) >> shift;
+ break;
+
+ case L2_SQ:
+ addr = PKO_L2_SQ_TOPOLOGY(node, queue);
+ data = oct_csr_read(addr);
+ mask = PKO_L2_DQ_TOPOLOGY_PARENT_MASK;
+ parent_q = (data & mask) >> shift;
+ break;
+
+ case PQ:
+ break;
+
+ default:
+ pr_err("%s: Invalid lvl=%d\n", __FILE__, level);
+ return -1;
+ }
+
+ free_queues(node, level, 1, &queue);
+ queue = parent_q;
+ }
+ }
+
+ return 0;
+}
+EXPORT_SYMBOL(octeon3_pko_interface_uninit);
diff --git a/drivers/net/ethernet/cavium/octeon/octeon3-pko.h b/drivers/net/ethernet/cavium/octeon/octeon3-pko.h
new file mode 100644
index 0000000..f1053a8
--- /dev/null
+++ b/drivers/net/ethernet/cavium/octeon/octeon3-pko.h
@@ -0,0 +1,159 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Octeon III Packet-Output Processing Unit (PKO)
+ *
+ * Copyright (C) 2018 Cavium, Inc.
+ */
+#ifndef _OCTEON3_PKO_H_
+#define _OCTEON3_PKO_H_
+
+#include <linux/bitops.h>
+
+#define PKO_DPFI_ENA_ENABLE BIT(0)
+#define PKO_DPFI_FLUSH_FLUSH_EN BIT(0)
+#define PKO_DPFI_FPA_AURA_NODE_SHIFT 10
+#define PKO_DPFI_STATUS_CACHE_FLUSHED BIT(0)
+#define PKO_DQ_SCHEDULE_PRIO_SHIFT 24
+#define PKO_DQ_SW_XOFF_SC_RAM_FLIP BIT(2)
+#define PKO_DQ_SW_XOFF_SC_RAM_CDIS BIT(1)
+#define PKO_DQ_TOPOLOGY_PARENT_MASK GENMASK_ULL(25, 16)
+#define PKO_DQ_TOPOLOGY_PARENT_SHIFT 16
+#define PKO_DQ_WM_CTL_KIND BIT(49)
+#define PKO_ENABLE_ENABLE BIT(0)
+#define PKO_L12_SQ_TOPOLOGY_PRIO_ANCHOR_MASK GENMASK_ULL(40, 32)
+#define PKO_L1_SQ_LINK_LINK_SHIFT 44
+#define PKO_L1_SQ_SHAPE_LINK_SHIFT 13
+#define PKO_L2_DQ_TOPOLOGY_PARENT_MASK GENMASK_ULL(20, 16)
+#define PKO_L1_SQ_TOPOLOGY_LINK_SHIFT 16
+#define PKO_L345_SQ_TOPOLOGY_PRIO_ANCHOR_MASK GENMASK_ULL(41, 32)
+#define PKO_L34_DQ_TOPOLOGY_PARENT_MASK GENMASK_ULL(24, 16)
+#define PKO_L3_L2_CHANNEL_CC_CHANNEL_MASK GENMASK_ULL(43, 32)
+#define PKO_L3_L2_CHANNEL_CC_CHANNEL_SHIFT 32
+#define PKO_LUT_PORT_MASK 0xf00
+#define PKO_LUT_PORT_TO_INDEX 0x800
+#define PKO_LUT_PQ_IDX_SHIFT 9
+#define PKO_LUT_QUEUE_NUMBER_MASK GENMASK_ULL(7, 0)
+#define PKO_LUT_QUEUE_NUMBER_SHIFT 8
+#define PKO_LUT_VALID BIT(15)
+#define PKO_MAC_CFG_FCS_ENA BIT(15)
+#define PKO_MAC_CFG_FCS_SOP_OFF_MASK GENMASK_ULL(14, 7)
+#define PKO_MAC_CFG_FCS_SOP_OFF_SHIFT 7
+#define PKO_MAC_CFG_FIFO_GRP_SHIFT 2
+#define PKO_MAC_CFG_FIFO_NUM_MASK GENMASK_ULL(4, 0)
+#define PKO_MAC_CFG_FIFO_NUM_UNDEFINED 0x1f
+#define PKO_MAC_CFG_FIFO_OFF_MASK GENMASK_ULL(1, 0)
+#define PKO_MAC_CFG_FIFO_UNDEFINED 0x1f
+#define PKO_MAC_CFG_MIN_PAD_ENA BIT(16)
+#define PKO_MAC_CFG_SKID_MAX_CNT_MASK GENMASK_ULL(6, 5)
+#define PKO_PDM_DQ_MINPAD_MINPAD BIT(0)
+#define PKO_PDM_PAD_MINLEN_SHIFT 3
+#define PKO_PDM_PAD_MINLEN_TYPICAL 60
+#define PKO_PTF_IOBP_CFG_MAX_READ_SIZE_MASK GENMASK_ULL(6, 0)
+#define PKO_PTF_STATUS_IN_FLIGHT_CNT_MASK GENMASK_ULL(11, 5)
+#define PKO_PTF_STATUS_MAC_NUM_MASK GENMASK_ULL(4, 0)
+#define PKO_PTGF_CFG_RATE_MASK GENMASK_ULL(5, 3)
+#define PKO_PTGF_CFG_RATE_SHIFT 3
+#define PKO_PTGF_CFG_RESET BIT(6)
+#define PKO_PTGF_CFG_SIZE_MASK GENMASK_ULL(2, 0)
+#define PKO_QUERY_DMA_SCRADDR_SHIFT 56
+#define PKO_QUERY_DMA_RTNLEN_SHIFT 48
+#define PKO_QUERY_DMA_DID_SHIFT 40
+#define PKO_QUERY_DMA_NODE_SHIFT 36
+#define PKO_QUERY_DMA_DQOP_SHIFT 32
+#define PKO_QUERY_DMA_DQ_SHIFT 16
+#define PKO_QUERY_RTN_DEPTH_MASK GENMASK_ULL(47, 0)
+#define PKO_QUERY_RTN_DQSTATUS_MASK GENMASK_ULL(63, 60)
+#define PKO_QUERY_RTN_DQSTATUS_SHIFT 60
+
+/* PKO_DQ_STATUS_E */
+enum pko_dq_status_e {
+ PKO_DQSTATUS_PASS = 0,
+ PKO_DQSTATUS_BADSTATE = 8,
+ PKO_DQSTATUS_NOFPABUF,
+ PKO_DQSTATUS_NOPKOBUF,
+ PKO_DQSTATUS_FAILRTNPTR,
+ PKO_DQSTATUS_ALREADYCREATED,
+ PKO_DQSTATUS_NOTCREATED,
+ PKO_DQSTATUS_NOTEMPTY,
+ PKO_DQSTATUS_SENDPKTDROP
+};
+
+#define PKO_SHAPER_CFG_COLOR_AWARE BIT(1)
+#define PKO_SQ_TOPOLOGY_LINK_SHIFT 16
+#define PKO_SQ_TOPOLOGY_PRIO_ANCHOR_SHIFT 32
+#define PKO_SQ_TOPOLOGY_RR_PRIO_MASK GENMASK_ULL(4, 1)
+#define PKO_SQ_TOPOLOGY_RR_PRIO_SHAPER 0xf
+#define PKO_SQ_TOPOLOGY_RR_PRIO_SHIFT 1
+#define PKO_STATUS_PKO_RDY BIT(63)
+
+#define PKO_SEND_HDR_AURA_SHIFT 48
+#define PKO_SEND_HDR_CKL4_SHIFT 46
+#define PKO_SEND_HDR_CKL3 BIT(45)
+#define PKO_SEND_HDR_LE BIT(43)
+#define PKO_SEND_HDR_N2 BIT(42)
+#define PKO_SEND_HDR_DF BIT(40)
+#define PKO_SEND_HDR_L4PTR_MASK GENMASK_ULL(31, 24)
+#define PKO_SEND_HDR_L4PTR_SHIFT 24
+#define PKO_SEND_HDR_L3PTR_SHIFT 16
+
+#define PKO_SEND_SUBDC4_SHIFT 44
+#define PKO_SEND_EXT_RA_SHIFT 40
+#define PKO_SEND_EXT_TSTMP BIT(39)
+#define PKO_SEND_EXT_MARKPTR_SHIFT 16
+
+/* PKO_REDALG_E */
+enum pko_redalg_e {
+ PKO_REDALG_E_STD,
+ PKO_REDALG_E_SEND,
+ PKO_REDALG_E_STALL,
+ PKO_REDALG_E_DISCARD
+};
+
+#define PKO_SEND_TSO_L2LEN_SHIFT 56
+#define PKO_SEND_TSO_SB_SHIFT 24
+#define PKO_SEND_TSO_MSS_SHIFT 8
+
+#define PKO_SEND_MEM_WMEM_SHIFT 62
+#define PKO_SEND_MEM_DSZ_SHIFT 60
+#define PKO_SEND_MEM_ALG_SHIFT 56
+#define PKO_SEND_MEM_OFFSET_SHIFT 48
+
+/* PKO_MEMALG_E */
+enum pko_memalg_e {
+ PKO_MEMALG_SET = 0,
+ PKO_MEMALG_SETTSTMP,
+ PKO_MEMALG_SETRSLT,
+ PKO_MEMALG_ADD = 8,
+ PKO_MEMALG_SUB,
+ PKO_MEMALG_ADDLEN,
+ PKO_MEMALG_SUBLEN,
+ PKO_MEMALG_ADDMBUF,
+ PKO_MEMALG_SUBMBUF
+};
+
+/* PKO_MEMDSZ_E */
+enum pko_memdsz_e {
+ PKO_MEMDSZ_B64,
+ PKO_MEMDSZ_B32,
+ PKO_MEMDSZ_B16,
+ PKO_MEMDSZ_B8
+};
+
+#define PKO_SEND_WORK_GRP_SHIFT 52
+#define PKO_SEND_WORK_TT_SHIFT 50
+
+#define PKO_SEND_GATHER_SIZE_MASK GENMASK_ULL(63, 48)
+#define PKO_SEND_GATHER_SIZE_SHIFT 48
+#define PKO_SEND_GATHER_SUBDC_SHIFT 45
+#define PKO_SEND_GATHER_ADDR_MASK GENMASK_ULL(41, 0)
+
+/* Pko sub-command three bit codes (SUBDC3) */
+#define PKO_SENDSUBDC_GATHER 0x1
+
+/* Pko sub-command four bit codes (SUBDC4) */
+#define PKO_SENDSUBDC_TSO 0x8
+#define PKO_SENDSUBDC_FREE 0x9
+#define PKO_SENDSUBDC_WORK 0xa
+#define PKO_SENDSUBDC_MEM 0xc
+#define PKO_SENDSUBDC_EXT 0xd
+
+#endif /* _OCTEON3_PKO_H_ */
--
2.1.4
^ permalink raw reply related
* [PATCH v12 07/10] netdev: cavium: octeon: Add Octeon III SSO Support
From: Steven J. Hill @ 2018-06-27 21:25 UTC (permalink / raw)
To: netdev; +Cc: Carlos Munoz, Chandrakala Chavva, Steven J. Hill
In-Reply-To: <1530134719-19407-1-git-send-email-steven.hill@cavium.com>
From: Carlos Munoz <cmunoz@cavium.com>
Add support for Octeon III SSO logic block for BGX Ethernet.
Signed-off-by: Carlos Munoz <cmunoz@cavium.com>
Signed-off-by: Steven J. Hill <Steven.Hill@cavium.com>
---
drivers/net/ethernet/cavium/octeon/octeon3-sso.c | 221 +++++++++++++++++++++++
drivers/net/ethernet/cavium/octeon/octeon3-sso.h | 89 +++++++++
2 files changed, 310 insertions(+)
create mode 100644 drivers/net/ethernet/cavium/octeon/octeon3-sso.c
create mode 100644 drivers/net/ethernet/cavium/octeon/octeon3-sso.h
diff --git a/drivers/net/ethernet/cavium/octeon/octeon3-sso.c b/drivers/net/ethernet/cavium/octeon/octeon3-sso.c
new file mode 100644
index 0000000..73afad0
--- /dev/null
+++ b/drivers/net/ethernet/cavium/octeon/octeon3-sso.c
@@ -0,0 +1,221 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Octeon III Schedule/Synchronize/Order Unit (SSO)
+ *
+ * Copyright (C) 2018 Cavium, Inc.
+ */
+
+#include "octeon3.h"
+
+static int octeon3_sso_get_num_groups(void)
+{
+ if (OCTEON_IS_MODEL(OCTEON_CN78XX))
+ return 256;
+ if (OCTEON_IS_MODEL(OCTEON_CNF75XX) || OCTEON_IS_MODEL(OCTEON_CN73XX))
+ return 64;
+ return 0;
+}
+
+void octeon3_sso_irq_set(int node, int group, bool enable)
+{
+ if (enable)
+ oct_csr_write(1, SSO_GRP_INT_THR(node, group));
+ else
+ oct_csr_write(0, SSO_GRP_INT_THR(node, group));
+
+ oct_csr_write(SSO_GRP_INT_EXE_INT, SSO_GRP_INT(node, group));
+}
+EXPORT_SYMBOL(octeon3_sso_irq_set);
+
+/* octeon3_sso_alloc_groups - Allocate a range of SSO groups.
+ * @node: Node where SSO resides.
+ * @groups: Pointer to allocated groups.
+ * @cnt: Number of groups to allocate.
+ * @start: Group number to start sequential allocation from. -1 for don't care.
+ *
+ * Returns 0 if successful, error code otherwise..
+ */
+int octeon3_sso_alloc_groups(int node, int *groups, int cnt, int start)
+{
+ struct global_resource_tag tag;
+ int group, ret;
+ char buf[16];
+
+ strncpy((char *)&tag.lo, "cvm_sso_", 8);
+ snprintf(buf, 16, "0%d......", node);
+ memcpy(&tag.hi, buf, 8);
+
+ res_mgr_create_resource(tag, octeon3_sso_get_num_groups());
+
+ if (!groups)
+ ret = res_mgr_alloc_range(tag, start, cnt, false, &group);
+ if (!ret)
+ ret = group;
+ else
+ ret = res_mgr_alloc_range(tag, start, cnt, false, groups);
+
+ return ret;
+}
+EXPORT_SYMBOL(octeon3_sso_alloc_groups);
+
+/* octeon3_sso_free_groups - Free SSO groups.
+ * @node: Node where SSO resides.
+ * @groups: Array of groups to free.
+ * @cnt: Number of groups to free.
+ */
+void octeon3_sso_free_groups(int node, int *groups, int cnt)
+{
+ struct global_resource_tag tag;
+ char buf[16];
+
+ /* Allocate the requested groups. */
+ strncpy((char *)&tag.lo, "cvm_sso_", 8);
+ snprintf(buf, 16, "0%d......", node);
+ memcpy(&tag.hi, buf, 8);
+
+ res_mgr_free_range(tag, groups, cnt);
+}
+EXPORT_SYMBOL(octeon3_sso_free_groups);
+
+/* octeon3_sso_pass1_limit - When the Transitory Admission Queue (TAQ) is
+ * almost full, it is possible for the SSo to hang. We work around this
+ * by ensuring that the sum of SSO_GRP(0..255)_TAQ_THR[MAX_THR] of all
+ * used groups is <= 1264. This may reduce single group performance when
+ * many groups are in use.
+ * @node: Node to update.
+ * @grp: SSO group to update.
+ */
+void octeon3_sso_pass1_limit(int node, int group)
+{
+ u64 max_thr, rsvd_thr, taq_add, taq_thr;
+
+ /* Ideally we would like to divide the maximum number of TAQ buffers
+ * (1264) among the SSO groups in use. However, since we do not know
+ * how many SSO groups are used by code outside this driver, we take
+ * the worst case approach.
+ */
+ max_thr = 1264 / octeon3_sso_get_num_groups();
+ if (max_thr < 4)
+ max_thr = 4;
+ rsvd_thr = max_thr - 1;
+
+ /* Changes to SSO_GRP_TAQ_THR[rsvd_thr] must also update
+ * SSO_TAQ_ADD[RSVD_FREE].
+ */
+ taq_thr = oct_csr_read(SSO_GRP_TAQ_THR(node, group));
+ taq_add = (rsvd_thr - (taq_thr & SSO_GRP_TAQ_THR_RSVD_THR_MASK)) <<
+ SSO_TAQ_ADD_RSVD_FREE_SHIFT;
+
+ taq_thr &= ~(SSO_GRP_TAQ_THR_MAX_THR_MASK |
+ SSO_GRP_TAQ_THR_RSVD_THR_MASK);
+ taq_thr |= max_thr << SSO_GRP_TAQ_THR_RSVD_THR_SHIFT;
+ taq_thr |= rsvd_thr;
+
+ oct_csr_write(taq_thr, SSO_GRP_TAQ_THR(node, group));
+ oct_csr_write(taq_add, SSO_TAQ_ADD(node));
+}
+EXPORT_SYMBOL(octeon3_sso_pass1_limit);
+
+/* octeon3_sso_shutdown - Shutdown the SSO.
+ * @node: Node where SSO to disable is.
+ * @aura: Aura used for the SSO buffers.
+ */
+void octeon3_sso_shutdown(int node, int aura)
+{
+ int i, max_grps, timeout;
+ u64 data, head, tail;
+ void *ptr;
+
+ /* Disable SSO. */
+ data = oct_csr_read(SSO_AW_CFG(node));
+ data |= SSO_AW_CFG_XAQ_ALOC_DIS | SSO_AW_CFG_XAQ_BYP_DIS;
+ data &= ~SSO_AW_CFG_RWEN;
+ oct_csr_write(data, SSO_AW_CFG(node));
+
+ /* Extract the FPA buffers. */
+ max_grps = octeon3_sso_get_num_groups();
+ for (i = 0; i < max_grps; i++) {
+ head = oct_csr_read(SSO_XAQ_HEAD_PTR(node, i));
+ tail = oct_csr_read(SSO_XAQ_TAIL_PTR(node, i));
+ data = oct_csr_read(SSO_GRP_AQ_CNT(node, i));
+
+ /* Verify pointers. */
+ head &= SSO_XAQ_PTR_MASK;
+ tail &= SSO_XAQ_PTR_MASK;
+ if (head != tail) {
+ pr_err("%s: Bad pointer\n", __func__);
+ continue;
+ }
+
+ /* This SSO group should have no pending entries. */
+ if (data & SSO_GRP_AQ_CNT_AQ_CNT_MASK)
+ pr_err("%s: Group not empty\n", __func__);
+
+ ptr = phys_to_virt(head);
+ octeon_fpa3_free(node, aura, ptr);
+
+ /* Clear pointers. */
+ oct_csr_write(0, SSO_XAQ_HEAD_PTR(node, i));
+ oct_csr_write(0, SSO_XAQ_HEAD_NEXT(node, i));
+ oct_csr_write(0, SSO_XAQ_TAIL_PTR(node, i));
+ oct_csr_write(0, SSO_XAQ_TAIL_NEXT(node, i));
+ }
+
+ /* Make sure all buffers are drained. */
+ timeout = 10000;
+ do {
+ data = oct_csr_read(SSO_AW_STATUS(node));
+ if ((data & SSO_AW_STATUS_XAQ_BU_CACHED_MASK) == 0)
+ break;
+ timeout--;
+ udelay(1);
+ } while (timeout);
+ if (!timeout)
+ pr_err("%s: Timed out draining buffers\n", __func__);
+}
+EXPORT_SYMBOL(octeon3_sso_shutdown);
+
+/* octeon3_sso_init - Initialize the SSO.
+ * @node: Node where SSO resides.
+ * @aura: Aura used for the SSO buffers.
+ */
+int octeon3_sso_init(int node, int aura)
+{
+ int i, max_grps, err = 0;
+ u64 data, phys;
+ void *mem;
+
+ data = SSO_AW_CFG_STT | SSO_AW_CFG_LDT | SSO_AW_CFG_LDWB;
+ oct_csr_write(data, SSO_AW_CFG(node));
+
+ data = (node << SSO_XAQ_AURA_NODE_SHIFT) | aura;
+ oct_csr_write(data, SSO_XAQ_AURA(node));
+
+ max_grps = octeon3_sso_get_num_groups();
+ for (i = 0; i < max_grps; i++) {
+ mem = octeon_fpa3_alloc(node, aura);
+ if (!mem) {
+ err = -ENOMEM;
+ goto out;
+ }
+
+ phys = virt_to_phys(mem);
+ oct_csr_write(phys, SSO_XAQ_HEAD_PTR(node, i));
+ oct_csr_write(phys, SSO_XAQ_HEAD_NEXT(node, i));
+ oct_csr_write(phys, SSO_XAQ_TAIL_PTR(node, i));
+ oct_csr_write(phys, SSO_XAQ_TAIL_NEXT(node, i));
+
+ /* SSO-18678 */
+ data = SSO_GRP_PRI_WEIGHT_MAXIMUM << SSO_GRP_PRI_WEIGHT_SHIFT;
+ oct_csr_write(data, SSO_GRP_PRI(node, i));
+ }
+
+ data = SSO_ERR0_FPE;
+ oct_csr_write(data, SSO_ERR0(node));
+
+ data = SSO_AW_CFG_STT | SSO_AW_CFG_LDT | SSO_AW_CFG_LDWB |
+ SSO_AW_CFG_RWEN;
+ oct_csr_write(data, SSO_AW_CFG(node));
+out:
+ return err;
+}
+EXPORT_SYMBOL(octeon3_sso_init);
diff --git a/drivers/net/ethernet/cavium/octeon/octeon3-sso.h b/drivers/net/ethernet/cavium/octeon/octeon3-sso.h
new file mode 100644
index 0000000..dc68c4b
--- /dev/null
+++ b/drivers/net/ethernet/cavium/octeon/octeon3-sso.h
@@ -0,0 +1,89 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Octeon III Schedule/Synchronize/Order Unit (SSO)
+ *
+ * Copyright (C) 2018 Cavium, Inc.
+ */
+#ifndef _OCTEON3_SSO_H_
+#define _OCTEON3_SSO_H_
+
+#include <linux/bitops.h>
+
+#define SSO_BASE 0x1670000000000ull
+#define SSO_ADDR(n) (SSO_BASE + SET_XKPHYS + NODE_OFFSET(n))
+#define SSO_AQ_ADDR(n, a) (SSO_ADDR(n) + ((a) << 3))
+#define SSO_GRP_ADDR(n, g) (SSO_ADDR(n) + ((g) << 16))
+
+#define SSO_AW_STATUS(n) (SSO_ADDR(n) + 0x000010e0)
+#define SSO_AW_CFG(n) (SSO_ADDR(n) + 0x000010f0)
+#define SSO_ERR0(n) (SSO_ADDR(n) + 0x00001240)
+#define SSO_TAQ_ADD(n) (SSO_ADDR(n) + 0x000020e0)
+#define SSO_XAQ_AURA(n) (SSO_ADDR(n) + 0x00002100)
+
+#define SSO_XAQ_HEAD_PTR(n, a) (SSO_AQ_ADDR(n, a) + 0x00080000)
+#define SSO_XAQ_TAIL_PTR(n, a) (SSO_AQ_ADDR(n, a) + 0x00090000)
+#define SSO_XAQ_HEAD_NEXT(n, a) (SSO_AQ_ADDR(n, a) + 0x000a0000)
+#define SSO_XAQ_TAIL_NEXT(n, a) (SSO_AQ_ADDR(n, a) + 0x000b0000)
+
+#define SSO_GRP_TAQ_THR(n, g) (SSO_GRP_ADDR(n, g) + 0x20000100)
+#define SSO_GRP_PRI(n, g) (SSO_GRP_ADDR(n, g) + 0x20000200)
+#define SSO_GRP_INT(n, g) (SSO_GRP_ADDR(n, g) + 0x20000400)
+#define SSO_GRP_INT_THR(n, g) (SSO_GRP_ADDR(n, g) + 0x20000500)
+#define SSO_GRP_AQ_CNT(n, g) (SSO_GRP_ADDR(n, g) + 0x20000700)
+
+/* SSO interrupt numbers start here */
+#define SSO_IRQ_START 0x61000
+
+#define SSO_AW_STATUS_XAQ_BU_CACHED_MASK GENMASK_ULL(5, 0)
+
+#define SSO_AW_CFG_XAQ_ALOC_DIS BIT(6)
+#define SSO_AW_CFG_XAQ_BYP_DIS BIT(4)
+#define SSO_AW_CFG_STT BIT(3)
+#define SSO_AW_CFG_LDT BIT(2)
+#define SSO_AW_CFG_LDWB BIT(1)
+#define SSO_AW_CFG_RWEN BIT(0)
+
+#define SSO_ERR0_FPE BIT(0)
+
+#define SSO_TAQ_ADD_RSVD_FREE_SHIFT 16
+
+#define SSO_XAQ_AURA_NODE_SHIFT 10
+
+#define SSO_XAQ_PTR_MASK GENMASK_ULL(41, 7)
+
+#define SSO_GRP_TAQ_THR_MAX_THR_MASK GENMASK_ULL(42, 32)
+#define SSO_GRP_TAQ_THR_RSVD_THR_MASK GENMASK_ULL(10, 0)
+#define SSO_GRP_TAQ_THR_RSVD_THR_SHIFT 32
+
+#define SSO_GRP_PRI_WEIGHT_MAXIMUM 63
+#define SSO_GRP_PRI_WEIGHT_SHIFT 16
+
+#define SSO_GRP_INT_EXE_INT BIT(1)
+
+#define SSO_GRP_AQ_CNT_AQ_CNT_MASK GENMASK_ULL(32, 0)
+
+/* SSO tag types */
+#define SSO_TAG_TYPE_ORDERED 0ull
+#define SSO_TAG_TYPE_ATOMIC 1ull
+#define SSO_TAG_TYPE_UNTAGGED 2ull
+#define SSO_TAG_TYPE_EMPTY 3ull
+#define SSO_TAG_SWDID 0x60ull
+
+
+/* SSO work queue bitfields */
+#define SSO_GET_WORK_DID_SHIFT 40
+#define SSO_GET_WORK_NODE_SHIFT 36
+#define SSO_GET_WORK_GROUPED BIT(30)
+#define SSO_GET_WORK_RTNGRP BIT(29)
+#define SSO_GET_WORK_IDX_GRP_MASK_SHIFT 4
+#define SSO_GET_WORK_WAITW_WAIT BIT(3)
+#define SSO_GET_WORK_WAITW_NO_WAIT 0ull
+
+#define SSO_GET_WORK_DMA_S_SCRADDR BIT(63)
+#define SSO_GET_WORK_DMA_S_LEN_SHIFT 48
+#define SSO_GET_WORK_LD_S_IO BIT(48)
+#define SSO_GET_WORK_RTN_S_NO_WORK BIT(63)
+#define SSO_GET_WORK_RTN_S_GRP_MASK GENMASK_ULL(57, 48)
+#define SSO_GET_WORK_RTN_S_GRP_SHIFT 48
+#define SSO_GET_WORK_RTN_S_WQP_MASK GENMASK_ULL(41, 0)
+
+#endif /* _OCTEON3_SSO_H_ */
--
2.1.4
^ permalink raw reply related
* [PATCH v12 08/10] netdev: cavium: octeon: Add Octeon III BGX Ethernet core
From: Steven J. Hill @ 2018-06-27 21:25 UTC (permalink / raw)
To: netdev; +Cc: Carlos Munoz, Chandrakala Chavva, Steven J. Hill
In-Reply-To: <1530134719-19407-1-git-send-email-steven.hill@cavium.com>
From: Carlos Munoz <cmunoz@cavium.com>
This is the main core of the BGX Ethernet driver.
Signed-off-by: Carlos Munoz <cmunoz@cavium.com>
Signed-off-by: Steven J. Hill <Steven.Hill@cavium.com>
---
drivers/net/ethernet/cavium/octeon/octeon3-core.c | 2363 +++++++++++++++++++++
1 file changed, 2363 insertions(+)
create mode 100644 drivers/net/ethernet/cavium/octeon/octeon3-core.c
diff --git a/drivers/net/ethernet/cavium/octeon/octeon3-core.c b/drivers/net/ethernet/cavium/octeon/octeon3-core.c
new file mode 100644
index 0000000..b0dfacb
--- /dev/null
+++ b/drivers/net/ethernet/cavium/octeon/octeon3-core.c
@@ -0,0 +1,2363 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Octeon III BGX Nexus Ethernet driver core
+ *
+ * Copyright (C) 2018 Cavium, Inc.
+ */
+#include <linux/etherdevice.h>
+#include <linux/if_vlan.h>
+#include <linux/ip.h>
+#include <linux/ipv6.h>
+#include <linux/kthread.h>
+#include <linux/net_tstamp.h>
+#include <linux/ptp_clock_kernel.h>
+#include <linux/timecounter.h>
+
+#include <asm/octeon/cvmx-mio-defs.h>
+
+#include "octeon3.h"
+
+/* First buffer:
+ *
+ * +---SKB---------+
+ * | |
+ * | |
+ * +--+--*data |
+ * | | |
+ * | | |
+ * | +---------------+
+ * | /|\
+ * | |
+ * | |
+ * \|/ |
+ * WQE - 128 -+-----> +-------------+-------+ -+-
+ * | | *skb ----+ | |
+ * | | | |
+ * | | | |
+ * WQE_SKIP = 128 | | |
+ * | | | |
+ * | | | |
+ * | | | |
+ * | | | First Skip
+ * WQE -----+-----> +---------------------+ |
+ * | word 0 | |
+ * | word 1 | |
+ * | word 2 | |
+ * | word 3 | |
+ * | word 4 | |
+ * +---------------------+ -+-
+ * +----+- packet link |
+ * | | packet data |
+ * | | |
+ * | | |
+ * | | . |
+ * | | . |
+ * | | . |
+ * | +---------------------+
+ * |
+ * |
+ * Later buffers:|
+ * |
+ * |
+ * |
+ * |
+ * |
+ * | +---SKB---------+
+ * | | |
+ * | | |
+ * | +--+--*data |
+ * | | | |
+ * | | | |
+ * | | +---------------+
+ * | | /|\
+ * | | |
+ * | | |
+ * | \|/ |
+ * WQE - 128 ----+--> +-------------+-------+ -+-
+ * | | *skb ----+ | |
+ * | | | |
+ * | | | |
+ * | | | |
+ * | | | LATER_SKIP = 128
+ * | | | |
+ * | | | |
+ * | | | |
+ * | +---------------------+ -+-
+ * | | packet link |
+ * +--> | packet data |
+ * | |
+ * | |
+ * | . |
+ * | . |
+ * | . |
+ * +---------------------+
+ */
+
+#define FPA3_NUM_AURAS 1024
+#define MAX_TX_QUEUE_DEPTH 512
+#define MAX_RX_CONTEXTS 32
+#define USE_ASYNC_IOBDMA 1 /* Always 1 */
+
+#define SKB_AURA_MAGIC 0xbadc0ffee4dad000ULL
+#define SKB_AURA_OFFSET 1
+#define SKB_PTR_OFFSET 0
+
+/* PTP registers and bits */
+#define MIO_PTP_CLOCK_HI(n) (CVMX_MIO_PTP_CLOCK_HI + NODE_OFFSET(n))
+#define MIO_PTP_CLOCK_CFG(n) (CVMX_MIO_PTP_CLOCK_CFG + NODE_OFFSET(n))
+#define MIO_PTP_CLOCK_COMP(n) (CVMX_MIO_PTP_CLOCK_COMP + NODE_OFFSET(n))
+
+/* Misc. bitfields */
+#define MIO_PTP_CLOCK_CFG_PTP_EN BIT(0)
+#define BGX_GMP_GMI_RX_FRM_CTL_PTP_MODE BIT(12)
+
+/* Up to 2 napis per core are supported */
+#define MAX_NAPI_PER_CPU 2
+#define MAX_NAPIS_PER_NODE (MAX_CORES * MAX_NAPI_PER_CPU)
+
+struct octeon3_napi_wrapper {
+ struct napi_struct napi;
+ int available;
+ int idx;
+ int cpu;
+ struct octeon3_rx *cxt;
+} ____cacheline_aligned_in_smp;
+
+static struct octeon3_napi_wrapper
+napi_wrapper[MAX_NODES][MAX_NAPIS_PER_NODE]
+__cacheline_aligned_in_smp;
+
+struct octeon3_ethernet;
+
+struct octeon3_rx {
+ struct octeon3_napi_wrapper *napiw;
+ DECLARE_BITMAP(napi_idx_bitmap, MAX_CORES);
+ spinlock_t napi_idx_lock; /* Protect the napi index bitmap */
+ struct octeon3_ethernet *parent;
+ int rx_grp;
+ int rx_irq;
+ cpumask_t rx_affinity_hint;
+};
+
+struct octeon3_ethernet {
+ struct bgx_port_netdev_priv bgx_priv; /* Must be first element. */
+ struct list_head list;
+ struct net_device *netdev;
+ enum octeon3_mac_type mac_type;
+ struct octeon3_rx rx_cxt[MAX_RX_CONTEXTS];
+ struct ptp_clock_info ptp_info;
+ struct ptp_clock *ptp_clock;
+ struct cyclecounter cc;
+ struct timecounter tc;
+ spinlock_t ptp_lock; /* Serialize ptp clock adjustments */
+ int num_rx_cxt;
+ int pki_aura;
+ int pknd;
+ int pko_queue;
+ int node;
+ int interface;
+ int index;
+ int rx_buf_count;
+ int tx_complete_grp;
+ int rx_timestamp_hw:1;
+ int tx_timestamp_hw:1;
+ spinlock_t stat_lock; /* Protects stats counters */
+ u64 last_packets;
+ u64 last_octets;
+ u64 last_dropped;
+ atomic64_t rx_packets;
+ atomic64_t rx_octets;
+ atomic64_t rx_dropped;
+ atomic64_t rx_errors;
+ atomic64_t rx_length_errors;
+ atomic64_t rx_crc_errors;
+ atomic64_t tx_packets;
+ atomic64_t tx_octets;
+ atomic64_t tx_dropped;
+ /* The following two fields need to be on a different cache line as
+ * they are updated by pko which invalidates the cache every time it
+ * updates them. The idea is to prevent other fields from being
+ * invalidated unnecessarily.
+ */
+ char cacheline_pad1[CVMX_CACHE_LINE_SIZE];
+ atomic64_t buffers_needed;
+ atomic64_t tx_backlog;
+ char cacheline_pad2[CVMX_CACHE_LINE_SIZE];
+};
+
+static DEFINE_MUTEX(octeon3_eth_init_mutex);
+
+struct octeon3_ethernet_node;
+
+struct octeon3_ethernet_worker {
+ wait_queue_head_t queue;
+ struct task_struct *task;
+ struct octeon3_ethernet_node *oen;
+ atomic_t kick;
+ int order;
+};
+
+struct octeon3_ethernet_node {
+ bool init_done;
+ bool napi_init_done;
+ int next_cpu_irq_affinity;
+ int node;
+ int pki_packet_pool;
+ int sso_pool;
+ int pko_pool;
+ void *sso_pool_stack;
+ void *pko_pool_stack;
+ void *pki_packet_pool_stack;
+ int sso_aura;
+ int pko_aura;
+ int tx_complete_grp;
+ int tx_irq;
+ cpumask_t tx_affinity_hint;
+ struct octeon3_ethernet_worker workers[8];
+ struct mutex device_list_lock; /* Protects the device list */
+ struct list_head device_list;
+ spinlock_t napi_alloc_lock; /* Protects napi allocations */
+};
+
+/* This array keeps track of the number of napis running on each cpu */
+static u8 octeon3_cpu_napi_cnt[NR_CPUS] __cacheline_aligned_in_smp;
+
+static int use_tx_queues;
+module_param(use_tx_queues, int, 0644);
+MODULE_PARM_DESC(use_tx_queues, "Use network layer transmit queues.");
+
+static int wait_pko_response;
+module_param(wait_pko_response, int, 0644);
+MODULE_PARM_DESC(wait_pko_response, "Wait for response after each pko command.");
+
+static int num_packet_buffers = 768;
+module_param(num_packet_buffers, int, 0444);
+MODULE_PARM_DESC(num_packet_buffers, "Number of packet buffers to allocate per port.");
+
+static int packet_buffer_size = 2048;
+module_param(packet_buffer_size, int, 0444);
+MODULE_PARM_DESC(packet_buffer_size, "Size of each RX packet buffer.");
+
+static int rx_contexts = 1;
+module_param(rx_contexts, int, 0444);
+MODULE_PARM_DESC(rx_contexts, "Number of RX threads per port.");
+
+int ilk0_lanes = 1;
+module_param(ilk0_lanes, int, 0444);
+MODULE_PARM_DESC(ilk0_lanes, "Number of SerDes lanes used by ILK link 0.");
+
+int ilk1_lanes = 1;
+module_param(ilk1_lanes, int, 0444);
+MODULE_PARM_DESC(ilk1_lanes, "Number of SerDes lanes used by ILK link 1.");
+
+static struct octeon3_ethernet_node octeon3_eth_node[MAX_NODES];
+static struct kmem_cache *octeon3_eth_sso_pko_cache;
+
+/* Reads a 64 bit value from the processor local scratchpad memory.
+ *
+ * @param offset byte offset into scratch pad to read
+ *
+ * @return value read
+ */
+static inline u64 scratch_read64(u64 offset)
+{
+ return *(u64 *)((long)SCRATCH_BASE_ADDR + offset);
+}
+
+/* Write a 64 bit value to the processor local scratchpad memory.
+ *
+ * @param offset byte offset into scratch pad to write
+ @ @praram value to write
+ */
+static inline void scratch_write64(u64 offset, u64 value)
+{
+ *(u64 *)((long)SCRATCH_BASE_ADDR + offset) = value;
+}
+
+static int get_pki_chan(int node, int interface, int index)
+{
+ int pki_chan;
+
+ pki_chan = node << 12;
+
+ if (OCTEON_IS_MODEL(OCTEON_CNF75XX) &&
+ (interface == 1 || interface == 2)) {
+ /* SRIO */
+ pki_chan |= 0x240 + (2 * (interface - 1)) + index;
+ } else {
+ /* BGX */
+ pki_chan |= 0x800 + (0x100 * interface) + (0x10 * index);
+ }
+
+ return pki_chan;
+}
+
+/* Map auras to the field priv->buffers_needed. Used to speed up packet
+ * transmission.
+ */
+static void *aura2bufs_needed[MAX_NODES][FPA3_NUM_AURAS];
+
+static int octeon3_eth_lgrp_to_ggrp(int node, int grp)
+{
+ return (node << 8) | grp;
+}
+
+static void octeon3_eth_gen_affinity(int node, cpumask_t *mask)
+{
+ int cpu;
+
+ do {
+ cpu = cpumask_next(octeon3_eth_node[node].next_cpu_irq_affinity,
+ cpu_online_mask);
+ octeon3_eth_node[node].next_cpu_irq_affinity++;
+ if (cpu >= nr_cpu_ids) {
+ octeon3_eth_node[node].next_cpu_irq_affinity = -1;
+ continue;
+ }
+ } while (false);
+ cpumask_clear(mask);
+ cpumask_set_cpu(cpu, mask);
+}
+
+struct wr_ret {
+ void *work;
+ u16 grp;
+};
+
+static inline struct wr_ret octeon3_core_get_work_sync(int grp)
+{
+ u64 node = cvmx_get_node_num();
+ u64 addr, response;
+ struct wr_ret r;
+
+ /* See SSO_GET_WORK_LD_S for the address to read */
+ addr = SSO_GET_WORK_DMA_S_SCRADDR;
+ addr |= SSO_GET_WORK_LD_S_IO;
+ addr |= SSO_TAG_SWDID << SSO_GET_WORK_DID_SHIFT;
+ addr |= node << SSO_GET_WORK_NODE_SHIFT;
+ addr |= SSO_GET_WORK_GROUPED;
+ addr |= SSO_GET_WORK_RTNGRP;
+ addr |= octeon3_eth_lgrp_to_ggrp(node, grp) <<
+ SSO_GET_WORK_IDX_GRP_MASK_SHIFT;
+ addr |= SSO_GET_WORK_WAITW_NO_WAIT;
+ response = __raw_readq((void __iomem *)addr);
+
+ /* See SSO_GET_WORK_RTN_S for the format of the response */
+ r.grp = (response & SSO_GET_WORK_RTN_S_GRP_MASK) >>
+ SSO_GET_WORK_RTN_S_GRP_SHIFT;
+ if (response & SSO_GET_WORK_RTN_S_NO_WORK)
+ r.work = NULL;
+ else
+ r.work = phys_to_virt(response & SSO_GET_WORK_RTN_S_WQP_MASK);
+
+ return r;
+}
+
+/* octeon3_core_get_work_async - Request work via a iobdma command. Doesn't wait
+ * for the response.
+ *
+ * @grp: Group to request work for.
+ */
+static inline void octeon3_core_get_work_async(unsigned int grp)
+{
+ u64 data, node = cvmx_get_node_num();
+
+ /* See SSO_GET_WORK_DMA_S for the command structure */
+ data = 1ull << SSO_GET_WORK_DMA_S_LEN_SHIFT;
+ data |= SSO_TAG_SWDID << SSO_GET_WORK_DID_SHIFT;
+ data |= node << SSO_GET_WORK_NODE_SHIFT;
+ data |= SSO_GET_WORK_GROUPED;
+ data |= SSO_GET_WORK_RTNGRP;
+ data |= octeon3_eth_lgrp_to_ggrp(node, grp) <<
+ SSO_GET_WORK_IDX_GRP_MASK_SHIFT;
+ data |= SSO_GET_WORK_WAITW_NO_WAIT;
+
+ __raw_writeq(data, (void __iomem *)IOBDMA_ORDERED_IO_ADDR);
+}
+
+/* octeon3_core_get_response_async - Read the request work response. Must be
+ * called after calling
+ * octeon3_core_get_work_async().
+ *
+ * Returns work queue entry.
+ */
+static inline struct wr_ret octeon3_core_get_response_async(void)
+{
+ struct wr_ret r;
+ u64 response;
+
+ CVMX_SYNCIOBDMA;
+ response = scratch_read64(0);
+
+ /* See SSO_GET_WORK_RTN_S for the format of the response */
+ r.grp = (response & SSO_GET_WORK_RTN_S_GRP_MASK) >>
+ SSO_GET_WORK_RTN_S_GRP_SHIFT;
+ if (response & SSO_GET_WORK_RTN_S_NO_WORK)
+ r.work = NULL;
+ else
+ r.work = phys_to_virt(response & SSO_GET_WORK_RTN_S_WQP_MASK);
+
+ return r;
+}
+
+static void octeon3_eth_replenish_rx(struct octeon3_ethernet *priv, int count)
+{
+ struct sk_buff *skb;
+ int i;
+
+ for (i = 0; i < count; i++) {
+ void **buf;
+
+ skb = __alloc_skb(packet_buffer_size, GFP_ATOMIC, 0,
+ priv->node);
+ if (!skb)
+ break;
+ buf = (void **)PTR_ALIGN(skb->head, 128);
+ buf[SKB_PTR_OFFSET] = skb;
+ octeon_fpa3_free(priv->node, priv->pki_aura, buf);
+ }
+}
+
+static bool octeon3_eth_tx_done_runnable(struct octeon3_ethernet_worker *worker)
+{
+ return atomic_read(&worker->kick) != 0 || kthread_should_stop();
+}
+
+static int octeon3_eth_replenish_all(struct octeon3_ethernet_node *oen)
+{
+ int batch_size = 32, pending = 0;
+ struct octeon3_ethernet *priv;
+
+ rcu_read_lock();
+ list_for_each_entry_rcu(priv, &oen->device_list, list) {
+ int amount = atomic64_sub_if_positive(batch_size,
+ &priv->buffers_needed);
+
+ if (amount >= 0) {
+ octeon3_eth_replenish_rx(priv, batch_size);
+ pending += amount;
+ }
+ }
+ rcu_read_unlock();
+ return pending;
+}
+
+static int octeon3_eth_tx_complete_hwtstamp(struct octeon3_ethernet *priv,
+ struct sk_buff *skb)
+{
+ struct skb_shared_hwtstamps shts;
+ u64 hwts, ns;
+
+ hwts = *((u64 *)(skb->cb) + 1);
+ ns = timecounter_cyc2time(&priv->tc, hwts);
+ memset(&shts, 0, sizeof(shts));
+ shts.hwtstamp = ns_to_ktime(ns);
+ skb_tstamp_tx(skb, &shts);
+
+ return 0;
+}
+
+static int octeon3_eth_tx_complete_worker(void *data)
+{
+ int backlog, backlog_stop_thresh, i, order, tx_complete_stop_thresh;
+ struct octeon3_ethernet_worker *worker = data;
+ struct octeon3_ethernet_node *oen = worker->oen;
+ u64 aq_cnt;
+
+ order = worker->order;
+ backlog_stop_thresh = (order == 0 ? 31 : order * 80);
+ tx_complete_stop_thresh = (order * 100);
+
+ while (!kthread_should_stop()) {
+ /* Replaced by wait_event to avoid warnings like
+ * "task oct3_eth/0:2:1250 blocked for more than 120 seconds."
+ */
+ wait_event_interruptible(worker->queue,
+ octeon3_eth_tx_done_runnable(worker));
+ atomic_dec_if_positive(&worker->kick); /* clear the flag */
+
+ do {
+ backlog = octeon3_eth_replenish_all(oen);
+ for (i = 0; i < 100; i++) {
+ void **work;
+ struct net_device *tx_netdev;
+ struct octeon3_ethernet *tx_priv;
+ struct sk_buff *skb;
+ struct wr_ret r;
+
+ r = octeon3_core_get_work_sync(oen->tx_complete_grp);
+ work = r.work;
+ if (!work)
+ break;
+ tx_netdev = work[0];
+ tx_priv = netdev_priv(tx_netdev);
+ if (unlikely(netif_queue_stopped(tx_netdev)) && atomic64_read(&tx_priv->tx_backlog) < MAX_TX_QUEUE_DEPTH)
+ netif_wake_queue(tx_netdev);
+ skb = container_of((void *)work,
+ struct sk_buff, cb);
+ if (unlikely(tx_priv->tx_timestamp_hw) && unlikely(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS))
+ octeon3_eth_tx_complete_hwtstamp(tx_priv, skb);
+ dev_kfree_skb(skb);
+ }
+
+ aq_cnt = oct_csr_read(SSO_GRP_AQ_CNT(oen->node, oen->tx_complete_grp));
+ aq_cnt &= SSO_GRP_AQ_CNT_AQ_CNT_MASK;
+ if ((backlog > backlog_stop_thresh ||
+ aq_cnt > tx_complete_stop_thresh) &&
+ order < ARRAY_SIZE(oen->workers) - 1) {
+ atomic_set(&oen->workers[order + 1].kick, 1);
+ wake_up(&oen->workers[order + 1].queue);
+ }
+ } while (!need_resched() && (backlog > backlog_stop_thresh ||
+ aq_cnt > tx_complete_stop_thresh));
+
+ cond_resched();
+
+ if (!octeon3_eth_tx_done_runnable(worker))
+ octeon3_sso_irq_set(oen->node, oen->tx_complete_grp,
+ true);
+ }
+
+ return 0;
+}
+
+static irqreturn_t octeon3_eth_tx_handler(int irq, void *info)
+{
+ struct octeon3_ethernet_node *oen = info;
+
+ /* Disarm the irq. */
+ octeon3_sso_irq_set(oen->node, oen->tx_complete_grp, false);
+ atomic_set(&oen->workers[0].kick, 1);
+ wake_up(&oen->workers[0].queue);
+ return IRQ_HANDLED;
+}
+
+static int octeon3_eth_global_init(unsigned int node,
+ struct platform_device *pdev)
+{
+ struct octeon3_ethernet_node *oen;
+ unsigned int sso_intsn;
+ int i, rv = 0;
+
+ mutex_lock(&octeon3_eth_init_mutex);
+
+ oen = octeon3_eth_node + node;
+
+ if (oen->init_done)
+ goto done;
+
+ /* CN78XX-P1.0 cannot un-initialize PKO, so get a module
+ * reference to prevent it from being unloaded.
+ */
+ if (OCTEON_IS_MODEL(OCTEON_CN78XX_PASS1_0))
+ if (!try_module_get(THIS_MODULE))
+ dev_err(&pdev->dev, "ERROR: Could not obtain module reference for CN78XX-P1.0\n");
+
+ INIT_LIST_HEAD(&oen->device_list);
+ mutex_init(&oen->device_list_lock);
+ spin_lock_init(&oen->napi_alloc_lock);
+
+ oen->node = node;
+
+ octeon_fpa3_init(node);
+ rv = octeon_fpa3_pool_init(node, -1, &oen->sso_pool,
+ &oen->sso_pool_stack, 40960);
+ if (rv)
+ goto done;
+
+ rv = octeon_fpa3_pool_init(node, -1, &oen->pko_pool,
+ &oen->pko_pool_stack, 40960);
+ if (rv)
+ goto done;
+
+ rv = octeon_fpa3_pool_init(node, -1, &oen->pki_packet_pool,
+ &oen->pki_packet_pool_stack,
+ 64 * num_packet_buffers);
+ if (rv)
+ goto done;
+
+ rv = octeon_fpa3_aura_init(node, oen->sso_pool, -1,
+ &oen->sso_aura, num_packet_buffers, 20480);
+ if (rv)
+ goto done;
+
+ rv = octeon_fpa3_aura_init(node, oen->pko_pool, -1,
+ &oen->pko_aura, num_packet_buffers, 20480);
+ if (rv)
+ goto done;
+
+ dev_info(&pdev->dev, "SSO:%d:%d, PKO:%d:%d\n", oen->sso_pool,
+ oen->sso_aura, oen->pko_pool, oen->pko_aura);
+
+ if (!octeon3_eth_sso_pko_cache) {
+ octeon3_eth_sso_pko_cache = kmem_cache_create("sso_pko", 4096,
+ 128, 0, NULL);
+ if (!octeon3_eth_sso_pko_cache) {
+ rv = -ENOMEM;
+ goto done;
+ }
+ }
+
+ rv = octeon_fpa3_mem_fill(node, octeon3_eth_sso_pko_cache,
+ oen->sso_aura, 1024);
+ if (rv)
+ goto done;
+
+ rv = octeon_fpa3_mem_fill(node, octeon3_eth_sso_pko_cache,
+ oen->pko_aura, 1024);
+ if (rv)
+ goto done;
+
+ rv = octeon3_sso_init(node, oen->sso_aura);
+ if (rv)
+ goto done;
+
+ oen->tx_complete_grp = octeon3_sso_alloc_groups(node, NULL, 1, -1);
+ if (oen->tx_complete_grp < 0)
+ goto done;
+
+ sso_intsn = SSO_IRQ_START | oen->tx_complete_grp;
+ oen->tx_irq = irq_create_mapping(NULL, sso_intsn);
+ if (!oen->tx_irq) {
+ rv = -ENODEV;
+ goto done;
+ }
+
+ rv = octeon3_pko_init_global(node, oen->pko_aura);
+ if (rv) {
+ rv = -ENODEV;
+ goto done;
+ }
+
+ octeon3_pki_vlan_init(node);
+ octeon3_pki_cluster_init(node, pdev);
+ octeon3_pki_ltype_init(node);
+ octeon3_pki_enable(node);
+
+ for (i = 0; i < ARRAY_SIZE(oen->workers); i++) {
+ oen->workers[i].oen = oen;
+ init_waitqueue_head(&oen->workers[i].queue);
+ oen->workers[i].order = i;
+ }
+ for (i = 0; i < ARRAY_SIZE(oen->workers); i++) {
+ oen->workers[i].task =
+ kthread_create_on_node(octeon3_eth_tx_complete_worker,
+ oen->workers + i, node,
+ "oct3_eth/%d:%d", node, i);
+ if (IS_ERR(oen->workers[i].task)) {
+ rv = PTR_ERR(oen->workers[i].task);
+ goto done;
+ } else {
+#ifdef CONFIG_NUMA
+ set_cpus_allowed_ptr(oen->workers[i].task,
+ cpumask_of_node(node));
+#endif
+ wake_up_process(oen->workers[i].task);
+ }
+ }
+
+ if (OCTEON_IS_MODEL(OCTEON_CN78XX_PASS1_X))
+ octeon3_sso_pass1_limit(node, oen->tx_complete_grp);
+
+ rv = request_irq(oen->tx_irq, octeon3_eth_tx_handler,
+ IRQ_TYPE_EDGE_RISING, "oct3_eth_tx_done", oen);
+ if (rv)
+ goto done;
+ octeon3_eth_gen_affinity(node, &oen->tx_affinity_hint);
+ irq_set_affinity_hint(oen->tx_irq, &oen->tx_affinity_hint);
+
+ octeon3_sso_irq_set(node, oen->tx_complete_grp, true);
+
+ oen->init_done = true;
+done:
+ mutex_unlock(&octeon3_eth_init_mutex);
+ return rv;
+}
+
+static struct sk_buff *octeon3_eth_work_to_skb(void *w)
+{
+ struct sk_buff *skb;
+ void **f = w;
+
+ skb = f[-16];
+ return skb;
+}
+
+/* octeon3_napi_alloc_cpu - Find an available cpu. This function must be called
+ * with the napi_alloc_lock lock held.
+ * @node: Node to allocate cpu from.
+ * @cpu: Cpu to bind the napi to:
+ * < 0: use any cpu.
+ * >= 0: use requested cpu.
+ *
+ * Returns cpu number.
+ * Returns <0 for error codes.
+ */
+static int octeon3_napi_alloc_cpu(int node, int cpu)
+{
+ int min_cnt = MAX_NAPIS_PER_NODE;
+ int min_cpu = -EBUSY;
+
+ if (cpu >= 0) {
+ min_cpu = cpu;
+ } else {
+ for_each_cpu(cpu, cpumask_of_node(node)) {
+ if (octeon3_cpu_napi_cnt[cpu] == 0) {
+ min_cpu = cpu;
+ break;
+ } else if (octeon3_cpu_napi_cnt[cpu] < min_cnt) {
+ min_cnt = octeon3_cpu_napi_cnt[cpu];
+ min_cpu = cpu;
+ }
+ }
+ }
+
+ if (min_cpu < 0)
+ return min_cpu;
+
+ octeon3_cpu_napi_cnt[min_cpu]++;
+
+ return min_cpu;
+}
+
+/* octeon3_napi_alloc - Allocate a napi.
+ * @cxt: Receive context the napi will be added to.
+ * @idx: Napi index within the receive context.
+ * @cpu: Cpu to bind the napi to:
+ * < 0: use any cpu.
+ * >= 0: use requested cpu.
+ *
+ * Returns pointer to napi wrapper.
+ * Returns NULL on error.
+ */
+static struct octeon3_napi_wrapper *octeon3_napi_alloc(struct octeon3_rx *cxt,
+ int idx, int cpu)
+{
+ struct octeon3_ethernet *priv = cxt->parent;
+ struct octeon3_ethernet_node *oen;
+ int i, node = priv->node;
+ unsigned long flags;
+
+ oen = octeon3_eth_node + node;
+ spin_lock_irqsave(&oen->napi_alloc_lock, flags);
+
+ /* Find a free napi wrapper */
+ for (i = 0; i < MAX_NAPIS_PER_NODE; i++) {
+ if (napi_wrapper[node][i].available) {
+ /* Allocate a cpu to use */
+ cpu = octeon3_napi_alloc_cpu(node, cpu);
+ if (cpu < 0)
+ break;
+
+ napi_wrapper[node][i].available = 0;
+ napi_wrapper[node][i].idx = idx;
+ napi_wrapper[node][i].cpu = cpu;
+ napi_wrapper[node][i].cxt = cxt;
+ spin_unlock_irqrestore(&oen->napi_alloc_lock, flags);
+ return &napi_wrapper[node][i];
+ }
+ }
+
+ spin_unlock_irqrestore(&oen->napi_alloc_lock, flags);
+ return NULL;
+}
+
+/* octeon_cpu_napi_sched - Schedule a napi for execution. The napi will start
+ * executing on the cpu calling this function.
+ * @info: Pointer to the napi to schedule for execution.
+ */
+static void octeon_cpu_napi_sched(void *info)
+{
+ struct napi_struct *napi = info;
+
+ napi_schedule(napi);
+}
+
+/* octeon3_rm_napi_from_cxt - Remove a napi from a receive context.
+ * @node: Node napi belongs to.
+ * @napiw: Pointer to napi to remove.
+ *
+ * Returns 0 if successful.
+ * Returns <0 for error codes.
+ */
+static int octeon3_rm_napi_from_cxt(int node,
+ struct octeon3_napi_wrapper *napiw)
+{
+ struct octeon3_ethernet_node *oen;
+ struct octeon3_rx *cxt;
+ unsigned long flags;
+ int idx;
+
+ oen = octeon3_eth_node + node;
+ cxt = napiw->cxt;
+ idx = napiw->idx;
+
+ /* Free the napi block */
+ spin_lock_irqsave(&oen->napi_alloc_lock, flags);
+ octeon3_cpu_napi_cnt[napiw->cpu]--;
+ napiw->available = 1;
+ napiw->idx = -1;
+ napiw->cpu = -1;
+ napiw->cxt = NULL;
+ spin_unlock_irqrestore(&oen->napi_alloc_lock, flags);
+
+ /* Free the napi idx */
+ spin_lock_irqsave(&cxt->napi_idx_lock, flags);
+ bitmap_clear(cxt->napi_idx_bitmap, idx, 1);
+ spin_unlock_irqrestore(&cxt->napi_idx_lock, flags);
+
+ return 0;
+}
+
+/* octeon3_add_napi_to_cxt - Add a napi to a receive context.
+ * @cxt: Pointer to receive context.
+ *
+ * Returns 0 if successful.
+ * Returns <0 for error codes.
+ */
+static int octeon3_add_napi_to_cxt(struct octeon3_rx *cxt)
+{
+ struct octeon3_ethernet *priv = cxt->parent;
+ struct octeon3_napi_wrapper *napiw;
+ unsigned long flags;
+ int idx, rc;
+
+ /* Get a free napi idx */
+ spin_lock_irqsave(&cxt->napi_idx_lock, flags);
+ idx = find_first_zero_bit(cxt->napi_idx_bitmap, MAX_CORES);
+ if (unlikely(idx >= MAX_CORES)) {
+ spin_unlock_irqrestore(&cxt->napi_idx_lock, flags);
+ return -ENOMEM;
+ }
+ bitmap_set(cxt->napi_idx_bitmap, idx, 1);
+ spin_unlock_irqrestore(&cxt->napi_idx_lock, flags);
+
+ /* Get a free napi block */
+ napiw = octeon3_napi_alloc(cxt, idx, -1);
+ if (unlikely(!napiw)) {
+ spin_lock_irqsave(&cxt->napi_idx_lock, flags);
+ bitmap_clear(cxt->napi_idx_bitmap, idx, 1);
+ spin_unlock_irqrestore(&cxt->napi_idx_lock, flags);
+ return -ENOMEM;
+ }
+
+ rc = smp_call_function_single(napiw->cpu, octeon_cpu_napi_sched,
+ &napiw->napi, 0);
+ if (unlikely(rc))
+ octeon3_rm_napi_from_cxt(priv->node, napiw);
+
+ return rc;
+}
+
+/* Receive one packet.
+ * returns the number of RX buffers consumed.
+ */
+static int octeon3_eth_rx_one(struct octeon3_rx *rx, bool is_async,
+ bool req_next)
+{
+ struct octeon3_ethernet *priv = rx->parent;
+ int len_remaining, ret, segments;
+ union buf_ptr packet_ptr;
+ unsigned int packet_len;
+ struct sk_buff *skb;
+ struct wqe *work;
+ struct wr_ret r;
+ void **buf;
+ u64 gaura;
+ u8 *data;
+
+ if (is_async)
+ r = octeon3_core_get_response_async();
+ else
+ r = octeon3_core_get_work_sync(rx->rx_grp);
+ work = r.work;
+ if (!work)
+ return 0;
+
+ /* Request the next work so it'll be ready when we need it */
+ if (is_async && req_next)
+ octeon3_core_get_work_async(rx->rx_grp);
+
+ skb = octeon3_eth_work_to_skb(work);
+
+ /* Save the aura and node this skb came from to allow the pko to free
+ * the skb back to the correct aura. A magic number is also added to
+ * later verify the skb came from the fpa.
+ *
+ * 63 12 11 10 9 0
+ * ---------------------------------------------------------------------
+ * | magic | node | aura |
+ * ---------------------------------------------------------------------
+ */
+ buf = (void **)PTR_ALIGN(skb->head, 128);
+ gaura = SKB_AURA_MAGIC | work->word0.aura;
+ buf[SKB_AURA_OFFSET] = (void *)gaura;
+
+ segments = work->word0.bufs;
+ ret = segments;
+ packet_ptr = work->packet_ptr;
+ if (unlikely(work->word2.err_level <= PKI_ERRLEV_LA &&
+ work->word2.err_code != PKI_OPCODE_NONE)) {
+ atomic64_inc(&priv->rx_errors);
+ switch (work->word2.err_code) {
+ case PKI_OPCODE_JABBER:
+ atomic64_inc(&priv->rx_length_errors);
+ break;
+ case PKI_OPCODE_FCS:
+ atomic64_inc(&priv->rx_crc_errors);
+ break;
+ }
+ data = phys_to_virt(packet_ptr.addr);
+ for (;;) {
+ dev_kfree_skb_any(skb);
+ segments--;
+ if (segments <= 0)
+ break;
+ packet_ptr.u64 = *(u64 *)(data - 8);
+#ifndef __LITTLE_ENDIAN
+ if (OCTEON_IS_MODEL(OCTEON_CN78XX_PASS1_X)) {
+ /* PKI_BUFLINK_S's are endian-swapped */
+ packet_ptr.u64 = swab64(packet_ptr.u64);
+ }
+#endif
+ data = phys_to_virt(packet_ptr.addr);
+ skb = octeon3_eth_work_to_skb((void *)round_down((unsigned long)data, 128ull));
+ }
+ goto out;
+ }
+
+ packet_len = work->word1.len;
+ data = phys_to_virt(packet_ptr.addr);
+ skb->data = data;
+ skb->len = packet_len;
+ len_remaining = packet_len;
+ if (segments == 1) {
+ /* Strip the ethernet fcs */
+ skb->len -= 4;
+ skb_set_tail_pointer(skb, skb->len);
+ } else {
+ bool first_frag = true;
+ struct sk_buff *current_skb = skb;
+ struct sk_buff *next_skb = NULL;
+ unsigned int segment_size;
+
+ skb_frag_list_init(skb);
+ for (;;) {
+ segment_size = (segments == 1) ?
+ len_remaining : packet_ptr.size;
+ len_remaining -= segment_size;
+ if (!first_frag) {
+ current_skb->len = segment_size;
+ skb->data_len += segment_size;
+ skb->truesize += current_skb->truesize;
+ }
+ skb_set_tail_pointer(current_skb, segment_size);
+ segments--;
+ if (segments == 0)
+ break;
+ packet_ptr.u64 = *(u64 *)(data - 8);
+#ifndef __LITTLE_ENDIAN
+ if (OCTEON_IS_MODEL(OCTEON_CN78XX_PASS1_X)) {
+ /* PKI_BUFLINK_S's are endian-swapped */
+ packet_ptr.u64 = swab64(packet_ptr.u64);
+ }
+#endif
+ data = phys_to_virt(packet_ptr.addr);
+ next_skb = octeon3_eth_work_to_skb((void *)round_down((unsigned long)data, 128ull));
+ if (first_frag) {
+ next_skb->next =
+ skb_shinfo(current_skb)->frag_list;
+ skb_shinfo(current_skb)->frag_list = next_skb;
+ } else {
+ current_skb->next = next_skb;
+ next_skb->next = NULL;
+ }
+ current_skb = next_skb;
+ first_frag = false;
+ current_skb->data = data;
+ }
+
+ /* Strip the ethernet fcs */
+ pskb_trim(skb, skb->len - 4);
+ }
+
+ if (likely(priv->netdev->flags & IFF_UP)) {
+ skb_checksum_none_assert(skb);
+ if (unlikely(priv->rx_timestamp_hw)) {
+ /* The first 8 bytes are the timestamp */
+ u64 hwts = *(u64 *)skb->data;
+ u64 ns;
+ struct skb_shared_hwtstamps *shts;
+
+ ns = timecounter_cyc2time(&priv->tc, hwts);
+ shts = skb_hwtstamps(skb);
+ memset(shts, 0, sizeof(*shts));
+ shts->hwtstamp = ns_to_ktime(ns);
+ __skb_pull(skb, 8);
+ }
+
+ skb->protocol = eth_type_trans(skb, priv->netdev);
+ skb->dev = priv->netdev;
+ if (priv->netdev->features & NETIF_F_RXCSUM) {
+ if ((work->word2.lc_hdr_type == PKI_LTYPE_IP4 ||
+ work->word2.lc_hdr_type == PKI_LTYPE_IP6) &&
+ (work->word2.lf_hdr_type == PKI_LTYPE_TCP ||
+ work->word2.lf_hdr_type == PKI_LTYPE_UDP ||
+ work->word2.lf_hdr_type == PKI_LTYPE_SCTP))
+ if (work->word2.err_code == 0)
+ skb->ip_summed = CHECKSUM_UNNECESSARY;
+ }
+
+ netif_receive_skb(skb);
+ } else {
+ /* Drop any packet received for a device that isn't up */
+ atomic64_inc(&priv->rx_dropped);
+ dev_kfree_skb_any(skb);
+ }
+out:
+ return ret;
+}
+
+static int octeon3_eth_napi(struct napi_struct *napi, int budget)
+{
+ int idx, napis_inuse, n = 0, n_bufs = 0, rx_count = 0;
+ struct octeon3_napi_wrapper *napiw;
+ struct octeon3_ethernet *priv;
+ u64 aq_cnt, old_scratch;
+ struct octeon3_rx *cxt;
+
+ napiw = container_of(napi, struct octeon3_napi_wrapper, napi);
+ cxt = napiw->cxt;
+ priv = cxt->parent;
+
+ /* Get the amount of work pending */
+ aq_cnt = oct_csr_read(SSO_GRP_AQ_CNT(priv->node, cxt->rx_grp));
+ aq_cnt &= SSO_GRP_AQ_CNT_AQ_CNT_MASK;
+ /* Allow the last thread to add/remove threads if the work
+ * incremented/decremented by more than what the current number
+ * of threads can support.
+ */
+ idx = find_last_bit(cxt->napi_idx_bitmap, MAX_CORES);
+ napis_inuse = bitmap_weight(cxt->napi_idx_bitmap, MAX_CORES);
+
+ if (napiw->idx == idx) {
+ if (aq_cnt > napis_inuse * 128) {
+ octeon3_add_napi_to_cxt(cxt);
+ } else if (napiw->idx > 0 && aq_cnt < (napis_inuse - 1) * 128) {
+ napi_complete(napi);
+ octeon3_rm_napi_from_cxt(priv->node, napiw);
+ return 0;
+ }
+ }
+
+ if (likely(USE_ASYNC_IOBDMA)) {
+ /* Save scratch in case userspace is using it */
+ CVMX_SYNCIOBDMA;
+ old_scratch = scratch_read64(0);
+
+ octeon3_core_get_work_async(cxt->rx_grp);
+ }
+
+ while (rx_count < budget) {
+ n = 0;
+
+ if (likely(USE_ASYNC_IOBDMA)) {
+ bool req_next = rx_count < (budget - 1) ? true : false;
+
+ n = octeon3_eth_rx_one(cxt, true, req_next);
+ } else {
+ n = octeon3_eth_rx_one(cxt, false, false);
+ }
+
+ if (n == 0)
+ break;
+
+ n_bufs += n;
+ rx_count++;
+ }
+
+ /* Wake up worker threads */
+ n_bufs = atomic64_add_return(n_bufs, &priv->buffers_needed);
+ if (n_bufs >= 32) {
+ struct octeon3_ethernet_node *oen;
+
+ oen = octeon3_eth_node + priv->node;
+ atomic_set(&oen->workers[0].kick, 1);
+ wake_up(&oen->workers[0].queue);
+ }
+
+ /* Stop the thread when no work is pending */
+ if (rx_count < budget) {
+ napi_complete(napi);
+
+ if (napiw->idx > 0)
+ octeon3_rm_napi_from_cxt(priv->node, napiw);
+ else
+ octeon3_sso_irq_set(cxt->parent->node, cxt->rx_grp,
+ true);
+ }
+
+ if (likely(USE_ASYNC_IOBDMA)) {
+ /* Restore the scratch area */
+ scratch_write64(0, old_scratch);
+ }
+
+ return rx_count;
+}
+
+/* octeon3_napi_init_node - Initialize the node napis.
+ * @node: Node napis belong to.
+ * @netdev: Default network device used to initialize the napis.
+ *
+ * Returns 0 if successful.
+ * Returns <0 for error codes.
+ */
+static int octeon3_napi_init_node(int node, struct net_device *netdev)
+{
+ struct octeon3_ethernet_node *oen;
+ unsigned long flags;
+ int i;
+
+ oen = octeon3_eth_node + node;
+ spin_lock_irqsave(&oen->napi_alloc_lock, flags);
+
+ if (oen->napi_init_done)
+ goto done;
+
+ for (i = 0; i < MAX_NAPIS_PER_NODE; i++) {
+ netif_napi_add(netdev, &napi_wrapper[node][i].napi,
+ octeon3_eth_napi, 32);
+ napi_enable(&napi_wrapper[node][i].napi);
+ napi_wrapper[node][i].available = 1;
+ napi_wrapper[node][i].idx = -1;
+ napi_wrapper[node][i].cpu = -1;
+ napi_wrapper[node][i].cxt = NULL;
+ }
+
+ oen->napi_init_done = true;
+done:
+ spin_unlock_irqrestore(&oen->napi_alloc_lock, flags);
+ return 0;
+}
+
+#undef BROKEN_SIMULATOR_CSUM
+
+static void ethtool_get_drvinfo(struct net_device *netdev,
+ struct ethtool_drvinfo *info)
+{
+ strcpy(info->driver, "octeon3-ethernet");
+ strcpy(info->version, "1.0");
+ strcpy(info->bus_info, "Builtin");
+}
+
+static int ethtool_get_ts_info(struct net_device *ndev,
+ struct ethtool_ts_info *info)
+{
+ struct octeon3_ethernet *priv = netdev_priv(ndev);
+
+ info->so_timestamping = SOF_TIMESTAMPING_TX_HARDWARE |
+ SOF_TIMESTAMPING_RX_HARDWARE | SOF_TIMESTAMPING_RAW_HARDWARE;
+
+ if (priv->ptp_clock)
+ info->phc_index = ptp_clock_index(priv->ptp_clock);
+ else
+ info->phc_index = -1;
+
+ info->tx_types = (1 << HWTSTAMP_TX_OFF) | (1 << HWTSTAMP_TX_ON);
+
+ info->rx_filters = (1 << HWTSTAMP_FILTER_NONE) |
+ (1 << HWTSTAMP_FILTER_ALL);
+
+ return 0;
+}
+
+static const struct ethtool_ops octeon3_ethtool_ops = {
+ .get_drvinfo = ethtool_get_drvinfo,
+ .get_link_ksettings = bgx_port_ethtool_get_link_ksettings,
+ .set_settings = bgx_port_ethtool_set_settings,
+ .nway_reset = bgx_port_ethtool_nway_reset,
+ .get_link = ethtool_op_get_link,
+ .get_ts_info = ethtool_get_ts_info,
+};
+
+static int octeon3_eth_ndo_change_mtu(struct net_device *netdev, int new_mtu)
+{
+ if (OCTEON_IS_MODEL(OCTEON_CN78XX_PASS1_X)) {
+ struct octeon3_ethernet *priv = netdev_priv(netdev);
+ int fifo_size, max_mtu = 1500;
+
+ /* On 78XX-Pass1 the mtu must be limited. The PKO may
+ * to lock up when calculating the L4 checksum for
+ * large packets. How large the packets can be depends
+ * on the amount of pko fifo assigned to the port.
+ *
+ * FIFO size Max frame size
+ * 2.5 KB 1920
+ * 5.0 KB 4480
+ * 10.0 KB 9600
+ *
+ * The maximum mtu is set to the largest frame size minus the
+ * l2 header.
+ */
+ fifo_size = octeon3_pko_get_fifo_size(priv->node,
+ priv->interface,
+ priv->index,
+ priv->mac_type);
+
+ switch (fifo_size) {
+ case 2560:
+ max_mtu = 1920 - ETH_HLEN - ETH_FCS_LEN -
+ (2 * VLAN_HLEN);
+ break;
+
+ case 5120:
+ max_mtu = 4480 - ETH_HLEN - ETH_FCS_LEN -
+ (2 * VLAN_HLEN);
+ break;
+
+ case 10240:
+ max_mtu = 9600 - ETH_HLEN - ETH_FCS_LEN -
+ (2 * VLAN_HLEN);
+ break;
+
+ default:
+ break;
+ }
+ if (new_mtu > max_mtu) {
+ netdev_warn(netdev, "Maximum MTU supported is %d",
+ max_mtu);
+ return -EINVAL;
+ }
+ }
+ return bgx_port_change_mtu(netdev, new_mtu);
+}
+
+static int octeon3_eth_common_ndo_init(struct net_device *netdev,
+ int extra_skip)
+{
+ int aura, base_rx_grp[MAX_RX_CONTEXTS], dq, i, pki_chan, r;
+ struct octeon3_ethernet *priv = netdev_priv(netdev);
+ struct octeon3_ethernet_node *oen = octeon3_eth_node + priv->node;
+
+ netif_carrier_off(netdev);
+
+ netdev->features |=
+#ifndef BROKEN_SIMULATOR_CSUM
+ NETIF_F_IP_CSUM |
+ NETIF_F_IPV6_CSUM |
+#endif
+ NETIF_F_SG |
+ NETIF_F_FRAGLIST |
+ NETIF_F_RXCSUM |
+ NETIF_F_LLTX;
+
+ if (!OCTEON_IS_MODEL(OCTEON_CN78XX_PASS1_X))
+ netdev->features |= NETIF_F_SCTP_CRC;
+
+ netdev->features |= NETIF_F_TSO | NETIF_F_TSO6;
+
+ /* Set user changeable settings */
+ netdev->hw_features = netdev->features;
+
+ priv->rx_buf_count = num_packet_buffers;
+
+ pki_chan = get_pki_chan(priv->node, priv->interface, priv->index);
+
+ dq = octeon3_pko_interface_init(priv->node, priv->interface,
+ priv->index, priv->mac_type, pki_chan);
+ if (dq < 0) {
+ dev_err(netdev->dev.parent, "Failed to initialize pko\n");
+ return -ENODEV;
+ }
+
+ r = octeon3_pko_activate_dq(priv->node, dq, 1);
+ if (r < 0) {
+ dev_err(netdev->dev.parent, "Failed to activate dq\n");
+ return -ENODEV;
+ }
+
+ priv->pko_queue = dq;
+ octeon_fpa3_aura_init(priv->node, oen->pki_packet_pool, -1, &aura,
+ num_packet_buffers, num_packet_buffers * 2);
+ priv->pki_aura = aura;
+ aura2bufs_needed[priv->node][priv->pki_aura] = &priv->buffers_needed;
+
+ r = octeon3_sso_alloc_groups(priv->node, base_rx_grp, rx_contexts, -1);
+ if (r) {
+ dev_err(netdev->dev.parent, "Failed to allocated SSO group\n");
+ return -ENODEV;
+ }
+ for (i = 0; i < rx_contexts; i++) {
+ priv->rx_cxt[i].rx_grp = base_rx_grp[i];
+ priv->rx_cxt[i].parent = priv;
+
+ if (OCTEON_IS_MODEL(OCTEON_CN78XX_PASS1_X))
+ octeon3_sso_pass1_limit(priv->node,
+ priv->rx_cxt[i].rx_grp);
+ }
+ priv->num_rx_cxt = rx_contexts;
+
+ priv->tx_complete_grp = oen->tx_complete_grp;
+ dev_info(netdev->dev.parent,
+ "rx sso grp:%d..%d aura:%d pknd:%d pko_queue:%d\n",
+ *base_rx_grp, *(base_rx_grp + priv->num_rx_cxt - 1),
+ priv->pki_aura, priv->pknd, priv->pko_queue);
+
+ octeon3_pki_port_init(priv->node, priv->pki_aura, *base_rx_grp,
+ extra_skip, (packet_buffer_size - 128),
+ priv->pknd, priv->num_rx_cxt);
+
+ priv->last_packets = 0;
+ priv->last_octets = 0;
+ priv->last_dropped = 0;
+
+ octeon3_napi_init_node(priv->node, netdev);
+
+ /* Register ethtool methods */
+ netdev->ethtool_ops = &octeon3_ethtool_ops;
+
+ return 0;
+}
+
+static int octeon3_eth_bgx_ndo_init(struct net_device *netdev)
+{
+ struct octeon3_ethernet *priv = netdev_priv(netdev);
+ const u8 *mac;
+ int r;
+
+ priv->pknd = bgx_port_get_pknd(priv->node, priv->interface,
+ priv->index);
+ octeon3_eth_common_ndo_init(netdev, 0);
+
+ /* Padding and FCS are done in BGX */
+ r = octeon3_pko_set_mac_options(priv->node, priv->interface,
+ priv->index, priv->mac_type, false,
+ false, 0);
+ if (r)
+ return r;
+
+ mac = bgx_port_get_mac(netdev);
+ if (mac && is_valid_ether_addr(mac)) {
+ memcpy(netdev->dev_addr, mac, ETH_ALEN);
+ netdev->addr_assign_type &= ~NET_ADDR_RANDOM;
+ } else {
+ eth_hw_addr_random(netdev);
+ }
+
+ bgx_port_set_rx_filtering(netdev);
+ octeon3_eth_ndo_change_mtu(netdev, netdev->mtu);
+
+ return 0;
+}
+
+static void octeon3_eth_ndo_uninit(struct net_device *netdev)
+{
+ struct octeon3_ethernet *priv = netdev_priv(netdev);
+ int grp[MAX_RX_CONTEXTS], i;
+
+ /* Shutdwon pki for this interface */
+ octeon3_pki_port_shutdown(priv->node, priv->pknd);
+ octeon_fpa3_release_aura(priv->node, priv->pki_aura);
+ aura2bufs_needed[priv->node][priv->pki_aura] = NULL;
+
+ /* Shutdown pko for this interface */
+ octeon3_pko_interface_uninit(priv->node, &priv->pko_queue, 1);
+
+ /* Free the receive contexts sso groups */
+ for (i = 0; i < rx_contexts; i++)
+ grp[i] = priv->rx_cxt[i].rx_grp;
+ octeon3_sso_free_groups(priv->node, grp, rx_contexts);
+}
+
+static irqreturn_t octeon3_eth_rx_handler(int irq, void *info)
+{
+ struct octeon3_rx *rx = info;
+
+ /* Disarm the irq. */
+ octeon3_sso_irq_set(rx->parent->node, rx->rx_grp, false);
+
+ napi_schedule(&rx->napiw->napi);
+ return IRQ_HANDLED;
+}
+
+static int octeon3_eth_common_ndo_open(struct net_device *netdev)
+{
+ struct octeon3_ethernet *priv = netdev_priv(netdev);
+ struct octeon3_rx *rx;
+ int i, idx, r;
+
+ for (i = 0; i < priv->num_rx_cxt; i++) {
+ unsigned int sso_intsn;
+ int cpu;
+
+ rx = priv->rx_cxt + i;
+ sso_intsn = SSO_IRQ_START | rx->rx_grp;
+
+ spin_lock_init(&rx->napi_idx_lock);
+
+ rx->rx_irq = irq_create_mapping(NULL, sso_intsn);
+ if (!rx->rx_irq) {
+ netdev_err(netdev, "ERROR: Couldn't map hwirq: %x\n",
+ sso_intsn);
+ r = -EINVAL;
+ goto err1;
+ }
+ r = request_irq(rx->rx_irq, octeon3_eth_rx_handler,
+ IRQ_TYPE_EDGE_RISING, netdev_name(netdev), rx);
+ if (r) {
+ netdev_err(netdev, "ERROR: Couldn't request irq: %d\n",
+ rx->rx_irq);
+ r = -ENOMEM;
+ goto err2;
+ }
+
+ octeon3_eth_gen_affinity(priv->node, &rx->rx_affinity_hint);
+ irq_set_affinity_hint(rx->rx_irq, &rx->rx_affinity_hint);
+
+ /* Allocate a napi index for this receive context */
+ bitmap_zero(priv->rx_cxt[i].napi_idx_bitmap, MAX_CORES);
+ idx = find_first_zero_bit(priv->rx_cxt[i].napi_idx_bitmap,
+ MAX_CORES);
+ if (idx >= MAX_CORES) {
+ netdev_err(netdev, "ERROR: Couldn't get napi index\n");
+ r = -ENOMEM;
+ goto err3;
+ }
+ bitmap_set(priv->rx_cxt[i].napi_idx_bitmap, idx, 1);
+ cpu = cpumask_first(&rx->rx_affinity_hint);
+
+ priv->rx_cxt[i].napiw = octeon3_napi_alloc(&priv->rx_cxt[i],
+ idx, cpu);
+ if (!priv->rx_cxt[i].napiw) {
+ r = -ENOMEM;
+ goto err4;
+ }
+
+ /* Arm the irq. */
+ octeon3_sso_irq_set(priv->node, rx->rx_grp, true);
+ }
+ octeon3_eth_replenish_rx(priv, priv->rx_buf_count);
+
+ return 0;
+
+err4:
+ bitmap_clear(priv->rx_cxt[i].napi_idx_bitmap, idx, 1);
+err3:
+ irq_set_affinity_hint(rx->rx_irq, NULL);
+ free_irq(rx->rx_irq, rx);
+err2:
+ irq_dispose_mapping(rx->rx_irq);
+err1:
+ for (i--; i >= 0; i--) {
+ rx = priv->rx_cxt + i;
+ irq_dispose_mapping(rx->rx_irq);
+ free_irq(rx->rx_irq, rx);
+ octeon3_rm_napi_from_cxt(priv->node, priv->rx_cxt[i].napiw);
+ priv->rx_cxt[i].napiw = NULL;
+ }
+
+ return r;
+}
+
+static int octeon3_eth_bgx_ndo_open(struct net_device *netdev)
+{
+ int rc;
+
+ rc = octeon3_eth_common_ndo_open(netdev);
+ if (rc == 0)
+ rc = bgx_port_enable(netdev);
+
+ return rc;
+}
+
+static int octeon3_eth_common_ndo_stop(struct net_device *netdev)
+{
+ struct octeon3_ethernet *priv = netdev_priv(netdev);
+ struct octeon3_rx *rx;
+ struct sk_buff *skb;
+ void **w;
+ int i;
+
+ /* Allow enough time for ingress in transit packets to be drained */
+ msleep(20);
+
+ /* Wait until sso has no more work for this interface */
+ for (i = 0; i < priv->num_rx_cxt; i++) {
+ rx = priv->rx_cxt + i;
+ while (oct_csr_read(SSO_GRP_AQ_CNT(priv->node, rx->rx_grp)))
+ msleep(20);
+ }
+
+ /* Free the irq and napi context for each rx context */
+ for (i = 0; i < priv->num_rx_cxt; i++) {
+ rx = priv->rx_cxt + i;
+ octeon3_sso_irq_set(priv->node, rx->rx_grp, false);
+ irq_set_affinity_hint(rx->rx_irq, NULL);
+ free_irq(rx->rx_irq, rx);
+ irq_dispose_mapping(rx->rx_irq);
+ rx->rx_irq = 0;
+
+ octeon3_rm_napi_from_cxt(priv->node, rx->napiw);
+ rx->napiw = NULL;
+ WARN_ON(!bitmap_empty(rx->napi_idx_bitmap, MAX_CORES));
+ }
+
+ /* Free the packet buffers */
+ for (;;) {
+ w = octeon_fpa3_alloc(priv->node, priv->pki_aura);
+ if (!w)
+ break;
+ skb = w[0];
+ dev_kfree_skb(skb);
+ }
+
+ return 0;
+}
+
+static int octeon3_eth_bgx_ndo_stop(struct net_device *netdev)
+{
+ int r;
+
+ r = bgx_port_disable(netdev);
+ if (r)
+ return r;
+
+ return octeon3_eth_common_ndo_stop(netdev);
+}
+
+static inline u64 build_pko_send_hdr_desc(struct sk_buff *skb, int gaura)
+{
+ u64 checksum_alg, send_hdr = 0;
+ u8 l4_hdr = 0;
+
+ /* See PKO_SEND_HDR_S in the HRM for the send header descriptor
+ * format.
+ */
+#ifdef __LITTLE_ENDIAN
+ send_hdr |= PKO_SEND_HDR_LE;
+#endif
+
+ if (!OCTEON_IS_MODEL(OCTEON_CN78XX_PASS1_X)) {
+ /* Don't allocate to L2 */
+ send_hdr |= PKO_SEND_HDR_N2;
+ }
+
+ /* Don't automatically free to FPA */
+ send_hdr |= PKO_SEND_HDR_DF;
+
+ send_hdr |= skb->len;
+ send_hdr |= (u64)gaura << PKO_SEND_HDR_AURA_SHIFT;
+
+ if (skb->ip_summed != CHECKSUM_NONE &&
+ skb->ip_summed != CHECKSUM_UNNECESSARY) {
+#ifndef BROKEN_SIMULATOR_CSUM
+ switch (skb->protocol) {
+ case htons(ETH_P_IP):
+ send_hdr |= ETH_HLEN << PKO_SEND_HDR_L3PTR_SHIFT;
+ send_hdr |= PKO_SEND_HDR_CKL3;
+ l4_hdr = ip_hdr(skb)->protocol;
+ send_hdr |= (ETH_HLEN + (4 * ip_hdr(skb)->ihl)) <<
+ PKO_SEND_HDR_L4PTR_SHIFT;
+ break;
+
+ case htons(ETH_P_IPV6):
+ l4_hdr = ipv6_hdr(skb)->nexthdr;
+ send_hdr |= ETH_HLEN << PKO_SEND_HDR_L3PTR_SHIFT;
+ break;
+
+ default:
+ break;
+ }
+#endif
+
+ checksum_alg = 1; /* UDP == 1 */
+ switch (l4_hdr) {
+ case IPPROTO_SCTP:
+ if (OCTEON_IS_MODEL(OCTEON_CN78XX_PASS1_X))
+ break;
+ checksum_alg++; /* SCTP == 3 */
+ /* Fall through */
+ case IPPROTO_TCP: /* TCP == 2 */
+ checksum_alg++;
+ /* Fall through */
+ case IPPROTO_UDP:
+ if (skb_transport_header_was_set(skb)) {
+ int l4ptr = skb_transport_header(skb) -
+ skb->data;
+ send_hdr &= ~PKO_SEND_HDR_L4PTR_MASK;
+ send_hdr |= l4ptr << PKO_SEND_HDR_L4PTR_SHIFT;
+ send_hdr |= checksum_alg <<
+ PKO_SEND_HDR_CKL4_SHIFT;
+ }
+ break;
+
+ default:
+ break;
+ }
+ }
+
+ return send_hdr;
+}
+
+static inline u64 build_pko_send_ext_desc(struct sk_buff *skb)
+{
+ u64 send_ext;
+
+ /* See PKO_SEND_EXT_S in the HRM for the send extended descriptor
+ * format.
+ */
+ skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
+ send_ext = (u64)PKO_SENDSUBDC_EXT << PKO_SEND_SUBDC4_SHIFT;
+ send_ext |= (u64)PKO_REDALG_E_SEND << PKO_SEND_EXT_RA_SHIFT;
+ send_ext |= PKO_SEND_EXT_TSTMP;
+ send_ext |= ETH_HLEN << PKO_SEND_EXT_MARKPTR_SHIFT;
+
+ return send_ext;
+}
+
+static inline u64 build_pko_send_tso(struct sk_buff *skb, uint mtu)
+{
+ u64 send_tso;
+
+ /* See PKO_SEND_TSO_S in the HRM for the send tso descriptor format */
+ send_tso = 12ull << PKO_SEND_TSO_L2LEN_SHIFT;
+ send_tso |= (u64)PKO_SENDSUBDC_TSO << PKO_SEND_SUBDC4_SHIFT;
+ send_tso |= (skb_transport_offset(skb) + tcp_hdrlen(skb)) <<
+ PKO_SEND_TSO_SB_SHIFT;
+ send_tso |= (mtu + ETH_HLEN) << PKO_SEND_TSO_MSS_SHIFT;
+
+ return send_tso;
+}
+
+static inline u64 build_pko_send_mem_sub(u64 addr)
+{
+ u64 send_mem;
+
+ /* See PKO_SEND_MEM_S in the HRM for the send mem descriptor format */
+ send_mem = (u64)PKO_SENDSUBDC_MEM << PKO_SEND_SUBDC4_SHIFT;
+ send_mem |= (u64)PKO_MEMDSZ_B64 << PKO_SEND_MEM_DSZ_SHIFT;
+ send_mem |= (u64)PKO_MEMALG_SUB << PKO_SEND_MEM_ALG_SHIFT;
+ send_mem |= 1ull << PKO_SEND_MEM_OFFSET_SHIFT;
+ send_mem |= addr;
+
+ return send_mem;
+}
+
+static inline u64 build_pko_send_mem_ts(u64 addr)
+{
+ u64 send_mem;
+
+ /* See PKO_SEND_MEM_S in the HRM for the send mem descriptor format */
+ send_mem = 1ull << PKO_SEND_MEM_WMEM_SHIFT;
+ send_mem |= (u64)PKO_SENDSUBDC_MEM << PKO_SEND_SUBDC4_SHIFT;
+ send_mem |= (u64)PKO_MEMDSZ_B64 << PKO_SEND_MEM_DSZ_SHIFT;
+ send_mem |= (u64)PKO_MEMALG_SETTSTMP << PKO_SEND_MEM_ALG_SHIFT;
+ send_mem |= addr;
+
+ return send_mem;
+}
+
+static inline u64 build_pko_send_free(u64 addr)
+{
+ u64 send_free;
+
+ /* See PKO_SEND_FREE_S in the HRM for the send free descriptor format */
+ send_free = (u64)PKO_SENDSUBDC_FREE << PKO_SEND_SUBDC4_SHIFT;
+ send_free |= addr;
+
+ return send_free;
+}
+
+static inline u64 build_pko_send_work(int grp, u64 addr)
+{
+ u64 send_work;
+
+ /* See PKO_SEND_WORK_S in the HRM for the send work descriptor format */
+ send_work = (u64)PKO_SENDSUBDC_WORK << PKO_SEND_SUBDC4_SHIFT;
+ send_work |= (u64)grp << PKO_SEND_WORK_GRP_SHIFT;
+ send_work |= SSO_TAG_TYPE_UNTAGGED << PKO_SEND_WORK_TT_SHIFT;
+ send_work |= addr;
+
+ return send_work;
+}
+
+static int octeon3_eth_ndo_start_xmit(struct sk_buff *skb,
+ struct net_device *netdev)
+{
+ struct octeon3_ethernet *priv = netdev_priv(netdev);
+ u64 aq_cnt = 0, *dma_addr, head_len, lmtdma_data;
+ u64 pko_send_desc, scr_off = LMTDMA_SCR_OFFSET;
+ int frag_count, gaura = 0, grp, i;
+ struct octeon3_ethernet_node *oen;
+ struct sk_buff *skb_tmp;
+ unsigned int mss;
+ long backlog;
+ void **work;
+
+ frag_count = 0;
+ if (skb_has_frag_list(skb))
+ skb_walk_frags(skb, skb_tmp)
+ frag_count++;
+
+ /* Drop the packet if pko or sso are not keeping up */
+ oen = octeon3_eth_node + priv->node;
+ aq_cnt = oct_csr_read(SSO_GRP_AQ_CNT(oen->node, oen->tx_complete_grp));
+ aq_cnt &= SSO_GRP_AQ_CNT_AQ_CNT_MASK;
+ backlog = atomic64_inc_return(&priv->tx_backlog);
+ if (unlikely(backlog > MAX_TX_QUEUE_DEPTH || aq_cnt > 100000)) {
+ if (use_tx_queues) {
+ netif_stop_queue(netdev);
+ } else {
+ atomic64_dec(&priv->tx_backlog);
+ goto skip_xmit;
+ }
+ }
+
+ /* We have space for 11 segment pointers, If there will be
+ * more than that, we must linearize. The count is: 1 (base
+ * SKB) + frag_count + nr_frags.
+ */
+ if (unlikely(skb_shinfo(skb)->nr_frags + frag_count > 10)) {
+ if (unlikely(__skb_linearize(skb)))
+ goto skip_xmit;
+ frag_count = 0;
+ }
+
+ work = (void **)skb->cb;
+ work[0] = netdev;
+ work[1] = NULL;
+
+ /* Adjust the port statistics. */
+ atomic64_inc(&priv->tx_packets);
+ atomic64_add(skb->len, &priv->tx_octets);
+
+ /* Make sure packet data writes are committed before
+ * submitting the command below
+ */
+ wmb();
+
+ /* Build the pko command */
+ pko_send_desc = build_pko_send_hdr_desc(skb, gaura);
+ preempt_disable();
+ scratch_write64(scr_off, pko_send_desc);
+ scr_off += sizeof(pko_send_desc);
+
+ /* Request packet to be ptp timestamped */
+ if ((unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) &&
+ unlikely(priv->tx_timestamp_hw)) {
+ pko_send_desc = build_pko_send_ext_desc(skb);
+ scratch_write64(scr_off, pko_send_desc);
+ scr_off += sizeof(pko_send_desc);
+ }
+
+ /* Add the tso descriptor if needed */
+ mss = skb_shinfo(skb)->gso_size;
+ if (unlikely(mss)) {
+ pko_send_desc = build_pko_send_tso(skb, netdev->mtu);
+ scratch_write64(scr_off, pko_send_desc);
+ scr_off += sizeof(pko_send_desc);
+ }
+
+ /* Add a gather descriptor for each segment. See PKO_SEND_GATHER_S for
+ * the send gather descriptor format.
+ */
+ pko_send_desc = 0;
+ pko_send_desc |= (u64)PKO_SENDSUBDC_GATHER <<
+ PKO_SEND_GATHER_SUBDC_SHIFT;
+ head_len = skb_headlen(skb);
+ if (head_len > 0) {
+ pko_send_desc |= head_len << PKO_SEND_GATHER_SIZE_SHIFT;
+ pko_send_desc |= virt_to_phys(skb->data);
+ scratch_write64(scr_off, pko_send_desc);
+ scr_off += sizeof(pko_send_desc);
+ }
+ for (i = 1; i <= skb_shinfo(skb)->nr_frags; i++) {
+ struct skb_frag_struct *fs = skb_shinfo(skb)->frags + i - 1;
+
+ pko_send_desc &= ~(PKO_SEND_GATHER_SIZE_MASK |
+ PKO_SEND_GATHER_ADDR_MASK);
+ pko_send_desc |= (u64)fs->size << PKO_SEND_GATHER_SIZE_SHIFT;
+ pko_send_desc |= virt_to_phys((u8 *)page_address(fs->page.p) +
+ fs->page_offset);
+ scratch_write64(scr_off, pko_send_desc);
+ scr_off += sizeof(pko_send_desc);
+ }
+ skb_walk_frags(skb, skb_tmp) {
+ pko_send_desc &= ~(PKO_SEND_GATHER_SIZE_MASK |
+ PKO_SEND_GATHER_ADDR_MASK);
+ pko_send_desc |= (u64)skb_tmp->len <<
+ PKO_SEND_GATHER_SIZE_SHIFT;
+ pko_send_desc |= virt_to_phys(skb_tmp->data);
+ scratch_write64(scr_off, pko_send_desc);
+ scr_off += sizeof(pko_send_desc);
+ }
+
+ /* Subtract 1 from the tx_backlog. */
+ pko_send_desc = build_pko_send_mem_sub(virt_to_phys(&priv->tx_backlog));
+ scratch_write64(scr_off, pko_send_desc);
+ scr_off += sizeof(pko_send_desc);
+
+ /* Write the ptp timestamp in the skb itself */
+ if ((unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) &&
+ unlikely(priv->tx_timestamp_hw)) {
+ pko_send_desc = build_pko_send_mem_ts(virt_to_phys(&work[1]));
+ scratch_write64(scr_off, pko_send_desc);
+ scr_off += sizeof(pko_send_desc);
+ }
+
+ /* Send work when finished with the packet. */
+ grp = octeon3_eth_lgrp_to_ggrp(priv->node, priv->tx_complete_grp);
+ pko_send_desc = build_pko_send_work(grp, virt_to_phys(work));
+ scratch_write64(scr_off, pko_send_desc);
+ scr_off += sizeof(pko_send_desc);
+
+ /* See PKO_SEND_DMA_S in the HRM for the lmtdam data format */
+ lmtdma_data = (u64)(LMTDMA_SCR_OFFSET >> PKO_LMTDMA_SCRADDR_SHIFT) <<
+ PKO_QUERY_DMA_SCRADDR_SHIFT;
+ if (wait_pko_response)
+ lmtdma_data |= 1ull << PKO_QUERY_DMA_RTNLEN_SHIFT;
+ lmtdma_data |= 0x51ull << PKO_QUERY_DMA_DID_SHIFT;
+ lmtdma_data |= (u64)priv->node << PKO_QUERY_DMA_NODE_SHIFT;
+ lmtdma_data |= priv->pko_queue << PKO_QUERY_DMA_DQ_SHIFT;
+
+ dma_addr = (u64 *)(LMTDMA_ORDERED_IO_ADDR | ((scr_off & 0x78) - 8));
+ *dma_addr = lmtdma_data;
+
+ preempt_enable();
+
+ if (wait_pko_response) {
+ u64 query_rtn;
+
+ CVMX_SYNCIOBDMA;
+
+ /* See PKO_QUERY_RTN_S in the HRM for the return format */
+ query_rtn = scratch_read64(LMTDMA_SCR_OFFSET);
+ query_rtn >>= PKO_QUERY_RTN_DQSTATUS_SHIFT;
+ if (unlikely(query_rtn != PKO_DQSTATUS_PASS)) {
+ netdev_err(netdev, "PKO enqueue failed %llx\n",
+ (unsigned long long)query_rtn);
+ dev_kfree_skb_any(skb);
+ }
+ }
+
+ return NETDEV_TX_OK;
+skip_xmit:
+ atomic64_inc(&priv->tx_dropped);
+ dev_kfree_skb_any(skb);
+ return NETDEV_TX_OK;
+}
+
+static void octeon3_eth_ndo_get_stats64(struct net_device *netdev,
+ struct rtnl_link_stats64 *s)
+{
+ u64 delta_dropped, delta_octets, delta_packets, dropped;
+ struct octeon3_ethernet *priv = netdev_priv(netdev);
+ u64 octets, packets;
+
+ spin_lock(&priv->stat_lock);
+
+ octeon3_pki_get_stats(priv->node, priv->pknd, &packets, &octets,
+ &dropped);
+
+ delta_packets = (packets - priv->last_packets) & GENMASK_ULL(47, 0);
+ delta_octets = (octets - priv->last_octets) & GENMASK_ULL(47, 0);
+ delta_dropped = (dropped - priv->last_dropped) & GENMASK_ULL(47, 0);
+
+ priv->last_packets = packets;
+ priv->last_octets = octets;
+ priv->last_dropped = dropped;
+
+ spin_unlock(&priv->stat_lock);
+
+ atomic64_add(delta_packets, &priv->rx_packets);
+ atomic64_add(delta_octets, &priv->rx_octets);
+ atomic64_add(delta_dropped, &priv->rx_dropped);
+
+ s->rx_packets = atomic64_read(&priv->rx_packets);
+ s->rx_bytes = atomic64_read(&priv->rx_octets);
+ s->rx_dropped = atomic64_read(&priv->rx_dropped);
+ s->rx_errors = atomic64_read(&priv->rx_errors);
+ s->rx_length_errors = atomic64_read(&priv->rx_length_errors);
+ s->rx_crc_errors = atomic64_read(&priv->rx_crc_errors);
+
+ s->tx_packets = atomic64_read(&priv->tx_packets);
+ s->tx_bytes = atomic64_read(&priv->tx_octets);
+ s->tx_dropped = atomic64_read(&priv->tx_dropped);
+}
+
+static int octeon3_eth_set_mac_address(struct net_device *netdev, void *addr)
+{
+ int r = eth_mac_addr(netdev, addr);
+
+ if (r)
+ return r;
+
+ bgx_port_set_rx_filtering(netdev);
+
+ return 0;
+}
+
+static u64 octeon3_cyclecounter_read(const struct cyclecounter *cc)
+{
+ struct octeon3_ethernet *priv;
+ u64 count;
+
+ priv = container_of(cc, struct octeon3_ethernet, cc);
+ count = oct_csr_read(MIO_PTP_CLOCK_HI(priv->node));
+ return count;
+}
+
+static int octeon3_bgx_hwtstamp(struct net_device *netdev, int en)
+{
+ struct octeon3_ethernet *priv = netdev_priv(netdev);
+ u64 data;
+
+ switch (bgx_port_get_mode(priv->node, priv->interface, priv->index)) {
+ case PORT_MODE_RGMII:
+ case PORT_MODE_SGMII:
+ data = oct_csr_read(BGX_GMP_GMI_RX_FRM_CTL(priv->node,
+ priv->interface,
+ priv->index));
+ if (en)
+ data |= BGX_GMP_GMI_RX_FRM_CTL_PTP_MODE;
+ else
+ data &= ~BGX_GMP_GMI_RX_FRM_CTL_PTP_MODE;
+ oct_csr_write(data, BGX_GMP_GMI_RX_FRM_CTL(priv->node,
+ priv->interface,
+ priv->index));
+ break;
+
+ case PORT_MODE_XAUI:
+ case PORT_MODE_RXAUI:
+ case PORT_MODE_10G_KR:
+ case PORT_MODE_XLAUI:
+ case PORT_MODE_40G_KR4:
+ case PORT_MODE_XFI:
+ data = oct_csr_read(BGX_SMU_RX_FRM_CTL(priv->node,
+ priv->interface,
+ priv->index));
+ if (en)
+ data |= BGX_GMP_GMI_RX_FRM_CTL_PTP_MODE;
+ else
+ data &= ~BGX_GMP_GMI_RX_FRM_CTL_PTP_MODE;
+ oct_csr_write(data, BGX_SMU_RX_FRM_CTL(priv->node,
+ priv->interface,
+ priv->index));
+ break;
+
+ default:
+ /* No timestamp support*/
+ return -EOPNOTSUPP;
+ }
+
+ return 0;
+}
+
+static int octeon3_pki_hwtstamp(struct net_device *netdev, int en)
+{
+ struct octeon3_ethernet *priv = netdev_priv(netdev);
+ int skip = en ? 8 : 0;
+
+ octeon3_pki_set_ptp_skip(priv->node, priv->pknd, skip);
+
+ return 0;
+}
+
+static int octeon3_ioctl_hwtstamp(struct net_device *netdev, struct ifreq *rq,
+ int cmd)
+{
+ struct octeon3_ethernet *priv = netdev_priv(netdev);
+ struct hwtstamp_config config;
+ u64 data;
+ int en;
+
+ /* The PTP block should be enabled */
+ data = oct_csr_read(MIO_PTP_CLOCK_CFG(priv->node));
+ if (!(data & MIO_PTP_CLOCK_CFG_PTP_EN)) {
+ netdev_err(netdev, "Error: PTP clock not enabled\n");
+ return -EOPNOTSUPP;
+ }
+
+ if (copy_from_user(&config, rq->ifr_data, sizeof(config)))
+ return -EFAULT;
+
+ if (config.flags) /* reserved for future extensions */
+ return -EINVAL;
+
+ switch (config.tx_type) {
+ case HWTSTAMP_TX_OFF:
+ priv->tx_timestamp_hw = 0;
+ break;
+ case HWTSTAMP_TX_ON:
+ priv->tx_timestamp_hw = 1;
+ break;
+ default:
+ return -ERANGE;
+ }
+
+ switch (config.rx_filter) {
+ case HWTSTAMP_FILTER_NONE:
+ priv->rx_timestamp_hw = 0;
+ en = 0;
+ break;
+ case HWTSTAMP_FILTER_ALL:
+ case HWTSTAMP_FILTER_SOME:
+ case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
+ case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
+ case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
+ case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
+ case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
+ case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
+ case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
+ case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
+ case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
+ case HWTSTAMP_FILTER_PTP_V2_EVENT:
+ case HWTSTAMP_FILTER_PTP_V2_SYNC:
+ case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
+ priv->rx_timestamp_hw = 1;
+ en = 1;
+ break;
+ default:
+ return -ERANGE;
+ }
+
+ octeon3_bgx_hwtstamp(netdev, en);
+ octeon3_pki_hwtstamp(netdev, en);
+
+ priv->cc.read = octeon3_cyclecounter_read;
+ priv->cc.mask = CYCLECOUNTER_MASK(64);
+ /* Ptp counter is always in nsec */
+ priv->cc.mult = 1;
+ priv->cc.shift = 0;
+ timecounter_init(&priv->tc, &priv->cc, ktime_to_ns(ktime_get_real()));
+
+ return 0;
+}
+
+static int octeon3_adjfreq(struct ptp_clock_info *ptp, s32 ppb)
+{
+ struct octeon3_ethernet *priv;
+ int neg_ppb = 0;
+ u64 comp, diff;
+
+ priv = container_of(ptp, struct octeon3_ethernet, ptp_info);
+
+ if (ppb < 0) {
+ ppb = -ppb;
+ neg_ppb = 1;
+ }
+
+ /* The part per billion (ppb) is a delta from the base frequency */
+ comp = (NSEC_PER_SEC << 32) / octeon_get_io_clock_rate();
+
+ diff = comp;
+ diff *= ppb;
+ diff = div_u64(diff, 1000000000ULL);
+
+ comp = neg_ppb ? comp - diff : comp + diff;
+
+ oct_csr_write(comp, MIO_PTP_CLOCK_COMP(priv->node));
+
+ return 0;
+}
+
+static int octeon3_adjtime(struct ptp_clock_info *ptp, s64 delta)
+{
+ struct octeon3_ethernet *priv;
+ unsigned long flags;
+ s64 now;
+
+ priv = container_of(ptp, struct octeon3_ethernet, ptp_info);
+
+ spin_lock_irqsave(&priv->ptp_lock, flags);
+ now = timecounter_read(&priv->tc);
+ now += delta;
+ timecounter_init(&priv->tc, &priv->cc, now);
+ spin_unlock_irqrestore(&priv->ptp_lock, flags);
+
+ return 0;
+}
+
+static int octeon3_gettime(struct ptp_clock_info *ptp, struct timespec64 *ts)
+{
+ struct octeon3_ethernet *priv;
+ unsigned long flags;
+ u32 remainder;
+ u64 ns;
+
+ priv = container_of(ptp, struct octeon3_ethernet, ptp_info);
+
+ spin_lock_irqsave(&priv->ptp_lock, flags);
+ ns = timecounter_read(&priv->tc);
+ spin_unlock_irqrestore(&priv->ptp_lock, flags);
+ ts->tv_sec = div_u64_rem(ns, 1000000000ULL, &remainder);
+ ts->tv_nsec = remainder;
+
+ return 0;
+}
+
+static int octeon3_settime(struct ptp_clock_info *ptp,
+ const struct timespec64 *ts)
+{
+ struct octeon3_ethernet *priv;
+ unsigned long flags;
+ u64 ns;
+
+ priv = container_of(ptp, struct octeon3_ethernet, ptp_info);
+ ns = timespec64_to_ns(ts);
+
+ spin_lock_irqsave(&priv->ptp_lock, flags);
+ timecounter_init(&priv->tc, &priv->cc, ns);
+ spin_unlock_irqrestore(&priv->ptp_lock, flags);
+
+ return 0;
+}
+
+static int octeon3_enable(struct ptp_clock_info *ptp,
+ struct ptp_clock_request *rq, int on)
+{
+ return -EOPNOTSUPP;
+}
+
+static int octeon3_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
+{
+ int rc;
+
+ switch (cmd) {
+ case SIOCSHWTSTAMP:
+ rc = octeon3_ioctl_hwtstamp(netdev, ifr, cmd);
+ break;
+
+ default:
+ rc = bgx_port_do_ioctl(netdev, ifr, cmd);
+ break;
+ }
+
+ return rc;
+}
+
+static const struct net_device_ops octeon3_eth_netdev_ops = {
+ .ndo_init = octeon3_eth_bgx_ndo_init,
+ .ndo_uninit = octeon3_eth_ndo_uninit,
+ .ndo_open = octeon3_eth_bgx_ndo_open,
+ .ndo_stop = octeon3_eth_bgx_ndo_stop,
+ .ndo_start_xmit = octeon3_eth_ndo_start_xmit,
+ .ndo_get_stats64 = octeon3_eth_ndo_get_stats64,
+ .ndo_set_rx_mode = bgx_port_set_rx_filtering,
+ .ndo_set_mac_address = octeon3_eth_set_mac_address,
+ .ndo_change_mtu = octeon3_eth_ndo_change_mtu,
+ .ndo_do_ioctl = octeon3_ioctl,
+};
+
+static int octeon3_eth_probe(struct platform_device *pdev)
+{
+ struct octeon3_ethernet *priv;
+ struct net_device *netdev;
+ int r;
+
+ struct mac_platform_data *pd = dev_get_platdata(&pdev->dev);
+
+ r = octeon3_eth_global_init(pd->numa_node, pdev);
+ if (r)
+ return r;
+
+ dev_info(&pdev->dev, "Probing %d-%d:%d\n", pd->numa_node, pd->interface,
+ pd->port);
+ netdev = alloc_etherdev(sizeof(struct octeon3_ethernet));
+ if (!netdev) {
+ dev_err(&pdev->dev, "Failed to allocated ethernet device\n");
+ return -ENOMEM;
+ }
+
+ /* Using transmit queues degrades performance significantly */
+ if (!use_tx_queues)
+ netdev->tx_queue_len = 0;
+
+ SET_NETDEV_DEV(netdev, &pdev->dev);
+ dev_set_drvdata(&pdev->dev, netdev);
+
+ if (pd->mac_type == BGX_MAC)
+ bgx_port_set_netdev(pdev->dev.parent, netdev);
+ priv = netdev_priv(netdev);
+ priv->netdev = netdev;
+ priv->mac_type = pd->mac_type;
+ INIT_LIST_HEAD(&priv->list);
+ priv->node = pd->numa_node;
+
+ mutex_lock(&octeon3_eth_node[priv->node].device_list_lock);
+ list_add_tail_rcu(&priv->list,
+ &octeon3_eth_node[priv->node].device_list);
+ mutex_unlock(&octeon3_eth_node[priv->node].device_list_lock);
+
+ priv->index = pd->port;
+ priv->interface = pd->interface;
+ spin_lock_init(&priv->stat_lock);
+
+ if (pd->src_type == XCV)
+ snprintf(netdev->name, IFNAMSIZ, "rgmii%d", pd->port);
+
+ if (priv->mac_type == BGX_MAC)
+ netdev->netdev_ops = &octeon3_eth_netdev_ops;
+
+ if (register_netdev(netdev) < 0) {
+ dev_err(&pdev->dev, "Failed to register ethernet device\n");
+ list_del(&priv->list);
+ free_netdev(netdev);
+ }
+
+ spin_lock_init(&priv->ptp_lock);
+ priv->ptp_info.owner = THIS_MODULE;
+ snprintf(priv->ptp_info.name, 16, "octeon3 ptp");
+ priv->ptp_info.max_adj = 250000000;
+ priv->ptp_info.n_alarm = 0;
+ priv->ptp_info.n_ext_ts = 0;
+ priv->ptp_info.n_per_out = 0;
+ priv->ptp_info.pps = 0;
+ priv->ptp_info.adjfreq = octeon3_adjfreq;
+ priv->ptp_info.adjtime = octeon3_adjtime;
+ priv->ptp_info.gettime64 = octeon3_gettime;
+ priv->ptp_info.settime64 = octeon3_settime;
+ priv->ptp_info.enable = octeon3_enable;
+ priv->ptp_clock = ptp_clock_register(&priv->ptp_info, &pdev->dev);
+
+ netdev_info(netdev, "Registered\n");
+ return 0;
+}
+
+/* octeon3_eth_global_exit - Free all the used resources and restore the
+ * hardware to the default state.
+ * @node: Node to free/reset.
+ *
+ * Returns 0 if successful.
+ * Returns <0 for error codes.
+ */
+static int octeon3_eth_global_exit(int node)
+{
+ struct octeon3_ethernet_node *oen = octeon3_eth_node + node;
+ int i;
+
+ /* Free the tx_complete irq */
+ octeon3_sso_irq_set(node, oen->tx_complete_grp, false);
+ irq_set_affinity_hint(oen->tx_irq, NULL);
+ free_irq(oen->tx_irq, oen);
+ irq_dispose_mapping(oen->tx_irq);
+ oen->tx_irq = 0;
+
+ /* Stop the worker threads */
+ for (i = 0; i < ARRAY_SIZE(oen->workers); i++)
+ kthread_stop(oen->workers[i].task);
+
+ /* Shutdown pki */
+ octeon3_pki_shutdown(node);
+ octeon_fpa3_release_pool(node, oen->pki_packet_pool);
+ kfree(oen->pki_packet_pool_stack);
+
+ /* Shutdown pko */
+ octeon3_pko_exit_global(node);
+ for (;;) {
+ void **w;
+
+ w = octeon_fpa3_alloc(node, oen->pko_aura);
+ if (!w)
+ break;
+ kmem_cache_free(octeon3_eth_sso_pko_cache, w);
+ }
+ octeon_fpa3_release_aura(node, oen->pko_aura);
+ octeon_fpa3_release_pool(node, oen->pko_pool);
+ kfree(oen->pko_pool_stack);
+
+ /* Shutdown sso */
+ octeon3_sso_shutdown(node, oen->sso_aura);
+ octeon3_sso_free_groups(node, &oen->tx_complete_grp, 1);
+ for (;;) {
+ void **w;
+
+ w = octeon_fpa3_alloc(node, oen->sso_aura);
+ if (!w)
+ break;
+ kmem_cache_free(octeon3_eth_sso_pko_cache, w);
+ }
+ octeon_fpa3_release_aura(node, oen->sso_aura);
+ octeon_fpa3_release_pool(node, oen->sso_pool);
+ kfree(oen->sso_pool_stack);
+
+ return 0;
+}
+
+static int octeon3_eth_remove(struct platform_device *pdev)
+{
+ struct mac_platform_data *pd = dev_get_platdata(&pdev->dev);
+ struct net_device *netdev = dev_get_drvdata(&pdev->dev);
+ struct octeon3_ethernet *priv = netdev_priv(netdev);
+ struct octeon3_ethernet_node *oen;
+ int node = priv->node;
+
+ oen = octeon3_eth_node + node;
+
+ ptp_clock_unregister(priv->ptp_clock);
+ unregister_netdev(netdev);
+ if (pd->mac_type == BGX_MAC)
+ bgx_port_set_netdev(pdev->dev.parent, NULL);
+ dev_set_drvdata(&pdev->dev, NULL);
+
+ /* Free all resources when there are no more devices */
+ mutex_lock(&octeon3_eth_init_mutex);
+ mutex_lock(&oen->device_list_lock);
+ list_del_rcu(&priv->list);
+ if (oen->init_done && list_empty(&oen->device_list)) {
+ int i;
+
+ for (i = 0; i < MAX_NAPIS_PER_NODE; i++) {
+ napi_disable(&napi_wrapper[node][i].napi);
+ netif_napi_del(&napi_wrapper[node][i].napi);
+ }
+
+ oen->init_done = false;
+ oen->napi_init_done = false;
+ octeon3_eth_global_exit(node);
+ }
+
+ mutex_unlock(&oen->device_list_lock);
+ mutex_unlock(&octeon3_eth_init_mutex);
+ free_netdev(netdev);
+
+ return 0;
+}
+
+static void octeon3_eth_shutdown(struct platform_device *pdev)
+{
+ octeon3_eth_remove(pdev);
+}
+
+static struct platform_driver octeon3_eth_driver = {
+ .probe = octeon3_eth_probe,
+ .remove = octeon3_eth_remove,
+ .shutdown = octeon3_eth_shutdown,
+ .driver = {
+ .owner = THIS_MODULE,
+ .name = "ethernet-mac-pki",
+ },
+};
+
+static int __init octeon3_eth_init(void)
+{
+ if (rx_contexts <= 0)
+ rx_contexts = 1;
+ if (rx_contexts > MAX_RX_CONTEXTS)
+ rx_contexts = MAX_RX_CONTEXTS;
+
+ return platform_driver_register(&octeon3_eth_driver);
+}
+module_init(octeon3_eth_init);
+
+static void __exit octeon3_eth_exit(void)
+{
+ platform_driver_unregister(&octeon3_eth_driver);
+
+ /* Destroy the memory cache used by sso and pko */
+ kmem_cache_destroy(octeon3_eth_sso_pko_cache);
+}
+module_exit(octeon3_eth_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Cavium, Inc. <support@caviumnetworks.com>");
+MODULE_DESCRIPTION("Cavium, Inc. PKI/PKO Ethernet driver.");
--
2.1.4
^ permalink raw reply related
* [PATCH v12 09/10] netdev: cavium: octeon: Add Octeon III BGX Ethernet building
From: Steven J. Hill @ 2018-06-27 21:25 UTC (permalink / raw)
To: netdev; +Cc: Carlos Munoz, Chandrakala Chavva, Steven J. Hill
In-Reply-To: <1530134719-19407-1-git-send-email-steven.hill@cavium.com>
From: Carlos Munoz <cmunoz@cavium.com>
Add the build and configuration files for the BGX Ethernet.
Signed-off-by: Carlos Munoz <cmunoz@cavium.com>
Signed-off-by: Steven J. Hill <Steven.Hill@cavium.com>
---
drivers/net/ethernet/cavium/Kconfig | 22 +++++++++++++++++++++-
drivers/net/ethernet/cavium/octeon/Makefile | 8 +++++++-
2 files changed, 28 insertions(+), 2 deletions(-)
diff --git a/drivers/net/ethernet/cavium/Kconfig b/drivers/net/ethernet/cavium/Kconfig
index 043e3c1..3b9709d 100644
--- a/drivers/net/ethernet/cavium/Kconfig
+++ b/drivers/net/ethernet/cavium/Kconfig
@@ -4,7 +4,7 @@
config NET_VENDOR_CAVIUM
bool "Cavium ethernet drivers"
- depends on PCI
+ depends on PCI || CAVIUM_OCTEON_SOC
default y
---help---
Select this option if you want enable Cavium network support.
@@ -100,4 +100,24 @@ config LIQUIDIO_VF
will be called liquidio_vf. MSI-X interrupt support is required
for this driver to work correctly
+config OCTEON3_BGX_PORT
+ tristate "Cavium Octeon III BGX port support"
+ depends on CAVIUM_OCTEON_SOC
+ ---help---
+ This driver adds support for Cavium Octeon III BGX ports. BGX ports
+ support sgmii, rgmii, xaui, rxaui, xlaui, xfi, 10KR and 40KR modes.
+
+ Say Y to use the management port on Octeon III boards or to use
+ any other ethernet port.
+
+config OCTEON3_ETHERNET
+ tristate "Cavium OCTEON III PKI/PKO Ethernet support"
+ depends on CAVIUM_OCTEON_SOC
+ select OCTEON_BGX_PORT
+ select OCTEON_FPA3
+ select FW_LOADER
+ ---help---
+ Support for 'BGX' Ethernet via PKI/PKO units. No support for
+ cn70xx chips, use OCTEON_ETHERNET instead.
+
endif # NET_VENDOR_CAVIUM
diff --git a/drivers/net/ethernet/cavium/octeon/Makefile b/drivers/net/ethernet/cavium/octeon/Makefile
index efa41c1..1939c84 100644
--- a/drivers/net/ethernet/cavium/octeon/Makefile
+++ b/drivers/net/ethernet/cavium/octeon/Makefile
@@ -1,5 +1,11 @@
+# SPDX-License-Identifier: GPL-2.0
#
# Makefile for the Cavium network device drivers.
#
-obj-$(CONFIG_OCTEON_MGMT_ETHERNET) += octeon_mgmt.o
+obj-$(CONFIG_OCTEON_MGMT_ETHERNET) += octeon_mgmt.o
+obj-$(CONFIG_OCTEON3_BGX_PORT) += octeon3-bgx-nexus.o octeon3-bgx-port.o
+obj-$(CONFIG_OCTEON3_ETHERNET) += octeon3-ethernet.o
+
+octeon3-ethernet-objs += octeon3-core.o octeon3-pki.o octeon3-pko.o \
+ octeon3-sso.o
--
2.1.4
^ permalink raw reply related
* [PATCH v12 10/10] MAINTAINERS: Add entry for drivers/net/ethernet/cavium/octeon/octeon3-*
From: Steven J. Hill @ 2018-06-27 21:25 UTC (permalink / raw)
To: netdev; +Cc: David Daney, Chandrakala Chavva
In-Reply-To: <1530134719-19407-1-git-send-email-steven.hill@cavium.com>
From: David Daney <david.daney@cavium.com>
Signed-off-by: David Daney <david.daney@cavium.com>
---
MAINTAINERS | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/MAINTAINERS b/MAINTAINERS
index 99e5cef..378009c 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -3279,6 +3279,12 @@ W: http://www.cavium.com
S: Supported
F: drivers/mmc/host/cavium*
+CAVIUM OCTEON-III NETWORK DRIVER
+M: Steven J. Hill <Steven.Hill@cavium.com>
+L: netdev@vger.kernel.org
+S: Supported
+F: drivers/net/ethernet/cavium/octeon/octeon3-*
+
CAVIUM OCTEON-TX CRYPTO DRIVER
M: George Cherian <george.cherian@cavium.com>
L: linux-crypto@vger.kernel.org
--
2.1.4
^ permalink raw reply related
* Re: [PATCH rdma-next 08/12] overflow.h: Add arithmetic shift helper
From: Rasmus Villemoes @ 2018-06-27 21:35 UTC (permalink / raw)
To: Leon Romanovsky, Jason Gunthorpe
Cc: Doug Ledford, Kees Cook, RDMA mailing list, Hadar Hen Zion,
Matan Barak, Michael J Ruhl, Noa Osherovich, Raed Salem,
Yishai Hadas, Saeed Mahameed, linux-netdev, linux-kernel
In-Reply-To: <20180627182220.GV30877@mtr-leonro.mtl.com>
On 2018-06-27 20:22, Leon Romanovsky wrote:
> On Wed, Jun 27, 2018 at 12:10:12PM -0600, Jason Gunthorpe wrote:
>> On Wed, Jun 27, 2018 at 11:36:03AM +0200, Rasmus Villemoes wrote:
>>> Well, the types you can check at compile-time, the values not, so you
>>> still have to define the result, i.e. contents of *d, for negative
>>> values (even if we decide that "overflow" should always be signalled in
>>> that case).
>>
>> Why do a need to define a 'result' beyond whatever the not-undefined
>> behavior shift expression produces?
Well, perhaps you don't, it's just that the other check_*_overflow have
that behaviour (which they inherit from gcc's builtins), and it's a
nice-to-have. But I see that it's hard to come up with something
sensible in all the "doesn't make sense" cases. When writing the tests
for test_overflow.c, you can of course just omit the comparison to
the/an expected result in the overflow case.
>> /*
>> * Compute *d = (a << s)
>> *
>> * Returns true if '*d' cannot hold the result or 'a << s' doesn't make sense.
>> * - 'a << s' causes bits to be lost when stored in d
>> * - 's' is garbage (eg negative) or so large that a << s is guarenteed to be 0
>> * - 'a' is negative
>> * - 'a << s' sets the sign bit, if any, in '*d'
>> * *d is not defined if false is returned.
>> */
>> #define check_shift_overflow(a, s, d) \
>> ({ \
>> typeof(a) _a = a; \
>> typeof(s) _s = s; \
>> typeof(d) _d = d; \
>> u64 _a_full = _a; \
>> unsigned int _to_shift = \
>> _s >= 0 && _s < 8 * sizeof(*d) ? _s : 0; \
>> \
>> *_d = (_a_full << _to_shift); \
>> \
>> (_to_shift != _s || *_d < 0 || _a < 0 || \
>> (*_d >> _to_shift) != a); \
>> })
That last a still needs to be _a. Other than that, I don't see anything
wrong with this version.
>> int main(int argc, const char *argv[])
>> {
>> int32_t s32;
>> uint32_t u32;
>>
>> assert(check_shift_overflow(1, 0, &s32) == false && s32 == (1 << 0));
[...]>> assert(check_shift_overflow(0xFFFFFFFF, 1, &s32) == true);
Please add these in some form to test_overflow.c, but do also include
cases where a and *d have different width, e.g. check_shift_overflow(1,
32, &s64) should be ok, while check_shift_overflow(65432, 0, &s16)
should not.
Rasmus
^ permalink raw reply
* [PATCH net-next 0/4] ila: Cleanup
From: Tom Herbert @ 2018-06-27 21:38 UTC (permalink / raw)
To: davem, netdev; +Cc: Tom Herbert
Perform some cleanup in ILA code. This includes:
- Fix rhashtable walk for cases where nl dumps are done with muliple
function calls. Add a skip index to skip over entries in
a node that have been previously visitied. Call rhashtable_walk_peek
to avoid dropping items between calls to ila_nl_dump.
- Call alloc_bucket_spinlocks to create bucket locks.
- Split out module initialization and netlink definitions into
separate files.
- Add ILA_CMD_FLUSH netlink command to clear the ILA translation table.
Tom Herbert (4):
ila: Fix use of rhashtable walk in ila_xlat.c
ila: Call library function alloc_bucket_locks
ila: Create main ila source file
ila: Flush netlink command to clear xlat table
include/uapi/linux/ila.h | 1 +
net/ipv6/ila/Makefile | 2 +-
net/ipv6/ila/ila.h | 27 ++++-
net/ipv6/ila/ila_common.c | 30 -----
net/ipv6/ila/ila_main.c | 121 +++++++++++++++++++
net/ipv6/ila/ila_xlat.c | 291 +++++++++++++++++++++++-----------------------
6 files changed, 292 insertions(+), 180 deletions(-)
create mode 100644 net/ipv6/ila/ila_main.c
--
2.7.4
^ permalink raw reply
* [PATCH net-next 1/4] ila: Fix use of rhashtable walk in ila_xlat.c
From: Tom Herbert @ 2018-06-27 21:38 UTC (permalink / raw)
To: davem, netdev; +Cc: Tom Herbert
In-Reply-To: <1530135542-10372-1-git-send-email-tom@quantonium.net>
Perform better EAGAIN handling, handle case where ila_dump_info
fails and we missed objects in the dump, and add a skip index
to skip over ila entires in a list on a rhashtable node that have
already been visited (by a previous call to ila_nl_dump).
Signed-off-by: Tom Herbert <tom@quantonium.net>
---
net/ipv6/ila/ila_xlat.c | 70 ++++++++++++++++++++++++++++++++++++++-----------
1 file changed, 54 insertions(+), 16 deletions(-)
diff --git a/net/ipv6/ila/ila_xlat.c b/net/ipv6/ila/ila_xlat.c
index 10ae135..40f3f64 100644
--- a/net/ipv6/ila/ila_xlat.c
+++ b/net/ipv6/ila/ila_xlat.c
@@ -475,24 +475,31 @@ static int ila_nl_cmd_get_mapping(struct sk_buff *skb, struct genl_info *info)
struct ila_dump_iter {
struct rhashtable_iter rhiter;
+ int skip;
};
static int ila_nl_dump_start(struct netlink_callback *cb)
{
struct net *net = sock_net(cb->skb->sk);
struct ila_net *ilan = net_generic(net, ila_net_id);
- struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args[0];
+ struct ila_dump_iter *iter;
+ int ret;
- if (!iter) {
- iter = kmalloc(sizeof(*iter), GFP_KERNEL);
- if (!iter)
- return -ENOMEM;
+ iter = kmalloc(sizeof(*iter), GFP_KERNEL);
+ if (!iter)
+ return -ENOMEM;
- cb->args[0] = (long)iter;
+ ret = rhashtable_walk_init(&ilan->rhash_table, &iter->rhiter,
+ GFP_KERNEL);
+ if (ret) {
+ kfree(iter);
+ return ret;
}
- return rhashtable_walk_init(&ilan->rhash_table, &iter->rhiter,
- GFP_KERNEL);
+ iter->skip = 0;
+ cb->args[0] = (long)iter;
+
+ return ret;
}
static int ila_nl_dump_done(struct netlink_callback *cb)
@@ -510,20 +517,45 @@ static int ila_nl_dump(struct sk_buff *skb, struct netlink_callback *cb)
{
struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args[0];
struct rhashtable_iter *rhiter = &iter->rhiter;
+ int skip = iter->skip;
struct ila_map *ila;
int ret;
rhashtable_walk_start(rhiter);
- for (;;) {
- ila = rhashtable_walk_next(rhiter);
+ /* Get first entry */
+ ila = rhashtable_walk_peek(rhiter);
+
+ if (ila && !IS_ERR(ila) && skip) {
+ /* Skip over visited entries */
+
+ while (ila && skip) {
+ /* Skip over any ila entries in this list that we
+ * have already dumped.
+ */
+ ila = rcu_access_pointer(ila->next);
+ skip--;
+ }
+ }
+ skip = 0;
+
+ for (;;) {
if (IS_ERR(ila)) {
- if (PTR_ERR(ila) == -EAGAIN)
- continue;
ret = PTR_ERR(ila);
- goto done;
+ if (ret == -EAGAIN) {
+ /* Table has changed and iter has reset. Return
+ * -EAGAIN to the application even if we have
+ * written data to the skb. The application
+ * needs to deal with this.
+ */
+
+ goto out_ret;
+ } else {
+ break;
+ }
} else if (!ila) {
+ ret = 0;
break;
}
@@ -532,15 +564,21 @@ static int ila_nl_dump(struct sk_buff *skb, struct netlink_callback *cb)
cb->nlh->nlmsg_seq, NLM_F_MULTI,
skb, ILA_CMD_GET);
if (ret)
- goto done;
+ goto out;
+ skip++;
ila = rcu_access_pointer(ila->next);
}
+
+ skip = 0;
+ ila = rhashtable_walk_next(rhiter);
}
- ret = skb->len;
+out:
+ iter->skip = skip;
+ ret = (skb->len ? : ret);
-done:
+out_ret:
rhashtable_walk_stop(rhiter);
return ret;
}
--
2.7.4
^ permalink raw reply related
* [PATCH net-next 2/4] ila: Call library function alloc_bucket_locks
From: Tom Herbert @ 2018-06-27 21:39 UTC (permalink / raw)
To: davem, netdev; +Cc: Tom Herbert
In-Reply-To: <1530135542-10372-1-git-send-email-tom@quantonium.net>
To allocate the array of bucket locks for the hash table we now
call library function alloc_bucket_spinlocks.
Signed-off-by: Tom Herbert <tom@quantonium.net>
---
net/ipv6/ila/ila_xlat.c | 23 +++++------------------
1 file changed, 5 insertions(+), 18 deletions(-)
diff --git a/net/ipv6/ila/ila_xlat.c b/net/ipv6/ila/ila_xlat.c
index 40f3f64..9cc8bee 100644
--- a/net/ipv6/ila/ila_xlat.c
+++ b/net/ipv6/ila/ila_xlat.c
@@ -31,27 +31,14 @@ struct ila_net {
bool hooks_registered;
};
+#define MAX_LOCKS 1024
#define LOCKS_PER_CPU 10
static int alloc_ila_locks(struct ila_net *ilan)
{
- unsigned int i, size;
- unsigned int nr_pcpus = num_possible_cpus();
-
- nr_pcpus = min_t(unsigned int, nr_pcpus, 32UL);
- size = roundup_pow_of_two(nr_pcpus * LOCKS_PER_CPU);
-
- if (sizeof(spinlock_t) != 0) {
- ilan->locks = kvmalloc_array(size, sizeof(spinlock_t),
- GFP_KERNEL);
- if (!ilan->locks)
- return -ENOMEM;
- for (i = 0; i < size; i++)
- spin_lock_init(&ilan->locks[i]);
- }
- ilan->locks_mask = size - 1;
-
- return 0;
+ return alloc_bucket_spinlocks(&ilan->locks, &ilan->locks_mask,
+ MAX_LOCKS, LOCKS_PER_CPU,
+ GFP_KERNEL);
}
static u32 hashrnd __read_mostly;
@@ -640,7 +627,7 @@ static __net_exit void ila_exit_net(struct net *net)
rhashtable_free_and_destroy(&ilan->rhash_table, ila_free_cb, NULL);
- kvfree(ilan->locks);
+ free_bucket_spinlocks(ilan->locks);
if (ilan->hooks_registered)
nf_unregister_net_hooks(net, ila_nf_hook_ops,
--
2.7.4
^ permalink raw reply related
* [PATCH net-next 3/4] ila: Create main ila source file
From: Tom Herbert @ 2018-06-27 21:39 UTC (permalink / raw)
To: davem, netdev; +Cc: Tom Herbert
In-Reply-To: <1530135542-10372-1-git-send-email-tom@quantonium.net>
Create a main ila file that contains the module initialization functions
as well as netlink definitions. Previously these were defined in
ila_xlat and ila_common. This approach allows better extensibility.
Signed-off-by: Tom Herbert <tom@quantonium.net>
---
net/ipv6/ila/Makefile | 2 +-
net/ipv6/ila/ila.h | 26 ++++++++-
net/ipv6/ila/ila_common.c | 30 ----------
net/ipv6/ila/ila_main.c | 115 +++++++++++++++++++++++++++++++++++++
net/ipv6/ila/ila_xlat.c | 142 +++++++++-------------------------------------
5 files changed, 168 insertions(+), 147 deletions(-)
create mode 100644 net/ipv6/ila/ila_main.c
diff --git a/net/ipv6/ila/Makefile b/net/ipv6/ila/Makefile
index 4b32e59..b7739ab 100644
--- a/net/ipv6/ila/Makefile
+++ b/net/ipv6/ila/Makefile
@@ -4,4 +4,4 @@
obj-$(CONFIG_IPV6_ILA) += ila.o
-ila-objs := ila_common.o ila_lwt.o ila_xlat.o
+ila-objs := ila_main.o ila_common.o ila_lwt.o ila_xlat.o
diff --git a/net/ipv6/ila/ila.h b/net/ipv6/ila/ila.h
index 3c7a11b..faba782 100644
--- a/net/ipv6/ila/ila.h
+++ b/net/ipv6/ila/ila.h
@@ -19,6 +19,7 @@
#include <linux/skbuff.h>
#include <linux/types.h>
#include <net/checksum.h>
+#include <net/genetlink.h>
#include <net/ip.h>
#include <net/protocol.h>
#include <uapi/linux/ila.h>
@@ -104,9 +105,30 @@ void ila_update_ipv6_locator(struct sk_buff *skb, struct ila_params *p,
void ila_init_saved_csum(struct ila_params *p);
+struct ila_net {
+ struct {
+ struct rhashtable rhash_table;
+ spinlock_t *locks; /* Bucket locks for entry manipulation */
+ unsigned int locks_mask;
+ bool hooks_registered;
+ } xlat;
+};
+
int ila_lwt_init(void);
void ila_lwt_fini(void);
-int ila_xlat_init(void);
-void ila_xlat_fini(void);
+
+int ila_xlat_init_net(struct net *net);
+void ila_xlat_exit_net(struct net *net);
+
+int ila_xlat_nl_cmd_add_mapping(struct sk_buff *skb, struct genl_info *info);
+int ila_xlat_nl_cmd_del_mapping(struct sk_buff *skb, struct genl_info *info);
+int ila_xlat_nl_cmd_get_mapping(struct sk_buff *skb, struct genl_info *info);
+int ila_xlat_nl_dump_start(struct netlink_callback *cb);
+int ila_xlat_nl_dump_done(struct netlink_callback *cb);
+int ila_xlat_nl_dump(struct sk_buff *skb, struct netlink_callback *cb);
+
+extern unsigned int ila_net_id;
+
+extern struct genl_family ila_nl_family;
#endif /* __ILA_H */
diff --git a/net/ipv6/ila/ila_common.c b/net/ipv6/ila/ila_common.c
index 8c88ecf..5793104 100644
--- a/net/ipv6/ila/ila_common.c
+++ b/net/ipv6/ila/ila_common.c
@@ -154,33 +154,3 @@ void ila_update_ipv6_locator(struct sk_buff *skb, struct ila_params *p,
iaddr->loc = p->locator;
}
-static int __init ila_init(void)
-{
- int ret;
-
- ret = ila_lwt_init();
-
- if (ret)
- goto fail_lwt;
-
- ret = ila_xlat_init();
- if (ret)
- goto fail_xlat;
-
- return 0;
-fail_xlat:
- ila_lwt_fini();
-fail_lwt:
- return ret;
-}
-
-static void __exit ila_fini(void)
-{
- ila_xlat_fini();
- ila_lwt_fini();
-}
-
-module_init(ila_init);
-module_exit(ila_fini);
-MODULE_AUTHOR("Tom Herbert <tom@herbertland.com>");
-MODULE_LICENSE("GPL");
diff --git a/net/ipv6/ila/ila_main.c b/net/ipv6/ila/ila_main.c
new file mode 100644
index 0000000..f6ac6b1
--- /dev/null
+++ b/net/ipv6/ila/ila_main.c
@@ -0,0 +1,115 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <net/genetlink.h>
+#include <net/ila.h>
+#include <net/netns/generic.h>
+#include <uapi/linux/genetlink.h>
+#include "ila.h"
+
+static const struct nla_policy ila_nl_policy[ILA_ATTR_MAX + 1] = {
+ [ILA_ATTR_LOCATOR] = { .type = NLA_U64, },
+ [ILA_ATTR_LOCATOR_MATCH] = { .type = NLA_U64, },
+ [ILA_ATTR_IFINDEX] = { .type = NLA_U32, },
+ [ILA_ATTR_CSUM_MODE] = { .type = NLA_U8, },
+ [ILA_ATTR_IDENT_TYPE] = { .type = NLA_U8, },
+};
+
+static const struct genl_ops ila_nl_ops[] = {
+ {
+ .cmd = ILA_CMD_ADD,
+ .doit = ila_xlat_nl_cmd_add_mapping,
+ .policy = ila_nl_policy,
+ .flags = GENL_ADMIN_PERM,
+ },
+ {
+ .cmd = ILA_CMD_DEL,
+ .doit = ila_xlat_nl_cmd_del_mapping,
+ .policy = ila_nl_policy,
+ .flags = GENL_ADMIN_PERM,
+ },
+ {
+ .cmd = ILA_CMD_GET,
+ .doit = ila_xlat_nl_cmd_get_mapping,
+ .start = ila_xlat_nl_dump_start,
+ .dumpit = ila_xlat_nl_dump,
+ .done = ila_xlat_nl_dump_done,
+ .policy = ila_nl_policy,
+ },
+};
+
+unsigned int ila_net_id;
+
+struct genl_family ila_nl_family __ro_after_init = {
+ .hdrsize = 0,
+ .name = ILA_GENL_NAME,
+ .version = ILA_GENL_VERSION,
+ .maxattr = ILA_ATTR_MAX,
+ .netnsok = true,
+ .parallel_ops = true,
+ .module = THIS_MODULE,
+ .ops = ila_nl_ops,
+ .n_ops = ARRAY_SIZE(ila_nl_ops),
+};
+
+static __net_init int ila_init_net(struct net *net)
+{
+ int err;
+
+ err = ila_xlat_init_net(net);
+ if (err)
+ goto ila_xlat_init_fail;
+
+ return 0;
+
+ila_xlat_init_fail:
+ return err;
+}
+
+static __net_exit void ila_exit_net(struct net *net)
+{
+ ila_xlat_exit_net(net);
+}
+
+static struct pernet_operations ila_net_ops = {
+ .init = ila_init_net,
+ .exit = ila_exit_net,
+ .id = &ila_net_id,
+ .size = sizeof(struct ila_net),
+};
+
+static int __init ila_init(void)
+{
+ int ret;
+
+ ret = register_pernet_device(&ila_net_ops);
+ if (ret)
+ goto register_device_fail;
+
+ ret = genl_register_family(&ila_nl_family);
+ if (ret)
+ goto register_family_fail;
+
+ ret = ila_lwt_init();
+ if (ret)
+ goto fail_lwt;
+
+ return 0;
+
+fail_lwt:
+ genl_unregister_family(&ila_nl_family);
+register_family_fail:
+ unregister_pernet_device(&ila_net_ops);
+register_device_fail:
+ return ret;
+}
+
+static void __exit ila_fini(void)
+{
+ ila_lwt_fini();
+ genl_unregister_family(&ila_nl_family);
+ unregister_pernet_device(&ila_net_ops);
+}
+
+module_init(ila_init);
+module_exit(ila_fini);
+MODULE_AUTHOR("Tom Herbert <tom@herbertland.com>");
+MODULE_LICENSE("GPL");
diff --git a/net/ipv6/ila/ila_xlat.c b/net/ipv6/ila/ila_xlat.c
index 9cc8bee..d05de89 100644
--- a/net/ipv6/ila/ila_xlat.c
+++ b/net/ipv6/ila/ila_xlat.c
@@ -22,21 +22,12 @@ struct ila_map {
struct rcu_head rcu;
};
-static unsigned int ila_net_id;
-
-struct ila_net {
- struct rhashtable rhash_table;
- spinlock_t *locks; /* Bucket locks for entry manipulation */
- unsigned int locks_mask;
- bool hooks_registered;
-};
-
#define MAX_LOCKS 1024
#define LOCKS_PER_CPU 10
static int alloc_ila_locks(struct ila_net *ilan)
{
- return alloc_bucket_spinlocks(&ilan->locks, &ilan->locks_mask,
+ return alloc_bucket_spinlocks(&ilan->xlat.locks, &ilan->xlat.locks_mask,
MAX_LOCKS, LOCKS_PER_CPU,
GFP_KERNEL);
}
@@ -58,7 +49,7 @@ static inline u32 ila_locator_hash(struct ila_locator loc)
static inline spinlock_t *ila_get_lock(struct ila_net *ilan,
struct ila_locator loc)
{
- return &ilan->locks[ila_locator_hash(loc) & ilan->locks_mask];
+ return &ilan->xlat.locks[ila_locator_hash(loc) & ilan->xlat.locks_mask];
}
static inline int ila_cmp_wildcards(struct ila_map *ila,
@@ -102,16 +93,6 @@ static const struct rhashtable_params rht_params = {
.obj_cmpfn = ila_cmpfn,
};
-static struct genl_family ila_nl_family;
-
-static const struct nla_policy ila_nl_policy[ILA_ATTR_MAX + 1] = {
- [ILA_ATTR_LOCATOR] = { .type = NLA_U64, },
- [ILA_ATTR_LOCATOR_MATCH] = { .type = NLA_U64, },
- [ILA_ATTR_IFINDEX] = { .type = NLA_U32, },
- [ILA_ATTR_CSUM_MODE] = { .type = NLA_U8, },
- [ILA_ATTR_IDENT_TYPE] = { .type = NLA_U8, },
-};
-
static int parse_nl_config(struct genl_info *info,
struct ila_xlat_params *xp)
{
@@ -149,7 +130,7 @@ static inline struct ila_map *ila_lookup_wildcards(struct ila_addr *iaddr,
{
struct ila_map *ila;
- ila = rhashtable_lookup_fast(&ilan->rhash_table, &iaddr->loc,
+ ila = rhashtable_lookup_fast(&ilan->xlat.rhash_table, &iaddr->loc,
rht_params);
while (ila) {
if (!ila_cmp_wildcards(ila, iaddr, ifindex))
@@ -166,7 +147,7 @@ static inline struct ila_map *ila_lookup_by_params(struct ila_xlat_params *xp,
{
struct ila_map *ila;
- ila = rhashtable_lookup_fast(&ilan->rhash_table,
+ ila = rhashtable_lookup_fast(&ilan->xlat.rhash_table,
&xp->ip.locator_match,
rht_params);
while (ila) {
@@ -222,7 +203,7 @@ static int ila_add_mapping(struct net *net, struct ila_xlat_params *xp)
spinlock_t *lock = ila_get_lock(ilan, xp->ip.locator_match);
int err = 0, order;
- if (!ilan->hooks_registered) {
+ if (!ilan->xlat.hooks_registered) {
/* We defer registering net hooks in the namespace until the
* first mapping is added.
*/
@@ -231,7 +212,7 @@ static int ila_add_mapping(struct net *net, struct ila_xlat_params *xp)
if (err)
return err;
- ilan->hooks_registered = true;
+ ilan->xlat.hooks_registered = true;
}
ila = kzalloc(sizeof(*ila), GFP_KERNEL);
@@ -246,12 +227,12 @@ static int ila_add_mapping(struct net *net, struct ila_xlat_params *xp)
spin_lock(lock);
- head = rhashtable_lookup_fast(&ilan->rhash_table,
+ head = rhashtable_lookup_fast(&ilan->xlat.rhash_table,
&xp->ip.locator_match,
rht_params);
if (!head) {
/* New entry for the rhash_table */
- err = rhashtable_lookup_insert_fast(&ilan->rhash_table,
+ err = rhashtable_lookup_insert_fast(&ilan->xlat.rhash_table,
&ila->node, rht_params);
} else {
struct ila_map *tila = head, *prev = NULL;
@@ -277,7 +258,7 @@ static int ila_add_mapping(struct net *net, struct ila_xlat_params *xp)
} else {
/* Make this ila new head */
RCU_INIT_POINTER(ila->next, head);
- err = rhashtable_replace_fast(&ilan->rhash_table,
+ err = rhashtable_replace_fast(&ilan->xlat.rhash_table,
&head->node,
&ila->node, rht_params);
if (err)
@@ -303,7 +284,7 @@ static int ila_del_mapping(struct net *net, struct ila_xlat_params *xp)
spin_lock(lock);
- head = rhashtable_lookup_fast(&ilan->rhash_table,
+ head = rhashtable_lookup_fast(&ilan->xlat.rhash_table,
&xp->ip.locator_match, rht_params);
ila = head;
@@ -333,15 +314,15 @@ static int ila_del_mapping(struct net *net, struct ila_xlat_params *xp)
* table
*/
err = rhashtable_replace_fast(
- &ilan->rhash_table, &ila->node,
+ &ilan->xlat.rhash_table, &ila->node,
&head->node, rht_params);
if (err)
goto out;
} else {
/* Entry no longer used */
- err = rhashtable_remove_fast(&ilan->rhash_table,
- &ila->node,
- rht_params);
+ err = rhashtable_remove_fast(
+ &ilan->xlat.rhash_table,
+ &ila->node, rht_params);
}
}
@@ -356,7 +337,7 @@ static int ila_del_mapping(struct net *net, struct ila_xlat_params *xp)
return err;
}
-static int ila_nl_cmd_add_mapping(struct sk_buff *skb, struct genl_info *info)
+int ila_xlat_nl_cmd_add_mapping(struct sk_buff *skb, struct genl_info *info)
{
struct net *net = genl_info_net(info);
struct ila_xlat_params p;
@@ -369,7 +350,7 @@ static int ila_nl_cmd_add_mapping(struct sk_buff *skb, struct genl_info *info)
return ila_add_mapping(net, &p);
}
-static int ila_nl_cmd_del_mapping(struct sk_buff *skb, struct genl_info *info)
+int ila_xlat_nl_cmd_del_mapping(struct sk_buff *skb, struct genl_info *info)
{
struct net *net = genl_info_net(info);
struct ila_xlat_params xp;
@@ -421,7 +402,7 @@ static int ila_dump_info(struct ila_map *ila,
return -EMSGSIZE;
}
-static int ila_nl_cmd_get_mapping(struct sk_buff *skb, struct genl_info *info)
+int ila_xlat_nl_cmd_get_mapping(struct sk_buff *skb, struct genl_info *info)
{
struct net *net = genl_info_net(info);
struct ila_net *ilan = net_generic(net, ila_net_id);
@@ -465,7 +446,7 @@ struct ila_dump_iter {
int skip;
};
-static int ila_nl_dump_start(struct netlink_callback *cb)
+int ila_xlat_nl_dump_start(struct netlink_callback *cb)
{
struct net *net = sock_net(cb->skb->sk);
struct ila_net *ilan = net_generic(net, ila_net_id);
@@ -476,7 +457,7 @@ static int ila_nl_dump_start(struct netlink_callback *cb)
if (!iter)
return -ENOMEM;
- ret = rhashtable_walk_init(&ilan->rhash_table, &iter->rhiter,
+ ret = rhashtable_walk_init(&ilan->xlat.rhash_table, &iter->rhiter,
GFP_KERNEL);
if (ret) {
kfree(iter);
@@ -489,7 +470,7 @@ static int ila_nl_dump_start(struct netlink_callback *cb)
return ret;
}
-static int ila_nl_dump_done(struct netlink_callback *cb)
+int ila_xlat_nl_dump_done(struct netlink_callback *cb)
{
struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args[0];
@@ -500,7 +481,7 @@ static int ila_nl_dump_done(struct netlink_callback *cb)
return 0;
}
-static int ila_nl_dump(struct sk_buff *skb, struct netlink_callback *cb)
+int ila_xlat_nl_dump(struct sk_buff *skb, struct netlink_callback *cb)
{
struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args[0];
struct rhashtable_iter *rhiter = &iter->rhiter;
@@ -570,77 +551,35 @@ static int ila_nl_dump(struct sk_buff *skb, struct netlink_callback *cb)
return ret;
}
-static const struct genl_ops ila_nl_ops[] = {
- {
- .cmd = ILA_CMD_ADD,
- .doit = ila_nl_cmd_add_mapping,
- .policy = ila_nl_policy,
- .flags = GENL_ADMIN_PERM,
- },
- {
- .cmd = ILA_CMD_DEL,
- .doit = ila_nl_cmd_del_mapping,
- .policy = ila_nl_policy,
- .flags = GENL_ADMIN_PERM,
- },
- {
- .cmd = ILA_CMD_GET,
- .doit = ila_nl_cmd_get_mapping,
- .start = ila_nl_dump_start,
- .dumpit = ila_nl_dump,
- .done = ila_nl_dump_done,
- .policy = ila_nl_policy,
- },
-};
-
-static struct genl_family ila_nl_family __ro_after_init = {
- .hdrsize = 0,
- .name = ILA_GENL_NAME,
- .version = ILA_GENL_VERSION,
- .maxattr = ILA_ATTR_MAX,
- .netnsok = true,
- .parallel_ops = true,
- .module = THIS_MODULE,
- .ops = ila_nl_ops,
- .n_ops = ARRAY_SIZE(ila_nl_ops),
-};
-
#define ILA_HASH_TABLE_SIZE 1024
-static __net_init int ila_init_net(struct net *net)
+int ila_xlat_init_net(struct net *net)
{
- int err;
struct ila_net *ilan = net_generic(net, ila_net_id);
+ int err;
err = alloc_ila_locks(ilan);
if (err)
return err;
- rhashtable_init(&ilan->rhash_table, &rht_params);
+ rhashtable_init(&ilan->xlat.rhash_table, &rht_params);
return 0;
}
-static __net_exit void ila_exit_net(struct net *net)
+void ila_xlat_exit_net(struct net *net)
{
struct ila_net *ilan = net_generic(net, ila_net_id);
- rhashtable_free_and_destroy(&ilan->rhash_table, ila_free_cb, NULL);
+ rhashtable_free_and_destroy(&ilan->xlat.rhash_table, ila_free_cb, NULL);
- free_bucket_spinlocks(ilan->locks);
+ free_bucket_spinlocks(ilan->xlat.locks);
- if (ilan->hooks_registered)
+ if (ilan->xlat.hooks_registered)
nf_unregister_net_hooks(net, ila_nf_hook_ops,
ARRAY_SIZE(ila_nf_hook_ops));
}
-static struct pernet_operations ila_net_ops = {
- .init = ila_init_net,
- .exit = ila_exit_net,
- .id = &ila_net_id,
- .size = sizeof(struct ila_net),
-};
-
static int ila_xlat_addr(struct sk_buff *skb, bool sir2ila)
{
struct ila_map *ila;
@@ -667,28 +606,3 @@ static int ila_xlat_addr(struct sk_buff *skb, bool sir2ila)
return 0;
}
-int __init ila_xlat_init(void)
-{
- int ret;
-
- ret = register_pernet_device(&ila_net_ops);
- if (ret)
- goto exit;
-
- ret = genl_register_family(&ila_nl_family);
- if (ret < 0)
- goto unregister;
-
- return 0;
-
-unregister:
- unregister_pernet_device(&ila_net_ops);
-exit:
- return ret;
-}
-
-void ila_xlat_fini(void)
-{
- genl_unregister_family(&ila_nl_family);
- unregister_pernet_device(&ila_net_ops);
-}
--
2.7.4
^ permalink raw reply related
* [PATCH net-next 4/4] ila: Flush netlink command to clear xlat table
From: Tom Herbert @ 2018-06-27 21:39 UTC (permalink / raw)
To: davem, netdev; +Cc: Tom Herbert
In-Reply-To: <1530135542-10372-1-git-send-email-tom@quantonium.net>
Add ILA_CMD_FLUSH netlink command to clear the ILA translation table.
Signed-off-by: Tom Herbert <tom@quantonium.net>
---
include/uapi/linux/ila.h | 1 +
net/ipv6/ila/ila.h | 1 +
net/ipv6/ila/ila_main.c | 6 +++++
net/ipv6/ila/ila_xlat.c | 62 ++++++++++++++++++++++++++++++++++++++++++++++--
4 files changed, 68 insertions(+), 2 deletions(-)
diff --git a/include/uapi/linux/ila.h b/include/uapi/linux/ila.h
index 483b77af..db45d3e 100644
--- a/include/uapi/linux/ila.h
+++ b/include/uapi/linux/ila.h
@@ -30,6 +30,7 @@ enum {
ILA_CMD_ADD,
ILA_CMD_DEL,
ILA_CMD_GET,
+ ILA_CMD_FLUSH,
__ILA_CMD_MAX,
};
diff --git a/net/ipv6/ila/ila.h b/net/ipv6/ila/ila.h
index faba782..1f747bc 100644
--- a/net/ipv6/ila/ila.h
+++ b/net/ipv6/ila/ila.h
@@ -123,6 +123,7 @@ void ila_xlat_exit_net(struct net *net);
int ila_xlat_nl_cmd_add_mapping(struct sk_buff *skb, struct genl_info *info);
int ila_xlat_nl_cmd_del_mapping(struct sk_buff *skb, struct genl_info *info);
int ila_xlat_nl_cmd_get_mapping(struct sk_buff *skb, struct genl_info *info);
+int ila_xlat_nl_cmd_flush(struct sk_buff *skb, struct genl_info *info);
int ila_xlat_nl_dump_start(struct netlink_callback *cb);
int ila_xlat_nl_dump_done(struct netlink_callback *cb);
int ila_xlat_nl_dump(struct sk_buff *skb, struct netlink_callback *cb);
diff --git a/net/ipv6/ila/ila_main.c b/net/ipv6/ila/ila_main.c
index f6ac6b1..18fac76 100644
--- a/net/ipv6/ila/ila_main.c
+++ b/net/ipv6/ila/ila_main.c
@@ -27,6 +27,12 @@ static const struct genl_ops ila_nl_ops[] = {
.flags = GENL_ADMIN_PERM,
},
{
+ .cmd = ILA_CMD_FLUSH,
+ .doit = ila_xlat_nl_cmd_flush,
+ .policy = ila_nl_policy,
+ .flags = GENL_ADMIN_PERM,
+ },
+ {
.cmd = ILA_CMD_GET,
.doit = ila_xlat_nl_cmd_get_mapping,
.start = ila_xlat_nl_dump_start,
diff --git a/net/ipv6/ila/ila_xlat.c b/net/ipv6/ila/ila_xlat.c
index d05de89..51a15ce 100644
--- a/net/ipv6/ila/ila_xlat.c
+++ b/net/ipv6/ila/ila_xlat.c
@@ -164,9 +164,9 @@ static inline void ila_release(struct ila_map *ila)
kfree_rcu(ila, rcu);
}
-static void ila_free_cb(void *ptr, void *arg)
+static void ila_free_node(struct ila_map *ila)
{
- struct ila_map *ila = (struct ila_map *)ptr, *next;
+ struct ila_map *next;
/* Assume rcu_readlock held */
while (ila) {
@@ -176,6 +176,11 @@ static void ila_free_cb(void *ptr, void *arg)
}
}
+static void ila_free_cb(void *ptr, void *arg)
+{
+ ila_free_node((struct ila_map *)ptr);
+}
+
static int ila_xlat_addr(struct sk_buff *skb, bool sir2ila);
static unsigned int
@@ -365,6 +370,59 @@ int ila_xlat_nl_cmd_del_mapping(struct sk_buff *skb, struct genl_info *info)
return 0;
}
+static inline spinlock_t *lock_from_ila_map(struct ila_net *ilan,
+ struct ila_map *ila)
+{
+ return ila_get_lock(ilan, ila->xp.ip.locator_match);
+}
+
+int ila_xlat_nl_cmd_flush(struct sk_buff *skb, struct genl_info *info)
+{
+ struct net *net = genl_info_net(info);
+ struct ila_net *ilan = net_generic(net, ila_net_id);
+ struct rhashtable_iter iter;
+ struct ila_map *ila;
+ spinlock_t *lock;
+ int ret;
+
+ ret = rhashtable_walk_init(&ilan->xlat.rhash_table, &iter, GFP_KERNEL);
+ if (ret)
+ goto done;
+
+ rhashtable_walk_start(&iter);
+
+ for (;;) {
+ ila = rhashtable_walk_next(&iter);
+
+ if (IS_ERR(ila)) {
+ if (PTR_ERR(ila) == -EAGAIN)
+ continue;
+ ret = PTR_ERR(ila);
+ goto done;
+ } else if (!ila) {
+ break;
+ }
+
+ lock = lock_from_ila_map(ilan, ila);
+
+ spin_lock(lock);
+
+ ret = rhashtable_remove_fast(&ilan->xlat.rhash_table,
+ &ila->node, rht_params);
+ if (!ret)
+ ila_free_node(ila);
+
+ spin_unlock(lock);
+
+ if (ret)
+ break;
+ }
+
+done:
+ rhashtable_walk_stop(&iter);
+ return ret;
+}
+
static int ila_fill_info(struct ila_map *ila, struct sk_buff *msg)
{
if (nla_put_u64_64bit(msg, ILA_ATTR_LOCATOR,
--
2.7.4
^ permalink raw reply related
* Re: [RFC PATCH] net, mm: account sock objects to kmemcg
From: Eric Dumazet @ 2018-06-27 21:51 UTC (permalink / raw)
To: Shakeel Butt, Andrew Morton
Cc: Johannes Weiner, Vladimir Davydov, Greg Thelen, Roman Gushchin,
David S . Miller, Eric Dumazet, Kirill Tkhai, linux-kernel,
netdev, linux-mm
In-Reply-To: <20180627204139.225988-1-shakeelb@google.com>
On 06/27/2018 01:41 PM, Shakeel Butt wrote:
> Currently the kernel accounts the memory for network traffic through
> mem_cgroup_[un]charge_skmem() interface. However the memory accounted
> only includes the truesize of sk_buff which does not include the size of
> sock objects. In our production environment, with opt-out kmem
> accounting, the sock kmem caches (TCP[v6], UDP[v6], RAW[v6], UNIX) are
> among the top most charged kmem caches and consume a significant amount
> of memory which can not be left as system overhead. So, this patch
> converts the kmem caches of more important sock objects to SLAB_ACCOUNT.
>
> Signed-off-by: Shakeel Butt <shakeelb@google.com>
> ---
> net/ipv4/raw.c | 1 +
> net/ipv4/tcp_ipv4.c | 2 +-
> net/ipv4/udp.c | 1 +
> net/ipv6/raw.c | 1 +
> net/ipv6/tcp_ipv6.c | 2 +-
> net/ipv6/udp.c | 1 +
> net/unix/af_unix.c | 1 +
> 7 files changed, 7 insertions(+), 2 deletions(-)
Hey, you just disclosed we do not use DCCP ;)
Joke aside, what about simply factorizing this stuff ?
diff --git a/net/core/sock.c b/net/core/sock.c
index bcc41829a16d50714bdd3c25c976c0b7296fab84..b6714f8d7e9ba313723a6f619799c56230ff5fd4 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -3243,7 +3243,8 @@ static int req_prot_init(const struct proto *prot)
rsk_prot->slab = kmem_cache_create(rsk_prot->slab_name,
rsk_prot->obj_size, 0,
- prot->slab_flags, NULL);
+ SLAB_ACCOUNT | prot->slab_flags,
+ NULL);
if (!rsk_prot->slab) {
pr_crit("%s: Can't create request sock SLAB cache!\n",
@@ -3258,7 +3259,8 @@ int proto_register(struct proto *prot, int alloc_slab)
if (alloc_slab) {
prot->slab = kmem_cache_create_usercopy(prot->name,
prot->obj_size, 0,
- SLAB_HWCACHE_ALIGN | prot->slab_flags,
+ SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT |
+ prot->slab_flags,
prot->useroffset, prot->usersize,
NULL);
^ permalink raw reply related
* Re: [RFC PATCH] net, mm: account sock objects to kmemcg
From: Shakeel Butt @ 2018-06-27 22:03 UTC (permalink / raw)
To: eric.dumazet
Cc: Andrew Morton, Johannes Weiner, Vladimir Davydov, Greg Thelen,
Roman Gushchin, davem, Eric Dumazet, Kirill Tkhai, LKML, netdev,
Linux MM
In-Reply-To: <f08b2e2c-d4c6-7a80-10d9-104c0aab593b@gmail.com>
On Wed, Jun 27, 2018 at 2:51 PM Eric Dumazet <eric.dumazet@gmail.com> wrote:
>
>
>
> On 06/27/2018 01:41 PM, Shakeel Butt wrote:
> > Currently the kernel accounts the memory for network traffic through
> > mem_cgroup_[un]charge_skmem() interface. However the memory accounted
> > only includes the truesize of sk_buff which does not include the size of
> > sock objects. In our production environment, with opt-out kmem
> > accounting, the sock kmem caches (TCP[v6], UDP[v6], RAW[v6], UNIX) are
> > among the top most charged kmem caches and consume a significant amount
> > of memory which can not be left as system overhead. So, this patch
> > converts the kmem caches of more important sock objects to SLAB_ACCOUNT.
> >
> > Signed-off-by: Shakeel Butt <shakeelb@google.com>
> > ---
> > net/ipv4/raw.c | 1 +
> > net/ipv4/tcp_ipv4.c | 2 +-
> > net/ipv4/udp.c | 1 +
> > net/ipv6/raw.c | 1 +
> > net/ipv6/tcp_ipv6.c | 2 +-
> > net/ipv6/udp.c | 1 +
> > net/unix/af_unix.c | 1 +
> > 7 files changed, 7 insertions(+), 2 deletions(-)
>
>
> Hey, you just disclosed we do not use DCCP ;)
>
Oops.
>
> Joke aside, what about simply factorizing this stuff ?
>
This will opt-in all the sock kmem_caches which I think is better and
much smaller change. Should I resend this or do you want to send the
patch?
> diff --git a/net/core/sock.c b/net/core/sock.c
> index bcc41829a16d50714bdd3c25c976c0b7296fab84..b6714f8d7e9ba313723a6f619799c56230ff5fd4 100644
> --- a/net/core/sock.c
> +++ b/net/core/sock.c
> @@ -3243,7 +3243,8 @@ static int req_prot_init(const struct proto *prot)
>
> rsk_prot->slab = kmem_cache_create(rsk_prot->slab_name,
> rsk_prot->obj_size, 0,
> - prot->slab_flags, NULL);
> + SLAB_ACCOUNT | prot->slab_flags,
> + NULL);
>
> if (!rsk_prot->slab) {
> pr_crit("%s: Can't create request sock SLAB cache!\n",
> @@ -3258,7 +3259,8 @@ int proto_register(struct proto *prot, int alloc_slab)
> if (alloc_slab) {
> prot->slab = kmem_cache_create_usercopy(prot->name,
> prot->obj_size, 0,
> - SLAB_HWCACHE_ALIGN | prot->slab_flags,
> + SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT |
> + prot->slab_flags,
> prot->useroffset, prot->usersize,
> NULL);
>
>
Shakeel
^ permalink raw reply
* [PATCH v1 net-next 01/14] net: Clear skb->tstamp only on the forwarding path
From: Jesus Sanchez-Palencia @ 2018-06-27 21:59 UTC (permalink / raw)
To: netdev
Cc: tglx, jan.altenberg, vinicius.gomes, kurt.kanzenbach, henrik,
richardcochran, levi.pearson, ilias.apalodimas, ivan.khoronzhuk,
mlichvar, willemb, jhs, xiyou.wangcong, jiri,
Jesus Sanchez-Palencia
In-Reply-To: <20180627215950.6719-1-jesus.sanchez-palencia@intel.com>
This is done in preparation for the upcoming time based transmission
patchset. Now that skb->tstamp will be used to hold packet's txtime,
we must ensure that it is being cleared when traversing namespaces.
Also, doing that from skb_scrub_packet() before the early return would
break our feature when tunnels are used.
Signed-off-by: Jesus Sanchez-Palencia <jesus.sanchez-palencia@intel.com>
---
net/core/skbuff.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index b1f274f22d85..236802b35203 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -4898,7 +4898,6 @@ EXPORT_SYMBOL(skb_try_coalesce);
*/
void skb_scrub_packet(struct sk_buff *skb, bool xnet)
{
- skb->tstamp = 0;
skb->pkt_type = PACKET_HOST;
skb->skb_iif = 0;
skb->ignore_df = 0;
@@ -4913,6 +4912,7 @@ void skb_scrub_packet(struct sk_buff *skb, bool xnet)
ipvs_reset(skb);
skb_orphan(skb);
skb->mark = 0;
+ skb->tstamp = 0;
}
EXPORT_SYMBOL_GPL(skb_scrub_packet);
--
2.17.1
^ permalink raw reply related
* [PATCH v1 net-next 07/14] net/sched: Add HW offloading capability to ETF
From: Jesus Sanchez-Palencia @ 2018-06-27 21:59 UTC (permalink / raw)
To: netdev
Cc: tglx, jan.altenberg, vinicius.gomes, kurt.kanzenbach, henrik,
richardcochran, levi.pearson, ilias.apalodimas, ivan.khoronzhuk,
mlichvar, willemb, jhs, xiyou.wangcong, jiri,
Jesus Sanchez-Palencia
In-Reply-To: <20180627215950.6719-1-jesus.sanchez-palencia@intel.com>
Add infra so etf qdisc supports HW offload of time-based transmission.
For hw offload, the time sorted list is still used, so packets are
dequeued always in order of txtime.
Example:
$ tc qdisc replace dev enp2s0 parent root handle 100 mqprio num_tc 3 \
map 2 2 1 0 2 2 2 2 2 2 2 2 2 2 2 2 queues 1@0 1@1 2@2 hw 0
$ tc qdisc add dev enp2s0 parent 100:1 etf offload delta 100000 \
clockid CLOCK_REALTIME
In this example, the Qdisc will use HW offload for the control of the
transmission time through the network adapter. The hrtimer used for
packets scheduling inside the qdisc will use the clockid CLOCK_REALTIME
as reference and packets leave the Qdisc "delta" (100000) nanoseconds
before their transmission time. Because this will be using HW offload and
since dynamic clocks are not supported by the hrtimer, the system clock
and the PHC clock must be synchronized for this mode to behave as
expected.
Signed-off-by: Jesus Sanchez-Palencia <jesus.sanchez-palencia@intel.com>
---
include/net/pkt_sched.h | 5 +++
include/uapi/linux/pkt_sched.h | 1 +
net/sched/sch_etf.c | 71 +++++++++++++++++++++++++++++++++-
3 files changed, 76 insertions(+), 1 deletion(-)
diff --git a/include/net/pkt_sched.h b/include/net/pkt_sched.h
index 2466ea143d01..7dc769e5452b 100644
--- a/include/net/pkt_sched.h
+++ b/include/net/pkt_sched.h
@@ -155,4 +155,9 @@ struct tc_cbs_qopt_offload {
s32 sendslope;
};
+struct tc_etf_qopt_offload {
+ u8 enable;
+ s32 queue;
+};
+
#endif
diff --git a/include/uapi/linux/pkt_sched.h b/include/uapi/linux/pkt_sched.h
index 9d6fd2004a03..efad482e69d2 100644
--- a/include/uapi/linux/pkt_sched.h
+++ b/include/uapi/linux/pkt_sched.h
@@ -941,6 +941,7 @@ struct tc_etf_qopt {
__s32 clockid;
__u32 flags;
#define TC_ETF_DEADLINE_MODE_ON BIT(0)
+#define TC_ETF_OFFLOAD_ON BIT(1)
};
enum {
diff --git a/net/sched/sch_etf.c b/net/sched/sch_etf.c
index 5f01a285f399..cd6cb5b69228 100644
--- a/net/sched/sch_etf.c
+++ b/net/sched/sch_etf.c
@@ -20,8 +20,10 @@
#include <net/sock.h>
#define DEADLINE_MODE_IS_ON(x) ((x)->flags & TC_ETF_DEADLINE_MODE_ON)
+#define OFFLOAD_IS_ON(x) ((x)->flags & TC_ETF_OFFLOAD_ON)
struct etf_sched_data {
+ bool offload;
bool deadline_mode;
int clockid;
int queue;
@@ -45,6 +47,9 @@ static inline int validate_input_params(struct tc_etf_qopt *qopt,
* * Dynamic clockids are not supported.
*
* * Delta must be a positive integer.
+ *
+ * Also note that for the HW offload case, we must
+ * expect that system clocks have been synchronized to PHC.
*/
if (qopt->clockid < 0) {
NL_SET_ERR_MSG(extack, "Dynamic clockids are not supported");
@@ -226,6 +231,56 @@ static struct sk_buff *etf_dequeue_timesortedlist(struct Qdisc *sch)
return skb;
}
+static void etf_disable_offload(struct net_device *dev,
+ struct etf_sched_data *q)
+{
+ struct tc_etf_qopt_offload etf = { };
+ const struct net_device_ops *ops;
+ int err;
+
+ if (!q->offload)
+ return;
+
+ ops = dev->netdev_ops;
+ if (!ops->ndo_setup_tc)
+ return;
+
+ etf.queue = q->queue;
+ etf.enable = 0;
+
+ err = ops->ndo_setup_tc(dev, TC_SETUP_QDISC_ETF, &etf);
+ if (err < 0)
+ pr_warn("Couldn't disable ETF offload for queue %d\n",
+ etf.queue);
+}
+
+static int etf_enable_offload(struct net_device *dev, struct etf_sched_data *q,
+ struct netlink_ext_ack *extack)
+{
+ const struct net_device_ops *ops = dev->netdev_ops;
+ struct tc_etf_qopt_offload etf = { };
+ int err;
+
+ if (q->offload)
+ return 0;
+
+ if (!ops->ndo_setup_tc) {
+ NL_SET_ERR_MSG(extack, "Specified device does not support ETF offload");
+ return -EOPNOTSUPP;
+ }
+
+ etf.queue = q->queue;
+ etf.enable = 1;
+
+ err = ops->ndo_setup_tc(dev, TC_SETUP_QDISC_ETF, &etf);
+ if (err < 0) {
+ NL_SET_ERR_MSG(extack, "Specified device failed to setup ETF hardware offload");
+ return err;
+ }
+
+ return 0;
+}
+
static int etf_init(struct Qdisc *sch, struct nlattr *opt,
struct netlink_ext_ack *extack)
{
@@ -252,8 +307,9 @@ static int etf_init(struct Qdisc *sch, struct nlattr *opt,
qopt = nla_data(tb[TCA_ETF_PARMS]);
- pr_debug("delta %d clockid %d deadline %s\n",
+ pr_debug("delta %d clockid %d offload %s deadline %s\n",
qopt->delta, qopt->clockid,
+ OFFLOAD_IS_ON(qopt) ? "on" : "off",
DEADLINE_MODE_IS_ON(qopt) ? "on" : "off");
err = validate_input_params(qopt, extack);
@@ -262,9 +318,16 @@ static int etf_init(struct Qdisc *sch, struct nlattr *opt,
q->queue = sch->dev_queue - netdev_get_tx_queue(dev, 0);
+ if (OFFLOAD_IS_ON(qopt)) {
+ err = etf_enable_offload(dev, q, extack);
+ if (err < 0)
+ return err;
+ }
+
/* Everything went OK, save the parameters used. */
q->delta = qopt->delta;
q->clockid = qopt->clockid;
+ q->offload = OFFLOAD_IS_ON(qopt);
q->deadline_mode = DEADLINE_MODE_IS_ON(qopt);
switch (q->clockid) {
@@ -327,10 +390,13 @@ static void etf_reset(struct Qdisc *sch)
static void etf_destroy(struct Qdisc *sch)
{
struct etf_sched_data *q = qdisc_priv(sch);
+ struct net_device *dev = qdisc_dev(sch);
/* Only cancel watchdog if it's been initialized. */
if (q->watchdog.qdisc == sch)
qdisc_watchdog_cancel(&q->watchdog);
+
+ etf_disable_offload(dev, q);
}
static int etf_dump(struct Qdisc *sch, struct sk_buff *skb)
@@ -345,6 +411,9 @@ static int etf_dump(struct Qdisc *sch, struct sk_buff *skb)
opt.delta = q->delta;
opt.clockid = q->clockid;
+ if (q->offload)
+ opt.flags |= TC_ETF_OFFLOAD_ON;
+
if (q->deadline_mode)
opt.flags |= TC_ETF_DEADLINE_MODE_ON;
--
2.17.1
^ permalink raw reply related
* [PATCH v1 net-next 05/14] net/sched: Allow creating a Qdisc watchdog with other clocks
From: Jesus Sanchez-Palencia @ 2018-06-27 21:59 UTC (permalink / raw)
To: netdev
Cc: tglx, jan.altenberg, vinicius.gomes, kurt.kanzenbach, henrik,
richardcochran, levi.pearson, ilias.apalodimas, ivan.khoronzhuk,
mlichvar, willemb, jhs, xiyou.wangcong, jiri
In-Reply-To: <20180627215950.6719-1-jesus.sanchez-palencia@intel.com>
From: Vinicius Costa Gomes <vinicius.gomes@intel.com>
This adds 'qdisc_watchdog_init_clockid()' that allows a clockid to be
passed, this allows other time references to be used when scheduling
the Qdisc to run.
Signed-off-by: Vinicius Costa Gomes <vinicius.gomes@intel.com>
---
include/net/pkt_sched.h | 2 ++
net/sched/sch_api.c | 11 +++++++++--
2 files changed, 11 insertions(+), 2 deletions(-)
diff --git a/include/net/pkt_sched.h b/include/net/pkt_sched.h
index 815b92a23936..2466ea143d01 100644
--- a/include/net/pkt_sched.h
+++ b/include/net/pkt_sched.h
@@ -72,6 +72,8 @@ struct qdisc_watchdog {
struct Qdisc *qdisc;
};
+void qdisc_watchdog_init_clockid(struct qdisc_watchdog *wd, struct Qdisc *qdisc,
+ clockid_t clockid);
void qdisc_watchdog_init(struct qdisc_watchdog *wd, struct Qdisc *qdisc);
void qdisc_watchdog_schedule_ns(struct qdisc_watchdog *wd, u64 expires);
diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c
index 54eca685420f..98541c6399db 100644
--- a/net/sched/sch_api.c
+++ b/net/sched/sch_api.c
@@ -596,12 +596,19 @@ static enum hrtimer_restart qdisc_watchdog(struct hrtimer *timer)
return HRTIMER_NORESTART;
}
-void qdisc_watchdog_init(struct qdisc_watchdog *wd, struct Qdisc *qdisc)
+void qdisc_watchdog_init_clockid(struct qdisc_watchdog *wd, struct Qdisc *qdisc,
+ clockid_t clockid)
{
- hrtimer_init(&wd->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED);
+ hrtimer_init(&wd->timer, clockid, HRTIMER_MODE_ABS_PINNED);
wd->timer.function = qdisc_watchdog;
wd->qdisc = qdisc;
}
+EXPORT_SYMBOL(qdisc_watchdog_init_clockid);
+
+void qdisc_watchdog_init(struct qdisc_watchdog *wd, struct Qdisc *qdisc)
+{
+ qdisc_watchdog_init_clockid(wd, qdisc, CLOCK_MONOTONIC);
+}
EXPORT_SYMBOL(qdisc_watchdog_init);
void qdisc_watchdog_schedule_ns(struct qdisc_watchdog *wd, u64 expires)
--
2.17.1
^ permalink raw reply related
* [PATCH v1 net-next 04/14] net: packet: Hook into time based transmission.
From: Jesus Sanchez-Palencia @ 2018-06-27 21:59 UTC (permalink / raw)
To: netdev
Cc: tglx, jan.altenberg, vinicius.gomes, kurt.kanzenbach, henrik,
richardcochran, levi.pearson, ilias.apalodimas, ivan.khoronzhuk,
mlichvar, willemb, jhs, xiyou.wangcong, jiri, Richard Cochran,
Jesus Sanchez-Palencia
In-Reply-To: <20180627215950.6719-1-jesus.sanchez-palencia@intel.com>
From: Richard Cochran <rcochran@linutronix.de>
For raw layer-2 packets, copy the desired future transmit time from
the CMSG cookie into the skb.
Signed-off-by: Richard Cochran <rcochran@linutronix.de>
Signed-off-by: Jesus Sanchez-Palencia <jesus.sanchez-palencia@intel.com>
---
net/packet/af_packet.c | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c
index ff8e7e245c37..255c0164e0aa 100644
--- a/net/packet/af_packet.c
+++ b/net/packet/af_packet.c
@@ -1951,6 +1951,7 @@ static int packet_sendmsg_spkt(struct socket *sock, struct msghdr *msg,
goto out_unlock;
}
+ sockc.transmit_time = 0;
sockc.tsflags = sk->sk_tsflags;
if (msg->msg_controllen) {
err = sock_cmsg_send(sk, msg, &sockc);
@@ -1962,6 +1963,7 @@ static int packet_sendmsg_spkt(struct socket *sock, struct msghdr *msg,
skb->dev = dev;
skb->priority = sk->sk_priority;
skb->mark = sk->sk_mark;
+ skb->tstamp = sockc.transmit_time;
sock_tx_timestamp(sk, sockc.tsflags, &skb_shinfo(skb)->tx_flags);
@@ -2457,6 +2459,7 @@ static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
skb->dev = dev;
skb->priority = po->sk.sk_priority;
skb->mark = po->sk.sk_mark;
+ skb->tstamp = sockc->transmit_time;
sock_tx_timestamp(&po->sk, sockc->tsflags, &skb_shinfo(skb)->tx_flags);
skb_shinfo(skb)->destructor_arg = ph.raw;
@@ -2633,6 +2636,7 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
if (unlikely(!(dev->flags & IFF_UP)))
goto out_put;
+ sockc.transmit_time = 0;
sockc.tsflags = po->sk.sk_tsflags;
if (msg->msg_controllen) {
err = sock_cmsg_send(&po->sk, msg, &sockc);
@@ -2829,6 +2833,7 @@ static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len)
if (unlikely(!(dev->flags & IFF_UP)))
goto out_unlock;
+ sockc.transmit_time = 0;
sockc.tsflags = sk->sk_tsflags;
sockc.mark = sk->sk_mark;
if (msg->msg_controllen) {
@@ -2903,6 +2908,7 @@ static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len)
skb->dev = dev;
skb->priority = sk->sk_priority;
skb->mark = sockc.mark;
+ skb->tstamp = sockc.transmit_time;
if (has_vnet_hdr) {
err = virtio_net_hdr_to_skb(skb, &vnet_hdr, vio_le());
--
2.17.1
^ permalink raw reply related
* [PATCH v1 net-next 00/14] Scheduled packet Transmission: ETF
From: Jesus Sanchez-Palencia @ 2018-06-27 21:59 UTC (permalink / raw)
To: netdev
Cc: tglx, jan.altenberg, vinicius.gomes, kurt.kanzenbach, henrik,
richardcochran, levi.pearson, ilias.apalodimas, ivan.khoronzhuk,
mlichvar, willemb, jhs, xiyou.wangcong, jiri
Overview
========
This work consists of a set of kernel interfaces that can be used by
applications that require (time-based) Scheduled Tx of packets.
It is comprised by 3 new components to the kernel:
- SO_TXTIME: socket option + cmsg programming interfaces.
- etf: the "earliest txtime first" qdisc, that provides per-queue
TxTime-based scheduling. This has been renamed from 'tbs' to
'etf' to better describe its functionality.
- taprio: the "time-aware priority scheduler" qdisc, that provides
per-port Time-Aware scheduling;
This patchset is providing the first 2 components, which have been
developed for longer. The taprio qdisc will be shared as an RFC separately
(shortly).
Note that this series is a follow up of the "Time based packet
transmission" RFCv3 [1].
etf (formerly known as 'tbs')
=============================
Changes since the RFC v3:
- removed patch adding CLOCKID_INVALID;
- now we report packet drops through the socket's error queue;
- the usage of CLOCK_TAI is enforced by the qdisc;
- fixed bug on igb driver to avoid timestamps from being overwritten;
- simplified queueing modes by making 'sorting' mandatory;
- renamed qdisc from 'tbs' to 'etf'.
For applications/systems that the concept of time slices isn't precise
enough, the etf qdisc allows applications to control the instant when
a packet should leave the network controller. When used in conjunction
with taprio, it can also be used in case the application needs to
control with greater guarantee the offset into each time slice a packet
will be sent. Another use case of etf, is when only a small number of
applications on a system are time sensitive, so it can then be used
with a more traditional root qdisc (like mqprio).
The etf qdisc is designed so it buffers packets until a configurable
time before their deadline (Tx time). The qdisc uses a rbtree internally
so the buffered packets are always 'ordered' by their txtime (deadline)
and will be dequeued following the earliest txtime first.
The qdisc will drop any packets with a Tx time in the past, or if a
packet expires while waiting for being dequeued. Drops can be reported
as errors back to userspace through the socket's error queue.
Example configuration:
$ tc qdisc add dev enp2s0 parent 100:1 etf offload delta 200000 \
clockid CLOCK_TAI
Here, the Qdisc will use HW offload for the txtime control.
Packets will be dequeued by the qdisc "delta" (200000) nanoseconds before
their transmission time. Because this will be using HW offload and
since dynamic clocks are not supported by hrtimers, the system clock
and the PHC clock must be synchronized for this mode to behave as expected.
A more complete example can be found here, with instructions of how to
test it:
https://gist.github.com/jeez/bd3afeff081ba64a695008dd8215866f [2]
Note that we haven't modified the qdisc so it uses a timerqueue because
the modification needed was increasing the number of cachelines of a sk_buff.
SO_TXTIME
=========
Changes since the RFC v3:
- skb->tstamp is now cleared in skb_scrub_packet();
- transmit time is now set for other send paths, and not only the
"fast" ones as before;
- removed the per-packet parameters (clockid and drop_if_late).
Now just the skb->tstamp is used;
- flags and clockid_t are now set per-socket as a parameter of
SO_TXTIME.
This series is also hosted on github and can be found at [3].
The companion iproute2 patches can be found at [4].
[1] https://patchwork.ozlabs.org/cover/882342/
[2] github doesn't make it clear, but the gist can be cloned like this:
$ git clone https://gist.github.com/jeez/bd3afeff081ba64a695008dd8215866f scheduled-tx-tests
[3] https://github.com/jeez/linux/tree/etf-v1
[4] https://github.com/jeez/iproute2/tree/etf-v1
Jesus Sanchez-Palencia (10):
net: Clear skb->tstamp only on the forwarding path
net: ipv4: Hook into time based transmission
net/sched: Add HW offloading capability to ETF
igb: Refactor igb_configure_cbs()
igb: Only change Tx arbitration when CBS is on
igb: Refactor igb_offload_cbs()
igb: Add support for ETF offload
igb: Only call skb_tx_timestamp after descriptors are ready
net/sched: Enforce usage of CLOCK_TAI for sch_etf
net/sched: Make etf report drops on error_queue
Richard Cochran (2):
net: Add a new socket option for a future transmit time.
net: packet: Hook into time based transmission.
Vinicius Costa Gomes (2):
net/sched: Allow creating a Qdisc watchdog with other clocks
net/sched: Introduce the ETF Qdisc
arch/alpha/include/uapi/asm/socket.h | 3 +
arch/ia64/include/uapi/asm/socket.h | 3 +
arch/mips/include/uapi/asm/socket.h | 3 +
arch/parisc/include/uapi/asm/socket.h | 3 +
arch/s390/include/uapi/asm/socket.h | 3 +
arch/sparc/include/uapi/asm/socket.h | 3 +
arch/xtensa/include/uapi/asm/socket.h | 3 +
.../net/ethernet/intel/igb/e1000_defines.h | 16 +
drivers/net/ethernet/intel/igb/igb.h | 1 +
drivers/net/ethernet/intel/igb/igb_main.c | 257 ++++++---
include/linux/netdevice.h | 1 +
include/linux/socket.h | 7 +
include/net/inet_sock.h | 1 +
include/net/pkt_sched.h | 7 +
include/net/sock.h | 9 +
include/uapi/asm-generic/socket.h | 3 +
include/uapi/linux/errqueue.h | 2 +
include/uapi/linux/pkt_sched.h | 18 +
net/core/skbuff.c | 2 +-
net/core/sock.c | 32 ++
net/ipv4/ip_output.c | 3 +
net/ipv4/raw.c | 2 +
net/ipv4/udp.c | 1 +
net/packet/af_packet.c | 6 +
net/sched/Kconfig | 11 +
net/sched/Makefile | 1 +
net/sched/sch_api.c | 11 +-
net/sched/sch_etf.c | 487 ++++++++++++++++++
28 files changed, 829 insertions(+), 70 deletions(-)
create mode 100644 net/sched/sch_etf.c
--
2.17.1
^ permalink raw reply
* [PATCH v1 net-next 02/14] net: Add a new socket option for a future transmit time.
From: Jesus Sanchez-Palencia @ 2018-06-27 21:59 UTC (permalink / raw)
To: netdev
Cc: tglx, jan.altenberg, vinicius.gomes, kurt.kanzenbach, henrik,
richardcochran, levi.pearson, ilias.apalodimas, ivan.khoronzhuk,
mlichvar, willemb, jhs, xiyou.wangcong, jiri, Richard Cochran,
Jesus Sanchez-Palencia
In-Reply-To: <20180627215950.6719-1-jesus.sanchez-palencia@intel.com>
From: Richard Cochran <rcochran@linutronix.de>
This patch introduces SO_TXTIME. User space enables this option in
order to pass a desired future transmit time in a CMSG when calling
sendmsg(2). The argument to this socket option is a 6-bytes long struct
defined as:
struct sock_txtime {
clockid_t clockid;
u16 flags;
};
Note that two new fields were added to struct sock by filling a 4-bytes
hole found in the struct. For that reason, neither the struct size or
number of cachelines were altered.
Signed-off-by: Richard Cochran <rcochran@linutronix.de>
Signed-off-by: Jesus Sanchez-Palencia <jesus.sanchez-palencia@intel.com>
---
arch/alpha/include/uapi/asm/socket.h | 3 +++
arch/ia64/include/uapi/asm/socket.h | 3 +++
arch/mips/include/uapi/asm/socket.h | 3 +++
arch/parisc/include/uapi/asm/socket.h | 3 +++
arch/s390/include/uapi/asm/socket.h | 3 +++
arch/sparc/include/uapi/asm/socket.h | 3 +++
arch/xtensa/include/uapi/asm/socket.h | 3 +++
include/linux/socket.h | 5 +++++
include/net/sock.h | 8 +++++++
include/uapi/asm-generic/socket.h | 3 +++
net/core/sock.c | 32 +++++++++++++++++++++++++++
11 files changed, 69 insertions(+)
diff --git a/arch/alpha/include/uapi/asm/socket.h b/arch/alpha/include/uapi/asm/socket.h
index be14f16149d5..065fb372e355 100644
--- a/arch/alpha/include/uapi/asm/socket.h
+++ b/arch/alpha/include/uapi/asm/socket.h
@@ -112,4 +112,7 @@
#define SO_ZEROCOPY 60
+#define SO_TXTIME 61
+#define SCM_TXTIME SO_TXTIME
+
#endif /* _UAPI_ASM_SOCKET_H */
diff --git a/arch/ia64/include/uapi/asm/socket.h b/arch/ia64/include/uapi/asm/socket.h
index 3efba40adc54..c872c4e6bafb 100644
--- a/arch/ia64/include/uapi/asm/socket.h
+++ b/arch/ia64/include/uapi/asm/socket.h
@@ -114,4 +114,7 @@
#define SO_ZEROCOPY 60
+#define SO_TXTIME 61
+#define SCM_TXTIME SO_TXTIME
+
#endif /* _ASM_IA64_SOCKET_H */
diff --git a/arch/mips/include/uapi/asm/socket.h b/arch/mips/include/uapi/asm/socket.h
index 49c3d4795963..71370fb3ceef 100644
--- a/arch/mips/include/uapi/asm/socket.h
+++ b/arch/mips/include/uapi/asm/socket.h
@@ -123,4 +123,7 @@
#define SO_ZEROCOPY 60
+#define SO_TXTIME 61
+#define SCM_TXTIME SO_TXTIME
+
#endif /* _UAPI_ASM_SOCKET_H */
diff --git a/arch/parisc/include/uapi/asm/socket.h b/arch/parisc/include/uapi/asm/socket.h
index 1d0fdc3b5d22..061b9cf2a779 100644
--- a/arch/parisc/include/uapi/asm/socket.h
+++ b/arch/parisc/include/uapi/asm/socket.h
@@ -104,4 +104,7 @@
#define SO_ZEROCOPY 0x4035
+#define SO_TXTIME 0x4036
+#define SCM_TXTIME SO_TXTIME
+
#endif /* _UAPI_ASM_SOCKET_H */
diff --git a/arch/s390/include/uapi/asm/socket.h b/arch/s390/include/uapi/asm/socket.h
index 3510c0fd06f4..39d901476ee5 100644
--- a/arch/s390/include/uapi/asm/socket.h
+++ b/arch/s390/include/uapi/asm/socket.h
@@ -111,4 +111,7 @@
#define SO_ZEROCOPY 60
+#define SO_TXTIME 61
+#define SCM_TXTIME SO_TXTIME
+
#endif /* _ASM_SOCKET_H */
diff --git a/arch/sparc/include/uapi/asm/socket.h b/arch/sparc/include/uapi/asm/socket.h
index d58520c2e6ff..7ea35e5601b6 100644
--- a/arch/sparc/include/uapi/asm/socket.h
+++ b/arch/sparc/include/uapi/asm/socket.h
@@ -101,6 +101,9 @@
#define SO_ZEROCOPY 0x003e
+#define SO_TXTIME 0x003f
+#define SCM_TXTIME SO_TXTIME
+
/* Security levels - as per NRL IPv6 - don't actually do anything */
#define SO_SECURITY_AUTHENTICATION 0x5001
#define SO_SECURITY_ENCRYPTION_TRANSPORT 0x5002
diff --git a/arch/xtensa/include/uapi/asm/socket.h b/arch/xtensa/include/uapi/asm/socket.h
index 75a07b8119a9..1de07a7f7680 100644
--- a/arch/xtensa/include/uapi/asm/socket.h
+++ b/arch/xtensa/include/uapi/asm/socket.h
@@ -116,4 +116,7 @@
#define SO_ZEROCOPY 60
+#define SO_TXTIME 61
+#define SCM_TXTIME SO_TXTIME
+
#endif /* _XTENSA_SOCKET_H */
diff --git a/include/linux/socket.h b/include/linux/socket.h
index 7ed4713d5337..ca476b7a8ff0 100644
--- a/include/linux/socket.h
+++ b/include/linux/socket.h
@@ -83,6 +83,11 @@ struct cmsghdr {
int cmsg_type; /* protocol-specific type */
};
+struct sock_txtime {
+ clockid_t clockid; /* reference clockid */
+ u16 flags; /* bit 0: txtime in deadline_mode */
+};
+
/*
* Ancillary data object information MACROS
* Table 5-14 of POSIX 1003.1g
diff --git a/include/net/sock.h b/include/net/sock.h
index b3b75419eafe..73f4404e49e4 100644
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -315,6 +315,7 @@ struct sock_common {
* @sk_destruct: called at sock freeing time, i.e. when all refcnt == 0
* @sk_reuseport_cb: reuseport group container
* @sk_rcu: used during RCU grace period
+ * @sk_txtime: used by time-based scheduling
*/
struct sock {
/*
@@ -468,6 +469,11 @@ struct sock {
u8 sk_shutdown;
u32 sk_tskey;
atomic_t sk_zckey;
+
+ u16 sk_clockid;
+ u16 sk_txtime_flags;
+#define SK_TXTIME_DEADLINE_MASK BIT(0)
+
struct socket *sk_socket;
void *sk_user_data;
#ifdef CONFIG_SECURITY
@@ -783,6 +789,7 @@ enum sock_flags {
SOCK_FILTER_LOCKED, /* Filter cannot be changed anymore */
SOCK_SELECT_ERR_QUEUE, /* Wake select on error queue */
SOCK_RCU_FREE, /* wait rcu grace period in sk_destruct() */
+ SOCK_TXTIME,
};
#define SK_FLAGS_TIMESTAMP ((1UL << SOCK_TIMESTAMP) | (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE))
@@ -1578,6 +1585,7 @@ void sock_kzfree_s(struct sock *sk, void *mem, int size);
void sk_send_sigurg(struct sock *sk);
struct sockcm_cookie {
+ u64 transmit_time;
u32 mark;
u16 tsflags;
};
diff --git a/include/uapi/asm-generic/socket.h b/include/uapi/asm-generic/socket.h
index 0ae758c90e54..a12692e5f7a8 100644
--- a/include/uapi/asm-generic/socket.h
+++ b/include/uapi/asm-generic/socket.h
@@ -107,4 +107,7 @@
#define SO_ZEROCOPY 60
+#define SO_TXTIME 61
+#define SCM_TXTIME SO_TXTIME
+
#endif /* __ASM_GENERIC_SOCKET_H */
diff --git a/net/core/sock.c b/net/core/sock.c
index bcc41829a16d..ef934d808941 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -91,6 +91,7 @@
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+#include <asm/unaligned.h>
#include <linux/capability.h>
#include <linux/errno.h>
#include <linux/errqueue.h>
@@ -697,6 +698,7 @@ EXPORT_SYMBOL(sk_mc_loop);
int sock_setsockopt(struct socket *sock, int level, int optname,
char __user *optval, unsigned int optlen)
{
+ struct sock_txtime sk_txtime;
struct sock *sk = sock->sk;
int val;
int valbool;
@@ -1070,6 +1072,22 @@ int sock_setsockopt(struct socket *sock, int level, int optname,
}
break;
+ case SO_TXTIME:
+ if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) {
+ ret = -EPERM;
+ } else if (optlen != sizeof(struct sock_txtime)) {
+ ret = -EINVAL;
+ } else if (copy_from_user(&sk_txtime, optval,
+ sizeof(struct sock_txtime))) {
+ ret = -EFAULT;
+ sock_valbool_flag(sk, SOCK_TXTIME, false);
+ } else {
+ sock_valbool_flag(sk, SOCK_TXTIME, true);
+ sk->sk_clockid = sk_txtime.clockid;
+ sk->sk_txtime_flags = sk_txtime.flags;
+ }
+ break;
+
default:
ret = -ENOPROTOOPT;
break;
@@ -1115,6 +1133,7 @@ int sock_getsockopt(struct socket *sock, int level, int optname,
u64 val64;
struct linger ling;
struct timeval tm;
+ struct sock_txtime txtime;
} v;
int lv = sizeof(int);
@@ -1403,6 +1422,12 @@ int sock_getsockopt(struct socket *sock, int level, int optname,
v.val = sock_flag(sk, SOCK_ZEROCOPY);
break;
+ case SO_TXTIME:
+ lv = sizeof(v.txtime);
+ v.txtime.clockid = sk->sk_clockid;
+ v.txtime.flags = sk->sk_txtime_flags;
+ break;
+
default:
/* We implement the SO_SNDLOWAT etc to not be settable
* (1003.1g 7).
@@ -2137,6 +2162,13 @@ int __sock_cmsg_send(struct sock *sk, struct msghdr *msg, struct cmsghdr *cmsg,
sockc->tsflags &= ~SOF_TIMESTAMPING_TX_RECORD_MASK;
sockc->tsflags |= tsflags;
break;
+ case SCM_TXTIME:
+ if (!sock_flag(sk, SOCK_TXTIME))
+ return -EINVAL;
+ if (cmsg->cmsg_len != CMSG_LEN(sizeof(u64)))
+ return -EINVAL;
+ sockc->transmit_time = get_unaligned((u64 *)CMSG_DATA(cmsg));
+ break;
/* SCM_RIGHTS and SCM_CREDENTIALS are semantically in SOL_UNIX. */
case SCM_RIGHTS:
case SCM_CREDENTIALS:
--
2.17.1
^ permalink raw reply related
* [PATCH v1 net-next 10/14] igb: Refactor igb_offload_cbs()
From: Jesus Sanchez-Palencia @ 2018-06-27 21:59 UTC (permalink / raw)
To: netdev
Cc: tglx, jan.altenberg, vinicius.gomes, kurt.kanzenbach, henrik,
richardcochran, levi.pearson, ilias.apalodimas, ivan.khoronzhuk,
mlichvar, willemb, jhs, xiyou.wangcong, jiri,
Jesus Sanchez-Palencia
In-Reply-To: <20180627215950.6719-1-jesus.sanchez-palencia@intel.com>
Split code into a separate function (igb_offload_apply()) that will be
used by ETF offload implementation.
Signed-off-by: Jesus Sanchez-Palencia <jesus.sanchez-palencia@intel.com>
---
drivers/net/ethernet/intel/igb/igb_main.c | 23 ++++++++++++++---------
1 file changed, 14 insertions(+), 9 deletions(-)
diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c
index 8c90f1e51add..c30ab7b260cc 100644
--- a/drivers/net/ethernet/intel/igb/igb_main.c
+++ b/drivers/net/ethernet/intel/igb/igb_main.c
@@ -2474,6 +2474,19 @@ igb_features_check(struct sk_buff *skb, struct net_device *dev,
return features;
}
+static void igb_offload_apply(struct igb_adapter *adapter, s32 queue)
+{
+ if (!is_fqtss_enabled(adapter)) {
+ enable_fqtss(adapter, true);
+ return;
+ }
+
+ igb_config_tx_modes(adapter, queue);
+
+ if (!is_any_cbs_enabled(adapter))
+ enable_fqtss(adapter, false);
+}
+
static int igb_offload_cbs(struct igb_adapter *adapter,
struct tc_cbs_qopt_offload *qopt)
{
@@ -2494,15 +2507,7 @@ static int igb_offload_cbs(struct igb_adapter *adapter,
if (err)
return err;
- if (is_fqtss_enabled(adapter)) {
- igb_config_tx_modes(adapter, qopt->queue);
-
- if (!is_any_cbs_enabled(adapter))
- enable_fqtss(adapter, false);
-
- } else {
- enable_fqtss(adapter, true);
- }
+ igb_offload_apply(adapter, qopt->queue);
return 0;
}
--
2.17.1
^ permalink raw reply related
* [PATCH v1 net-next 11/14] igb: Add support for ETF offload
From: Jesus Sanchez-Palencia @ 2018-06-27 21:59 UTC (permalink / raw)
To: netdev
Cc: tglx, jan.altenberg, vinicius.gomes, kurt.kanzenbach, henrik,
richardcochran, levi.pearson, ilias.apalodimas, ivan.khoronzhuk,
mlichvar, willemb, jhs, xiyou.wangcong, jiri,
Jesus Sanchez-Palencia
In-Reply-To: <20180627215950.6719-1-jesus.sanchez-palencia@intel.com>
Implement HW offload support for SO_TXTIME through igb's Launchtime
feature. This is done by extending igb_setup_tc() so it supports
TC_SETUP_QDISC_ETF and configuring i210 so time based transmit
arbitration is enabled.
The FQTSS transmission mode added before is extended so strict
priority (SP) queues wait for stream reservation (SR) ones.
igb_config_tx_modes() is extended so it can support enabling/disabling
Launchtime following the previous approach used for the credit-based
shaper (CBS).
As the previous flow, FQTSS transmission mode is enabled automatically
by the driver once Launchtime (or CBS, as before) is enabled.
Similarly, it's automatically disabled when the feature is disabled
for the last queue that had it setup on.
The driver just consumes the transmit times from the skbuffs directly,
so no special handling is done in case an 'invalid' time is provided.
We assume this has been handled by the ETF qdisc already.
Signed-off-by: Jesus Sanchez-Palencia <jesus.sanchez-palencia@intel.com>
---
.../net/ethernet/intel/igb/e1000_defines.h | 16 ++
drivers/net/ethernet/intel/igb/igb.h | 1 +
drivers/net/ethernet/intel/igb/igb_main.c | 139 +++++++++++++++---
3 files changed, 139 insertions(+), 17 deletions(-)
diff --git a/drivers/net/ethernet/intel/igb/e1000_defines.h b/drivers/net/ethernet/intel/igb/e1000_defines.h
index 252440a418dc..8a28f3388f69 100644
--- a/drivers/net/ethernet/intel/igb/e1000_defines.h
+++ b/drivers/net/ethernet/intel/igb/e1000_defines.h
@@ -1048,6 +1048,22 @@
#define E1000_TQAVCTRL_XMIT_MODE BIT(0)
#define E1000_TQAVCTRL_DATAFETCHARB BIT(4)
#define E1000_TQAVCTRL_DATATRANARB BIT(8)
+#define E1000_TQAVCTRL_DATATRANTIM BIT(9)
+#define E1000_TQAVCTRL_SP_WAIT_SR BIT(10)
+/* Fetch Time Delta - bits 31:16
+ *
+ * This field holds the value to be reduced from the launch time for
+ * fetch time decision. The FetchTimeDelta value is defined in 32 ns
+ * granularity.
+ *
+ * This field is 16 bits wide, and so the maximum value is:
+ *
+ * 65535 * 32 = 2097120 ~= 2.1 msec
+ *
+ * XXX: We are configuring the max value here since we couldn't come up
+ * with a reason for not doing so.
+ */
+#define E1000_TQAVCTRL_FETCHTIME_DELTA (0xFFFF << 16)
/* TX Qav Credit Control fields */
#define E1000_TQAVCC_IDLESLOPE_MASK 0xFFFF
diff --git a/drivers/net/ethernet/intel/igb/igb.h b/drivers/net/ethernet/intel/igb/igb.h
index 9643b5b3d444..ca54e268d157 100644
--- a/drivers/net/ethernet/intel/igb/igb.h
+++ b/drivers/net/ethernet/intel/igb/igb.h
@@ -262,6 +262,7 @@ struct igb_ring {
u16 count; /* number of desc. in the ring */
u8 queue_index; /* logical index of the ring*/
u8 reg_idx; /* physical index of the ring */
+ bool launchtime_enable; /* true if LaunchTime is enabled */
bool cbs_enable; /* indicates if CBS is enabled */
s32 idleslope; /* idleSlope in kbps */
s32 sendslope; /* sendSlope in kbps */
diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c
index c30ab7b260cc..9b9a6a6227e0 100644
--- a/drivers/net/ethernet/intel/igb/igb_main.c
+++ b/drivers/net/ethernet/intel/igb/igb_main.c
@@ -1666,13 +1666,26 @@ static bool is_any_cbs_enabled(struct igb_adapter *adapter)
return false;
}
+static bool is_any_txtime_enabled(struct igb_adapter *adapter)
+{
+ int i;
+
+ for (i = 0; i < adapter->num_tx_queues; i++) {
+ if (adapter->tx_ring[i]->launchtime_enable)
+ return true;
+ }
+
+ return false;
+}
+
/**
* igb_config_tx_modes - Configure "Qav Tx mode" features on igb
* @adapter: pointer to adapter struct
* @queue: queue number
*
- * Configure CBS for a given hardware queue. Parameters are retrieved
- * from the correct Tx ring, so igb_save_cbs_params() should be used
+ * Configure CBS and Launchtime for a given hardware queue.
+ * Parameters are retrieved from the correct Tx ring, so
+ * igb_save_cbs_params() and igb_save_txtime_params() should be used
* for setting those correctly prior to this function being called.
**/
static void igb_config_tx_modes(struct igb_adapter *adapter, int queue)
@@ -1686,6 +1699,19 @@ static void igb_config_tx_modes(struct igb_adapter *adapter, int queue)
WARN_ON(hw->mac.type != e1000_i210);
WARN_ON(queue < 0 || queue > 1);
+ /* If any of the Qav features is enabled, configure queues as SR and
+ * with HIGH PRIO. If none is, then configure them with LOW PRIO and
+ * as SP.
+ */
+ if (ring->cbs_enable || ring->launchtime_enable) {
+ set_tx_desc_fetch_prio(hw, queue, TX_QUEUE_PRIO_HIGH);
+ set_queue_mode(hw, queue, QUEUE_MODE_STREAM_RESERVATION);
+ } else {
+ set_tx_desc_fetch_prio(hw, queue, TX_QUEUE_PRIO_LOW);
+ set_queue_mode(hw, queue, QUEUE_MODE_STRICT_PRIORITY);
+ }
+
+ /* If CBS is enabled, set DataTranARB and config its parameters. */
if (ring->cbs_enable || queue == 0) {
/* i210 does not allow the queue 0 to be in the Strict
* Priority mode while the Qav mode is enabled, so,
@@ -1702,9 +1728,6 @@ static void igb_config_tx_modes(struct igb_adapter *adapter, int queue)
ring->hicredit = ETH_FRAME_LEN;
}
- set_tx_desc_fetch_prio(hw, queue, TX_QUEUE_PRIO_HIGH);
- set_queue_mode(hw, queue, QUEUE_MODE_STREAM_RESERVATION);
-
/* Always set data transfer arbitration to credit-based
* shaper algorithm on TQAVCTRL if CBS is enabled for any of
* the queues.
@@ -1780,8 +1803,6 @@ static void igb_config_tx_modes(struct igb_adapter *adapter, int queue)
wr32(E1000_I210_TQAVHC(queue),
0x80000000 + ring->hicredit * 0x7735);
} else {
- set_tx_desc_fetch_prio(hw, queue, TX_QUEUE_PRIO_LOW);
- set_queue_mode(hw, queue, QUEUE_MODE_STRICT_PRIORITY);
/* Set idleSlope to zero. */
tqavcc = rd32(E1000_I210_TQAVCC(queue));
@@ -1802,17 +1823,61 @@ static void igb_config_tx_modes(struct igb_adapter *adapter, int queue)
}
}
+ /* If LaunchTime is enabled, set DataTranTIM. */
+ if (ring->launchtime_enable) {
+ /* Always set DataTranTIM on TQAVCTRL if LaunchTime is enabled
+ * for any of the SR queues, and configure fetchtime delta.
+ * XXX NOTE:
+ * - LaunchTime will be enabled for all SR queues.
+ * - A fixed offset can be added relative to the launch
+ * time of all packets if configured at reg LAUNCH_OS0.
+ * We are keeping it as 0 for now (default value).
+ */
+ tqavctrl = rd32(E1000_I210_TQAVCTRL);
+ tqavctrl |= E1000_TQAVCTRL_DATATRANTIM |
+ E1000_TQAVCTRL_FETCHTIME_DELTA;
+ wr32(E1000_I210_TQAVCTRL, tqavctrl);
+ } else {
+ /* If Launchtime is not enabled for any SR queues anymore,
+ * then clear DataTranTIM on TQAVCTRL and clear fetchtime delta,
+ * effectively disabling Launchtime.
+ */
+ if (!is_any_txtime_enabled(adapter)) {
+ tqavctrl = rd32(E1000_I210_TQAVCTRL);
+ tqavctrl &= ~E1000_TQAVCTRL_DATATRANTIM;
+ tqavctrl &= ~E1000_TQAVCTRL_FETCHTIME_DELTA;
+ wr32(E1000_I210_TQAVCTRL, tqavctrl);
+ }
+ }
+
/* XXX: In i210 controller the sendSlope and loCredit parameters from
* CBS are not configurable by software so we don't do any 'controller
* configuration' in respect to these parameters.
*/
- netdev_dbg(netdev, "CBS %s: queue %d idleslope %d sendslope %d hiCredit %d locredit %d\n",
- (ring->cbs_enable) ? "enabled" : "disabled", queue,
+ netdev_dbg(netdev, "Qav Tx mode: cbs %s, launchtime %s, queue %d \
+ idleslope %d sendslope %d hiCredit %d \
+ locredit %d\n",
+ (ring->cbs_enable) ? "enabled" : "disabled",
+ (ring->launchtime_enable) ? "enabled" : "disabled", queue,
ring->idleslope, ring->sendslope, ring->hicredit,
ring->locredit);
}
+static int igb_save_txtime_params(struct igb_adapter *adapter, int queue,
+ bool enable)
+{
+ struct igb_ring *ring;
+
+ if (queue < 0 || queue > adapter->num_tx_queues)
+ return -EINVAL;
+
+ ring = adapter->tx_ring[queue];
+ ring->launchtime_enable = enable;
+
+ return 0;
+}
+
static int igb_save_cbs_params(struct igb_adapter *adapter, int queue,
bool enable, int idleslope, int sendslope,
int hicredit, int locredit)
@@ -1856,10 +1921,11 @@ static void igb_setup_tx_mode(struct igb_adapter *adapter)
int i, max_queue;
/* Configure TQAVCTRL register: set transmit mode to 'Qav',
- * set data fetch arbitration to 'round robin'.
+ * set data fetch arbitration to 'round robin', set SP_WAIT_SR
+ * so SP queues wait for SR ones.
*/
val = rd32(E1000_I210_TQAVCTRL);
- val |= E1000_TQAVCTRL_XMIT_MODE;
+ val |= E1000_TQAVCTRL_XMIT_MODE | E1000_TQAVCTRL_SP_WAIT_SR;
val &= ~E1000_TQAVCTRL_DATAFETCHARB;
wr32(E1000_I210_TQAVCTRL, val);
@@ -2483,7 +2549,7 @@ static void igb_offload_apply(struct igb_adapter *adapter, s32 queue)
igb_config_tx_modes(adapter, queue);
- if (!is_any_cbs_enabled(adapter))
+ if (!is_any_cbs_enabled(adapter) && !is_any_txtime_enabled(adapter))
enable_fqtss(adapter, false);
}
@@ -2756,6 +2822,30 @@ static int igb_setup_tc_block(struct igb_adapter *adapter,
}
}
+static int igb_offload_txtime(struct igb_adapter *adapter,
+ struct tc_etf_qopt_offload *qopt)
+{
+ struct e1000_hw *hw = &adapter->hw;
+ int err;
+
+ /* Launchtime offloading is only supported by i210 controller. */
+ if (hw->mac.type != e1000_i210)
+ return -EOPNOTSUPP;
+
+ /* Launchtime offloading is only supported by queues 0 and 1. */
+ if (qopt->queue < 0 || qopt->queue > 1)
+ return -EINVAL;
+
+ err = igb_save_txtime_params(adapter, qopt->queue, qopt->enable);
+
+ if (err)
+ return err;
+
+ igb_offload_apply(adapter, qopt->queue);
+
+ return 0;
+}
+
static int igb_setup_tc(struct net_device *dev, enum tc_setup_type type,
void *type_data)
{
@@ -2766,6 +2856,8 @@ static int igb_setup_tc(struct net_device *dev, enum tc_setup_type type,
return igb_offload_cbs(adapter, type_data);
case TC_SETUP_BLOCK:
return igb_setup_tc_block(adapter, type_data);
+ case TC_SETUP_QDISC_ETF:
+ return igb_offload_txtime(adapter, type_data);
default:
return -EOPNOTSUPP;
@@ -5586,11 +5678,14 @@ static void igb_set_itr(struct igb_q_vector *q_vector)
}
}
-static void igb_tx_ctxtdesc(struct igb_ring *tx_ring, u32 vlan_macip_lens,
- u32 type_tucmd, u32 mss_l4len_idx)
+static void igb_tx_ctxtdesc(struct igb_ring *tx_ring,
+ struct igb_tx_buffer *first,
+ u32 vlan_macip_lens, u32 type_tucmd,
+ u32 mss_l4len_idx)
{
struct e1000_adv_tx_context_desc *context_desc;
u16 i = tx_ring->next_to_use;
+ struct timespec64 ts;
context_desc = IGB_TX_CTXTDESC(tx_ring, i);
@@ -5605,9 +5700,18 @@ static void igb_tx_ctxtdesc(struct igb_ring *tx_ring, u32 vlan_macip_lens,
mss_l4len_idx |= tx_ring->reg_idx << 4;
context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
- context_desc->seqnum_seed = 0;
context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd);
context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
+
+ /* We assume there is always a valid tx time available. Invalid times
+ * should have been handled by the upper layers.
+ */
+ if (tx_ring->launchtime_enable) {
+ ts = ns_to_timespec64(first->skb->tstamp);
+ context_desc->seqnum_seed = cpu_to_le32(ts.tv_nsec / 32);
+ } else {
+ context_desc->seqnum_seed = 0;
+ }
}
static int igb_tso(struct igb_ring *tx_ring,
@@ -5690,7 +5794,8 @@ static int igb_tso(struct igb_ring *tx_ring,
vlan_macip_lens |= (ip.hdr - skb->data) << E1000_ADVTXD_MACLEN_SHIFT;
vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
- igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
+ igb_tx_ctxtdesc(tx_ring, first, vlan_macip_lens,
+ type_tucmd, mss_l4len_idx);
return 1;
}
@@ -5745,7 +5850,7 @@ static void igb_tx_csum(struct igb_ring *tx_ring, struct igb_tx_buffer *first)
vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
- igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, 0);
+ igb_tx_ctxtdesc(tx_ring, first, vlan_macip_lens, type_tucmd, 0);
}
#define IGB_SET_FLAG(_input, _flag, _result) \
--
2.17.1
^ permalink raw reply related
* [PATCH v1 net-next 12/14] igb: Only call skb_tx_timestamp after descriptors are ready
From: Jesus Sanchez-Palencia @ 2018-06-27 21:59 UTC (permalink / raw)
To: netdev
Cc: tglx, jan.altenberg, vinicius.gomes, kurt.kanzenbach, henrik,
richardcochran, levi.pearson, ilias.apalodimas, ivan.khoronzhuk,
mlichvar, willemb, jhs, xiyou.wangcong, jiri,
Jesus Sanchez-Palencia
In-Reply-To: <20180627215950.6719-1-jesus.sanchez-palencia@intel.com>
Currently, skb_tx_timestamp() is being called before the DMA
descriptors are prepared in igb_xmit_frame_ring(), which happens
during either the igb_tso() or igb_tx_csum() calls.
Given that now the skb->tstamp might be used to carry the timestamp
for SO_TXTIME, we must only call skb_tx_timestamp() after the
information has been copied into the DMA tx_ring.
Signed-off-by: Jesus Sanchez-Palencia <jesus.sanchez-palencia@intel.com>
---
drivers/net/ethernet/intel/igb/igb_main.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c
index 9b9a6a6227e0..0d72f2417143 100644
--- a/drivers/net/ethernet/intel/igb/igb_main.c
+++ b/drivers/net/ethernet/intel/igb/igb_main.c
@@ -6138,8 +6138,6 @@ netdev_tx_t igb_xmit_frame_ring(struct sk_buff *skb,
}
}
- skb_tx_timestamp(skb);
-
if (skb_vlan_tag_present(skb)) {
tx_flags |= IGB_TX_FLAGS_VLAN;
tx_flags |= (skb_vlan_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
@@ -6155,6 +6153,8 @@ netdev_tx_t igb_xmit_frame_ring(struct sk_buff *skb,
else if (!tso)
igb_tx_csum(tx_ring, first);
+ skb_tx_timestamp(skb);
+
if (igb_tx_map(tx_ring, first, hdr_len))
goto cleanup_tx_tstamp;
--
2.17.1
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox